mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-10-14 19:27:09 -04:00
Integrate the stream engine with the lookahead engine and owned staging
The stream engine (mode=adaptive) stays the default; the lookahead engine from this PR is selectable as mode=lookahead and takes over when the stream engine's space plan does not fit. Once the library is loaded an engine failure fails the install instead of switching to Java downloads. nativeStream stages files in the owned .droiddeck-epic-staging namespace, and its handle carries the worker thread that release joins. Each mode logs its own summary line; docs describe all four writers.
This commit is contained in:
The412Banner committed
1 parent
f2ecda239a
commit
26dec85c1d
8 files changed
+256
-191
No files matched your search
@@ -89,9 +89,10 @@ object SessionPrefs {
|
||||
|
||||
/**
|
||||
* Hidden A/B switch for the Epic installer: "adaptive" (default: native parallel fetch written
|
||||
* straight into staged files), "batch" (the older whole-cache native fetch, then native
|
||||
* assembly; needs about twice the game's size free) or "java" (the bounded one-chunk-at-a-time
|
||||
* Java writer). No UI; set it in the `session` preferences for benchmarks.
|
||||
* straight into staged files), "lookahead" (the native ordered-lookahead engine that caches each
|
||||
* chunk and copies it into staged files), "batch" (the older whole-cache native fetch, then
|
||||
* native assembly; needs about twice the game's size free) or "java" (the bounded Java writer).
|
||||
* No UI; set it in the `session` preferences for benchmarks.
|
||||
*/
|
||||
fun storesEpicMode(context: Context): String = prefs(context).getString("storesEpicMode", "adaptive") ?: "adaptive"
|
||||
|
||||
|
||||
@@ -51,10 +51,12 @@ import java.util.zip.Inflater;
|
||||
* set plus the device's language, a delta pass that skips files already on disk with the right
|
||||
* size and SHA-1, then the writer. When the native library loads, the writer is the adaptive
|
||||
* native install ({@link EpicNative#stream}: chunks fetched in parallel and written straight into
|
||||
* staged files, shared chunks in a budgeted cache); otherwise, or when its space plan does not
|
||||
* fit, the bounded-cache Java writer, one chunk at a time. Both replace each file the same safe
|
||||
* way. A hidden preference ({@code storesEpicMode}) can pick the older whole-cache native batch
|
||||
* engine or the Java writer instead, for side-by-side benchmarks.
|
||||
* staged files, shared chunks in a budgeted cache); when its space plan does not fit, the native
|
||||
* lookahead install ({@link EpicNative#installAdaptive}), which sizes its window to the free space.
|
||||
* Builds without the library use the bounded Java writer (parallel prefetch, one staged writer).
|
||||
* Once the library is loaded, an engine failure fails the install; it never silently switches to
|
||||
* Java downloads. A hidden preference ({@code storesEpicMode}) picks a mode explicitly for
|
||||
* side-by-side benchmarks: adaptive, lookahead, batch (the older whole-cache engine) or java.
|
||||
*
|
||||
* Chunk URLs carry no auth tokens; the subfolder is the decimal group number; the Cloudflare
|
||||
* mirror is skipped (it refuses chunks). Blocking; the download queue runs it on its own thread.
|
||||
@@ -87,10 +89,11 @@ public final class EpicDownloadManager {
|
||||
static final long ADAPTIVE_OPEN_WINDOW_BYTES = 256L * 1024 * 1024;
|
||||
|
||||
static final String MODE_ADAPTIVE = "adaptive";
|
||||
static final String MODE_LOOKAHEAD = "lookahead";
|
||||
static final String MODE_BATCH = "batch";
|
||||
static final String MODE_JAVA = "java";
|
||||
|
||||
/** How {@link #install} may use the native engine; null = the bounded Java writer only. */
|
||||
/** Which writer {@link #install} asks for and how wide the native fetch may go. */
|
||||
static final class NativeOptions {
|
||||
final String mode;
|
||||
final int maxWorkers, processWorkers;
|
||||
@@ -98,6 +101,8 @@ public final class EpicDownloadManager {
|
||||
NativeOptions(String mode, int maxWorkers, int processWorkers, String tierId) {
|
||||
this.mode = mode; this.maxWorkers = maxWorkers; this.processWorkers = processWorkers; this.tierId = tierId;
|
||||
}
|
||||
/** The default: the adaptive engine at the Fast tier's window. */
|
||||
static NativeOptions defaults() { return new NativeOptions(MODE_ADAPTIVE, 32, 2, "fast"); }
|
||||
}
|
||||
|
||||
public interface Callback {
|
||||
@@ -220,7 +225,7 @@ public final class EpicDownloadManager {
|
||||
|
||||
static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath,
|
||||
List<String> installTags, AtomicBoolean cancel, Callback cb) throws InstallException {
|
||||
return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, null);
|
||||
return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, NativeOptions.defaults());
|
||||
}
|
||||
|
||||
static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath,
|
||||
@@ -275,32 +280,43 @@ public final class EpicDownloadManager {
|
||||
File output = new File(installDir, f.filename.replace("\\", "/"));
|
||||
if (output.isFile()) oldFileBytes.put(f.filename, output.length());
|
||||
}
|
||||
if (nativeOptions != null && !MODE_JAVA.equals(nativeOptions.mode)) {
|
||||
NativeAttempt attempt = MODE_BATCH.equals(nativeOptions.mode)
|
||||
? installBatch(manifest, pending, needed, pendingBytes, planned, installDir, cachePath, chunkCacheDir, nativeOptions, cancel, cb)
|
||||
: installAdaptive(manifest, pending, pendingBytes, oldFileBytes, planned, installDir, cachePath, chunkCacheDir, nativeOptions, cancel, cb);
|
||||
if (attempt.handled) return attempt.result;
|
||||
} else if (nativeOptions != null) {
|
||||
NativeOptions options = nativeOptions != null ? nativeOptions : NativeOptions.defaults();
|
||||
String mode = options.mode;
|
||||
if (!MODE_LOOKAHEAD.equals(mode) && !MODE_BATCH.equals(mode) && !MODE_JAVA.equals(mode)) mode = MODE_ADAPTIVE;
|
||||
boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded();
|
||||
if (!nativeAvailable && !MODE_JAVA.equals(mode)) {
|
||||
cb.onLog("epic: native engine unavailable; mode=java-bounded");
|
||||
mode = MODE_JAVA;
|
||||
} else if (MODE_JAVA.equals(mode)) {
|
||||
cb.onLog("epic: mode=java-bounded chosen by storesEpicMode");
|
||||
}
|
||||
if (MODE_ADAPTIVE.equals(mode)) {
|
||||
NativeAttempt attempt = installAdaptive(manifest, pending, pendingBytes, oldFileBytes, planned, installDir, cachePath, chunkCacheDir, stagingDir, options, cancel, cb);
|
||||
if (attempt.handled) return attempt.result;
|
||||
// Its space plan did not fit: the lookahead engine sizes its window to what is free.
|
||||
mode = MODE_LOOKAHEAD;
|
||||
} else if (MODE_BATCH.equals(mode)) {
|
||||
return installBatch(manifest, pending, needed, pendingBytes, planned, installDir, cachePath, chunkCacheDir, options, cancel, cb);
|
||||
}
|
||||
boolean lookahead = MODE_LOOKAHEAD.equals(mode);
|
||||
Map<String, Long> cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed);
|
||||
boolean cacheOnInstallVolume = cachePath.isEmpty();
|
||||
boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded();
|
||||
long peakGameAdditional = nativeAvailable ? 0 : cacheOnInstallVolume
|
||||
long peakGameAdditional = lookahead ? 0 : cacheOnInstallVolume
|
||||
? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true, oldFileBytes)
|
||||
: estimateStagedFilePeak(pending, oldFileBytes);
|
||||
long peakCacheAdditional = nativeAvailable || cacheOnInstallVolume
|
||||
long peakCacheAdditional = lookahead || cacheOnInstallVolume
|
||||
? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false);
|
||||
long freeGame = StoreInstallRoot.availableBytes(installDir);
|
||||
if (!nativeAvailable && cacheOnInstallVolume) {
|
||||
// The lookahead engine plans its own window against both free-space readings.
|
||||
if (!lookahead && cacheOnInstallVolume) {
|
||||
if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free");
|
||||
} else if (!nativeAvailable) {
|
||||
} else if (!lookahead) {
|
||||
long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir);
|
||||
if (StoreInstallRoot.insufficientSpace(peakCacheAdditional, freeCache)) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free");
|
||||
if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need " + fmt(peakGameAdditional) + ", only " + fmt(freeGame) + " free");
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
if (nativeAvailable) {
|
||||
if (lookahead) {
|
||||
JSONObject request = nativeRequest(manifest, pending, chunksByGuid, cachedChunkBytes,
|
||||
oldFileBytes, installDir, chunkCacheDir, stagingDir, cacheOnInstallVolume);
|
||||
request.put("free_game", StoreInstallRoot.availableBytes(installDir));
|
||||
@@ -310,8 +326,8 @@ public final class EpicDownloadManager {
|
||||
cb.onSizes(nativeTotal, planned);
|
||||
cb.onBytes(0, nativeTotal, 0);
|
||||
EpicNative.Result outcome = EpicNative.installAdaptive(request.toString(), cancel, new EpicNative.InstallListener() {
|
||||
@Override public void onPlan(int chunks, long bytes, String mode) {
|
||||
cb.onLog("epic: engine=adaptive-rust " + mode);
|
||||
@Override public void onPlan(int chunks, long bytes, String plan) {
|
||||
cb.onLog("epic: mode=lookahead engine=lookahead-rust " + plan);
|
||||
}
|
||||
@Override public void onProgress(long done, long total, int files, int filesTotal) {
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000);
|
||||
@@ -334,6 +350,9 @@ public final class EpicDownloadManager {
|
||||
});
|
||||
if (outcome.cancelled || cancel.get()) return null;
|
||||
if (!outcome.success) throw new InstallException("Native Epic install failed: " + outcome.error);
|
||||
long lookaheadMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000);
|
||||
cb.onLog("epic: summary mode=lookahead files=" + pending.size() + " bytes_written=" + nativeTotal + " elapsed_ms=" + lookaheadMs
|
||||
+ " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", nativeTotal * 1000.0 / lookaheadMs / 1048576.0));
|
||||
cb.onStage("install", nativeTotal, nativeTotal, pending.size(), pending.size());
|
||||
cb.onProgress("Complete", 100);
|
||||
return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest);
|
||||
@@ -501,17 +520,16 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* The adaptive native install. Falls back (nothing fetched or written) when the engine is not
|
||||
* there, refuses the plan, or its space plan does not fit while the bounded writer's might.
|
||||
* The adaptive native install; the library is loaded. Hands back {@code FALLBACK} (nothing
|
||||
* fetched or written) only when its space plan does not fit, so the caller can run the
|
||||
* lookahead engine, which narrows its window to the free space. Any other failure is final.
|
||||
*/
|
||||
private static NativeAttempt installAdaptive(Manifest manifest, List<FileInfo> pending, long pendingBytes, Map<String, Long> oldFileBytes,
|
||||
long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options,
|
||||
AtomicBoolean cancel, Callback cb) throws InstallException {
|
||||
long planned, File installDir, String cachePath, File chunkCacheDir, File stagingDir,
|
||||
NativeOptions options, AtomicBoolean cancel, Callback cb) throws Exception {
|
||||
int[] pendingIdx = pendingIndices(manifest, pending);
|
||||
if (manifest.rawBytes == null || pendingIdx == null) return NativeAttempt.FALLBACK;
|
||||
if (!StoresNative.INSTANCE.getAvailable()) {
|
||||
cb.onLog("epic: native engine unavailable; mode=java-bounded");
|
||||
return NativeAttempt.FALLBACK;
|
||||
if (manifest.rawBytes == null || pendingIdx == null) {
|
||||
throw new InstallException("Native Epic install failed: the manifest is not available to the native engine");
|
||||
}
|
||||
long stagedPeak = estimateStagedFilePeak(pending, oldFileBytes);
|
||||
long openAllowance = adaptiveOpenStagingAllowance(oldFileBytes);
|
||||
@@ -520,8 +538,7 @@ public final class EpicDownloadManager {
|
||||
long freeScratch = scratchRequested ? StoreInstallRoot.availableBytes(chunkCacheDir) : -1;
|
||||
AdaptiveSpacePlan space = planAdaptiveSpace(stagedPeak, openAllowance, freeGame, scratchRequested, freeScratch);
|
||||
if (space.error != null) {
|
||||
// The bounded writer needs less (one chunk at a time, sequential swaps): let it decide.
|
||||
cb.onLog("epic: adaptive space plan does not fit (" + space.error + "); mode=java-bounded");
|
||||
cb.onLog("epic: adaptive space plan does not fit (" + space.error + "); mode=lookahead");
|
||||
return NativeAttempt.FALLBACK;
|
||||
}
|
||||
String engineCache = space.cacheBesideGame ? "" : cachePath;
|
||||
@@ -540,7 +557,7 @@ public final class EpicDownloadManager {
|
||||
final Object lock = new Object();
|
||||
final long started = System.nanoTime();
|
||||
final long[] state = new long[] {0, 0, System.currentTimeMillis(), 0, 0}; // bytes, files, speed ms, speed bytes, speed bps
|
||||
EpicNative.Result res = EpicNative.stream(manifest.rawBytes, installDir.getPath(), engineCache, cdnPrefixes(manifest.cdnUrls), pendingIdx, pendingBytes,
|
||||
EpicNative.Result res = EpicNative.stream(manifest.rawBytes, installDir.getPath(), engineCache, stagingDir.getCanonicalPath(), cdnPrefixes(manifest.cdnUrls), pendingIdx, pendingBytes,
|
||||
space.cacheBudget, "", options.maxWorkers, options.processWorkers, cancel, new EpicNative.StreamListener() {
|
||||
@Override public void onStreamProgress(long bytesWritten, long bytesTotal, int filesDone, int filesTotal, long filesBytes) {
|
||||
synchronized (lock) {
|
||||
@@ -574,13 +591,10 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
@Override public void onComplete(boolean success, String error, long bytes) {}
|
||||
});
|
||||
if (!res.started) {
|
||||
cb.onLog("epic: adaptive engine not started (" + res.error + "); mode=java-bounded");
|
||||
return NativeAttempt.FALLBACK;
|
||||
}
|
||||
if (!res.started) throw new InstallException("Native Epic install failed: " + res.error);
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000);
|
||||
if (res.cancelled) return NativeAttempt.done(null);
|
||||
if (!res.success) throw new InstallException(res.error == null || res.error.isEmpty() ? "Epic download failed" : res.error);
|
||||
if (!res.success) throw new InstallException("Native Epic install failed: " + res.error);
|
||||
deleteDir(engineCacheDir);
|
||||
deleteDir(chunkCacheDir);
|
||||
cb.onLog("epic: summary mode=adaptive files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs
|
||||
@@ -596,14 +610,12 @@ public final class EpicDownloadManager {
|
||||
* once (about twice the game), and it writes files in place rather than staged, so an
|
||||
* existing install is marked updating before anything is fetched.
|
||||
*/
|
||||
private static NativeAttempt installBatch(Manifest manifest, List<FileInfo> pending, List<ChunkInfo> needed, long pendingBytes,
|
||||
long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options,
|
||||
AtomicBoolean cancel, Callback cb) throws InstallException {
|
||||
private static Result installBatch(Manifest manifest, List<FileInfo> pending, List<ChunkInfo> needed, long pendingBytes,
|
||||
long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options,
|
||||
AtomicBoolean cancel, Callback cb) throws InstallException {
|
||||
int[] pendingIdx = pendingIndices(manifest, pending);
|
||||
if (manifest.rawBytes == null || pendingIdx == null) return NativeAttempt.FALLBACK;
|
||||
if (!StoresNative.INSTANCE.getAvailable()) {
|
||||
cb.onLog("epic: native engine unavailable; mode=java-bounded");
|
||||
return NativeAttempt.FALLBACK;
|
||||
if (manifest.rawBytes == null || pendingIdx == null) {
|
||||
throw new InstallException("Native Epic install failed: the manifest is not available to the native engine");
|
||||
}
|
||||
long cacheMissing = 0, totalBytes = 0;
|
||||
for (ChunkInfo c : needed) {
|
||||
@@ -619,7 +631,7 @@ public final class EpicDownloadManager {
|
||||
if (StoreInstallRoot.insufficientSpace(cacheMissing, freeCache)) throw new InstallException("Not enough internal space for the batch download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free (short by " + fmt(cacheMissing - freeCache) + ")");
|
||||
if (StoreInstallRoot.insufficientSpace(pendingBytes, freeGame)) throw new InstallException("Not enough free space on the game's storage: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free (short by " + fmt(pendingBytes - freeGame) + ")");
|
||||
}
|
||||
if (cancel.get()) return NativeAttempt.done(null);
|
||||
if (cancel.get()) return null;
|
||||
final long fTotal = totalBytes;
|
||||
final int totalChunks = needed.size();
|
||||
final int totalFiles = pending.size();
|
||||
@@ -638,11 +650,8 @@ public final class EpicDownloadManager {
|
||||
@Override public void onLog(String line) { cb.onLog(line); }
|
||||
@Override public void onComplete(boolean success, String error, long bytes) {}
|
||||
});
|
||||
if (!fetch.started) {
|
||||
cb.onLog("epic: batch engine not started (" + fetch.error + "); mode=java-bounded");
|
||||
return NativeAttempt.FALLBACK;
|
||||
}
|
||||
if (fetch.cancelled) return NativeAttempt.done(null);
|
||||
if (!fetch.started) throw new InstallException("Native Epic install failed: " + fetch.error);
|
||||
if (fetch.cancelled) return null;
|
||||
if (!fetch.success) throw new InstallException("Epic batch download failed: " + fetch.error);
|
||||
// Assembly overwrites files in place: the old installed record stops being truthful now.
|
||||
try { StoreInstalls.markUpdating(installDir); } catch (Exception e) { throw new InstallException("Could not mark the install as updating: " + e.getMessage()); }
|
||||
@@ -655,7 +664,7 @@ public final class EpicDownloadManager {
|
||||
@Override public void onLog(String line) { cb.onLog(line); }
|
||||
@Override public void onComplete(boolean success, String error, long bytes) {}
|
||||
});
|
||||
if (asm.cancelled) return NativeAttempt.done(null);
|
||||
if (asm.cancelled) return null;
|
||||
if (!asm.started || !asm.success) throw new InstallException(asm.error == null || asm.error.isEmpty() ? "Epic batch assembly failed" : asm.error);
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000);
|
||||
cb.onLog("epic: summary mode=batch files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs
|
||||
@@ -663,7 +672,7 @@ public final class EpicDownloadManager {
|
||||
deleteDir(chunkCacheDir);
|
||||
cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles);
|
||||
cb.onProgress("Complete", 100);
|
||||
return NativeAttempt.done(new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest));
|
||||
return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest);
|
||||
}
|
||||
|
||||
private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) {
|
||||
|
||||
@@ -9,20 +9,23 @@ import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* The native Epic engines in `libdroiddeckstores.so`. [stream] is the install the manager uses
|
||||
* whenever the library loads: chunks fetched in parallel by the adaptive fetch core and written
|
||||
* straight into each file's staging file as they arrive, shared chunks kept in a cache of at most
|
||||
* the given budget, every file verified and swapped in with the same rules as the bounded Java
|
||||
* writer (see `stream.rs`). [run] and [assemble] are the older batch pair: [run] fills the chunk
|
||||
* cache with verified, decompressed chunks for the pending files, [assemble] writes the files from
|
||||
* it. They stay for callers that explicitly want a whole-game cache.
|
||||
* The native Epic engines in `libdroiddeckstores.so`:
|
||||
*
|
||||
* - [stream] (mode `adaptive`, the default): chunks fetched in parallel by the adaptive fetch core
|
||||
* and written straight into each file's staging file as they arrive, shared chunks kept in a
|
||||
* cache of at most the given budget, every file verified and swapped in (`stream.rs`).
|
||||
* - [installAdaptive] (mode `lookahead`): a JSON snapshot of the pending files; Rust chooses a
|
||||
* storage-safe ordered lookahead, reserves and caches each chunk, then copies it into staged
|
||||
* files (`adaptive.rs`). The manager also uses it when the stream engine's space plan does not
|
||||
* fit, since it narrows its window to the free space.
|
||||
* - [run] + [assemble] (mode `batch`): every chunk into the cache, then the files from it.
|
||||
*
|
||||
* The cache is `<installDir>/.chunks` when `chunkCacheDir` is `""`, else the scratch folder given -
|
||||
* the app's own cache when the game installs to a card.
|
||||
*
|
||||
* All block while polling [AtomicBoolean] `cancel` every 250 ms. [run] and [assemble] flip the
|
||||
* native flag on cancel and wait up to 5 s; [stream] waits for the run to finish, because its
|
||||
* cleanup (removing unfinished staging files) must be done before the manager returns.
|
||||
* All block while polling [AtomicBoolean] `cancel` every 250 ms. [stream] and [installAdaptive]
|
||||
* wait for the native run to finish after a cancel (their staging cleanup must be done before the
|
||||
* manager returns); [run] and [assemble] wait up to 5 s.
|
||||
*/
|
||||
object EpicNative {
|
||||
private const val TAG = "EpicNative"
|
||||
@@ -87,12 +90,12 @@ object EpicNative {
|
||||
|
||||
/**
|
||||
* The adaptive install of the files [pendingFileIdx] (indices into the manifest's file list),
|
||||
* with at most [cacheBudget] bytes of shared chunks cached. `started == false` means nothing
|
||||
* was fetched or written and the caller runs the bounded Java writer.
|
||||
* with at most [cacheBudget] bytes of shared chunks cached, staged in [stagingDir] (the
|
||||
* manager's owned staging directory). `started == false` means nothing was fetched or written.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun stream(
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedBytes: Long,
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedBytes: Long,
|
||||
cacheBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, cancel: AtomicBoolean?, listener: StreamListener,
|
||||
): Result {
|
||||
if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0)
|
||||
@@ -114,10 +117,10 @@ object EpicNative {
|
||||
}
|
||||
}
|
||||
val handle: Long = try {
|
||||
nativeStream(manifest, installDir, chunkCacheDir, cdnPrefixes, pendingFileIdx, expectedBytes, cacheBudget, caBundlePath, maxWorkers, processWorkers, inner)
|
||||
nativeStream(manifest, installDir, chunkCacheDir, stagingDir, cdnPrefixes, pendingFileIdx, expectedBytes, cacheBudget, caBundlePath, maxWorkers, processWorkers, inner)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}")
|
||||
return Result(false, false, false, "nativeStream: ${t.javaClass.simpleName}", 0L, 0, 0)
|
||||
return Result(false, false, false, "nativeStream: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0)
|
||||
}
|
||||
if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get())
|
||||
var cancelSent = false
|
||||
@@ -129,6 +132,7 @@ object EpicNative {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (completion.get() == null) nativeCancel(handle)
|
||||
nativeRelease(handle)
|
||||
}
|
||||
val c = completion.get()
|
||||
@@ -200,7 +204,7 @@ object EpicNative {
|
||||
|
||||
@JvmStatic
|
||||
private external fun nativeStream(
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedBytes: Long,
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedBytes: Long,
|
||||
cacheBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: StreamListener,
|
||||
): Long
|
||||
|
||||
|
||||
@@ -112,11 +112,14 @@ On-disk protocol (what a resumed run, by either engine, relies on): a file is as
|
||||
|
||||
## `com.droiddeck.launcher.stores.epic.EpicNative`
|
||||
|
||||
The manager's default Epic install is the **stream** run (`nativeStream`, below): the chunks
|
||||
are fetched in parallel and written straight into staged files as they arrive. The lookahead
|
||||
install and the batch pair stay selectable (the hidden `storesEpicMode` preference).
|
||||
The manager's default Epic install is the **stream** run (`nativeStream`, mode `adaptive`,
|
||||
below): the chunks are fetched in parallel and written straight into staged files as they arrive;
|
||||
on device it measured about 83 MB/s on a 71 GB install. The **lookahead** install
|
||||
(`nativeInstallAdaptive`, mode `lookahead`) is its fallback when the stream engine's space plan
|
||||
does not fit, and is selectable directly, as is the legacy batch pair (`nativeStart` +
|
||||
`nativeAssemble`, mode `batch`), through the hidden `storesEpicMode` preference.
|
||||
|
||||
### Production adaptive install (snapshot schema v1)
|
||||
### Lookahead install (snapshot schema v1)
|
||||
|
||||
`nativeInstallAdaptive(request: String, listener: EpicNative.Listener): Long`
|
||||
exports `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive`.
|
||||
@@ -142,9 +145,10 @@ return zero after `onComplete(false, error, 0)` and are **terminal**, not fallba
|
||||
and verified swapped file counts, not fetched wire bytes. Fetch and staged assembly overlap
|
||||
in one native run, using the same pure reservation/release policy as the peak planner.
|
||||
Cancellation/release joins native workers; no worker survives the terminal callback/handle
|
||||
release. Only `StoresNative.ensureLoaded() == false` selects bounded parallel Java.
|
||||
release. Only `StoresNative.ensureLoaded() == false` selects bounded parallel Java; the same rule
|
||||
holds for `nativeStream`.
|
||||
|
||||
### Legacy batch APIs (not the production installer)
|
||||
### Legacy batch APIs (mode `batch`, for A/B runs)
|
||||
|
||||
Two explicit calls provide the legacy batch interface:
|
||||
|
||||
@@ -262,24 +266,26 @@ Bannerlator's `BlEpicDownload.run(...)` wraps `nativeStart` in a blocking call t
|
||||
|
||||
| Method | JNI signature | Symbol |
|
||||
|---|---|---|
|
||||
| `nativeStream(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;[Ljava/lang/String;[IJJLjava/lang/String;IILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream` |
|
||||
| `nativeStream(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;[IJJLjava/lang/String;IILcom/droiddeck/launcher/stores/epic/EpicNative$StreamListener;)J` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream` |
|
||||
|
||||
```kotlin
|
||||
@JvmStatic private external fun nativeStream(
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array<String>,
|
||||
pendingFileIdx: IntArray, expectedBytes: Long, cacheBudget: Long, caBundlePath: String,
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String,
|
||||
cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedBytes: Long, cacheBudget: Long, caBundlePath: String,
|
||||
maxWorkers: Int, processWorkers: Int, listener: StreamListener,
|
||||
): Long
|
||||
```
|
||||
|
||||
Parameters as `nativeStart`, except `expectedBytes` (the manager's `Σ fileSize` of the pending
|
||||
Parameters as `nativeStart`, plus `stagingDir` (the manager's owned
|
||||
`<installDir>/.droiddeck-epic-staging`, which must exist; `""` = stage beside each file), and
|
||||
except `expectedBytes` (the manager's `Σ fileSize` of the pending
|
||||
files, `-1` = no cross-check; no chunk count) and `cacheBudget` (the most bytes of shared chunks
|
||||
the cache may hold; the manager sizes it from free space, `0` = never cache, refetch instead).
|
||||
`chunkCacheDir` is `""` (`<installDir>/.chunks`) or a scratch directory; the manager falls back to
|
||||
`""` for a card install when internal storage cannot hold the budget.
|
||||
|
||||
What the run does (`epic/stream.rs`): a chunk one file part uses is decompressed, verified and
|
||||
written at the part's offset into that file's unique sibling `.droiddeck-epic-*.stage` file; a
|
||||
written at the part's offset into a unique `stream-*.stage` file in `stagingDir`; a
|
||||
chunk several parts use is also kept in the cache, decompressed, as `<GUID>` (a later copy after
|
||||
an eviction as `<GUID>.<n>`) and deleted after its last covered part. The plan evicts by furthest
|
||||
next use when the budget would be exceeded and fetches the evicted chunk again later. Each file,
|
||||
@@ -289,8 +295,8 @@ renamed over its destination. Cancel or failure removes every unfinished stage f
|
||||
|
||||
Planning runs on the calling thread and also prepares the cache (keeps verified chunks this
|
||||
install needs, deletes everything else, as the Java writer does). A refusal returns `0` after a
|
||||
synchronous `onComplete(false, reason, 0)`; nothing was fetched or written, and the manager runs
|
||||
its bounded Java writer.
|
||||
synchronous `onComplete(false, reason, 0)`; nothing was fetched or written, and the manager fails
|
||||
the install with that reason (it does not switch engines once the library is loaded).
|
||||
|
||||
| `StreamListener` method | JNI signature | When |
|
||||
|---|---|---|
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
//! Epic Games Store download adapter for the shared fetch core (`crate::fetch_core`).
|
||||
//!
|
||||
//! Production installs use [`adaptive`]: the Java manifest/delta pass supplies a versioned
|
||||
//! snapshot, the shared pure storage schedule selects an ordered lookahead, and fetch_core
|
||||
//! runs concurrently with verified staged assembly. Old outputs survive until atomic swap;
|
||||
//! a synchronous JNI callback marks the sidecar updating before the first replacement.
|
||||
//! Legacy manifest/batch APIs remain available. A missing library uses the bounded Java
|
||||
//! installer; native plan/runtime failures never silently switch engines.
|
||||
//! Production installs use [`stream`] (mode `adaptive`): the manager hands over the manifest
|
||||
//! bytes and the pending files, fetch_core downloads the chunks in first-use order, and each chunk
|
||||
//! is written straight into the staged files that use it (shared ones kept in a budgeted cache).
|
||||
//! [`adaptive`] (mode `lookahead`) is the fallback when the stream engine's space plan does not
|
||||
//! fit: the Java manifest/delta pass supplies a versioned snapshot, the shared pure storage
|
||||
//! schedule selects an ordered lookahead, and fetch_core runs concurrently with verified staged
|
||||
//! assembly. Both keep old outputs until their atomic swap; a synchronous JNI callback marks the
|
||||
//! sidecar updating before the first replacement. Legacy batch APIs remain available. A missing
|
||||
//! library uses the bounded Java installer; native plan/runtime failures never silently switch
|
||||
//! engines.
|
||||
//!
|
||||
//! Submodules:
|
||||
//! - [`adaptive`] — budget planner and bounded overlapping fetch/assembly coordinator.
|
||||
|
||||
@@ -7,7 +7,8 @@
|
||||
//! listener: EpicNative.Listener) -> long handle (0 = not started)
|
||||
//! nativeAssemble(manifest: byte[], installDir: String, chunkCacheDir: String, fileIdx: int[],
|
||||
//! listener: EpicNative.Listener) -> long handle (0 = not started)
|
||||
//! nativeStream(manifest: byte[], installDir: String, chunkCacheDir: String, cdnPrefixes: String[],
|
||||
//! nativeStream(manifest: byte[], installDir: String, chunkCacheDir: String, stagingDir: String,
|
||||
//! cdnPrefixes: String[],
|
||||
//! pendingFileIdx: int[], expectedBytes: long, cacheBudget: long, caBundlePath: String,
|
||||
//! maxWorkers: int, processWorkers: int, listener: EpicNative.StreamListener)
|
||||
//! -> long handle (0 = not started)
|
||||
@@ -515,6 +516,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
manifest: JByteArray,
|
||||
install_dir: JString,
|
||||
chunk_cache_dir: JString,
|
||||
staging_dir: JString,
|
||||
cdn_prefixes: JObjectArray,
|
||||
pending_file_idx: JIntArray,
|
||||
expected_bytes: jlong,
|
||||
@@ -535,6 +537,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
};
|
||||
let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default();
|
||||
let chunk_cache_dir = jstring_to_string(&mut env, &chunk_cache_dir).unwrap_or_default();
|
||||
let staging_dir = jstring_to_string(&mut env, &staging_dir).unwrap_or_default();
|
||||
let cdn_prefixes = string_array_to_vec(&mut env, &cdn_prefixes);
|
||||
let pending_file_indices: Vec<usize> = int_array_to_vec(&env, &pending_file_idx)
|
||||
.into_iter()
|
||||
@@ -567,6 +570,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
manifest_bytes,
|
||||
install_dir,
|
||||
chunk_cache_dir,
|
||||
staging_dir,
|
||||
cdn_prefixes,
|
||||
pending_file_indices,
|
||||
expected_bytes: if expected_bytes >= 0 {
|
||||
@@ -601,9 +605,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let thread_cancel = Arc::clone(&cancel);
|
||||
let handle = Box::new(EpicDownloadHandle { cancel });
|
||||
|
||||
thread::spawn(move || {
|
||||
let worker = thread::spawn(move || {
|
||||
crate::priority::background();
|
||||
let progress = |p: &StreamProgress| {
|
||||
let Ok(mut env) = vm.attach_current_thread_as_daemon() else {
|
||||
@@ -641,6 +643,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
outcome.bytes_written,
|
||||
);
|
||||
});
|
||||
let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) });
|
||||
|
||||
Box::into_raw(handle) as jlong
|
||||
}
|
||||
|
||||
@@ -36,8 +36,10 @@
|
||||
//! new request goes out. Memory is that budget plus one decompressed chunk per pool thread.
|
||||
//!
|
||||
//! **Each file is replaced safely**, with the bounded Java writer's rules: written into a unique
|
||||
//! sibling `.droiddeck-epic-*.stage` file (never a manifest path, so a game file legitimately
|
||||
//! named `x.part` is safe), its SHA-1 computed as the bytes go out (parts written out of order are
|
||||
//! `.stage` file in the manager's owned staging directory (`<installDir>/.droiddeck-epic-staging`,
|
||||
//! whose leftovers the manager recovers before the next run; a sibling `.droiddeck-epic-*.stage`
|
||||
//! when no directory is given), never a manifest path, so a game file legitimately named `x.part`
|
||||
//! is safe. Its SHA-1 computed as the bytes go out (parts written out of order are
|
||||
//! read back once the bytes before them are in, while they are still in the page cache), then
|
||||
//! synced, its size and hash checked, the install marked `updating` once (the Java callback,
|
||||
//! before the first replacement), and the stage renamed over the destination. On cancel or failure
|
||||
@@ -211,6 +213,8 @@ pub struct StreamRequest {
|
||||
pub install_dir: String,
|
||||
/// `""` = `<installDir>/.chunks`; a path = scratch cache (internal storage for a card install).
|
||||
pub chunk_cache_dir: String,
|
||||
/// The manager's owned staging directory on the install volume; `""` = stage beside each file.
|
||||
pub staging_dir: String,
|
||||
pub cdn_prefixes: Vec<String>,
|
||||
/// Indices into `manifest.files` of the files still to write (Java's delta pass), in order.
|
||||
pub pending_file_indices: Vec<usize>,
|
||||
@@ -264,6 +268,8 @@ pub struct StreamPlan {
|
||||
pub event_first_copy: Vec<bool>,
|
||||
/// Every manifest file's destination: a staging name may never be one of them.
|
||||
pub reserved: HashSet<PathBuf>,
|
||||
/// Where staging files go; `None` = beside each destination.
|
||||
pub staging_dir: Option<PathBuf>,
|
||||
pub bytes_total: u64,
|
||||
pub budget: u64,
|
||||
pub hosts: Vec<String>,
|
||||
@@ -364,6 +370,15 @@ pub fn build_stream_plan(req: &StreamRequest) -> Result<StreamPlan, String> {
|
||||
}
|
||||
|
||||
let install_dir = PathBuf::from(&req.install_dir);
|
||||
let staging_dir = if req.staging_dir.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let dir = PathBuf::from(&req.staging_dir);
|
||||
if !dir.is_dir() {
|
||||
return Err(format!("stream: staging dir {} missing", dir.display()));
|
||||
}
|
||||
Some(dir)
|
||||
};
|
||||
let cache_dir = resolve_cache_dir(&req.install_dir, &req.chunk_cache_dir);
|
||||
fs::create_dir_all(&cache_dir)
|
||||
.map_err(|e| format!("stream: mkdirs {}: {e}", cache_dir.display()))?;
|
||||
@@ -440,6 +455,7 @@ pub fn build_stream_plan(req: &StreamRequest) -> Result<StreamPlan, String> {
|
||||
event_first_file,
|
||||
event_first_copy,
|
||||
reserved,
|
||||
staging_dir,
|
||||
bytes_total,
|
||||
budget: req.cache_budget,
|
||||
hosts,
|
||||
@@ -848,8 +864,8 @@ impl<'a> Engine<'a> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A unique sibling `.droiddeck-epic-*.stage` file, never a manifest path
|
||||
/// (`createStagingFile`).
|
||||
/// A unique `.stage` file in the owned staging directory (or beside the destination), never a
|
||||
/// manifest path (`createStagingFile`). The destination's directory is created either way.
|
||||
fn open_staging(&self, f: usize, w: &mut FileWriter) -> Result<(), SinkError> {
|
||||
let file = &self.plan.files[f];
|
||||
let dest = self.plan.install_dir.join(&file.rel_path);
|
||||
@@ -858,14 +874,18 @@ impl<'a> Engine<'a> {
|
||||
.map(Path::to_path_buf)
|
||||
.unwrap_or_else(|| self.plan.install_dir.clone());
|
||||
fs::create_dir_all(&parent).map_err(|err| fatal_write(&file.rel_path, err))?;
|
||||
let (stage_dir, prefix) = match &self.plan.staging_dir {
|
||||
Some(dir) => (dir.clone(), "stream-"),
|
||||
None => (parent.clone(), ".droiddeck-epic-"),
|
||||
};
|
||||
for _ in 0..64 {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.subsec_nanos())
|
||||
.unwrap_or(0);
|
||||
let seq = STAGE_SEQ.fetch_add(1, Ordering::Relaxed);
|
||||
let path = parent.join(format!(
|
||||
".droiddeck-epic-{}{seq}{nanos}.stage",
|
||||
let path = stage_dir.join(format!(
|
||||
"{prefix}{}-{seq}-{nanos}.stage",
|
||||
std::process::id()
|
||||
));
|
||||
if self.plan.reserved.contains(&path) {
|
||||
@@ -1330,6 +1350,7 @@ mod tests {
|
||||
manifest_bytes: self.manifest(),
|
||||
install_dir: self.install.to_str().unwrap().to_string(),
|
||||
chunk_cache_dir: self.dir.join("cache").to_str().unwrap().to_string(),
|
||||
staging_dir: String::new(),
|
||||
cdn_prefixes: vec!["http://127.0.0.1:9/Builds/x".to_string()],
|
||||
pending_file_indices: pending,
|
||||
expected_bytes: None,
|
||||
@@ -1608,6 +1629,33 @@ mod tests {
|
||||
assert!(fx.stages().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_owned_staging_dir_holds_the_stages_and_is_left_empty() {
|
||||
let fx = Fixture::new("stream-staging");
|
||||
let staging = fx.install.join(".droiddeck-epic-staging");
|
||||
fs::create_dir_all(&staging).unwrap();
|
||||
let mut req = fx.request(vec![0, 1, 2], 1 << 20);
|
||||
req.staging_dir = staging.to_str().unwrap().to_string();
|
||||
let plan = build_stream_plan(&req).unwrap();
|
||||
let cancel = AtomicBool::new(false);
|
||||
let rec = Recorder::new();
|
||||
with_engine(&plan, &cancel, &rec, |engine| {
|
||||
engine.start().unwrap();
|
||||
engine.process(&item(0), fx.body_for(&plan, 0)).unwrap();
|
||||
let stages: Vec<_> = fs::read_dir(&staging).unwrap().flatten().collect();
|
||||
assert_eq!(stages.len(), 1, "a.bin is staged in the owned directory");
|
||||
assert!(stages[0].file_name().to_string_lossy().ends_with(".stage"));
|
||||
for id in 1..plan.fetch_events.len() {
|
||||
engine.process(&item(id), fx.body_for(&plan, id)).unwrap();
|
||||
}
|
||||
});
|
||||
assert_eq!(fs::read_dir(&staging).unwrap().count(), 0);
|
||||
assert_eq!(fs::read(fx.install.join("dir/b.bin")).unwrap(), fx.expected(1));
|
||||
let mut missing = fx.request(vec![0], 1 << 20);
|
||||
missing.staging_dir = fx.dir.join("nope").to_str().unwrap().to_string();
|
||||
assert!(build_stream_plan(&missing).unwrap_err().contains("staging dir"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_manifest_file_named_like_a_stage_is_never_used_for_staging() {
|
||||
let fx = Fixture::new("stream-reserved");
|
||||
|
||||
+82
-92
@@ -45,45 +45,57 @@ with a placeholder job. Failures are kept in the Downloads UI, notification and
|
||||
duplicated as transient failure toasts. Active queue persistence and ordinary backend pause/retry
|
||||
are unchanged.
|
||||
|
||||
Epic installs have three writers, all with the same per-file safety rules (below):
|
||||
Epic installs have four writers, all with the same per-file safety rules (below). The hidden
|
||||
`storesEpicMode` string in the `session` preferences picks one: `adaptive` (default), `lookahead`,
|
||||
`batch` or `java`. There is no UI for it; set it on a test device to compare the modes on the same
|
||||
game. Every mode logs an `epic: summary mode=<mode>` line (files, bytes written, elapsed, average
|
||||
MB/s). When `libdroiddeckstores.so` does not load, every mode runs the Java writer. Once it is
|
||||
loaded, an engine failure (refused plan, ABI mismatch, network, hash or I/O failure, unsupported
|
||||
atomic swap) fails the install; it never silently reruns as Java downloads.
|
||||
|
||||
- **Adaptive (default when `libdroiddeckstores.so` loads)** - `EpicNative.stream`,
|
||||
`epic/stream.rs`. The chunks go through the shared fetch core in the order their files need
|
||||
them, with its full adaptivity (window ramp, per-host caps, in-flight byte budget, retries,
|
||||
host rotation, stall handoff). A chunk only one file part uses is verified and written straight
|
||||
into that file's staging file at the part's offset, in whatever order it arrives; no cache. A
|
||||
chunk several parts use is written into every part whose file the download has reached and kept,
|
||||
decompressed, in the cache until its last part is written. The cache has a byte budget; the plan
|
||||
simulates the install in file order and, when the budget would be exceeded, drops the cached
|
||||
chunk needed furthest ahead and downloads it again later. Files are opened only once the
|
||||
download reaches them, so the set of half-written files stays within the fetch window. All writes
|
||||
happen in the fetch core's process pool, whose completion frees the in-flight byte budget: a slow
|
||||
card throttles the download instead of growing memory. Each file's SHA-1 is computed as its
|
||||
parts are written (parts that arrived out of order are read back once the bytes before them are
|
||||
in). The run logs `stream-plan` (direct vs shared chunks, planned refetches, budget) and
|
||||
`stream-summary mode=adaptive` (peak cache, peak staging, refetches, bytes written from the
|
||||
download vs from the cache, average MB/s).
|
||||
- **Bounded Java** - when the library does not load or link, when the engine refuses the plan,
|
||||
or when the adaptive space plan does not fit while this one might. It fetches one chunk at a
|
||||
time, verifies it, and writes the required parts into the current game file. Chunks are deleted
|
||||
after their last use; reusable chunks are retained up to 64 MiB, with at most one additional
|
||||
incoming chunk before eviction. Evicted shared chunks may be downloaded again. It logs
|
||||
`epic: summary mode=java-bounded`.
|
||||
- **Batch** - the earlier native design, kept for A/B benchmarks: every chunk fetched into the
|
||||
cache in parallel, then the files assembled from it (`EpicNative.run` + `EpicNative.assemble`).
|
||||
It needs the cache and the files at once (about twice the game) and checks exactly that before
|
||||
starting; it writes files in place, so an existing install is marked `updating` before assembly.
|
||||
It logs `epic: summary mode=batch`.
|
||||
Epic installs use **native adaptive fetch/assembly** when `libdroiddeckstores.so` loads.
|
||||
The Java delta pass sends a versioned snapshot of pending files, chunk metadata, verified cache
|
||||
lengths, old output lengths and both measured free-space probes to Rust. The native planner and
|
||||
execution share one ordered moving-window policy. Before any chunk request, Rust chooses the
|
||||
largest safe lookahead (128, 64, 32, 16, 8, 4, 2 or 1 part references) and reusable-cache budget
|
||||
(64 MiB, 16 MiB or zero) whose simulated staged-file and cache peaks fit.
|
||||
Native device IDs also detect an explicitly named cache on the game volume, so two separate
|
||||
paths cannot incorrectly turn one filesystem's requirement into two independent checks.
|
||||
**Adaptive (default)** - `EpicNative.stream`, `epic/stream.rs`. Measured on device: about
|
||||
83 MB/s on a 71 GB install. The chunks go through the shared fetch core in the order their files
|
||||
need them, with its full adaptivity (window ramp, per-host caps, in-flight byte budget, retries,
|
||||
host rotation, stall handoff). A chunk only one file part uses is verified and written straight
|
||||
into that file's staging file at the part's offset, in whatever order it arrives; it is never
|
||||
cached, so each byte is written once. A chunk several parts use is written into every part whose
|
||||
file the download has reached and kept, decompressed, in the cache until its last part is written.
|
||||
The cache has a byte budget; the plan simulates the install in file order and, when the budget
|
||||
would be exceeded, drops the cached chunk needed furthest ahead and downloads it again later.
|
||||
Files are opened only once the download reaches them, so the set of half-written files stays
|
||||
within the fetch window. All writes happen in the fetch core's process pool, whose completion
|
||||
frees the in-flight byte budget: a slow card throttles the download instead of growing memory.
|
||||
Each file's SHA-1 is computed as its parts are written (parts that arrived out of order are read
|
||||
back once the bytes before them are in). Stage files go into the owned staging namespace below.
|
||||
The run logs `stream-plan` (direct vs shared chunks, planned refetches, budget) and
|
||||
`stream-summary mode=adaptive` (peak cache, peak staging, refetches, bytes written from the
|
||||
download vs from the cache).
|
||||
|
||||
| Storage situation | Native policy |
|
||||
Its space plan (`EpicDownloadManager.planAdaptiveSpace`, unit-tested in `EpicAdaptivePlanTest`)
|
||||
checks each volume for what lands on it. The game volume: the sequential staged-file peak
|
||||
(remaining file bytes, each replaced file's old copy credited only after its swap), plus, for an
|
||||
update, the largest old file plus 256 MiB for files finishing out of order within the fetch
|
||||
window, plus the cache when it is beside the game. The cache volume: the budget plus 64 MiB of
|
||||
chunks in transit. The budget is half the cache volume's free space after those needs, at most
|
||||
1 GiB (256 MiB when free space is unknown; 0 is allowed and means every reuse is a refetch). On a
|
||||
card install the cache stays on internal scratch unless internal storage cannot hold at least
|
||||
64 MiB of it; then the cache goes beside the game (still released chunk by chunk) instead of
|
||||
failing. A plan that does not fit logs the short storage and the shortfall and hands over to the
|
||||
lookahead engine, which narrows its window to the space there is (or fails its own preflight with
|
||||
an explicit error). Running out of space in the middle of an adaptive run is not destructive: a
|
||||
failed write fails the run, unfinished staging files are removed, the old files stay.
|
||||
|
||||
**Lookahead** - `EpicNative.installAdaptive`, `epic/adaptive.rs`; also the adaptive engine's
|
||||
fallback when its space plan does not fit. The Java delta pass sends a versioned snapshot of
|
||||
pending files, chunk metadata, verified cache lengths, old output lengths and both measured
|
||||
free-space probes to Rust. The native planner and execution share one ordered moving-window
|
||||
policy. Before any chunk request, Rust chooses the largest safe lookahead (128, 64, 32, 16, 8, 4,
|
||||
2 or 1 part references) and reusable-cache budget (64 MiB, 16 MiB or zero) whose simulated
|
||||
staged-file and cache peaks fit. Native device IDs also detect an explicitly named cache on the
|
||||
game volume, so two separate paths cannot incorrectly turn one filesystem's requirement into two
|
||||
independent checks.
|
||||
|
||||
| Storage situation | Lookahead policy |
|
||||
| --- | --- |
|
||||
| Plenty of room | Up to 128 references ahead, up to 32 network requests, 64 MiB reusable data |
|
||||
| Tight | Smaller lookahead and/or reusable budget; parallel fetching remains where space permits |
|
||||
@@ -91,68 +103,46 @@ paths cannot incorrectly turn one filesystem's requirement into two independent
|
||||
| Does not fit even the minimal schedule | Explicit preflight error before requests or replacement |
|
||||
| Free space unknown | Conservative bounded lookahead (at most eight), not an invented zero |
|
||||
|
||||
The native fetch coordinator reuses `fetch_core`'s host scheduling, retries, adaptive network
|
||||
window and decode pool. The staged writer runs concurrently, waiting only for its next required
|
||||
chunk, and reads parts through the native assembler's streaming copy primitive. GUIDs in the
|
||||
lookahead are reserved once before submission, at their full decoded size. Chunks are released
|
||||
at last use on **both** cache layouts; distant reusable data is oldest-first bounded and may
|
||||
be refetched. At most the selected reusable budget plus distinct lookahead decoded sizes is
|
||||
resident/reserved. New request admission re-probes both volumes and can shrink unsent lookahead
|
||||
or retention if headroom disappears; already in-flight reservations remain protected.
|
||||
If even existing reservations cannot fit, the run fails and joins its workers rather than
|
||||
waiting indefinitely or deleting old game files. An install to SD retains its internal scratch
|
||||
cache, so assembled output is the only game data written to the card.
|
||||
The lookahead fetch coordinator reuses `fetch_core`'s host scheduling, retries, adaptive network
|
||||
window and decode pool. Each chunk is cached, then its parts are copied into staged files by a
|
||||
writer that runs concurrently, waiting only for its next required chunk, through the native
|
||||
assembler's streaming copy primitive. GUIDs in the lookahead are reserved once before submission,
|
||||
at their full decoded size. Chunks are released at last use on **both** cache layouts; distant
|
||||
reusable data is oldest-first bounded and may be refetched. At most the selected reusable budget
|
||||
plus distinct lookahead decoded sizes is resident/reserved. New request admission re-probes both
|
||||
volumes and can shrink unsent lookahead or retention if headroom disappears; already in-flight
|
||||
reservations remain protected. If even existing reservations cannot fit, the run fails and joins
|
||||
its workers rather than waiting indefinitely or deleting old game files. An install to SD retains
|
||||
its internal scratch cache, so assembled output is the only game data written to the card.
|
||||
|
||||
Only failure to **load** the native library selects the Java fallback. An ABI mismatch, native
|
||||
planning/network/hash/I/O failure or unsupported atomic swap does not silently rerun Java.
|
||||
Eight fallback Java workers prefetch the next eight chunk-part references, across file boundaries,
|
||||
while one writer assembles and verifies files in manifest order. Shared GUIDs within that window
|
||||
are fetched only once. Chunks are deleted after their last use; other reusable chunks are retained
|
||||
up to 64 MiB. At most eight additional decoded chunks (complete or in flight) coexist with that
|
||||
retained cache. The size bound is 64 MiB plus the decoded sizes of the window's distinct GUIDs,
|
||||
not a fixed eight-MiB allowance: manifests can supply larger chunk windows. Evicted distant shared
|
||||
chunks may be downloaded again. No worker downloads outside the moving window; cancellation and
|
||||
failure disconnect and join all workers before returning or removing the cache.
|
||||
The legacy native batch APIs remain available but are not used by the production install path.
|
||||
**Java** - without the native library, or with `storesEpicMode=java`. Eight Java workers prefetch
|
||||
the next eight chunk-part references, across file boundaries, while one writer assembles and
|
||||
verifies files in manifest order. Shared GUIDs within that window are fetched only once. Chunks
|
||||
are deleted after their last use; other reusable chunks are retained up to 64 MiB. At most eight
|
||||
additional decoded chunks (complete or in flight) coexist with that retained cache. The size
|
||||
bound is 64 MiB plus the decoded sizes of the window's distinct GUIDs, not a fixed eight-MiB
|
||||
allowance: manifests can supply larger chunk windows. Evicted distant shared chunks may be
|
||||
downloaded again. No worker downloads outside the moving window; cancellation and failure
|
||||
disconnect and join all workers before returning or removing the cache. Its summary line is
|
||||
`mode=java-bounded`.
|
||||
|
||||
The earlier batch design intentionally prioritizes parallel-download throughput. Native
|
||||
`epic/assemble.rs::build_assemble_plan` enables `scratch_cache` only for a nonempty cache path.
|
||||
`run_assemble` decrements per-file chunk references and removes chunks at their last use **only
|
||||
when that flag is set**; the default beside-game cache is removed at the end. The native tests
|
||||
`default_cache_assembles_like_the_kotlin_loop_and_removes_the_cache_at_the_end` and
|
||||
the scratch-cache assembly tests cover that distinction. Extending last-use reclamation to
|
||||
internal native assembly could keep parallel batch throughput and reduce the overlapping
|
||||
cache/output peak, but batch fetching still fills
|
||||
the cache before assembly. Production native adaptive and Java fallback paths instead overlap
|
||||
network fetches with assembly without retaining a whole-game cache. No native or Java throughput
|
||||
benchmark is claimed.
|
||||
**Batch** - the earlier native design, kept for A/B benchmarks: every chunk fetched into the
|
||||
cache in parallel, then the files assembled from it (`EpicNative.run` + `EpicNative.assemble`).
|
||||
It needs the cache and the files at once (about twice the game) and checks exactly that before
|
||||
starting; it writes files in place, so an existing install is marked `updating` before assembly.
|
||||
`epic/assemble.rs::build_assemble_plan` enables `scratch_cache` only for a nonempty cache path:
|
||||
`run_assemble` then removes chunks at their last use; the default beside-game cache is removed at
|
||||
the end (`default_cache_assembles_like_the_kotlin_loop_and_removes_the_cache_at_the_end` and the
|
||||
scratch-cache assembly tests cover that distinction).
|
||||
|
||||
The writer is chosen by the hidden `storesEpicMode` string in the `session` preferences:
|
||||
`adaptive` (default), `batch` or `java`. There is no UI for it; set it on a test device to compare
|
||||
the modes on the same game.
|
||||
|
||||
The adaptive space plan (`EpicDownloadManager.planAdaptiveSpace`, unit-tested in
|
||||
`EpicAdaptivePlanTest`) checks each volume for what lands on it. The game volume: the sequential
|
||||
staged-file peak (remaining file bytes, each replaced file's old copy credited only after its
|
||||
swap), plus, for an update, the largest old file plus 256 MiB for files finishing out of order
|
||||
within the fetch window, plus the cache when it is beside the game. The cache volume: the budget
|
||||
plus 64 MiB of chunks in transit. The budget is half the cache volume's free space after those
|
||||
needs, at most 1 GiB (256 MiB when free space is unknown; 0 is allowed and means every reuse is a
|
||||
refetch). On a card install the cache stays on internal scratch unless internal storage cannot
|
||||
hold at least 64 MiB of it; then the cache goes beside the game (still released chunk by chunk)
|
||||
instead of failing. A plan that does not fit names the short storage and the shortfall in the log
|
||||
and hands over to the bounded writer, whose own checks decide. Running out of space in the middle
|
||||
of an adaptive run is not destructive: a failed write fails the run, unfinished staging files are
|
||||
removed, the old files stay.
|
||||
|
||||
The Epic download bar, measured rate and ETA (adaptive and bounded) count pending file bytes written, not unique
|
||||
The Epic download bar, measured rate and ETA (adaptive, lookahead and Java) count pending file bytes written, not unique
|
||||
compressed chunks fetched. Shared chunks therefore cannot complete the bar before later files
|
||||
are written, and an evicted chunk's refetch is reflected in the time spent doing that work.
|
||||
Streaming stays in the Download stage until all files are written and verified, then enters
|
||||
Install for registration; it does not bounce between stages for each file.
|
||||
|
||||
The space check follows the manifest delta pass and uses the same pure moving-window policy as
|
||||
execution, reserving all incoming decoded chunk sizes before writing parts and simulating
|
||||
For the lookahead and Java writers, the space check follows the manifest delta pass and uses the
|
||||
same pure moving-window policy as execution, reserving all incoming decoded chunk sizes before writing parts and simulating
|
||||
last-use releases and unpinned oldest-first evictions. Existing verified cache bytes count against
|
||||
current free space and are credited only at their scheduled release. This avoids adding the entire
|
||||
cache to the entire game without undercounting concurrent downloads. Each file is written to a
|
||||
@@ -167,7 +157,7 @@ verified cached chunks are accounted for on resume. On a card, game and cache re
|
||||
are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed
|
||||
or cancelled staging files are removed without deleting old outputs. A successful run removes the
|
||||
cache. A resume within a large incomplete file restarts that file, not its individual parts.
|
||||
Both writers use `<installDir>/.droiddeck-epic-staging/`, whose `owner` marker identifies this
|
||||
All three staged writers (adaptive, lookahead, Java) use `<installDir>/.droiddeck-epic-staging/`, whose `owner` marker identifies this
|
||||
namespace. Before the delta pass and free-space probes, the manager removes orphan `.stage`
|
||||
entries only from this owned namespace. A manifest naming that directory or any descendant
|
||||
fails without deletion; an unowned directory or unexpected entry also fails conservatively.
|
||||
|
||||
Reference in new issue
Block a user