From f2ecda239a5880b96bcef49a4261e82e08915832 Mon Sep 17 00:00:00 2001 From: The412Banner Date: Tue, 13 Oct 2026 20:31:22 -0400 Subject: [PATCH] Use the adaptive Epic engine by default, keep bounded and batch modes EpicDownloadManager runs EpicNative.stream when the library loads and its per-volume space plan fits (cache on internal scratch for card installs, beside the game when internal storage is full), and falls back to the bounded Java writer otherwise. A hidden storesEpicMode preference selects adaptive, batch (whole-cache fetch then native assembly, with its 2x space check) or java for A/B runs; each mode logs a summary line. --- .../launcher/session/SessionPrefs.kt | 8 + .../stores/epic/EpicDownloadManager.java | 334 +++++++++++++++++- .../launcher/stores/epic/EpicNative.kt | 90 ++++- .../stores/epic/EpicAdaptivePlanTest.kt | 57 +++ docs/development/stores.md | 49 ++- 5 files changed, 524 insertions(+), 14 deletions(-) create mode 100644 app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt index b941508d..4c9afe3e 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt @@ -87,6 +87,14 @@ object SessionPrefs { prefs(context).edit().putString("gameStoresSpeedTier", tier).apply() } + /** + * Hidden A/B switch for the Epic installer: "adaptive" (default: native parallel fetch written + * straight into staged files), "batch" (the older whole-cache native fetch, then native + * assembly; needs about twice the game's size free) or "java" (the bounded one-chunk-at-a-time + * Java writer). No UI; set it in the `session` preferences for benchmarks. + */ + fun storesEpicMode(context: Context): String = prefs(context).getString("storesEpicMode", "adaptive") ?: "adaptive" + /** Which tab a signed-in store opens on when its chip is picked: [STORES_OPEN_LIBRARY] or [STORES_OPEN_STORE]. */ fun storesOpenTab(context: Context): String = prefs(context).getString("storesOpenTab", STORES_OPEN_LIBRARY)?.takeIf { it == STORES_OPEN_STORE } ?: STORES_OPEN_LIBRARY diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java index d0ea636c..5eccc046 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java @@ -7,6 +7,7 @@ import com.droiddeck.launcher.stores.StoreLog; import com.droiddeck.launcher.stores.StoreInstallRoot; import com.droiddeck.launcher.stores.StoreInstalls; import com.droiddeck.launcher.stores.StoresNative; +import com.droiddeck.launcher.stores.download.StoreDownloadTier; import org.json.JSONArray; import org.json.JSONObject; @@ -41,15 +42,19 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import java.util.zip.Inflater; /** * The Epic install pipeline, ported from Bannerlator's EpicDownloadManager: the manifest API's CDN * list, the manifest binary (ChunksV4, or the older JSON form), the files narrowed to the base * set plus the device's language, a delta pass that skips files already on disk with the right - * size and SHA-1, then native adaptive parallel fetch/staged assembly when the library loads. - * Builds without the library use bounded Java parallel prefetch and one staged writer instead. - * Neither production path requires a whole-game chunk cache. + * size and SHA-1, then the writer. When the native library loads, the writer is the adaptive + * native install ({@link EpicNative#stream}: chunks fetched in parallel and written straight into + * staged files, shared chunks in a budgeted cache); otherwise, or when its space plan does not + * fit, the bounded-cache Java writer, one chunk at a time. Both replace each file the same safe + * way. A hidden preference ({@code storesEpicMode}) can pick the older whole-cache native batch + * engine or the Java writer instead, for side-by-side benchmarks. * * Chunk URLs carry no auth tokens; the subfolder is the decimal group number; the Cloudflare * mirror is skipped (it refuses chunks). Blocking; the download queue runs it on its own thread. @@ -63,6 +68,37 @@ public final class EpicDownloadManager { static final int PREFETCH_PARTS = 8; static final String STAGING_DIRECTORY = ".droiddeck-epic-staging"; private static final String STAGING_OWNER = "DroidDeck Epic staging v1\n"; + /** Most the adaptive engine caches of shared chunks; more only saves the occasional refetch. */ + static final long ADAPTIVE_CACHE_MAX_BYTES = 1024L * 1024 * 1024; + /** Its budget when free space cannot be read. */ + static final long ADAPTIVE_CACHE_DEFAULT_BYTES = 256L * 1024 * 1024; + /** + * Below this, internal scratch is too small to be worth a cache of its own: the shared chunks + * then go beside the game (still released at their last use) rather than failing the install. + */ + static final long ADAPTIVE_CACHE_MIN_SCRATCH_BYTES = MAX_RETAINED_CHUNK_CACHE_BYTES; + /** Chunks being decoded, written or released at any moment, beyond the budget itself. */ + static final long ADAPTIVE_TRANSIENT_BYTES = 64L * 1024 * 1024; + /** + * Files the fetch window has open beyond the sequential plan: the parallel writer can finish + * files a little out of order, so replaced files near the front of the window may not have + * released their old copies yet. + */ + static final long ADAPTIVE_OPEN_WINDOW_BYTES = 256L * 1024 * 1024; + + static final String MODE_ADAPTIVE = "adaptive"; + static final String MODE_BATCH = "batch"; + static final String MODE_JAVA = "java"; + + /** How {@link #install} may use the native engine; null = the bounded Java writer only. */ + static final class NativeOptions { + final String mode; + final int maxWorkers, processWorkers; + final String tierId; + NativeOptions(String mode, int maxWorkers, int processWorkers, String tierId) { + this.mode = mode; this.maxWorkers = maxWorkers; this.processWorkers = processWorkers; this.tierId = tierId; + } + } public interface Callback { void onProgress(String message, int pct); @@ -126,6 +162,8 @@ public final class EpicDownloadManager { /** The prerequisite fields were read (an empty name and path then mean the build names none). */ public boolean prereqRead = false; public List cdnUrls = new ArrayList<>(); + /** The manifest as downloaded, for the native engines (which parse it themselves); null = Java only. */ + byte[] rawBytes = null; } /** What an install leaves for the sidecar. */ @@ -166,8 +204,12 @@ public final class EpicDownloadManager { Manifest manifest = parseManifest(manifestBytes); if (manifest == null) throw new InstallException("The manifest could not be parsed"); manifest.cdnUrls = cdnUrls; + manifest.rawBytes = manifestBytes; cb.onLog("epic: chunkDir=" + manifest.chunkDir + " chunks=" + manifest.uniqueChunks.size() + " files=" + manifest.files.size() + " exe=" + manifest.launchExe); - return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb); + StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); + String mode = com.droiddeck.launcher.session.SessionPrefs.INSTANCE.storesEpicMode(ctx); + NativeOptions options = new NativeOptions(mode, Math.max(1, Math.min(128, tier.getNetworkWindow())), Math.max(2, tier.getProcessWorkers()), tier.getId()); + return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, options); } catch (InstallException e) { throw e; } catch (Exception e) { @@ -178,6 +220,12 @@ public final class EpicDownloadManager { static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, List installTags, AtomicBoolean cancel, Callback cb) throws InstallException { + return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, null); + } + + static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, + List installTags, AtomicBoolean cancel, Callback cb, + NativeOptions nativeOptions) throws InstallException { try { List cdnUrls = manifest.cdnUrls; File installDir = new File(installDirPath); @@ -220,7 +268,6 @@ public final class EpicDownloadManager { Map chunksByGuid = new LinkedHashMap<>(); for (ChunkInfo c : needed) chunksByGuid.put(c.guidStr(), c); if (!chunkCacheDir.isDirectory() && !chunkCacheDir.mkdirs()) throw new InstallException("Could not create Epic chunk cache"); - Map cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed); long pendingBytes = 0; Map oldFileBytes = new LinkedHashMap<>(); for (FileInfo f : pending) { @@ -228,6 +275,15 @@ public final class EpicDownloadManager { File output = new File(installDir, f.filename.replace("\\", "/")); if (output.isFile()) oldFileBytes.put(f.filename, output.length()); } + if (nativeOptions != null && !MODE_JAVA.equals(nativeOptions.mode)) { + NativeAttempt attempt = MODE_BATCH.equals(nativeOptions.mode) + ? installBatch(manifest, pending, needed, pendingBytes, planned, installDir, cachePath, chunkCacheDir, nativeOptions, cancel, cb) + : installAdaptive(manifest, pending, pendingBytes, oldFileBytes, planned, installDir, cachePath, chunkCacheDir, nativeOptions, cancel, cb); + if (attempt.handled) return attempt.result; + } else if (nativeOptions != null) { + cb.onLog("epic: mode=java-bounded chosen by storesEpicMode"); + } + Map cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed); boolean cacheOnInstallVolume = cachePath.isEmpty(); boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded(); long peakGameAdditional = nativeAvailable ? 0 : cacheOnInstallVolume @@ -345,6 +401,8 @@ public final class EpicDownloadManager { } } deleteDir(chunkCacheDir); + long boundedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); + cb.onLog("epic: summary mode=java-bounded files=" + doneFiles + " bytes_written=" + writtenBytes + " elapsed_ms=" + boundedMs + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", writtenBytes * 1000.0 / boundedMs / 1048576.0)); cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles); cb.onProgress("Complete", 100); return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); @@ -356,6 +414,272 @@ public final class EpicDownloadManager { } } + /** What a native attempt did: {@code handled} = the install ended there ({@code result} null when cancelled). */ + private static final class NativeAttempt { + final boolean handled; + final Result result; + private NativeAttempt(boolean handled, Result result) { this.handled = handled; this.result = result; } + static final NativeAttempt FALLBACK = new NativeAttempt(false, null); + static NativeAttempt done(Result result) { return new NativeAttempt(true, result); } + } + + /** The pending files as indices into the manifest's file list, what the native runs take; null when one is not in it. */ + static int[] pendingIndices(Manifest manifest, List pending) { + java.util.IdentityHashMap index = new java.util.IdentityHashMap<>(); + for (int i = 0; i < manifest.files.size(); i++) index.put(manifest.files.get(i), i); + int[] pendingIdx = new int[pending.size()]; + for (int i = 0; i < pending.size(); i++) { + Integer k = index.get(pending.get(i)); + if (k == null) return null; + pendingIdx[i] = k; + } + return pendingIdx; + } + + private static String[] cdnPrefixes(List cdnUrls) { + String[] prefixes = new String[cdnUrls.size()]; + for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir; + return prefixes; + } + + /** + * Where the adaptive engine's shared-chunk cache goes and how big it may get, and what each + * volume must hold. A card install keeps the cache on internal scratch (each byte hits the + * card once); when internal storage cannot hold a useful cache, the cache goes beside the game + * instead of failing. {@code error} names the storage that is short, and by how much. + */ + static final class AdaptiveSpacePlan { + boolean cacheBesideGame; + long cacheBudget; + long gameNeed; + long scratchNeed; + String error; + } + + /** The part of free space the shared cache may use, after {@code need}: half the headroom, capped. */ + static long adaptiveCacheBudget(long free, long need) { + if (free < 0) return ADAPTIVE_CACHE_DEFAULT_BYTES; + long headroom = free - need - ADAPTIVE_TRANSIENT_BYTES; + if (headroom <= 0) return 0; + return Math.min(ADAPTIVE_CACHE_MAX_BYTES, headroom / 2); + } + + /** + * Space the game volume needs beyond the sequential staged plan: none for a fresh install (a + * staged file becomes the file itself); for replaced files, the largest old copy that may not + * be released yet plus what the fetch window spans, never more than all old copies together. + */ + static long adaptiveOpenStagingAllowance(Map oldFileBytes) { + long total = 0, largest = 0; + for (long bytes : oldFileBytes.values()) { total += bytes; largest = Math.max(largest, bytes); } + return Math.min(total, largest + ADAPTIVE_OPEN_WINDOW_BYTES); + } + + static AdaptiveSpacePlan planAdaptiveSpace(long stagedPeak, long openAllowance, long freeGame, boolean scratchRequested, long freeScratch) { + AdaptiveSpacePlan plan = new AdaptiveSpacePlan(); + long files = stagedPeak + openAllowance; + long scratchBudget = scratchRequested ? adaptiveCacheBudget(freeScratch, 0) : 0; + plan.cacheBesideGame = !scratchRequested || (freeScratch >= 0 && scratchBudget < ADAPTIVE_CACHE_MIN_SCRATCH_BYTES); + if (plan.cacheBesideGame) { + plan.cacheBudget = adaptiveCacheBudget(freeGame, files); + plan.gameNeed = files + plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; + plan.scratchNeed = 0; + } else { + plan.cacheBudget = scratchBudget; + plan.gameNeed = files; + plan.scratchNeed = plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; + } + if (StoreInstallRoot.insufficientSpace(plan.gameNeed, freeGame)) { + plan.error = "Not enough free space on the game's storage: need " + fmt(plan.gameNeed) + + (plan.cacheBesideGame && scratchRequested ? " (download cache moved beside the game: internal storage is full)" : "") + + ", only " + fmt(freeGame) + " free (short by " + fmt(plan.gameNeed - freeGame) + ")"; + } else if (!plan.cacheBesideGame && StoreInstallRoot.insufficientSpace(plan.scratchNeed, freeScratch)) { + plan.error = "Not enough internal space for the download cache: need " + fmt(plan.scratchNeed) + ", only " + fmt(freeScratch) + + " free (short by " + fmt(plan.scratchNeed - freeScratch) + ")"; + } + return plan; + } + + /** + * The adaptive native install. Falls back (nothing fetched or written) when the engine is not + * there, refuses the plan, or its space plan does not fit while the bounded writer's might. + */ + private static NativeAttempt installAdaptive(Manifest manifest, List pending, long pendingBytes, Map oldFileBytes, + long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options, + AtomicBoolean cancel, Callback cb) throws InstallException { + int[] pendingIdx = pendingIndices(manifest, pending); + if (manifest.rawBytes == null || pendingIdx == null) return NativeAttempt.FALLBACK; + if (!StoresNative.INSTANCE.getAvailable()) { + cb.onLog("epic: native engine unavailable; mode=java-bounded"); + return NativeAttempt.FALLBACK; + } + long stagedPeak = estimateStagedFilePeak(pending, oldFileBytes); + long openAllowance = adaptiveOpenStagingAllowance(oldFileBytes); + long freeGame = StoreInstallRoot.availableBytes(installDir); + boolean scratchRequested = !cachePath.isEmpty(); + long freeScratch = scratchRequested ? StoreInstallRoot.availableBytes(chunkCacheDir) : -1; + AdaptiveSpacePlan space = planAdaptiveSpace(stagedPeak, openAllowance, freeGame, scratchRequested, freeScratch); + if (space.error != null) { + // The bounded writer needs less (one chunk at a time, sequential swaps): let it decide. + cb.onLog("epic: adaptive space plan does not fit (" + space.error + "); mode=java-bounded"); + return NativeAttempt.FALLBACK; + } + String engineCache = space.cacheBesideGame ? "" : cachePath; + File engineCacheDir = engineCache.isEmpty() ? new File(installDir, ".chunks") : chunkCacheDir; + if (space.cacheBesideGame && scratchRequested) { + cb.onLog("epic: internal storage cannot hold the shared-chunk cache (" + fmt(freeScratch) + " free); cache beside the game"); + } + if (cancel.get()) return NativeAttempt.done(null); + cb.onLog("epic: mode=adaptive tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers + + " cache=" + (engineCache.isEmpty() ? "beside the game" : "scratch") + " budget=" + fmt(space.cacheBudget) + + " game need=" + fmt(space.gameNeed) + " of " + fmt(freeGame) + (space.cacheBesideGame ? "" : ", scratch need=" + fmt(space.scratchNeed) + " of " + fmt(freeScratch))); + cb.onSizes(pendingBytes, planned); + cb.onBytes(0, pendingBytes, 0); + final int totalFiles = pending.size(); + final File fInstallDir = installDir; + final Object lock = new Object(); + final long started = System.nanoTime(); + final long[] state = new long[] {0, 0, System.currentTimeMillis(), 0, 0}; // bytes, files, speed ms, speed bytes, speed bps + EpicNative.Result res = EpicNative.stream(manifest.rawBytes, installDir.getPath(), engineCache, cdnPrefixes(manifest.cdnUrls), pendingIdx, pendingBytes, + space.cacheBudget, "", options.maxWorkers, options.processWorkers, cancel, new EpicNative.StreamListener() { + @Override public void onStreamProgress(long bytesWritten, long bytesTotal, int filesDone, int filesTotal, long filesBytes) { + synchronized (lock) { + if (bytesWritten < state[0] && filesDone <= state[1]) return; // a late, older sample + state[0] = Math.max(state[0], bytesWritten); + long nowMs = System.currentTimeMillis(); + if (nowMs - state[2] >= 500) { + state[4] = (state[0] - state[3]) * 1000L / Math.max(1, nowMs - state[2]); + state[2] = nowMs; + state[3] = state[0]; + } + cb.onBytes(state[0], pendingBytes, state[4]); + cb.onProgress("Downloading chunks (" + filesDone + "/" + totalFiles + " files)" + speedSuffix(state[4]), + (int) (state[0] * 80L / Math.max(1, pendingBytes))); + if (filesDone > state[1]) { + state[1] = filesDone; + cb.onStage("stream", filesBytes, pendingBytes, filesDone, totalFiles); + } + } + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public boolean onBeforeReplace() { + // Once a swap can mix versions, the old installed record is no longer truthful. + try { + StoreInstalls.markUpdating(fInstallDir); + return true; + } catch (Exception e) { + cb.onLog("epic: could not mark the install as updating: " + e.getMessage()); + return false; + } + } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (!res.started) { + cb.onLog("epic: adaptive engine not started (" + res.error + "); mode=java-bounded"); + return NativeAttempt.FALLBACK; + } + long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); + if (res.cancelled) return NativeAttempt.done(null); + if (!res.success) throw new InstallException(res.error == null || res.error.isEmpty() ? "Epic download failed" : res.error); + deleteDir(engineCacheDir); + deleteDir(chunkCacheDir); + cb.onLog("epic: summary mode=adaptive files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs + + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); + cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); + cb.onProgress("Complete", 100); + return NativeAttempt.done(new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest)); + } + + /** + * The older native batch install, kept for A/B benchmarks: every chunk fetched into the cache + * in parallel, then the files assembled from it. It needs the whole cache and the files at + * once (about twice the game), and it writes files in place rather than staged, so an + * existing install is marked updating before anything is fetched. + */ + private static NativeAttempt installBatch(Manifest manifest, List pending, List needed, long pendingBytes, + long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options, + AtomicBoolean cancel, Callback cb) throws InstallException { + int[] pendingIdx = pendingIndices(manifest, pending); + if (manifest.rawBytes == null || pendingIdx == null) return NativeAttempt.FALLBACK; + if (!StoresNative.INSTANCE.getAvailable()) { + cb.onLog("epic: native engine unavailable; mode=java-bounded"); + return NativeAttempt.FALLBACK; + } + long cacheMissing = 0, totalBytes = 0; + for (ChunkInfo c : needed) { + if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += chunkWindowSize(c); + totalBytes += Math.max(c.fileSize, 1); + } + long freeGame = StoreInstallRoot.availableBytes(installDir); + if (cachePath.isEmpty()) { + long need = cacheMissing + pendingBytes; + if (StoreInstallRoot.insufficientSpace(need, freeGame)) throw new InstallException("Not enough free space for the batch Epic download: need " + fmt(need) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(freeGame) + " free (short by " + fmt(need - freeGame) + ")"); + } else { + long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir); + if (StoreInstallRoot.insufficientSpace(cacheMissing, freeCache)) throw new InstallException("Not enough internal space for the batch download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free (short by " + fmt(cacheMissing - freeCache) + ")"); + if (StoreInstallRoot.insufficientSpace(pendingBytes, freeGame)) throw new InstallException("Not enough free space on the game's storage: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free (short by " + fmt(pendingBytes - freeGame) + ")"); + } + if (cancel.get()) return NativeAttempt.done(null); + final long fTotal = totalBytes; + final int totalChunks = needed.size(); + final int totalFiles = pending.size(); + cb.onLog("epic: mode=batch tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers + (cachePath.isEmpty() ? "" : " cache=scratch") + " cache_missing=" + fmt(cacheMissing)); + cb.onSizes(pendingBytes, planned); + final AtomicLong lastMs = new AtomicLong(System.currentTimeMillis()), lastBytes = new AtomicLong(), speed = new AtomicLong(); + final long started = System.nanoTime(); + EpicNative.Result fetch = EpicNative.run(manifest.rawBytes, installDir.getPath(), cachePath, cdnPrefixes(manifest.cdnUrls), pendingIdx, totalChunks, fTotal, "", + options.maxWorkers, options.processWorkers, cancel, new EpicNative.Listener() { + @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); } + @Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) { + sampleSpeed(bytesDone, lastMs, lastBytes, speed); + cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speed.get()), (int) (bytesDone * 80L / Math.max(1, fTotal))); + cb.onBytes(bytesDone, fTotal, speed.get()); + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (!fetch.started) { + cb.onLog("epic: batch engine not started (" + fetch.error + "); mode=java-bounded"); + return NativeAttempt.FALLBACK; + } + if (fetch.cancelled) return NativeAttempt.done(null); + if (!fetch.success) throw new InstallException("Epic batch download failed: " + fetch.error); + // Assembly overwrites files in place: the old installed record stops being truthful now. + try { StoreInstalls.markUpdating(installDir); } catch (Exception e) { throw new InstallException("Could not mark the install as updating: " + e.getMessage()); } + EpicNative.Result asm = EpicNative.assemble(manifest.rawBytes, installDir.getPath(), cachePath, pendingIdx, cancel, new EpicNative.Listener() { + @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {} + @Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) { + cb.onStage("stream", bytesDone, bytesTotal, done, Math.max(total, totalFiles)); + cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles)))); + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (asm.cancelled) return NativeAttempt.done(null); + if (!asm.started || !asm.success) throw new InstallException(asm.error == null || asm.error.isEmpty() ? "Epic batch assembly failed" : asm.error); + long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); + cb.onLog("epic: summary mode=batch files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs + + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); + deleteDir(chunkCacheDir); + cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); + cb.onProgress("Complete", 100); + return NativeAttempt.done(new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest)); + } + + private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { + long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); + if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { + long prevB = lastSpeedBytes.getAndSet(done); + long dt = nowMs - prevMs; + if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); + } + } + + private static String speedSuffix(long bps) { + if (bps <= 0) return ""; + return " " + (bps >= 1048576 ? String.format(java.util.Locale.ROOT, "%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); + } + static File recoverOwnedStaging(File installDir, List manifestFiles) throws IOException { File staging = new File(installDir, STAGING_DIRECTORY); Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY); diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt index eca9c973..ab079e81 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt @@ -9,17 +9,20 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference /** - * [installAdaptive] is the production native Epic fetch/assembly engine. Its snapshot is the - * manager's delta-selected pending files, verified cache and old output sizes. Rust chooses a - * storage-safe ordered lookahead, overlaps fetch_core with staged assembly and calls beforeSwap - * before the first verified replacement. A loaded-engine error fails explicitly. + * The native Epic engines in `libdroiddeckstores.so`. [stream] is the install the manager uses + * whenever the library loads: chunks fetched in parallel by the adaptive fetch core and written + * straight into each file's staging file as they arrive, shared chunks kept in a cache of at most + * the given budget, every file verified and swapped in with the same rules as the bounded Java + * writer (see `stream.rs`). [run] and [assemble] are the older batch pair: [run] fills the chunk + * cache with verified, decompressed chunks for the pending files, [assemble] writes the files from + * it. They stay for callers that explicitly want a whole-game cache. * * The cache is `/.chunks` when `chunkCacheDir` is `""`, else the scratch folder given - - * the app's own cache when the game installs to a card. Adaptive installs release last-use chunks - * in both layouts and remove the cache on success. [run]/[assemble] are legacy batch APIs. + * the app's own cache when the game installs to a card. * - * Cancellation flips the native flag; the native handle is joined before returning, so no - * cache/staging writer survives completion or cache cleanup. + * All block while polling [AtomicBoolean] `cancel` every 250 ms. [run] and [assemble] flip the + * native flag on cancel and wait up to 5 s; [stream] waits for the run to finish, because its + * cleanup (removing unfinished staging files) must be done before the manager returns. */ object EpicNative { private const val TAG = "EpicNative" @@ -48,6 +51,18 @@ object EpicNative { nativeInstallAdaptive(request, inner) } + /** Callbacks from a [stream] run; every method is called on a native thread. */ + interface StreamListener { + /** File bytes written so far (staging included), and the files verified and swapped in so far. */ + fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) + /** Engine log line (already in logcat under `EpicNative`). */ + fun onLog(line: String) + /** Once, before the first file replaces its destination; false (or a throw) fails the run with nothing replaced. */ + fun onBeforeReplace(): Boolean + /** Terminal; `bytes` = file bytes written. */ + fun onComplete(success: Boolean, error: String, bytes: Long) + } + /** Outcome of a run. `started == false`: nothing happened on disk, the caller runs its own loop. */ class Result( @JvmField val started: Boolean, @JvmField val success: Boolean, @JvmField val cancelled: Boolean, @@ -70,6 +85,59 @@ object EpicNative { fun assemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, cancel: AtomicBoolean?, listener: Listener): Result = drive("nativeAssemble", cancel, listener) { inner -> nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, inner) } + /** + * The adaptive install of the files [pendingFileIdx] (indices into the manifest's file list), + * with at most [cacheBudget] bytes of shared chunks cached. `started == false` means nothing + * was fetched or written and the caller runs the bounded Java writer. + */ + @JvmStatic + fun stream( + manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, + cacheBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, cancel: AtomicBoolean?, listener: StreamListener, + ): Result { + if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) + val latch = CountDownLatch(1) + val completion = AtomicReference(null) + val doneRef = AtomicReference(0) + val totalRef = AtomicReference(pendingFileIdx.size) + val inner = object : StreamListener { + override fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) { + doneRef.set(filesDone); totalRef.set(filesTotal) + listener.onStreamProgress(bytesWritten, bytesTotal, filesDone, filesTotal, filesBytes) + } + override fun onLog(line: String) = listener.onLog(line) + override fun onBeforeReplace(): Boolean = runCatching { listener.onBeforeReplace() }.getOrDefault(false) + override fun onComplete(success: Boolean, error: String, bytes: Long) { + completion.set(Completion(success, error, bytes)) + latch.countDown() + runCatching { listener.onComplete(success, error, bytes) } + } + } + val handle: Long = try { + nativeStream(manifest, installDir, chunkCacheDir, cdnPrefixes, pendingFileIdx, expectedBytes, cacheBudget, caBundlePath, maxWorkers, processWorkers, inner) + } catch (t: Throwable) { + Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}") + return Result(false, false, false, "nativeStream: ${t.javaClass.simpleName}", 0L, 0, 0) + } + if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) + var cancelSent = false + try { + while (!latch.await(250, TimeUnit.MILLISECONDS)) { + if (!cancelSent && cancel != null && cancel.get()) { + nativeCancel(handle) + cancelSent = true + } + } + } finally { + nativeRelease(handle) + } + val c = completion.get() + val cancelled = cancelSent || (c != null && !c.success && c.error == "cancelled") + val success = !cancelled && c != null && c.success + val error = when { cancelled -> "cancelled"; c == null -> "no completion"; else -> c.error } + return Result(true, success, cancelled, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get()) + } + private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, strict: Boolean = false, start: (Listener) -> Long): Result { if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) val latch = CountDownLatch(1) @@ -130,6 +198,12 @@ object EpicNative { @JvmStatic private external fun nativeAssemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, listener: Listener): Long + @JvmStatic + private external fun nativeStream( + manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, + cacheBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: StreamListener, + ): Long + @JvmStatic private external fun nativeCancel(handle: Long) diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt new file mode 100644 index 00000000..efb0f8b0 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt @@ -0,0 +1,57 @@ +package com.droiddeck.launcher.stores.epic + +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicAdaptivePlanTest { + private val mib = 1024L * 1024 + private val gib = 1024 * mib + + @Test fun freshInstallsNeedNoOpenStagingAllowance() { + assertEquals(0L, EpicDownloadManager.adaptiveOpenStagingAllowance(emptyMap())) + val old = mapOf("big.pak" to 10 * gib, "small.pak" to gib) + assertEquals(10 * gib + EpicDownloadManager.ADAPTIVE_OPEN_WINDOW_BYTES, EpicDownloadManager.adaptiveOpenStagingAllowance(old)) + assertEquals(100 * mib, EpicDownloadManager.adaptiveOpenStagingAllowance(mapOf("a" to 60 * mib, "b" to 40 * mib))) + } + + @Test fun cacheBudgetIsHalfTheHeadroomCappedAndDefaultsWhenSpaceIsUnknown() { + assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_DEFAULT_BYTES, EpicDownloadManager.adaptiveCacheBudget(-1, 0)) + assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_MAX_BYTES, EpicDownloadManager.adaptiveCacheBudget(100 * gib, 10 * gib)) + assertEquals(0L, EpicDownloadManager.adaptiveCacheBudget(gib, gib)) + val free = 1000 * mib + assertEquals((free - 400 * mib - EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES) / 2, EpicDownloadManager.adaptiveCacheBudget(free, 400 * mib)) + } + + @Test fun cardInstallKeepsTheCacheOnInternalScratchWhenItFits() { + val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 10 * gib) + assertNull(plan.error) + assertFalse(plan.cacheBesideGame) + assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_MAX_BYTES, plan.cacheBudget) + assertEquals(20 * gib, plan.gameNeed) + assertEquals(plan.cacheBudget + EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.scratchNeed) + } + + @Test fun fullInternalStorageMovesTheCacheBesideTheGameInsteadOfFailing() { + val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 100 * mib) + assertNull(plan.error) + assertTrue(plan.cacheBesideGame) + assertEquals(0L, plan.scratchNeed) + assertTrue(plan.cacheBudget > 0) + assertEquals(20 * gib + plan.cacheBudget + EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.gameNeed) + assertTrue(plan.gameNeed <= 30 * gib) + } + + @Test fun shortStorageIsNamedWithTheShortfall() { + val card = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 10 * gib, true, 10 * gib) + assertTrue(card.error!!.contains("game's storage")) + assertTrue(card.error!!.contains("short by 10.0 GB")) + val internal = EpicDownloadManager.planAdaptiveSpace(20 * gib, gib, 20 * gib, false, -1) + assertTrue(internal.error!!.contains("game's storage")) + assertNull(EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, -1, true, -1).error) + } +} diff --git a/docs/development/stores.md b/docs/development/stores.md index 8eb7233a..bf321c03 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -45,6 +45,35 @@ with a placeholder job. Failures are kept in the Downloads UI, notification and duplicated as transient failure toasts. Active queue persistence and ordinary backend pause/retry are unchanged. +Epic installs have three writers, all with the same per-file safety rules (below): + +- **Adaptive (default when `libdroiddeckstores.so` loads)** - `EpicNative.stream`, + `epic/stream.rs`. The chunks go through the shared fetch core in the order their files need + them, with its full adaptivity (window ramp, per-host caps, in-flight byte budget, retries, + host rotation, stall handoff). A chunk only one file part uses is verified and written straight + into that file's staging file at the part's offset, in whatever order it arrives; no cache. A + chunk several parts use is written into every part whose file the download has reached and kept, + decompressed, in the cache until its last part is written. The cache has a byte budget; the plan + simulates the install in file order and, when the budget would be exceeded, drops the cached + chunk needed furthest ahead and downloads it again later. Files are opened only once the + download reaches them, so the set of half-written files stays within the fetch window. All writes + happen in the fetch core's process pool, whose completion frees the in-flight byte budget: a slow + card throttles the download instead of growing memory. Each file's SHA-1 is computed as its + parts are written (parts that arrived out of order are read back once the bytes before them are + in). The run logs `stream-plan` (direct vs shared chunks, planned refetches, budget) and + `stream-summary mode=adaptive` (peak cache, peak staging, refetches, bytes written from the + download vs from the cache, average MB/s). +- **Bounded Java** - when the library does not load or link, when the engine refuses the plan, + or when the adaptive space plan does not fit while this one might. It fetches one chunk at a + time, verifies it, and writes the required parts into the current game file. Chunks are deleted + after their last use; reusable chunks are retained up to 64 MiB, with at most one additional + incoming chunk before eviction. Evicted shared chunks may be downloaded again. It logs + `epic: summary mode=java-bounded`. +- **Batch** - the earlier native design, kept for A/B benchmarks: every chunk fetched into the + cache in parallel, then the files assembled from it (`EpicNative.run` + `EpicNative.assemble`). + It needs the cache and the files at once (about twice the game) and checks exactly that before + starting; it writes files in place, so an existing install is marked `updating` before assembly. + It logs `epic: summary mode=batch`. Epic installs use **native adaptive fetch/assembly** when `libdroiddeckstores.so` loads. The Java delta pass sends a versioned snapshot of pending files, chunk metadata, verified cache lengths, old output lengths and both measured free-space probes to Rust. The native planner and @@ -98,7 +127,25 @@ the cache before assembly. Production native adaptive and Java fallback paths in network fetches with assembly without retaining a whole-game cache. No native or Java throughput benchmark is claimed. -The bounded Epic download bar, measured rate and ETA count pending file bytes written, not unique +The writer is chosen by the hidden `storesEpicMode` string in the `session` preferences: +`adaptive` (default), `batch` or `java`. There is no UI for it; set it on a test device to compare +the modes on the same game. + +The adaptive space plan (`EpicDownloadManager.planAdaptiveSpace`, unit-tested in +`EpicAdaptivePlanTest`) checks each volume for what lands on it. The game volume: the sequential +staged-file peak (remaining file bytes, each replaced file's old copy credited only after its +swap), plus, for an update, the largest old file plus 256 MiB for files finishing out of order +within the fetch window, plus the cache when it is beside the game. The cache volume: the budget +plus 64 MiB of chunks in transit. The budget is half the cache volume's free space after those +needs, at most 1 GiB (256 MiB when free space is unknown; 0 is allowed and means every reuse is a +refetch). On a card install the cache stays on internal scratch unless internal storage cannot +hold at least 64 MiB of it; then the cache goes beside the game (still released chunk by chunk) +instead of failing. A plan that does not fit names the short storage and the shortfall in the log +and hands over to the bounded writer, whose own checks decide. Running out of space in the middle +of an adaptive run is not destructive: a failed write fails the run, unfinished staging files are +removed, the old files stay. + +The Epic download bar, measured rate and ETA (adaptive and bounded) count pending file bytes written, not unique compressed chunks fetched. Shared chunks therefore cannot complete the bar before later files are written, and an evicted chunk's refetch is reflected in the time spent doing that work. Streaming stays in the Download stage until all files are written and verified, then enters