mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-10-14 19:27:09 -04:00
Bound Epic chunk cache during installation
This commit is contained in:
Juan Marti Mercado committed
1 parent
eaa095925c
commit
78b3f27816
7 files changed
+595
-210
No files matched your search
@@ -54,6 +54,9 @@ object StoreInstallRoot {
|
||||
fun checkStorage(context: Context, item: CatalogItem, targetRoot: File): StorageCheck {
|
||||
val size = estimatedBytesNeeded(item)
|
||||
val freeTarget = availableBytes(targetRoot)
|
||||
// Epic's manifest delta accounts for completed files and reusable chunks on resume.
|
||||
// Comparing the full catalog size to remaining free space would reject valid resumes.
|
||||
if (item.store == Store.EPIC) return StorageCheck(enough = true, requiredBytes = size, availableBytes = freeTarget)
|
||||
if (size <= 0L) return StorageCheck(enough = true, requiredBytes = 0L, availableBytes = freeTarget)
|
||||
if (freeTarget in 1 until size) {
|
||||
val msg = context.getString(com.droiddeck.launcher.R.string.stores_storage_insufficient, formatBytes(size), formatBytes(freeTarget))
|
||||
|
||||
@@ -4,13 +4,11 @@ import android.content.Context;
|
||||
import android.util.Log;
|
||||
|
||||
import com.droiddeck.launcher.stores.StoreLog;
|
||||
import com.droiddeck.launcher.stores.download.StoreDownloadTier;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
@@ -29,20 +27,16 @@ import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.zip.Inflater;
|
||||
|
||||
/**
|
||||
* The Epic install pipeline, ported from Bannerlator's EpicDownloadManager: the manifest API's CDN
|
||||
* list, the manifest binary (ChunksV4, or the older JSON form), the files narrowed to the base
|
||||
* set plus the device's language, a delta pass that skips files already on disk with the right
|
||||
* size and SHA-1, the chunk fetch into `.chunks/<GUID>` (the native engine when it is there, the
|
||||
* Java pool otherwise), and the assembly of files from those chunks.
|
||||
* size and SHA-1, then the bounded-cache chunk-to-file writer. The native batch engine remains
|
||||
* available for other callers, but this install path intentionally uses the same serial writer
|
||||
* whether or not native code is installed: a whole-game chunk cache can exceed available space.
|
||||
*
|
||||
* Chunk URLs carry no auth tokens; the subfolder is the decimal group number; the Cloudflare
|
||||
* mirror is skipped (it refuses chunks). Blocking; the download queue runs it on its own thread.
|
||||
@@ -51,6 +45,8 @@ public final class EpicDownloadManager {
|
||||
|
||||
private static final String TAG = "EpicDownload";
|
||||
private static final String UA = EpicAuthClient.USER_AGENT;
|
||||
/** Keep reusable chunk data bounded even when a manifest shares chunks across distant files. */
|
||||
static final long MAX_RETAINED_CHUNK_CACHE_BYTES = 64L * 1024 * 1024;
|
||||
|
||||
public interface Callback {
|
||||
void onProgress(String message, int pct);
|
||||
@@ -154,11 +150,23 @@ public final class EpicDownloadManager {
|
||||
if (manifest == null) throw new InstallException("The manifest could not be parsed");
|
||||
manifest.cdnUrls = cdnUrls;
|
||||
cb.onLog("epic: chunkDir=" + manifest.chunkDir + " chunks=" + manifest.uniqueChunks.size() + " files=" + manifest.files.size() + " exe=" + manifest.launchExe);
|
||||
return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb);
|
||||
} catch (InstallException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "install failed", e);
|
||||
throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : ""));
|
||||
}
|
||||
}
|
||||
|
||||
static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath,
|
||||
List<String> installTags, AtomicBoolean cancel, Callback cb) throws InstallException {
|
||||
try {
|
||||
List<CdnUrl> cdnUrls = manifest.cdnUrls;
|
||||
File installDir = new File(installDirPath);
|
||||
installDir.mkdirs();
|
||||
// "" = the cache beside the game; a path = the scratch cache (the app's own storage when
|
||||
// the game goes to a card). The engine and the Java loops read the same value.
|
||||
// the game goes to a card). The bounded writer uses this location on every build.
|
||||
final String cachePath = chunkCacheDirPath == null ? "" : chunkCacheDirPath;
|
||||
final File chunkCacheDir = cachePath.isEmpty() ? new File(installDir, ".chunks") : new File(cachePath);
|
||||
chunkCacheDir.mkdirs();
|
||||
@@ -191,125 +199,116 @@ public final class EpicDownloadManager {
|
||||
return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest);
|
||||
}
|
||||
List<ChunkInfo> needed = uniqueChunksForFiles(manifest, pending);
|
||||
// The cache holds whole chunk windows (~1 MiB each), and a window is shared with files
|
||||
// this device does not install (other tags, other builds' data): the cache is often
|
||||
// larger than the game itself (Metalstorm: 9.1 GB of chunks for 4.4 GB of files). Both
|
||||
// have to fit at once until assembly, so the check counts both, on their own volumes.
|
||||
long cacheMissing = 0, pendingBytes = 0;
|
||||
for (ChunkInfo c : needed) if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += Math.max(c.windowSize, 1);
|
||||
for (FileInfo f : pending) pendingBytes += f.fileSize();
|
||||
if (cachePath.isEmpty()) {
|
||||
long free = installDir.getUsableSpace();
|
||||
if (free > 0 && cacheMissing + pendingBytes > free) throw new InstallException("Not enough free space: need " + fmt(cacheMissing + pendingBytes) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(free) + " free");
|
||||
Map<String, ChunkInfo> chunksByGuid = new LinkedHashMap<>();
|
||||
for (ChunkInfo c : needed) chunksByGuid.put(c.guidStr(), c);
|
||||
if (!chunkCacheDir.isDirectory() && !chunkCacheDir.mkdirs()) throw new InstallException("Could not create Epic chunk cache");
|
||||
Map<String, Long> cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed);
|
||||
long pendingBytes = 0;
|
||||
for (FileInfo f : pending) {
|
||||
pendingBytes += f.fileSize();
|
||||
File output = new File(installDir, f.filename.replace("\\", "/"));
|
||||
File parent = output.getParentFile();
|
||||
if (parent != null) parent.mkdirs();
|
||||
// This file failed the delta check, so it cannot be resumed as a valid install.
|
||||
// Remove it before estimating/writing its replacement to avoid requiring space
|
||||
// for both the corrupt file and its verified replacement.
|
||||
if (output.isFile() && !output.delete() && output.exists()) throw new InstallException("Could not replace incomplete file " + output.getName());
|
||||
}
|
||||
boolean cacheOnInstallVolume = cachePath.isEmpty();
|
||||
long peakGameAdditional = cacheOnInstallVolume
|
||||
? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true)
|
||||
: pendingBytes;
|
||||
long peakCacheAdditional = cacheOnInstallVolume
|
||||
? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false);
|
||||
long freeGame = installDir.getUsableSpace();
|
||||
if (cacheOnInstallVolume) {
|
||||
if (freeGame > 0 && peakGameAdditional > freeGame) throw new InstallException("Not enough free space for the bounded Epic install: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free");
|
||||
} else {
|
||||
long freeCache = chunkCacheDir.getUsableSpace(), freeGame = installDir.getUsableSpace();
|
||||
if (freeCache > 0 && cacheMissing > freeCache) throw new InstallException("Not enough internal space for the download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free");
|
||||
long freeCache = chunkCacheDir.getUsableSpace();
|
||||
if (freeCache > 0 && peakCacheAdditional > freeCache) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free");
|
||||
if (freeGame > 0 && pendingBytes > freeGame) throw new InstallException("Not enough free space: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free");
|
||||
}
|
||||
cb.onLog("epic: cache " + (cachePath.isEmpty() ? "beside the game" : "scratch") + ", " + fmt(cacheMissing) + " to fetch for " + fmt(pendingBytes) + " of files");
|
||||
cb.onLog("epic: cache " + (cacheOnInstallVolume ? "beside the game" : "scratch") + ", peak game additional=" + fmt(peakGameAdditional) + (cacheOnInstallVolume ? "" : ", peak cache additional=" + fmt(peakCacheAdditional)) + " for " + fmt(pendingBytes) + " of files");
|
||||
long totalBytes = 0;
|
||||
for (ChunkInfo c : needed) totalBytes += Math.max(c.fileSize, 1);
|
||||
final long fTotalBytes = totalBytes;
|
||||
final int totalChunks = needed.size();
|
||||
final AtomicLong completedBytes = new AtomicLong(0);
|
||||
final AtomicInteger completedCount = new AtomicInteger(0);
|
||||
final AtomicInteger failCount = new AtomicInteger(0);
|
||||
final AtomicLong lastSpeedMs = new AtomicLong(System.currentTimeMillis());
|
||||
final AtomicLong lastSpeedBytes = new AtomicLong(0);
|
||||
final AtomicLong speedBps = new AtomicLong(0);
|
||||
cb.onSizes(fTotalBytes, planned);
|
||||
cb.onBytes(0, fTotalBytes, 0);
|
||||
cb.onLog("epic: " + fmt(fTotalBytes) + " in " + totalChunks + " chunks");
|
||||
|
||||
boolean javaPool = true;
|
||||
final int[] pendingIdx = pendingIndices(manifest, pending);
|
||||
boolean engine = com.droiddeck.launcher.stores.StoresNative.INSTANCE.getAvailable() && pendingIdx != null;
|
||||
String nativeError = null;
|
||||
if (engine) {
|
||||
NativeOutcome r = runNativePool(ctx, manifestBytes, pendingIdx, needed, fTotalBytes, installDirPath, cachePath, cdnUrls, cancel, cb,
|
||||
completedBytes, completedCount, lastSpeedMs, lastSpeedBytes, speedBps);
|
||||
if (r.started) {
|
||||
javaPool = false;
|
||||
if (r.cancelled) return null;
|
||||
if (!r.success) {
|
||||
cb.onLog("epic: engine failed: " + r.error);
|
||||
nativeError = r.error;
|
||||
failCount.incrementAndGet();
|
||||
}
|
||||
} else cb.onLog("epic: engine not started (" + r.error + "); using the built-in pool");
|
||||
}
|
||||
if (javaPool) {
|
||||
cb.onLog("epic: engine=built-in (8 threads)");
|
||||
ExecutorService pool = Executors.newFixedThreadPool(8, com.droiddeck.launcher.stores.download.DownloadQueue.workerFactory("epic-dl"));
|
||||
for (ChunkInfo chunk : needed) {
|
||||
final ChunkInfo fc = chunk;
|
||||
pool.submit(() -> {
|
||||
if (cancel.get()) return;
|
||||
File cached = new File(chunkCacheDir, fc.guidStr());
|
||||
if (!cached.exists() && !downloadChunkStreaming(fc, manifest.chunkDir, cdnUrls, cached)) {
|
||||
cb.onLog("FAIL chunk=" + fc.guidStr());
|
||||
failCount.incrementAndGet();
|
||||
return;
|
||||
}
|
||||
long done = completedBytes.addAndGet(Math.max(fc.fileSize, 1));
|
||||
int cnt = completedCount.incrementAndGet();
|
||||
sampleSpeed(done, lastSpeedMs, lastSpeedBytes, speedBps);
|
||||
cb.onProgress("Downloading chunks (" + cnt + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (done * 80L / Math.max(1, fTotalBytes)));
|
||||
cb.onBytes(done, fTotalBytes, speedBps.get());
|
||||
});
|
||||
}
|
||||
pool.shutdown();
|
||||
try {
|
||||
while (!pool.awaitTermination(250, TimeUnit.MILLISECONDS)) {
|
||||
if (cancel.get()) { pool.shutdownNow(); pool.awaitTermination(5, TimeUnit.SECONDS); return null; }
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
pool.shutdownNow();
|
||||
throw new InstallException("interrupted");
|
||||
}
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
if (failCount.get() > 0) {
|
||||
if (nativeError != null && !nativeError.isEmpty()) {
|
||||
throw new InstallException(nativeError);
|
||||
}
|
||||
throw new InstallException(failCount.get() + " chunk(s) failed to download");
|
||||
}
|
||||
|
||||
// The engine's assembler when it starts (it drops each chunk after its last use and the
|
||||
// cache at the end); the loop below when it does not.
|
||||
if (engine) {
|
||||
NativeOutcome r = runNativeAssembly(manifestBytes, pendingIdx, pending.size(), installDirPath, cachePath, cancel, cb);
|
||||
if (r.started) {
|
||||
if (r.cancelled) return null;
|
||||
if (!r.success) throw new InstallException(r.error.isEmpty() ? "assembly failed" : r.error);
|
||||
cb.onProgress("Complete", 100);
|
||||
return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest);
|
||||
}
|
||||
cb.onLog("epic: assembler not started (" + r.error + "); writing the files here");
|
||||
}
|
||||
int totalFiles = pending.size(), doneFiles = 0;
|
||||
long writtenBytes = 0, writeTotal = 0;
|
||||
for (FileInfo f : pending) writeTotal += f.fileSize();
|
||||
cb.onLog("epic: engine=bounded-java (shared path; native availability does not change cache semantics)");
|
||||
Map<String, Integer> remainingRefs = chunkReferences(pending);
|
||||
Set<String> creditedChunks = new LinkedHashSet<>();
|
||||
long completedBytes = 0, writtenBytes = 0;
|
||||
long fetchedBytes = 0;
|
||||
long transferStarted = System.nanoTime();
|
||||
int completedCount = 0, doneFiles = 0, totalFiles = pending.size();
|
||||
for (FileInfo file : pending) {
|
||||
if (cancel.get()) return null;
|
||||
String relPath = file.filename.replace("\\", "/");
|
||||
File outFile = new File(installDir, relPath);
|
||||
File parent = outFile.getParentFile();
|
||||
if (parent != null) parent.mkdirs();
|
||||
cb.onProgress("Writing: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), 80 + (int) (doneFiles * 20L / totalFiles));
|
||||
try (FileOutputStream fos = new FileOutputStream(outFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) {
|
||||
// Like the native assembler, write to the final path and remove it on failure.
|
||||
// A sibling "<name>.part" can itself be a legitimate file in the manifest.
|
||||
File partFile = outFile;
|
||||
MessageDigest fileSha = expectedDigest(file.sha1);
|
||||
try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) {
|
||||
for (ChunkPart part : file.parts) {
|
||||
File cachedChunk = new File(chunkCacheDir, part.guidStr());
|
||||
if (!cachedChunk.exists()) throw new InstallException("Missing chunk " + part.guidStr() + " for " + relPath);
|
||||
byte[] data = readFile(cachedChunk);
|
||||
bos.write(data, part.offset, part.size);
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
String guid = part.guidStr();
|
||||
ChunkInfo chunk = chunksByGuid.get(guid);
|
||||
if (chunk == null) throw new InstallException("Missing chunk metadata " + guid + " for " + relPath);
|
||||
File cachedChunk = new File(chunkCacheDir, guid);
|
||||
if (!cachedChunk.isFile() || !isValidCachedChunk(chunk, cachedChunk)) {
|
||||
if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) {
|
||||
throw new InstallException("Could not clear invalid Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.remove(guid);
|
||||
cachedChunk = new File(chunkCacheDir, guid);
|
||||
if (!downloadChunkStreaming(chunk, manifest.chunkDir, cdnUrls, cachedChunk, cancel)) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
throw new InstallException("Failed to download Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.put(guid, cachedChunk.length());
|
||||
fetchedBytes += Math.max(chunk.fileSize, 1);
|
||||
}
|
||||
if (creditedChunks.add(guid)) {
|
||||
completedBytes += Math.max(chunk.fileSize, 1);
|
||||
completedCount++;
|
||||
cb.onProgress("Downloading chunks (" + completedCount + "/" + totalChunks + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes)));
|
||||
}
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000);
|
||||
cb.onBytes(completedBytes, fTotalBytes, fetchedBytes * 1000 / elapsedMs);
|
||||
copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath);
|
||||
Integer refs = remainingRefs.get(guid);
|
||||
if (refs != null) {
|
||||
if (refs <= 1) {
|
||||
remainingRefs.remove(guid);
|
||||
if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) {
|
||||
throw new InstallException("Could not release consumed Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.remove(guid);
|
||||
} else remainingRefs.put(guid, refs - 1);
|
||||
}
|
||||
trimCacheToBound(chunkCacheDir, cachedChunkBytes);
|
||||
}
|
||||
bos.flush();
|
||||
} catch (IOException e) {
|
||||
partFile.delete();
|
||||
if (cancel.get()) return null;
|
||||
throw new InstallException("Could not assemble " + relPath + ": " + e.getMessage());
|
||||
} catch (InstallException e) {
|
||||
partFile.delete();
|
||||
throw e;
|
||||
}
|
||||
if (cancel.get()) { partFile.delete(); return null; }
|
||||
if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) {
|
||||
partFile.delete();
|
||||
throw new InstallException("SHA-1 mismatch for " + relPath);
|
||||
}
|
||||
doneFiles++;
|
||||
writtenBytes += file.fileSize();
|
||||
cb.onStage("install", writtenBytes, writeTotal, doneFiles, totalFiles);
|
||||
cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles);
|
||||
}
|
||||
deleteDir(chunkCacheDir);
|
||||
cb.onProgress("Complete", 100);
|
||||
@@ -322,75 +321,154 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
}
|
||||
|
||||
private static final class NativeOutcome { boolean started, success, cancelled; String error = ""; }
|
||||
|
||||
/** The pending files as indices into the manifest's file list, what both native runs take; null when one is not in it. */
|
||||
private static int[] pendingIndices(Manifest manifest, List<FileInfo> pending) {
|
||||
java.util.IdentityHashMap<FileInfo, Integer> index = new java.util.IdentityHashMap<>();
|
||||
for (int i = 0; i < manifest.files.size(); i++) index.put(manifest.files.get(i), i);
|
||||
int[] pendingIdx = new int[pending.size()];
|
||||
for (int i = 0; i < pending.size(); i++) {
|
||||
Integer k = index.get(pending.get(i));
|
||||
if (k == null) return null;
|
||||
pendingIdx[i] = k;
|
||||
/**
|
||||
* Discard abandoned/orphaned cache entries and retain only verified chunks this install uses.
|
||||
* A bounded run consumes each chunk at its last part reference, so stale chunks from an older
|
||||
* all-at-once run must not silently occupy the space needed by the game files.
|
||||
*/
|
||||
static Map<String, Long> prepareBoundedCache(File cacheDir, List<ChunkInfo> needed) throws InstallException {
|
||||
Map<String, ChunkInfo> byGuid = new LinkedHashMap<>();
|
||||
for (ChunkInfo chunk : needed) byGuid.put(chunk.guidStr(), chunk);
|
||||
Map<String, Long> valid = new LinkedHashMap<>();
|
||||
File[] entries = cacheDir.listFiles();
|
||||
if (entries == null) throw new InstallException("Could not inspect Epic chunk cache");
|
||||
for (File entry : entries) {
|
||||
ChunkInfo chunk = byGuid.get(entry.getName());
|
||||
if (chunk == null || !isValidCachedChunk(chunk, entry)) {
|
||||
deleteDir(entry);
|
||||
if (entry.exists()) throw new InstallException("Could not clear stale Epic cache entry " + entry.getName());
|
||||
continue;
|
||||
}
|
||||
valid.put(entry.getName(), entry.length());
|
||||
}
|
||||
return pendingIdx;
|
||||
trimCacheToBound(cacheDir, valid);
|
||||
return valid;
|
||||
}
|
||||
|
||||
/** The native chunk pool on the same counters; never throws. */
|
||||
private static NativeOutcome runNativePool(Context ctx, byte[] manifestBytes, int[] pendingIdx, List<ChunkInfo> needed, long totalBytes,
|
||||
String installDirPath, String cachePath, List<CdnUrl> cdnUrls, AtomicBoolean cancel, Callback cb,
|
||||
AtomicLong completedBytes, AtomicInteger completedCount, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) {
|
||||
NativeOutcome r = new NativeOutcome();
|
||||
try {
|
||||
String[] prefixes = new String[cdnUrls.size()];
|
||||
for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir;
|
||||
final int totalChunks = needed.size();
|
||||
StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx);
|
||||
int workers = Math.max(1, Math.min(128, tier.getNetworkWindow()));
|
||||
int process = Math.max(2, tier.getProcessWorkers());
|
||||
cb.onLog("epic: engine=native workers=" + workers + " process_workers=" + process + " tier=" + tier.getId() + (cachePath.isEmpty() ? "" : " cache=scratch"));
|
||||
EpicNative.Result res = EpicNative.run(manifestBytes, installDirPath, cachePath, prefixes, pendingIdx, totalChunks, totalBytes, "", workers, process, cancel,
|
||||
new EpicNative.Listener() {
|
||||
@Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); }
|
||||
@Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) {
|
||||
completedBytes.set(bytesDone);
|
||||
completedCount.set(chunksDone);
|
||||
sampleSpeed(bytesDone, lastSpeedMs, lastSpeedBytes, speedBps);
|
||||
cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (bytesDone * 80L / Math.max(1, totalBytes)));
|
||||
cb.onBytes(bytesDone, totalBytes, speedBps.get());
|
||||
}
|
||||
@Override public void onLog(String line) { cb.onLog(line); }
|
||||
@Override public void onComplete(boolean success, String error, long bytesCredited) {}
|
||||
});
|
||||
r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error;
|
||||
} catch (Throwable t) {
|
||||
r.started = false;
|
||||
r.error = t.getClass().getSimpleName() + ": " + t.getMessage();
|
||||
/** Evict oldest reusable chunks until retained cache data is under the fixed disk ceiling. */
|
||||
private static void trimCacheToBound(File cacheDir, Map<String, Long> cacheSizes) throws InstallException {
|
||||
long retained = 0;
|
||||
for (long bytes : cacheSizes.values()) retained += bytes;
|
||||
java.util.Iterator<Map.Entry<String, Long>> iterator = cacheSizes.entrySet().iterator();
|
||||
while (retained > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) {
|
||||
Map.Entry<String, Long> entry = iterator.next();
|
||||
File chunk = new File(cacheDir, entry.getKey());
|
||||
long bytes = entry.getValue();
|
||||
if (chunk.exists() && !chunk.delete() && chunk.exists()) {
|
||||
throw new InstallException("Could not release Epic cache chunk " + entry.getKey());
|
||||
}
|
||||
retained -= bytes;
|
||||
iterator.remove();
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
/** The native assembler: the pending files written from the cache, the cache removed on success; never throws. */
|
||||
private static NativeOutcome runNativeAssembly(byte[] manifestBytes, int[] pendingIdx, int totalFiles, String installDirPath, String cachePath, AtomicBoolean cancel, Callback cb) {
|
||||
NativeOutcome r = new NativeOutcome();
|
||||
try {
|
||||
cb.onLog("epic: assembler=native files=" + totalFiles + (cachePath.isEmpty() ? "" : " cache=scratch"));
|
||||
EpicNative.Result res = EpicNative.assemble(manifestBytes, installDirPath, cachePath, pendingIdx, cancel, new EpicNative.Listener() {
|
||||
@Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {}
|
||||
@Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) {
|
||||
cb.onStage("install", bytesDone, bytesTotal, done, Math.max(total, totalFiles));
|
||||
cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles))));
|
||||
static boolean isValidCachedChunk(ChunkInfo chunk, File file) {
|
||||
if (!file.isFile()) return false;
|
||||
if (chunk.windowSize > 0 && file.length() != chunk.windowSize) return false;
|
||||
byte[] expected = chunk.sha1;
|
||||
if (expected == null || expected.length != 20) return true;
|
||||
boolean allZero = true;
|
||||
for (byte b : expected) if (b != 0) { allZero = false; break; }
|
||||
if (allZero) return true;
|
||||
try (FileInputStream in = new FileInputStream(file)) {
|
||||
MessageDigest sha = MessageDigest.getInstance("SHA-1");
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int count;
|
||||
while ((count = in.read(buffer)) != -1) sha.update(buffer, 0, count);
|
||||
return MessageDigest.isEqual(sha.digest(), expected);
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static MessageDigest expectedDigest(byte[] expected) throws InstallException {
|
||||
if (expected == null || expected.length != 20) return null;
|
||||
for (byte b : expected) if (b != 0) {
|
||||
try { return MessageDigest.getInstance("SHA-1"); }
|
||||
catch (java.security.NoSuchAlgorithmException e) { throw new InstallException("SHA-1 verification unavailable"); }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static Map<String, Integer> chunkReferences(List<FileInfo> files) {
|
||||
Map<String, Integer> refs = new LinkedHashMap<>();
|
||||
for (FileInfo file : files) for (ChunkPart part : file.parts) {
|
||||
String guid = part.guidStr();
|
||||
refs.put(guid, refs.containsKey(guid) ? refs.get(guid) + 1 : 1);
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulates the serial fetch → append part → last-reference eviction order. The returned
|
||||
* additional-space peak is measured against the current free-space reading: existing verified
|
||||
* chunks are already reflected in that reading and count as reclaimed only when actually freed.
|
||||
*/
|
||||
static long estimatePeakAdditionalBytes(List<FileInfo> files, Map<String, ChunkInfo> chunksByGuid,
|
||||
Map<String, Long> cachedChunkBytes, boolean includeFileWrites) {
|
||||
Map<String, Integer> refs = chunkReferences(files);
|
||||
Map<String, Long> residentChunks = new LinkedHashMap<>(cachedChunkBytes);
|
||||
long residentDelta = 0, peak = 0;
|
||||
long cacheResident = 0;
|
||||
for (long bytes : residentChunks.values()) cacheResident += bytes;
|
||||
for (FileInfo file : files) {
|
||||
for (ChunkPart part : file.parts) {
|
||||
String guid = part.guidStr();
|
||||
ChunkInfo chunk = chunksByGuid.get(guid);
|
||||
if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid);
|
||||
Long knownBytes = residentChunks.get(guid);
|
||||
long chunkBytes = knownBytes != null ? knownBytes : chunkWindowSize(chunk);
|
||||
if (knownBytes == null) {
|
||||
residentChunks.put(guid, chunkBytes);
|
||||
cacheResident += chunkBytes;
|
||||
residentDelta += chunkBytes;
|
||||
peak = Math.max(peak, residentDelta);
|
||||
}
|
||||
@Override public void onLog(String line) { cb.onLog(line); }
|
||||
@Override public void onComplete(boolean success, String error, long bytes) {}
|
||||
});
|
||||
r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error;
|
||||
} catch (Throwable t) {
|
||||
r.started = false;
|
||||
r.error = t.getClass().getSimpleName() + ": " + t.getMessage();
|
||||
if (includeFileWrites) {
|
||||
residentDelta += part.size & 0xFFFFFFFFL;
|
||||
peak = Math.max(peak, residentDelta);
|
||||
}
|
||||
int count = refs.get(guid);
|
||||
if (count <= 1) {
|
||||
refs.remove(guid);
|
||||
Long removed = residentChunks.remove(guid);
|
||||
if (removed != null) { cacheResident -= removed; residentDelta -= removed; }
|
||||
} else refs.put(guid, count - 1);
|
||||
java.util.Iterator<Map.Entry<String, Long>> iterator = residentChunks.entrySet().iterator();
|
||||
while (cacheResident > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) {
|
||||
Map.Entry<String, Long> entry = iterator.next();
|
||||
if (refs.containsKey(entry.getKey())) {
|
||||
long removed = entry.getValue();
|
||||
cacheResident -= removed;
|
||||
residentDelta -= removed;
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.max(0, peak);
|
||||
}
|
||||
|
||||
private static void copyChunkPart(File chunkFile, ChunkPart part, BufferedOutputStream out,
|
||||
MessageDigest fileSha, AtomicBoolean cancel, String relPath) throws IOException {
|
||||
long offset = part.offset & 0xFFFFFFFFL;
|
||||
long remaining = part.size & 0xFFFFFFFFL;
|
||||
long length = chunkFile.length();
|
||||
if (offset > length || remaining > length - offset) {
|
||||
throw new IOException("Chunk " + part.guidStr() + " is too short for " + relPath);
|
||||
}
|
||||
try (FileInputStream in = new FileInputStream(chunkFile)) {
|
||||
in.getChannel().position(offset);
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
while (remaining > 0) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
int count = in.read(buffer, 0, (int) Math.min(buffer.length, remaining));
|
||||
if (count < 0) throw new IOException("Unexpected end of chunk " + part.guidStr());
|
||||
out.write(buffer, 0, count);
|
||||
if (fileSha != null) fileSha.update(buffer, 0, count);
|
||||
remaining -= count;
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---- CDN list and manifest download ---------------------------------------------------------
|
||||
@@ -667,11 +745,12 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
|
||||
/** One chunk streamed from the first CDN that serves it, SHA-1 checked, published by rename. */
|
||||
private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List<CdnUrl> cdnUrls, File outFile) {
|
||||
private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List<CdnUrl> cdnUrls, File outFile, AtomicBoolean cancel) {
|
||||
String chunkPath = chunk.getPath(chunkDir);
|
||||
File tmp = new File(outFile.getPath() + ".part");
|
||||
tmp.delete();
|
||||
for (CdnUrl cdn : cdnUrls) {
|
||||
if (cancel.get()) return false;
|
||||
HttpURLConnection conn = null;
|
||||
try {
|
||||
conn = (HttpURLConnection) new URL(cdn.baseUrl + cdn.cloudDir + "/" + chunkPath).openConnection();
|
||||
@@ -697,12 +776,14 @@ public final class EpicDownloadManager {
|
||||
int storedAs = hdr.get() & 0xFF;
|
||||
if (headerSize > 41) skipFully(in, headerSize - 41);
|
||||
byte[] iobuf = new byte[131072];
|
||||
long outputBytes = 0;
|
||||
if ((storedAs & 1) != 0) {
|
||||
Inflater inflater = new Inflater();
|
||||
byte[] obuf = new byte[131072];
|
||||
int remaining = compressedSize;
|
||||
try {
|
||||
while (remaining > 0 && !inflater.finished()) {
|
||||
if (cancel.get()) break;
|
||||
if (inflater.needsInput()) {
|
||||
int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining));
|
||||
if (n <= 0) break;
|
||||
@@ -710,29 +791,47 @@ public final class EpicDownloadManager {
|
||||
inflater.setInput(iobuf, 0, n);
|
||||
}
|
||||
int out = inflater.inflate(obuf);
|
||||
if (out > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); }
|
||||
if (out > 0) {
|
||||
outputBytes = checkedChunkWriteSize(chunk, outputBytes, out);
|
||||
fos.write(obuf, 0, out);
|
||||
if (sha != null) sha.update(obuf, 0, out);
|
||||
}
|
||||
if (out == 0 && !inflater.needsInput() && !inflater.finished()) {
|
||||
throw new IOException("invalid compressed Epic chunk");
|
||||
}
|
||||
}
|
||||
int out;
|
||||
while ((out = inflater.inflate(obuf)) > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); }
|
||||
while ((out = inflater.inflate(obuf)) > 0) {
|
||||
outputBytes = checkedChunkWriteSize(chunk, outputBytes, out);
|
||||
fos.write(obuf, 0, out);
|
||||
if (sha != null) sha.update(obuf, 0, out);
|
||||
}
|
||||
if (!cancel.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk");
|
||||
} finally {
|
||||
inflater.end();
|
||||
}
|
||||
} else {
|
||||
int remaining = compressedSize;
|
||||
while (remaining > 0) {
|
||||
if (cancel.get()) break;
|
||||
int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining));
|
||||
if (n <= 0) break;
|
||||
outputBytes = checkedChunkWriteSize(chunk, outputBytes, n);
|
||||
fos.write(iobuf, 0, n);
|
||||
if (sha != null) sha.update(iobuf, 0, n);
|
||||
remaining -= n;
|
||||
}
|
||||
if (!cancel.get() && remaining != 0) throw new IOException("truncated Epic chunk");
|
||||
}
|
||||
}
|
||||
if (cancel.get()) { tmp.delete(); conn.disconnect(); return false; }
|
||||
if (chunk.windowSize > 0 && tmp.length() != chunk.windowSize) { tmp.delete(); conn.disconnect(); continue; }
|
||||
if (sha != null && !MessageDigest.isEqual(sha.digest(), chunk.sha1)) { tmp.delete(); conn.disconnect(); continue; }
|
||||
if (!tmp.renameTo(outFile)) { tmp.delete(); conn.disconnect(); continue; }
|
||||
conn.disconnect();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "chunk " + chunk.guidStr() + " failed: " + e.getMessage());
|
||||
tmp.delete();
|
||||
if (conn != null) conn.disconnect();
|
||||
}
|
||||
@@ -742,6 +841,17 @@ public final class EpicDownloadManager {
|
||||
return false;
|
||||
}
|
||||
|
||||
private static long checkedChunkWriteSize(ChunkInfo chunk, long written, int count) throws IOException {
|
||||
long next = written + count;
|
||||
if (next > chunkWindowSize(chunk)) throw new IOException("chunk exceeds expected window size");
|
||||
return next;
|
||||
}
|
||||
|
||||
private static long chunkWindowSize(ChunkInfo chunk) {
|
||||
// Legacy JSON manifests omit window sizes; their chunks use 1 MiB windows.
|
||||
return chunk.windowSize > 0 ? chunk.windowSize : 1024L * 1024;
|
||||
}
|
||||
|
||||
private static void readFully(InputStream in, byte[] buf) throws IOException {
|
||||
int off = 0;
|
||||
while (off < buf.length) { int n = in.read(buf, off, buf.length - off); if (n < 0) throw new IOException("stream ended"); off += n; }
|
||||
@@ -835,15 +945,6 @@ public final class EpicDownloadManager {
|
||||
|
||||
// ---- helpers --------------------------------------------------------------------------------
|
||||
|
||||
static byte[] readFile(File f) throws IOException {
|
||||
try (FileInputStream fis = new FileInputStream(f)) {
|
||||
byte[] data = new byte[(int) f.length()];
|
||||
int off = 0;
|
||||
while (off < data.length) { int r = fis.read(data, off, data.length - off); if (r < 0) break; off += r; }
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
static void deleteDir(File dir) {
|
||||
if (!dir.exists()) return;
|
||||
File[] files = dir.listFiles();
|
||||
@@ -851,20 +952,6 @@ public final class EpicDownloadManager {
|
||||
dir.delete();
|
||||
}
|
||||
|
||||
private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) {
|
||||
long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get();
|
||||
if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) {
|
||||
long prevB = lastSpeedBytes.getAndSet(done);
|
||||
long dt = nowMs - prevMs;
|
||||
if (dt > 0) speedBps.set((done - prevB) * 1000L / dt);
|
||||
}
|
||||
}
|
||||
|
||||
private static String speedSuffix(long bps) {
|
||||
if (bps <= 0) return "";
|
||||
return " " + (bps >= 1048576 ? String.format("%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s");
|
||||
}
|
||||
|
||||
private static String fmt(long bytes) {
|
||||
if (bytes >= 1073741824L) return String.format("%.1f GB", bytes / 1073741824.0);
|
||||
if (bytes >= 1048576L) return String.format("%.1f MB", bytes / 1048576.0);
|
||||
|
||||
@@ -9,17 +9,18 @@ import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* The native half of an Epic install in `libdroiddeckstores.so`: [run] fills the chunk cache with
|
||||
* The native batch APIs for Epic in `libdroiddeckstores.so`: [run] fills the chunk cache with
|
||||
* verified, decompressed chunks for the files the manager says are pending, [assemble] writes those
|
||||
* files from the cache. Everything around them (manifest, install tags, delta pass, post-install)
|
||||
* stays in the manager.
|
||||
* files from the cache. The production installer intentionally uses one serial Java
|
||||
* chunk-to-file path instead, so cache bounds and resume behavior do not depend on native
|
||||
* availability. These APIs remain for callers that explicitly need batch fetch/assembly.
|
||||
*
|
||||
* The cache is `<installDir>/.chunks` when `chunkCacheDir` is `""`, else the scratch folder given -
|
||||
* the app's own cache when the game installs to a card, where the assembler drops each chunk right
|
||||
* after its last use and the folder at the end. Both calls take the same value for one install.
|
||||
*
|
||||
* Both block like the Java loops they replace and poll [AtomicBoolean] `cancel` every 250 ms: on
|
||||
* cancel they flip the native flag, wait up to 5 s for the run to wind down, then return.
|
||||
* Both block while polling [AtomicBoolean] `cancel` every 250 ms: on cancel they flip the native
|
||||
* flag, wait up to 5 s for the run to wind down, then return.
|
||||
*/
|
||||
object EpicNative {
|
||||
private const val TAG = "EpicNative"
|
||||
|
||||
@@ -73,4 +73,11 @@ class StoreInstallRootTest {
|
||||
val check = StoreInstallRoot.checkStorage(app, zeroItem, StoreInstallRoot.internalRoot(app))
|
||||
assertTrue(check.enough)
|
||||
}
|
||||
|
||||
@Test fun epicDefersFullSizeChecksToItsManifestDeltaForResume() {
|
||||
val item = CatalogItem(Store.EPIC, "large", "Large", imageUrl = null, owned = true, sizeBytes = Long.MAX_VALUE)
|
||||
val check = StoreInstallRoot.checkStorage(app, item, StoreInstallRoot.internalRoot(app))
|
||||
assertTrue(check.enough)
|
||||
assertEquals(Long.MAX_VALUE, check.requiredBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package com.droiddeck.launcher.stores.epic
|
||||
|
||||
import java.io.File
|
||||
import java.net.InetAddress
|
||||
import java.net.ServerSocket
|
||||
import java.net.SocketException
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.security.MessageDigest
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import org.junit.After
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Before
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.rules.TemporaryFolder
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [33])
|
||||
class EpicBoundedInstallTest {
|
||||
@get:Rule val temp = TemporaryFolder()
|
||||
private lateinit var server: ServerSocket
|
||||
private lateinit var worker: Thread
|
||||
private lateinit var manifest: EpicDownloadManager.Manifest
|
||||
private lateinit var root: File
|
||||
private val requests = AtomicInteger()
|
||||
private val reject = AtomicBoolean()
|
||||
private val bytes = ByteArray(256 * 1024) { (it % 251).toByte() }
|
||||
|
||||
@Before fun setUp() {
|
||||
root = temp.newFolder("game")
|
||||
server = ServerSocket(0, 8, InetAddress.getByName("127.0.0.1"))
|
||||
worker = Thread {
|
||||
while (!server.isClosed) {
|
||||
val socket = try { server.accept() } catch (e: SocketException) {
|
||||
if (server.isClosed) break else throw e
|
||||
}
|
||||
socket.use {
|
||||
it.soTimeout = 5000
|
||||
val reader = it.getInputStream().bufferedReader()
|
||||
while (!reader.readLine().isNullOrEmpty()) { }
|
||||
requests.incrementAndGet()
|
||||
val out = it.getOutputStream()
|
||||
if (reject.get()) {
|
||||
out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
} else {
|
||||
val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN)
|
||||
.putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size)
|
||||
.put(ByteArray(24)).put(0).array()
|
||||
out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
out.write(header)
|
||||
out.write(bytes)
|
||||
}
|
||||
out.flush()
|
||||
}
|
||||
}
|
||||
}.apply { start() }
|
||||
val chunk = EpicDownloadManager.ChunkInfo().apply {
|
||||
guid = intArrayOf(1, 0, 0, 0)
|
||||
windowSize = bytes.size
|
||||
fileSize = bytes.size.toLong()
|
||||
sha1 = sha(bytes)
|
||||
}
|
||||
manifest = EpicDownloadManager.Manifest().apply {
|
||||
chunkDir = "chunks"
|
||||
uniqueChunks = arrayListOf(chunk)
|
||||
cdnUrls = listOf(EpicDownloadManager.CdnUrl("http://127.0.0.1:${server.localPort}", "", ""))
|
||||
files = arrayListOf(gameFile("game.bin", 0, bytes.size / 2), gameFile("game.bin.part", bytes.size / 2, bytes.size / 2))
|
||||
}
|
||||
}
|
||||
|
||||
@After fun tearDown() { server.close(); worker.join(6000); assertFalse(worker.isAlive) }
|
||||
|
||||
private fun sha(data: ByteArray) = MessageDigest.getInstance("SHA-1").digest(data)
|
||||
|
||||
private fun gameFile(name: String, offset: Int, size: Int) = EpicDownloadManager.FileInfo().apply {
|
||||
filename = name
|
||||
sha1 = sha(bytes.copyOfRange(offset, offset + size))
|
||||
parts.add(EpicDownloadManager.ChunkPart().apply {
|
||||
guid = intArrayOf(1, 0, 0, 0)
|
||||
this.offset = offset
|
||||
this.size = size
|
||||
})
|
||||
}
|
||||
|
||||
private fun install(cancel: AtomicBoolean = AtomicBoolean(), callback: EpicDownloadManager.Callback =
|
||||
EpicDownloadManager.Callback { _, _ -> }, cache: String = "") =
|
||||
EpicDownloadManager.installManifest(manifest, root.path, cache, null, cancel, callback)
|
||||
|
||||
@Test fun installsSharedChunksAndPreservesLegitimatePartFilename() {
|
||||
assertNotNull(install())
|
||||
assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes())
|
||||
assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes())
|
||||
assertEquals(1, requests.get())
|
||||
assertFalse(File(root, ".chunks").exists())
|
||||
assertNotNull(install())
|
||||
assertEquals("Verified files must not be downloaded again", 1, requests.get())
|
||||
}
|
||||
|
||||
@Test fun pauseKeepsCompletedFilesAndResumeReusesSharedCache() {
|
||||
val cancel = AtomicBoolean()
|
||||
val callback = object : EpicDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {}
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
if (stage == "install" && items == 1) cancel.set(true)
|
||||
}
|
||||
}
|
||||
assertNull(install(cancel, callback))
|
||||
assertTrue(File(root, "game.bin").exists())
|
||||
assertFalse(File(root, "game.bin.part").exists())
|
||||
assertEquals(1, requests.get())
|
||||
assertNotNull(install())
|
||||
assertEquals(1, requests.get())
|
||||
}
|
||||
|
||||
@Test fun cancellationDuringChunkDeliveryRemovesIncompleteOutput() {
|
||||
val cancel = AtomicBoolean()
|
||||
val callback = object : EpicDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {
|
||||
if (message.startsWith("Downloading chunks")) cancel.set(true)
|
||||
}
|
||||
}
|
||||
assertNull(install(cancel, callback))
|
||||
assertFalse(File(root, "game.bin").exists())
|
||||
assertNotNull(install())
|
||||
assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes())
|
||||
}
|
||||
|
||||
@Test fun fileHashFailureIsExplicitAndRetryRepairsIt() {
|
||||
val expected = manifest.files[0].sha1
|
||||
manifest.files[0].sha1 = ByteArray(20) { 1 }
|
||||
val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertTrue(error.message.orEmpty().contains("SHA-1 mismatch"))
|
||||
assertFalse(File(root, "game.bin").exists())
|
||||
manifest.files[0].sha1 = expected
|
||||
assertNotNull(install())
|
||||
}
|
||||
|
||||
@Test fun corruptCacheIsRefetchedAndScratchCacheIsRemovedOnSuccess() {
|
||||
val cache = temp.newFolder("scratch")
|
||||
File(cache, manifest.uniqueChunks[0].guidStr()).writeBytes(ByteArray(bytes.size))
|
||||
assertNotNull(install(cache = cache.path))
|
||||
assertEquals(1, requests.get())
|
||||
assertFalse(cache.exists())
|
||||
}
|
||||
|
||||
@Test fun networkFailureLeavesNoCompletedFileAndCanBeRetried() {
|
||||
reject.set(true)
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertFalse(File(root, "game.bin").exists())
|
||||
reject.set(false)
|
||||
assertNotNull(install())
|
||||
}
|
||||
|
||||
@Test fun hogwartsSizedSequentialPlanFitsAvailableSpaceWithoutFullChunkCache() {
|
||||
val gib = 1024L * 1024 * 1024
|
||||
val file = EpicDownloadManager.FileInfo()
|
||||
val chunks = LinkedHashMap<String, EpicDownloadManager.ChunkInfo>()
|
||||
repeat(72) { index ->
|
||||
val chunk = EpicDownloadManager.ChunkInfo().apply {
|
||||
guid = intArrayOf(index + 1, 0, 0, 0)
|
||||
windowSize = gib.toInt()
|
||||
}
|
||||
chunks[chunk.guidStr()] = chunk
|
||||
file.parts.add(EpicDownloadManager.ChunkPart().apply {
|
||||
guid = chunk.guid
|
||||
size = chunk.windowSize
|
||||
})
|
||||
}
|
||||
val peak = EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true)
|
||||
assertEquals(73 * gib, peak)
|
||||
assertTrue(peak < 96 * gib)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
package com.droiddeck.launcher.stores.epic
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Assert.assertThrows
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
@@ -12,6 +14,103 @@ import org.robolectric.annotation.Config
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [33])
|
||||
class EpicJsonManifestTest {
|
||||
@Test
|
||||
fun boundedInstallPeakCountsOnlyOneLiveChunkForSequentialChunks() {
|
||||
val first = chunk(1)
|
||||
val second = chunk(2)
|
||||
val file = file(part(1, 0, 100), part(2, 0, 100))
|
||||
val chunks = mapOf(first.guidStr() to first, second.guidStr() to second)
|
||||
|
||||
assertEquals(300L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true))
|
||||
assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun sharedCachedChunkIsReclaimedOnlyAfterItsLastPart() {
|
||||
val shared = chunk(1)
|
||||
val files = listOf(file(part(1, 0, 60)), file(part(1, 60, 40)))
|
||||
|
||||
assertEquals(
|
||||
100L,
|
||||
EpicDownloadManager.estimatePeakAdditionalBytes(
|
||||
files, mapOf(shared.guidStr() to shared), mapOf(shared.guidStr() to 100L), true,
|
||||
),
|
||||
)
|
||||
assertEquals(
|
||||
200L,
|
||||
EpicDownloadManager.estimatePeakAdditionalBytes(
|
||||
files, mapOf(shared.guidStr() to shared), emptyMap(), true,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun distantSharedChunksAreEvictedAndRefetchedWithinTheCacheCeiling() {
|
||||
val first = chunk(1).apply { windowSize = 40 * 1024 * 1024 }
|
||||
val second = chunk(2).apply { windowSize = 40 * 1024 * 1024 }
|
||||
val files = listOf(
|
||||
file(part(1, 0, 40 * 1024 * 1024)),
|
||||
file(part(2, 0, 40 * 1024 * 1024)),
|
||||
file(part(1, 0, 40 * 1024 * 1024)),
|
||||
file(part(2, 0, 40 * 1024 * 1024)),
|
||||
)
|
||||
val chunks = mapOf(first.guidStr() to first, second.guidStr() to second)
|
||||
|
||||
val gamePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true)
|
||||
val cachePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false)
|
||||
|
||||
assertEquals(200L * 1024 * 1024, gamePeak)
|
||||
assertTrue(cachePeak <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 40L * 1024 * 1024)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun resumeCacheDropsCorruptAndUnneededEntriesButKeepsVerifiedChunks() {
|
||||
val dir = java.io.File(System.getProperty("user.dir"), "build/epic-cache-test-${System.nanoTime()}")
|
||||
assertTrue(dir.mkdirs())
|
||||
try {
|
||||
val bytes = ByteArray(100) { (it % 251).toByte() }
|
||||
val expectedSha = java.security.MessageDigest.getInstance("SHA-1").digest(bytes)
|
||||
val valid = chunk(1).apply { sha1 = expectedSha }
|
||||
val corrupt = chunk(2).apply { sha1 = expectedSha }
|
||||
java.io.File(dir, valid.guidStr()).writeBytes(bytes)
|
||||
java.io.File(dir, corrupt.guidStr()).writeBytes(ByteArray(100) { 1 })
|
||||
java.io.File(dir, "orphan").writeBytes(bytes)
|
||||
|
||||
val retained = EpicDownloadManager.prepareBoundedCache(dir, listOf(valid, corrupt))
|
||||
|
||||
assertEquals(mapOf(valid.guidStr() to 100L), retained)
|
||||
assertTrue(java.io.File(dir, valid.guidStr()).exists())
|
||||
assertFalse(java.io.File(dir, corrupt.guidStr()).exists())
|
||||
assertFalse(java.io.File(dir, "orphan").exists())
|
||||
} finally {
|
||||
EpicDownloadManager.deleteDir(dir)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun referenceCountsIncludeRepeatedPartsSoCancelledFilesCanBeReplanned() {
|
||||
val pending = listOf(file(part(1, 0, 20), part(1, 20, 20)), file(part(1, 40, 60)))
|
||||
|
||||
assertEquals(3, EpicDownloadManager.chunkReferences(pending)["00000001000000000000000000000000"])
|
||||
}
|
||||
|
||||
private fun chunk(id: Int) = EpicDownloadManager.ChunkInfo().apply {
|
||||
guid = intArrayOf(id, 0, 0, 0)
|
||||
windowSize = 100
|
||||
fileSize = 100
|
||||
}
|
||||
|
||||
private fun part(id: Int, offset: Int, size: Int) = EpicDownloadManager.ChunkPart().apply {
|
||||
guid = intArrayOf(id, 0, 0, 0)
|
||||
this.offset = offset
|
||||
this.size = size
|
||||
}
|
||||
|
||||
private fun file(vararg parts: EpicDownloadManager.ChunkPart) = EpicDownloadManager.FileInfo().apply {
|
||||
filename = "test.bin"
|
||||
this.parts.addAll(parts)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun blobNumbersAreDecimalBytesLeastSignificantFirst() {
|
||||
assertEquals(1L, EpicDownloadManager.blobToNum("001000000000"))
|
||||
|
||||
@@ -24,15 +24,25 @@ scanned (`StoreInstallRoot.roots`): internal, every card, and the Steam library'
|
||||
installs an earlier build put there. `<Store>` is `GOG`, `Epic` or `Amazon`; `<title>` is the title
|
||||
with unsafe characters dropped, at most 60 characters (`StoreInstallRoot.folderName`), chosen once;
|
||||
a rerun (repair, update) lands on the folder whose sidecar carries the game's id, wherever it is.
|
||||
An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`: the
|
||||
`chunkCacheDir` of both native calls (`EpicNative.run` fetches into it, `EpicNative.assemble`
|
||||
writes the files from it and drops each chunk after its last use, then the folder), and of the
|
||||
manager's own loops when the engine is not there; `""` keeps the cache beside the game, as an
|
||||
internal install has it. The scratch folder is removed on cancel-with-delete and on uninstall.
|
||||
The cache holds whole ~1 MiB chunk windows, shared with files this device does not install, so it
|
||||
is often larger than the game (Metalstorm: 9.1 GB of chunks for 4.4 GB of files); the free-space
|
||||
check, made after the delta pass, counts the missing chunks and the missing files on their own
|
||||
volumes. A successful run removes the cache.
|
||||
An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`;
|
||||
`""` keeps the cache beside the game, as an internal install has it. The scratch folder is
|
||||
removed on cancel-with-delete and on uninstall.
|
||||
|
||||
Epic installs use the same bounded Java writer whether the native engine is available or not.
|
||||
It fetches one chunk at a time, verifies it, and writes the required parts into the current
|
||||
game file. Chunks are deleted after their last use; reusable chunks are retained up to 64 MiB,
|
||||
with at most one additional incoming chunk before eviction. Evicted shared chunks may be
|
||||
downloaded again. This trades parallel-download throughput and potentially extra network
|
||||
traffic for bounded temporary disk usage. The native batch APIs remain available but are not
|
||||
used by this install path.
|
||||
|
||||
The space check follows the manifest delta pass and simulates file writes, chunk fetches,
|
||||
and evictions, rather than adding the entire cache to the entire game. Completed files and
|
||||
verified cached chunks are accounted for on resume. On a card, game and cache requirements
|
||||
are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed
|
||||
or cancelled output files are removed, and an interrupted process is repaired by the next
|
||||
delta pass. A successful run removes the cache. A resume within a large incomplete file
|
||||
restarts that file, not its individual parts.
|
||||
|
||||
## The sidecar
|
||||
|
||||
|
||||
Reference in new issue
Block a user