mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-10-14 19:27:09 -04:00
Add adaptive parallel Epic installation with safe staged recovery
This commit is contained in:
1 parent
05d1795a51
commit
cb7f745fcb
11 files changed
+2492
-182
No files matched your search
@@ -6,6 +6,7 @@ import android.util.Log;
|
||||
import com.droiddeck.launcher.stores.StoreLog;
|
||||
import com.droiddeck.launcher.stores.StoreInstallRoot;
|
||||
import com.droiddeck.launcher.stores.StoreInstalls;
|
||||
import com.droiddeck.launcher.stores.StoresNative;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
@@ -22,6 +23,8 @@ import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
@@ -32,15 +35,21 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.zip.Inflater;
|
||||
|
||||
/**
|
||||
* The Epic install pipeline, ported from Bannerlator's EpicDownloadManager: the manifest API's CDN
|
||||
* list, the manifest binary (ChunksV4, or the older JSON form), the files narrowed to the base
|
||||
* set plus the device's language, a delta pass that skips files already on disk with the right
|
||||
* size and SHA-1, then the bounded-cache chunk-to-file writer. The native batch engine remains
|
||||
* available for other callers, but this install path intentionally uses the same serial writer
|
||||
* whether or not native code is installed: a whole-game chunk cache can exceed available space.
|
||||
* size and SHA-1, then native adaptive parallel fetch/staged assembly when the library loads.
|
||||
* Builds without the library use bounded Java parallel prefetch and one staged writer instead.
|
||||
* Neither production path requires a whole-game chunk cache.
|
||||
*
|
||||
* Chunk URLs carry no auth tokens; the subfolder is the decimal group number; the Cloudflare
|
||||
* mirror is skipped (it refuses chunks). Blocking; the download queue runs it on its own thread.
|
||||
@@ -51,6 +60,9 @@ public final class EpicDownloadManager {
|
||||
private static final String UA = EpicAuthClient.USER_AGENT;
|
||||
/** Keep reusable chunk data bounded even when a manifest shares chunks across distant files. */
|
||||
static final long MAX_RETAINED_CHUNK_CACHE_BYTES = 64L * 1024 * 1024;
|
||||
static final int PREFETCH_PARTS = 8;
|
||||
static final String STAGING_DIRECTORY = ".droiddeck-epic-staging";
|
||||
private static final String STAGING_OWNER = "DroidDeck Epic staging v1\n";
|
||||
|
||||
public interface Callback {
|
||||
void onProgress(String message, int pct);
|
||||
@@ -175,6 +187,7 @@ public final class EpicDownloadManager {
|
||||
final String cachePath = chunkCacheDirPath == null ? "" : chunkCacheDirPath;
|
||||
final File chunkCacheDir = cachePath.isEmpty() ? new File(installDir, ".chunks") : new File(cachePath);
|
||||
chunkCacheDir.mkdirs();
|
||||
File stagingDir = recoverOwnedStaging(installDir, manifest.files);
|
||||
|
||||
List<FileInfo> selected = resolveInstallFiles(manifest, installTags);
|
||||
cb.onLog("epic: " + selected.size() + "/" + manifest.files.size() + " files for tags " + (installTags == null ? "(all)" : installTags.toString()));
|
||||
@@ -216,99 +229,120 @@ public final class EpicDownloadManager {
|
||||
if (output.isFile()) oldFileBytes.put(f.filename, output.length());
|
||||
}
|
||||
boolean cacheOnInstallVolume = cachePath.isEmpty();
|
||||
long peakGameAdditional = cacheOnInstallVolume
|
||||
boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded();
|
||||
long peakGameAdditional = nativeAvailable ? 0 : cacheOnInstallVolume
|
||||
? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true, oldFileBytes)
|
||||
: estimateStagedFilePeak(pending, oldFileBytes);
|
||||
long peakCacheAdditional = cacheOnInstallVolume
|
||||
long peakCacheAdditional = nativeAvailable || cacheOnInstallVolume
|
||||
? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false);
|
||||
long freeGame = StoreInstallRoot.availableBytes(installDir);
|
||||
if (cacheOnInstallVolume) {
|
||||
if (!nativeAvailable && cacheOnInstallVolume) {
|
||||
if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free");
|
||||
} else {
|
||||
} else if (!nativeAvailable) {
|
||||
long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir);
|
||||
if (StoreInstallRoot.insufficientSpace(peakCacheAdditional, freeCache)) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free");
|
||||
if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need " + fmt(peakGameAdditional) + ", only " + fmt(freeGame) + " free");
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
if (nativeAvailable) {
|
||||
JSONObject request = nativeRequest(manifest, pending, chunksByGuid, cachedChunkBytes,
|
||||
oldFileBytes, installDir, chunkCacheDir, stagingDir, cacheOnInstallVolume);
|
||||
request.put("free_game", StoreInstallRoot.availableBytes(installDir));
|
||||
request.put("free_cache", StoreInstallRoot.availableBytes(chunkCacheDir));
|
||||
final long nativeTotal = pendingBytes;
|
||||
final long nativeStarted = System.nanoTime();
|
||||
cb.onSizes(nativeTotal, planned);
|
||||
cb.onBytes(0, nativeTotal, 0);
|
||||
EpicNative.Result outcome = EpicNative.installAdaptive(request.toString(), cancel, new EpicNative.InstallListener() {
|
||||
@Override public void onPlan(int chunks, long bytes, String mode) {
|
||||
cb.onLog("epic: engine=adaptive-rust " + mode);
|
||||
}
|
||||
@Override public void onProgress(long done, long total, int files, int filesTotal) {
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000);
|
||||
cb.onBytes(done, total, done * 1000 / elapsedMs);
|
||||
cb.onProgress("Downloading chunks…", (int) (done * 80L / Math.max(1, total)));
|
||||
cb.onStage("stream", done, total, files, filesTotal);
|
||||
}
|
||||
@Override public void onLog(String line) { cb.onLog(line); }
|
||||
@Override public void onComplete(boolean success, String error, long bytes) {}
|
||||
@Override public boolean beforeSwap() {
|
||||
try {
|
||||
if (cancel.get()) return false;
|
||||
StoreInstalls.markUpdating(installDir);
|
||||
return !cancel.get();
|
||||
} catch (Exception e) {
|
||||
cb.onLog("epic: cannot mark updating: " + e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
});
|
||||
if (outcome.cancelled || cancel.get()) return null;
|
||||
if (!outcome.success) throw new InstallException("Native Epic install failed: " + outcome.error);
|
||||
cb.onStage("install", nativeTotal, nativeTotal, pending.size(), pending.size());
|
||||
cb.onProgress("Complete", 100);
|
||||
return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest);
|
||||
}
|
||||
cb.onLog("epic: cache " + (cacheOnInstallVolume ? "beside the game" : "scratch") + ", peak game additional=" + fmt(peakGameAdditional) + (cacheOnInstallVolume ? "" : ", peak cache additional=" + fmt(peakCacheAdditional)) + " for " + fmt(pendingBytes) + " of files");
|
||||
final long fTotalBytes = pendingBytes;
|
||||
final int totalChunks = needed.size();
|
||||
cb.onSizes(fTotalBytes, planned);
|
||||
cb.onBytes(0, fTotalBytes, 0);
|
||||
cb.onLog("epic: " + fmt(fTotalBytes) + " to write using " + totalChunks + " unique chunks");
|
||||
cb.onLog("epic: engine=bounded-java (shared path; native availability does not change cache semantics)");
|
||||
Map<String, Integer> remainingRefs = chunkReferences(pending);
|
||||
cb.onLog("epic: engine=bounded-java (8-worker prefetch; native availability does not change cache semantics)");
|
||||
long completedBytes = 0, writtenBytes = 0;
|
||||
long transferStarted = System.nanoTime();
|
||||
int completedCount = 0, doneFiles = 0, totalFiles = pending.size();
|
||||
Set<String> reservedPaths = new LinkedHashSet<>();
|
||||
for (FileInfo file : manifest.files) reservedPaths.add(new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath());
|
||||
boolean updateRecorded = false;
|
||||
for (FileInfo file : pending) {
|
||||
if (cancel.get()) return null;
|
||||
String relPath = file.filename.replace("\\", "/");
|
||||
File outFile = new File(installDir, relPath);
|
||||
cb.onProgress("Downloading chunks: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), (int) (completedBytes * 80L / Math.max(1, fTotalBytes)));
|
||||
MessageDigest fileSha = expectedDigest(file.sha1);
|
||||
File partFile = createStagingFile(outFile, reservedPaths);
|
||||
try {
|
||||
try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) {
|
||||
for (ChunkPart part : file.parts) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
String guid = part.guidStr();
|
||||
ChunkInfo chunk = chunksByGuid.get(guid);
|
||||
if (chunk == null) throw new InstallException("Missing chunk metadata " + guid + " for " + relPath);
|
||||
File cachedChunk = new File(chunkCacheDir, guid);
|
||||
if (!cachedChunk.isFile() || !isValidCachedChunk(chunk, cachedChunk)) {
|
||||
if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) {
|
||||
throw new InstallException("Could not clear invalid Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.remove(guid);
|
||||
cachedChunk = new File(chunkCacheDir, guid);
|
||||
if (!downloadChunkStreaming(chunk, manifest.chunkDir, cdnUrls, cachedChunk, cancel)) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
throw new InstallException("Failed to download Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.put(guid, cachedChunk.length());
|
||||
try (ChunkPrefetch prefetch = new ChunkPrefetch(pending, chunksByGuid, cachedChunkBytes,
|
||||
chunkCacheDir, manifest.chunkDir, cdnUrls, cancel)) {
|
||||
for (FileInfo file : pending) {
|
||||
if (cancel.get()) return null;
|
||||
String relPath = file.filename.replace("\\", "/");
|
||||
File outFile = new File(installDir, relPath);
|
||||
cb.onProgress("Downloading chunks: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), (int) (completedBytes * 80L / Math.max(1, fTotalBytes)));
|
||||
MessageDigest fileSha = expectedDigest(file.sha1);
|
||||
File partFile = createStagingFile(outFile, reservedPaths, stagingDir);
|
||||
try {
|
||||
try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) {
|
||||
for (ChunkPart part : file.parts) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
String guid = part.guidStr();
|
||||
File cachedChunk = prefetch.await(guid);
|
||||
completedCount++;
|
||||
cb.onProgress("Downloading chunks (part " + completedCount + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes)));
|
||||
copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath);
|
||||
completedBytes += part.size & 0xFFFFFFFFL;
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000);
|
||||
cb.onBytes(completedBytes, fTotalBytes, completedBytes * 1000 / elapsedMs);
|
||||
prefetch.consume();
|
||||
}
|
||||
completedCount++;
|
||||
cb.onProgress("Downloading chunks (part " + completedCount + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes)));
|
||||
copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath);
|
||||
completedBytes += part.size & 0xFFFFFFFFL;
|
||||
long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000);
|
||||
cb.onBytes(completedBytes, fTotalBytes, completedBytes * 1000 / elapsedMs);
|
||||
if (consumeChunkReference(remainingRefs, guid)) {
|
||||
if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) {
|
||||
throw new InstallException("Could not release consumed Epic chunk " + guid);
|
||||
}
|
||||
cachedChunkBytes.remove(guid);
|
||||
}
|
||||
trimCacheToBound(chunkCacheDir, cachedChunkBytes);
|
||||
bos.flush();
|
||||
fos.getFD().sync();
|
||||
}
|
||||
bos.flush();
|
||||
fos.getFD().sync();
|
||||
if (cancel.get()) return null;
|
||||
if (partFile.length() != file.fileSize()) throw new InstallException("Size mismatch for " + relPath);
|
||||
if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) {
|
||||
throw new InstallException("SHA-1 mismatch for " + relPath);
|
||||
}
|
||||
// Once a swap can mix versions, the old installed record is no longer truthful.
|
||||
if (!updateRecorded) {
|
||||
StoreInstalls.markUpdating(installDir);
|
||||
updateRecorded = true;
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
Files.move(partFile.toPath(), outFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (IOException e) {
|
||||
if (cancel.get()) return null;
|
||||
throw new InstallException("Could not assemble or atomically replace " + relPath + ": " + e.getMessage());
|
||||
} finally {
|
||||
partFile.delete();
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
if (partFile.length() != file.fileSize()) throw new InstallException("Size mismatch for " + relPath);
|
||||
if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) {
|
||||
throw new InstallException("SHA-1 mismatch for " + relPath);
|
||||
}
|
||||
// Once a swap can mix versions, the old installed record is no longer truthful.
|
||||
if (!updateRecorded) {
|
||||
StoreInstalls.markUpdating(installDir);
|
||||
updateRecorded = true;
|
||||
}
|
||||
if (cancel.get()) return null;
|
||||
Files.move(partFile.toPath(), outFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (IOException e) {
|
||||
if (cancel.get()) return null;
|
||||
throw new InstallException("Could not assemble or atomically replace " + relPath + ": " + e.getMessage());
|
||||
} finally {
|
||||
partFile.delete();
|
||||
doneFiles++;
|
||||
writtenBytes += file.fileSize();
|
||||
cb.onStage("stream", writtenBytes, pendingBytes, doneFiles, totalFiles);
|
||||
}
|
||||
doneFiles++;
|
||||
writtenBytes += file.fileSize();
|
||||
cb.onStage("stream", writtenBytes, pendingBytes, doneFiles, totalFiles);
|
||||
}
|
||||
deleteDir(chunkCacheDir);
|
||||
cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles);
|
||||
@@ -322,12 +356,108 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
}
|
||||
|
||||
private static File createStagingFile(File output, Set<String> reservedPaths) throws IOException {
|
||||
static File recoverOwnedStaging(File installDir, List<FileInfo> manifestFiles) throws IOException {
|
||||
File staging = new File(installDir, STAGING_DIRECTORY);
|
||||
Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY);
|
||||
if (!staging.getCanonicalFile().toPath().equals(expected) ||
|
||||
(Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS) &&
|
||||
!Files.isDirectory(staging.toPath(), LinkOption.NOFOLLOW_LINKS))) {
|
||||
throw new IOException("Unsafe Epic staging namespace; refusing to traverse it");
|
||||
}
|
||||
String namespace = expected.toString();
|
||||
for (FileInfo file : manifestFiles) {
|
||||
String path = new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath();
|
||||
if (path.equals(namespace) || path.startsWith(namespace + File.separator)) {
|
||||
throw new IOException("Manifest conflicts with Epic staging namespace");
|
||||
}
|
||||
}
|
||||
File owner = new File(staging, "owner");
|
||||
if (Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS)) {
|
||||
if (!Files.isRegularFile(owner.toPath(), LinkOption.NOFOLLOW_LINKS) ||
|
||||
!owner.getCanonicalFile().toPath().equals(expected.resolve("owner")) ||
|
||||
owner.length() != STAGING_OWNER.length() ||
|
||||
!STAGING_OWNER.equals(new String(Files.readAllBytes(owner.toPath()), StandardCharsets.UTF_8))) {
|
||||
throw new IOException("Unowned Epic staging namespace; refusing to delete it");
|
||||
}
|
||||
File[] files = staging.listFiles();
|
||||
if (files == null) throw new IOException("Could not inspect Epic staging");
|
||||
for (File file : files) {
|
||||
if (file.getName().equals("owner")) continue;
|
||||
if (!Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS) ||
|
||||
!file.getCanonicalFile().toPath().equals(expected.resolve(file.getName())) ||
|
||||
!file.getName().endsWith(".stage")) throw new IOException("Unexpected Epic staging entry");
|
||||
}
|
||||
for (File file : files) {
|
||||
if (file.getName().equals("owner")) continue;
|
||||
if (!file.delete()) throw new IOException("Could not recover orphan Epic staging");
|
||||
}
|
||||
} else {
|
||||
if (!staging.mkdir()) throw new IOException("Could not create Epic staging namespace");
|
||||
try (FileOutputStream out = new FileOutputStream(owner)) {
|
||||
out.write(STAGING_OWNER.getBytes(StandardCharsets.UTF_8));
|
||||
out.getFD().sync();
|
||||
}
|
||||
}
|
||||
return staging;
|
||||
}
|
||||
|
||||
private static JSONObject nativeRequest(Manifest manifest, List<FileInfo> files,
|
||||
Map<String, ChunkInfo> chunks, Map<String, Long> cached, Map<String, Long> old,
|
||||
File install, File cache, File staging, boolean sameVolume) throws Exception {
|
||||
JSONObject request = new JSONObject();
|
||||
request.put("schema_version", 1);
|
||||
request.put("install_dir", install.getCanonicalPath());
|
||||
request.put("cache_dir", cache.getCanonicalPath());
|
||||
request.put("staging_dir", staging.getCanonicalPath());
|
||||
request.put("same_volume", sameVolume);
|
||||
request.put("chunk_dir", manifest.chunkDir);
|
||||
request.put("max_workers", 32);
|
||||
JSONArray hosts = new JSONArray();
|
||||
for (CdnUrl cdn : manifest.cdnUrls) hosts.put(cdn.baseUrl + cdn.cloudDir);
|
||||
request.put("hosts", hosts);
|
||||
JSONArray chunkList = new JSONArray();
|
||||
for (ChunkInfo chunk : chunks.values()) {
|
||||
JSONObject item = new JSONObject();
|
||||
item.put("guid", chunk.guidStr());
|
||||
item.put("path", chunk.getPath(manifest.chunkDir));
|
||||
item.put("size", chunkWindowSize(chunk));
|
||||
item.put("exact_size", chunk.windowSize > 0);
|
||||
item.put("wire_size", Math.max(1, chunk.fileSize));
|
||||
item.put("sha1", hexDigest(chunk.sha1));
|
||||
item.put("cached", cached.containsKey(chunk.guidStr()) ? cached.get(chunk.guidStr()) : -1);
|
||||
chunkList.put(item);
|
||||
}
|
||||
request.put("chunks", chunkList);
|
||||
JSONArray fileList = new JSONArray();
|
||||
for (FileInfo file : files) {
|
||||
JSONObject item = new JSONObject();
|
||||
item.put("path", file.filename.replace("\\", "/"));
|
||||
item.put("sha1", hexDigest(file.sha1));
|
||||
item.put("old_size", old.containsKey(file.filename) ? old.get(file.filename) : 0);
|
||||
JSONArray parts = new JSONArray();
|
||||
for (ChunkPart part : file.parts) {
|
||||
parts.put(new JSONObject().put("guid", part.guidStr()).put("offset", part.offset & 0xFFFFFFFFL).put("size", part.size & 0xFFFFFFFFL));
|
||||
}
|
||||
item.put("parts", parts);
|
||||
fileList.put(item);
|
||||
}
|
||||
request.put("files", fileList);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static String hexDigest(byte[] digest) {
|
||||
if (digest == null || digest.length != 20) return "";
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (byte b : digest) out.append(String.format("%02x", b & 255));
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private static File createStagingFile(File output, Set<String> reservedPaths, File staging) throws IOException {
|
||||
File parent = output.getParentFile();
|
||||
if (!parent.isDirectory() && !parent.mkdirs()) throw new IOException("Could not create output directory");
|
||||
// Unique sibling staging is on the output volume, never a manifest's literal "<name>.part".
|
||||
// Owned staging is on the output volume, never a manifest's literal "<name>.part".
|
||||
while (true) {
|
||||
File stage = File.createTempFile(".droiddeck-epic-", ".stage", parent);
|
||||
File stage = File.createTempFile("file-", ".stage", staging);
|
||||
if (!reservedPaths.contains(stage.getCanonicalPath())) return stage;
|
||||
if (!stage.delete()) throw new IOException("Could not clear staging collision");
|
||||
}
|
||||
@@ -407,6 +537,135 @@ public final class EpicDownloadManager {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure disk schedule shared by execution and preflight. The next eight part references pin
|
||||
* at most eight GUIDs (including across file boundaries); other reusable data is oldest-first
|
||||
* capped at 64 MiB. Incoming downloads reserve their entire decoded size before submission.
|
||||
*/
|
||||
static final class PrefetchSchedule {
|
||||
final List<String> parts = new ArrayList<>();
|
||||
final Map<String, Long> resident;
|
||||
final Map<String, Integer> refs;
|
||||
final Map<String, ChunkInfo> chunks;
|
||||
int cursor;
|
||||
|
||||
PrefetchSchedule(List<FileInfo> files, Map<String, ChunkInfo> chunks, Map<String, Long> cached) {
|
||||
for (FileInfo file : files) for (ChunkPart part : file.parts) parts.add(part.guidStr());
|
||||
this.chunks = chunks;
|
||||
resident = new LinkedHashMap<>(cached);
|
||||
refs = chunkReferences(files);
|
||||
}
|
||||
|
||||
Set<String> window() {
|
||||
return new LinkedHashSet<>(parts.subList(cursor, Math.min(parts.size(), cursor + PREFETCH_PARTS)));
|
||||
}
|
||||
|
||||
List<String> fill() {
|
||||
List<String> fetch = new ArrayList<>();
|
||||
for (String guid : window()) if (!resident.containsKey(guid)) {
|
||||
ChunkInfo chunk = chunks.get(guid);
|
||||
if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid);
|
||||
resident.put(guid, chunkWindowSize(chunk));
|
||||
fetch.add(guid);
|
||||
}
|
||||
return fetch;
|
||||
}
|
||||
|
||||
List<String> consume() {
|
||||
List<String> release = new ArrayList<>();
|
||||
String guid = parts.get(cursor++);
|
||||
if (consumeChunkReference(refs, guid)) {
|
||||
resident.remove(guid);
|
||||
release.add(guid);
|
||||
}
|
||||
Set<String> pinned = window();
|
||||
long reusable = 0;
|
||||
for (Map.Entry<String, Long> entry : resident.entrySet()) {
|
||||
if (!pinned.contains(entry.getKey())) reusable += entry.getValue();
|
||||
}
|
||||
Iterator<Map.Entry<String, Long>> it = resident.entrySet().iterator();
|
||||
while (reusable > MAX_RETAINED_CHUNK_CACHE_BYTES && it.hasNext()) {
|
||||
Map.Entry<String, Long> entry = it.next();
|
||||
if (pinned.contains(entry.getKey())) continue;
|
||||
reusable -= entry.getValue();
|
||||
release.add(entry.getKey());
|
||||
it.remove();
|
||||
}
|
||||
return release;
|
||||
}
|
||||
|
||||
long bytes() {
|
||||
long bytes = 0;
|
||||
for (long size : resident.values()) bytes += size;
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ChunkPrefetch implements AutoCloseable {
|
||||
final PrefetchSchedule schedule;
|
||||
final File cacheDir;
|
||||
final AtomicBoolean cancel;
|
||||
final AtomicBoolean stopped = new AtomicBoolean();
|
||||
final ExecutorService workers = Executors.newFixedThreadPool(PREFETCH_PARTS);
|
||||
final Map<String, Future<Boolean>> jobs = new LinkedHashMap<>();
|
||||
final Set<HttpURLConnection> connections = ConcurrentHashMap.newKeySet();
|
||||
final String chunkDir;
|
||||
final List<CdnUrl> cdns;
|
||||
|
||||
ChunkPrefetch(List<FileInfo> files, Map<String, ChunkInfo> chunks, Map<String, Long> cached,
|
||||
File cacheDir, String chunkDir, List<CdnUrl> cdns, AtomicBoolean cancel) {
|
||||
schedule = new PrefetchSchedule(files, chunks, cached);
|
||||
this.cacheDir = cacheDir;
|
||||
this.chunkDir = chunkDir;
|
||||
this.cdns = cdns;
|
||||
this.cancel = cancel;
|
||||
}
|
||||
|
||||
File await(String guid) throws Exception {
|
||||
for (String fetch : schedule.fill()) {
|
||||
jobs.put(fetch, workers.submit(() -> downloadChunkStreaming(schedule.chunks.get(fetch),
|
||||
chunkDir, cdns, new File(cacheDir, fetch), cancel, stopped, connections)));
|
||||
}
|
||||
Future<Boolean> job = jobs.get(guid);
|
||||
if (job != null) while (true) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
try {
|
||||
if (!job.get(100, TimeUnit.MILLISECONDS)) {
|
||||
if (cancel.get()) throw new IOException("cancelled");
|
||||
throw new InstallException("Failed to download Epic chunk " + guid);
|
||||
}
|
||||
break;
|
||||
} catch (TimeoutException ignored) { }
|
||||
}
|
||||
return new File(cacheDir, guid);
|
||||
}
|
||||
|
||||
void consume() throws InstallException {
|
||||
for (String guid : schedule.consume()) {
|
||||
// Only the writer advances the schedule, after closing its chunk reader.
|
||||
jobs.remove(guid);
|
||||
File chunk = new File(cacheDir, guid);
|
||||
if (chunk.exists() && !chunk.delete() && chunk.exists()) {
|
||||
throw new InstallException("Could not release Epic cache chunk " + guid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override public void close() {
|
||||
stopped.set(true);
|
||||
for (HttpURLConnection connection : connections) connection.disconnect();
|
||||
workers.shutdownNow();
|
||||
boolean interrupted = false;
|
||||
while (true) {
|
||||
try {
|
||||
if (workers.awaitTermination(100, TimeUnit.MILLISECONDS)) break;
|
||||
for (HttpURLConnection connection : connections) connection.disconnect();
|
||||
} catch (InterruptedException e) { interrupted = true; }
|
||||
}
|
||||
if (interrupted) Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
static boolean isValidCachedChunk(ChunkInfo chunk, File file) {
|
||||
if (!file.isFile()) return false;
|
||||
if (chunk.windowSize > 0 && file.length() != chunk.windowSize) return false;
|
||||
@@ -445,7 +704,7 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulates the serial fetch → append part → last-reference eviction order. The returned
|
||||
* Simulates reserved parallel prefetch → append part → last-reference eviction order. The returned
|
||||
* additional-space peak is measured against the current free-space reading: existing verified
|
||||
* chunks are already reflected in that reading and count as reclaimed only when actually freed.
|
||||
* Old output bytes are reclaimed after the complete new file is staged and verified, never
|
||||
@@ -459,33 +718,21 @@ public final class EpicDownloadManager {
|
||||
static long estimatePeakAdditionalBytes(List<FileInfo> files, Map<String, ChunkInfo> chunksByGuid,
|
||||
Map<String, Long> cachedChunkBytes, boolean includeFileWrites,
|
||||
Map<String, Long> oldFileBytes) {
|
||||
Map<String, Integer> refs = chunkReferences(files);
|
||||
Map<String, Long> residentChunks = new LinkedHashMap<>(cachedChunkBytes);
|
||||
long residentDelta = 0, peak = 0;
|
||||
PrefetchSchedule schedule = new PrefetchSchedule(files, chunksByGuid, cachedChunkBytes);
|
||||
long initialCache = schedule.bytes();
|
||||
long fileDelta = 0, peak = 0;
|
||||
for (FileInfo file : files) {
|
||||
for (ChunkPart part : file.parts) {
|
||||
String guid = part.guidStr();
|
||||
ChunkInfo chunk = chunksByGuid.get(guid);
|
||||
if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid);
|
||||
Long knownBytes = residentChunks.get(guid);
|
||||
long chunkBytes = knownBytes != null ? knownBytes : chunkWindowSize(chunk);
|
||||
if (knownBytes == null) {
|
||||
residentChunks.put(guid, chunkBytes);
|
||||
residentDelta += chunkBytes;
|
||||
peak = Math.max(peak, residentDelta);
|
||||
}
|
||||
schedule.fill();
|
||||
peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache);
|
||||
if (includeFileWrites) {
|
||||
residentDelta += part.size & 0xFFFFFFFFL;
|
||||
peak = Math.max(peak, residentDelta);
|
||||
fileDelta += part.size & 0xFFFFFFFFL;
|
||||
peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache);
|
||||
}
|
||||
if (consumeChunkReference(refs, guid)) {
|
||||
Long removed = residentChunks.remove(guid);
|
||||
if (removed != null) residentDelta -= removed;
|
||||
}
|
||||
for (long bytes : evictOldestReusableChunks(residentChunks).values()) residentDelta -= bytes;
|
||||
schedule.consume();
|
||||
}
|
||||
// The verified staging file coexists with the old output until its atomic swap.
|
||||
if (includeFileWrites) residentDelta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0;
|
||||
if (includeFileWrites) fileDelta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0;
|
||||
}
|
||||
return Math.max(0, peak);
|
||||
}
|
||||
@@ -786,15 +1033,19 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
|
||||
/** One chunk streamed from the first CDN that serves it, SHA-1 checked, published by rename. */
|
||||
private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List<CdnUrl> cdnUrls, File outFile, AtomicBoolean cancel) {
|
||||
private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List<CdnUrl> cdnUrls,
|
||||
File outFile, AtomicBoolean cancel, AtomicBoolean stopped,
|
||||
Set<HttpURLConnection> connections) {
|
||||
String chunkPath = chunk.getPath(chunkDir);
|
||||
File tmp = new File(outFile.getPath() + ".part");
|
||||
tmp.delete();
|
||||
for (CdnUrl cdn : cdnUrls) {
|
||||
if (cancel.get()) return false;
|
||||
if (cancel.get() || stopped.get()) return false;
|
||||
HttpURLConnection conn = null;
|
||||
try {
|
||||
conn = (HttpURLConnection) new URL(cdn.baseUrl + cdn.cloudDir + "/" + chunkPath).openConnection();
|
||||
connections.add(conn);
|
||||
if (cancel.get() || stopped.get()) return false;
|
||||
conn.setConnectTimeout(30000);
|
||||
conn.setReadTimeout(60000);
|
||||
conn.setRequestProperty("User-Agent", UA);
|
||||
@@ -824,7 +1075,7 @@ public final class EpicDownloadManager {
|
||||
int remaining = compressedSize;
|
||||
try {
|
||||
while (remaining > 0 && !inflater.finished()) {
|
||||
if (cancel.get()) break;
|
||||
if (cancel.get() || stopped.get()) break;
|
||||
if (inflater.needsInput()) {
|
||||
int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining));
|
||||
if (n <= 0) break;
|
||||
@@ -843,18 +1094,19 @@ public final class EpicDownloadManager {
|
||||
}
|
||||
int out;
|
||||
while ((out = inflater.inflate(obuf)) > 0) {
|
||||
if (cancel.get() || stopped.get()) break;
|
||||
outputBytes = checkedChunkWriteSize(chunk, outputBytes, out);
|
||||
fos.write(obuf, 0, out);
|
||||
if (sha != null) sha.update(obuf, 0, out);
|
||||
}
|
||||
if (!cancel.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk");
|
||||
if (!cancel.get() && !stopped.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk");
|
||||
} finally {
|
||||
inflater.end();
|
||||
}
|
||||
} else {
|
||||
int remaining = compressedSize;
|
||||
while (remaining > 0) {
|
||||
if (cancel.get()) break;
|
||||
if (cancel.get() || stopped.get()) break;
|
||||
int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining));
|
||||
if (n <= 0) break;
|
||||
outputBytes = checkedChunkWriteSize(chunk, outputBytes, n);
|
||||
@@ -862,10 +1114,10 @@ public final class EpicDownloadManager {
|
||||
if (sha != null) sha.update(iobuf, 0, n);
|
||||
remaining -= n;
|
||||
}
|
||||
if (!cancel.get() && remaining != 0) throw new IOException("truncated Epic chunk");
|
||||
if (!cancel.get() && !stopped.get() && remaining != 0) throw new IOException("truncated Epic chunk");
|
||||
}
|
||||
}
|
||||
if (cancel.get()) { tmp.delete(); conn.disconnect(); return false; }
|
||||
if (cancel.get() || stopped.get()) { tmp.delete(); conn.disconnect(); return false; }
|
||||
if (chunk.windowSize > 0 && tmp.length() != chunk.windowSize) { tmp.delete(); conn.disconnect(); continue; }
|
||||
if (sha != null && !MessageDigest.isEqual(sha.digest(), chunk.sha1)) { tmp.delete(); conn.disconnect(); continue; }
|
||||
if (!tmp.renameTo(outFile)) { tmp.delete(); conn.disconnect(); continue; }
|
||||
@@ -875,6 +1127,12 @@ public final class EpicDownloadManager {
|
||||
Log.w(TAG, "chunk " + chunk.guidStr() + " failed: " + e.getMessage());
|
||||
tmp.delete();
|
||||
if (conn != null) conn.disconnect();
|
||||
} finally {
|
||||
if (conn != null) {
|
||||
connections.remove(conn);
|
||||
conn.disconnect();
|
||||
}
|
||||
tmp.delete();
|
||||
}
|
||||
}
|
||||
tmp.delete();
|
||||
|
||||
@@ -9,18 +9,17 @@ import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* The native batch APIs for Epic in `libdroiddeckstores.so`: [run] fills the chunk cache with
|
||||
* verified, decompressed chunks for the files the manager says are pending, [assemble] writes those
|
||||
* files from the cache. The production installer intentionally uses one serial Java
|
||||
* chunk-to-file path instead, so cache bounds and resume behavior do not depend on native
|
||||
* availability. These APIs remain for callers that explicitly need batch fetch/assembly.
|
||||
* [installAdaptive] is the production native Epic fetch/assembly engine. Its snapshot is the
|
||||
* manager's delta-selected pending files, verified cache and old output sizes. Rust chooses a
|
||||
* storage-safe ordered lookahead, overlaps fetch_core with staged assembly and calls beforeSwap
|
||||
* before the first verified replacement. A loaded-engine error fails explicitly.
|
||||
*
|
||||
* The cache is `<installDir>/.chunks` when `chunkCacheDir` is `""`, else the scratch folder given -
|
||||
* the app's own cache when the game installs to a card, where the assembler drops each chunk right
|
||||
* after its last use and the folder at the end. Both calls take the same value for one install.
|
||||
* the app's own cache when the game installs to a card. Adaptive installs release last-use chunks
|
||||
* in both layouts and remove the cache on success. [run]/[assemble] are legacy batch APIs.
|
||||
*
|
||||
* Both block while polling [AtomicBoolean] `cancel` every 250 ms: on cancel they flip the native
|
||||
* flag, wait up to 5 s for the run to wind down, then return.
|
||||
* Cancellation flips the native flag; the native handle is joined before returning, so no
|
||||
* cache/staging writer survives completion or cache cleanup.
|
||||
*/
|
||||
object EpicNative {
|
||||
private const val TAG = "EpicNative"
|
||||
@@ -37,6 +36,18 @@ object EpicNative {
|
||||
fun onComplete(success: Boolean, error: String, bytes: Long)
|
||||
}
|
||||
|
||||
interface InstallListener : Listener {
|
||||
/** Synchronous approval before the first verified atomic replacement. False aborts. */
|
||||
fun beforeSwap(): Boolean
|
||||
}
|
||||
|
||||
/** One native budgeted fetch/assembly run. A loaded-engine failure is never a Java retry. */
|
||||
@JvmStatic
|
||||
fun installAdaptive(request: String, cancel: AtomicBoolean, listener: InstallListener): Result =
|
||||
drive("nativeInstallAdaptive", cancel, listener, strict = true) { inner ->
|
||||
nativeInstallAdaptive(request, inner)
|
||||
}
|
||||
|
||||
/** Outcome of a run. `started == false`: nothing happened on disk, the caller runs its own loop. */
|
||||
class Result(
|
||||
@JvmField val started: Boolean, @JvmField val success: Boolean, @JvmField val cancelled: Boolean,
|
||||
@@ -59,13 +70,14 @@ object EpicNative {
|
||||
fun assemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, cancel: AtomicBoolean?, listener: Listener): Result =
|
||||
drive("nativeAssemble", cancel, listener) { inner -> nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, inner) }
|
||||
|
||||
private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, start: (Listener) -> Long): Result {
|
||||
private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, strict: Boolean = false, start: (Listener) -> Long): Result {
|
||||
if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0)
|
||||
val latch = CountDownLatch(1)
|
||||
val completion = AtomicReference<Completion?>(null)
|
||||
val doneRef = AtomicReference(0)
|
||||
val totalRef = AtomicReference(0)
|
||||
val inner = object : Listener {
|
||||
val inner = object : InstallListener {
|
||||
override fun beforeSwap(): Boolean = (listener as? InstallListener)?.beforeSwap() ?: false
|
||||
override fun onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String) { totalRef.set(chunksTotal); listener.onPlan(chunksTotal, bytesTotal, chunkDir) }
|
||||
override fun onProgress(bytesDone: Long, bytesTotal: Long, done: Int, total: Int) { doneRef.set(done); totalRef.set(total); listener.onProgress(bytesDone, bytesTotal, done, total) }
|
||||
override fun onLog(line: String) = listener.onLog(line)
|
||||
@@ -78,21 +90,26 @@ object EpicNative {
|
||||
val handle: Long = try {
|
||||
start(inner)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}")
|
||||
return Result(false, false, false, "$what: ${t.javaClass.simpleName}", 0L, 0, 0)
|
||||
Log.w(TAG, "${if (strict) "engine start failed (no fallback)" else "engine unavailable"}: ${t.javaClass.simpleName}: ${t.message}")
|
||||
return Result(strict, false, false, "$what: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0)
|
||||
}
|
||||
if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get())
|
||||
if (handle == 0L) return Result(strict, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get())
|
||||
var cancelSent = false
|
||||
try {
|
||||
while (!latch.await(250, TimeUnit.MILLISECONDS)) {
|
||||
if (cancel != null && cancel.get()) {
|
||||
nativeCancel(handle)
|
||||
cancelSent = true
|
||||
latch.await(5, TimeUnit.SECONDS)
|
||||
if (strict) {
|
||||
while (!latch.await(250, TimeUnit.MILLISECONDS)) { }
|
||||
} else {
|
||||
latch.await(5, TimeUnit.SECONDS)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (completion.get() == null) nativeCancel(handle)
|
||||
nativeRelease(handle)
|
||||
}
|
||||
val c = completion.get()
|
||||
@@ -101,6 +118,9 @@ object EpicNative {
|
||||
return Result(true, success, cancelSent, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get())
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
private external fun nativeInstallAdaptive(request: String, listener: Listener): Long
|
||||
|
||||
@JvmStatic
|
||||
private external fun nativeStart(
|
||||
manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array<String>, pendingFileIdx: IntArray, expectedChunks: Int, expectedBytes: Long,
|
||||
|
||||
@@ -112,7 +112,37 @@ On-disk protocol (what a resumed run, by either engine, relies on): a file is as
|
||||
|
||||
## `com.droiddeck.launcher.stores.epic.EpicNative`
|
||||
|
||||
Two runs make up the native half of an Epic install:
|
||||
### Production adaptive install (snapshot schema v1)
|
||||
|
||||
`nativeInstallAdaptive(request: String, listener: EpicNative.Listener): Long`
|
||||
exports `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive`.
|
||||
Its exact Kotlin descriptor is
|
||||
`(Ljava/lang/String;Lcom/droiddeck/launcher/stores/epic/EpicNative$Listener;)J`.
|
||||
The listener must also implement `EpicNative.InstallListener.beforeSwap(): Boolean` (`()Z`).
|
||||
Rust invokes it synchronously after staging/hash/sync and before the first atomic replacement;
|
||||
false or a Java exception aborts without replacing that output. Java marks the sidecar updating
|
||||
there. Later verified swaps preserve mixed-update/resume semantics.
|
||||
|
||||
The UTF-8 JSON request contains:
|
||||
|
||||
- `schema_version: 1`, absolute `install_dir`, `cache_dir`, owned `staging_dir`,
|
||||
`same_volume`, signed `free_game`/`free_cache` (`-1` unknown), `max_workers`, CDN `hosts`.
|
||||
- `chunks`: distinct `guid`, relative CDN `path`, decoded `size`, `exact_size`, `wire_size`,
|
||||
SHA-1 hex `sha1` (empty/all-zero means absent), `cached` verified length (`-1` absent).
|
||||
- `files`: pending manifest-order relative `path`, `sha1`, current `old_size`, ordered `parts`
|
||||
containing `guid`, unsigned `offset` and `size`.
|
||||
|
||||
Parse, owned-namespace validation and adaptive storage planning happen synchronously. Errors
|
||||
return zero after `onComplete(false, error, 0)` and are **terminal**, not fallback signals.
|
||||
`onPlan` reports the selected lookahead/retention/peaks; `onProgress` counts staged file bytes
|
||||
and verified swapped file counts, not fetched wire bytes. Fetch and staged assembly overlap
|
||||
in one native run, using the same pure reservation/release policy as the peak planner.
|
||||
Cancellation/release joins native workers; no worker survives the terminal callback/handle
|
||||
release. Only `StoresNative.ensureLoaded() == false` selects bounded parallel Java.
|
||||
|
||||
### Legacy batch APIs (not the production installer)
|
||||
|
||||
Two explicit calls provide the legacy batch interface:
|
||||
|
||||
1. **Fetch** (`nativeStart`): fills the chunk cache with verified, decompressed chunks for the
|
||||
files the manager says are pending. The Kotlin manager still fetches and parses the manifest
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
//! Epic Games Store download adapter for the shared fetch core (`crate::fetch_core`).
|
||||
//!
|
||||
//! Scope = the chunk-fetch inner loop of `EpicDownloadManager.java` (the fixed 8-thread pool that
|
||||
//! fills `<installDir>/.chunks/<GUID>`), nothing more. The Java manager still parses the manifest
|
||||
//! API JSON, downloads the manifest, selects files (install tags), runs the delta/verify pass,
|
||||
//! assembles files from the chunk cache and does every post-install step. The adapter re-parses
|
||||
//! the same manifest bytes, rebuilds the same chunk plan for the pending files Java hands it,
|
||||
//! skips chunks already in the cache and writes verified chunks with the same `.part` + rename
|
||||
//! protocol, so the on-disk state is identical whichever engine ran (see
|
||||
//! Bannerlator's `docs/RUST_EPIC_PARITY.md` for the rule-by-rule table).
|
||||
//! Production installs use [`adaptive`]: the Java manifest/delta pass supplies a versioned
|
||||
//! snapshot, the shared pure storage schedule selects an ordered lookahead, and fetch_core
|
||||
//! runs concurrently with verified staged assembly. Old outputs survive until atomic swap;
|
||||
//! a synchronous JNI callback marks the sidecar updating before the first replacement.
|
||||
//! Legacy manifest/batch APIs remain available. A missing library uses the bounded Java
|
||||
//! installer; native plan/runtime failures never silently switch engines.
|
||||
//!
|
||||
//! Submodules:
|
||||
//! - [`adaptive`] — budget planner and bounded overlapping fetch/assembly coordinator.
|
||||
//! - [`manifest`] — ChunksV4 binary manifest + legacy JSON manifest parsers (1:1 with the Java
|
||||
//! `parseManifest` / `parseJsonManifest`).
|
||||
//! - [`plan`] — install-tag selection, unique-chunk plan, chunk paths / CDN URLs.
|
||||
@@ -26,6 +25,7 @@
|
||||
//! carries the same licence (see `Cargo.toml`).
|
||||
|
||||
pub mod assemble;
|
||||
pub mod adaptive;
|
||||
pub mod chunk;
|
||||
pub mod driver;
|
||||
pub mod jni;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -24,7 +24,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::io::{BufWriter, Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
@@ -33,6 +33,33 @@ use sha1::{Digest, Sha1};
|
||||
use super::manifest::{parse_manifest, FileInfo, Manifest};
|
||||
use super::plan::resolve_cache_dir;
|
||||
|
||||
/// Streaming assembly primitive used by the budgeted installer; never holds a decoded chunk
|
||||
/// in memory and closes the reader before the caller releases the cache reservation.
|
||||
pub fn copy_chunk_part(
|
||||
cache_path: &Path, offset: u64, size: u64, output: &mut impl Write,
|
||||
sha: &mut Sha1, cancel: &AtomicBool,
|
||||
) -> Result<u64, String> {
|
||||
let mut input = File::open(cache_path).map_err(|e| format!("read chunk: {e}"))?;
|
||||
let length = input.metadata().map_err(|e| e.to_string())?.len();
|
||||
if offset.checked_add(size).filter(|end| *end <= length).is_none() {
|
||||
return Err("chunk part exceeds cached data".into());
|
||||
}
|
||||
input.seek(SeekFrom::Start(offset)).map_err(|e| e.to_string())?;
|
||||
let mut left = size;
|
||||
let mut buffer = [0u8; 64 * 1024];
|
||||
while left > 0 {
|
||||
if cancel.load(Ordering::Acquire) { return Err("cancelled".into()); }
|
||||
let take = left.min(buffer.len() as u64) as usize;
|
||||
let count = input.read(&mut buffer[..take])
|
||||
.map_err(|e| e.to_string())?;
|
||||
if count == 0 { return Err("short cached chunk".into()); }
|
||||
output.write_all(&buffer[..count]).map_err(|e| e.to_string())?;
|
||||
sha.update(&buffer[..count]);
|
||||
left -= count as u64;
|
||||
}
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
/// Everything the JNI layer hands over for one assembly run.
|
||||
pub struct AssembleRequest {
|
||||
pub manifest_bytes: Vec<u8>,
|
||||
|
||||
@@ -93,10 +93,14 @@ struct HashingWriter<W: Write> {
|
||||
inner: W,
|
||||
sha: Option<Sha1>,
|
||||
written: u64,
|
||||
limit: u64,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for HashingWriter<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
if self.written.saturating_add(buf.len() as u64) > self.limit {
|
||||
return Err(io::Error::other("chunk exceeds reserved decoded window"));
|
||||
}
|
||||
let n = self.inner.write(buf)?;
|
||||
if let Some(sha) = self.sha.as_mut() {
|
||||
sha.update(&buf[..n]);
|
||||
@@ -129,10 +133,21 @@ pub fn write_verified_chunk(
|
||||
// `tmp.delete()` — clear any stale partial from a prior interrupted attempt.
|
||||
let _ = fs::remove_file(&tmp);
|
||||
|
||||
let result = write_part(body, expected_sha1, &tmp, final_path);
|
||||
let result = write_part(body, expected_sha1, &tmp, final_path, u64::MAX, false);
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn write_verified_chunk_bounded(
|
||||
body: &[u8], expected_sha1: Option<&[u8; 20]>, final_path: &Path,
|
||||
decoded_limit: u64, exact_size: bool,
|
||||
) -> Result<u64, String> {
|
||||
let tmp = part_path(final_path);
|
||||
let result = write_part(body, expected_sha1, &tmp, final_path, decoded_limit, exact_size);
|
||||
if result.is_err() { let _ = fs::remove_file(tmp); }
|
||||
result
|
||||
}
|
||||
|
||||
@@ -141,23 +156,46 @@ fn write_part(
|
||||
expected_sha1: Option<&[u8; 20]>,
|
||||
tmp: &Path,
|
||||
final_path: &Path,
|
||||
limit: u64,
|
||||
exact_size: bool,
|
||||
) -> Result<u64, String> {
|
||||
let hdr = parse_chunk_header(body)?;
|
||||
let payload = chunk_payload(body, &hdr)?;
|
||||
if limit != u64::MAX && (hdr.compressed_size < 0 || payload.len() != hdr.compressed_size as usize) {
|
||||
return Err("truncated chunk body".into());
|
||||
}
|
||||
|
||||
let file = File::create(tmp).map_err(|e| format!("open {}: {e}", tmp.display()))?;
|
||||
let mut out = HashingWriter {
|
||||
inner: BufWriter::with_capacity(128 * 1024, file),
|
||||
sha: expected_sha1.map(|_| Sha1::new()),
|
||||
written: 0,
|
||||
limit,
|
||||
};
|
||||
|
||||
if hdr.is_compressed() {
|
||||
// Java: inflate until the stream finishes or the input runs out; a corrupt stream throws
|
||||
// (DataFormatException) → this CDN attempt fails. flate2 behaves the same way: EOF on a
|
||||
// truncated stream is a clean end (partial output), corruption is an error.
|
||||
let mut dec = flate2::read::ZlibDecoder::new(payload);
|
||||
io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?;
|
||||
if limit == u64::MAX {
|
||||
let mut dec = flate2::read::ZlibDecoder::new(payload);
|
||||
io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?;
|
||||
} else {
|
||||
let mut dec = flate2::Decompress::new(true);
|
||||
let mut buffer = [0u8; 128 * 1024];
|
||||
loop {
|
||||
let input = dec.total_in() as usize;
|
||||
let before = dec.total_out();
|
||||
let status = dec.decompress(&payload[input..], &mut buffer, flate2::FlushDecompress::None)
|
||||
.map_err(|e| format!("inflate: {e}"))?;
|
||||
let count = (dec.total_out() - before) as usize;
|
||||
out.write_all(&buffer[..count]).map_err(|e| format!("inflate write: {e}"))?;
|
||||
if status == flate2::Status::StreamEnd { break; }
|
||||
if count == 0 && dec.total_in() as usize == input {
|
||||
return Err("truncated compressed Epic chunk".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.write_all(payload).map_err(|e| format!("write: {e}"))?;
|
||||
}
|
||||
@@ -165,6 +203,9 @@ fn write_part(
|
||||
let written = out.written;
|
||||
let digest = out.sha.take().map(|s| s.finalize());
|
||||
drop(out); // close the file before the rename
|
||||
if exact_size && written != limit {
|
||||
return Err("chunk decoded size mismatch".into());
|
||||
}
|
||||
|
||||
if let (Some(expected), Some(actual)) = (expected_sha1, digest) {
|
||||
if actual.as_slice() != &expected[..] {
|
||||
|
||||
@@ -63,6 +63,7 @@ fn android_log(message: &str) {
|
||||
/// assembly run, so one `nativeCancel` / `nativeRelease` pair serves both.
|
||||
pub struct EpicDownloadHandle {
|
||||
cancel: Arc<AtomicBool>,
|
||||
worker: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
fn clear_pending_exception(env: &mut JNIEnv) {
|
||||
@@ -314,12 +315,11 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let thread_cancel = Arc::clone(&cancel);
|
||||
let handle = Box::new(EpicDownloadHandle { cancel });
|
||||
|
||||
thread::spawn(move || {
|
||||
let worker = thread::spawn(move || {
|
||||
crate::priority::background();
|
||||
run_on_thread(vm, listener, plan, req, thread_cancel);
|
||||
});
|
||||
let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) });
|
||||
|
||||
Box::into_raw(handle) as jlong
|
||||
}
|
||||
@@ -425,9 +425,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let thread_cancel = Arc::clone(&cancel);
|
||||
let handle = Box::new(EpicDownloadHandle { cancel });
|
||||
|
||||
thread::spawn(move || {
|
||||
let worker = thread::spawn(move || {
|
||||
crate::priority::background();
|
||||
let progress = |bytes_done: u64, bytes_total: u64, files_done: u64, files_total: u64| {
|
||||
let Ok(mut env) = vm.attach_current_thread_as_daemon() else {
|
||||
@@ -454,6 +452,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
outcome.bytes_written,
|
||||
);
|
||||
});
|
||||
let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) });
|
||||
|
||||
Box::into_raw(handle) as jlong
|
||||
}
|
||||
@@ -483,5 +482,73 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
}
|
||||
// The worker thread owns its own Arc clone of the cancel flag and its own GlobalRef, so
|
||||
// freeing the handle is safe even if a (cancelled) run is still winding down.
|
||||
drop(unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) });
|
||||
let mut handle = unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) };
|
||||
if let Some(worker) = handle.worker.take() { let _ = worker.join(); }
|
||||
}
|
||||
|
||||
/// ABI v1 snapshot: selected chunks/files, verified cache lengths, old output lengths and both
|
||||
/// measured free-space probes. Parse/budget failures are terminal, never fallback requests.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive(
|
||||
mut env: JNIEnv, _class: JClass, request: JString, listener: JObject,
|
||||
) -> jlong {
|
||||
if listener.is_null() { return 0; }
|
||||
let parsed = jstring_to_string(&mut env, &request).ok_or("unreadable adaptive request".to_string())
|
||||
.and_then(|s| super::adaptive::Request::parse(&s))
|
||||
.and_then(|req| super::adaptive::choose_budget(&req).map(|budget| (req, budget)));
|
||||
let (req, budget) = match parsed {
|
||||
Ok(plan) => plan,
|
||||
Err(error) => {
|
||||
call_on_complete(&mut env, &listener, false, &error, 0);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
call_on_plan(&mut env, &listener, req.chunks.len() as u64, 0,
|
||||
&format!("lookahead={} retained={} game_peak={} cache_peak={}",
|
||||
budget.lookahead, budget.retained, budget.game_peak, budget.cache_peak));
|
||||
let Ok(vm) = env.get_java_vm() else {
|
||||
call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0);
|
||||
return 0;
|
||||
};
|
||||
let Ok(listener) = env.new_global_ref(&listener) else {
|
||||
call_on_complete(&mut env, &listener, false, "listener ref failed", 0);
|
||||
return 0;
|
||||
};
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let thread_cancel = cancel.clone();
|
||||
let worker = thread::spawn(move || {
|
||||
crate::priority::background();
|
||||
let progress = |done, total, files, files_total| {
|
||||
if let Ok(mut env) = vm.attach_current_thread_as_daemon() {
|
||||
call_on_progress(&mut env, listener.as_obj(), done, total, files, files_total);
|
||||
}
|
||||
};
|
||||
let before_swap = || -> Result<(), String> {
|
||||
let mut env = vm.attach_current_thread_as_daemon().map_err(|e| e.to_string())?;
|
||||
match env.call_method(listener.as_obj(), "beforeSwap", "()Z", &[]) {
|
||||
Ok(value) => if value.z().unwrap_or(false) { Ok(()) } else { Err("Could not mark installed record updating before replacement".into()) },
|
||||
_ => {
|
||||
clear_pending_exception(&mut env);
|
||||
Err("Could not mark installed record updating before replacement".into())
|
||||
}
|
||||
}
|
||||
};
|
||||
let log = |line: &str| {
|
||||
android_log(line);
|
||||
if let Ok(mut env) = vm.attach_current_thread_as_daemon() {
|
||||
call_on_log(&mut env, listener.as_obj(), line);
|
||||
}
|
||||
};
|
||||
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(||
|
||||
super::adaptive::run(&req, budget, &thread_cancel, &before_swap, &progress, &log)))
|
||||
.unwrap_or_else(|_| Err("Native Epic install panicked".into()));
|
||||
let (success, error, bytes) = match outcome {
|
||||
Ok(bytes) => (true, String::new(), bytes),
|
||||
Err(error) => (false, error, 0),
|
||||
};
|
||||
if let Ok(mut env) = vm.attach_current_thread_as_daemon() {
|
||||
call_on_complete(&mut env, listener.as_obj(), success, &error, bytes);
|
||||
}
|
||||
});
|
||||
Box::into_raw(Box::new(EpicDownloadHandle { cancel, worker: Some(worker) })) as jlong
|
||||
}
|
||||
@@ -12,6 +12,11 @@ import java.nio.ByteOrder
|
||||
import java.security.MessageDigest
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.net.Socket
|
||||
import org.junit.After
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Before
|
||||
@@ -33,6 +38,11 @@ class EpicBoundedInstallTest {
|
||||
private lateinit var root: File
|
||||
private val requests = AtomicInteger()
|
||||
private val reject = AtomicBoolean()
|
||||
private val rejectGuid = AtomicInteger()
|
||||
private val handlers = Executors.newCachedThreadPool()
|
||||
private val sockets = ConcurrentHashMap.newKeySet<Socket>()
|
||||
private var requestBarrier: CountDownLatch? = null
|
||||
private var deliveryGate: CountDownLatch? = null
|
||||
private var bytes = ByteArray(256 * 1024) { (it % 251).toByte() }
|
||||
|
||||
@Before fun setUp() {
|
||||
@@ -44,23 +54,38 @@ class EpicBoundedInstallTest {
|
||||
val socket = try { server.accept() } catch (e: SocketException) {
|
||||
if (server.isClosed) break else throw e
|
||||
}
|
||||
socket.use {
|
||||
it.soTimeout = 5000
|
||||
val reader = it.getInputStream().bufferedReader()
|
||||
while (!reader.readLine().isNullOrEmpty()) { }
|
||||
requests.incrementAndGet()
|
||||
val out = it.getOutputStream()
|
||||
if (reject.get()) {
|
||||
out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
} else {
|
||||
val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN)
|
||||
.putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size)
|
||||
.put(ByteArray(24)).put(0).array()
|
||||
out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
out.write(header)
|
||||
out.write(bytes)
|
||||
}
|
||||
out.flush()
|
||||
sockets.add(socket)
|
||||
handlers.submit {
|
||||
try {
|
||||
socket.use {
|
||||
it.soTimeout = 5000
|
||||
val reader = it.getInputStream().bufferedReader()
|
||||
val request = reader.readLine()
|
||||
while (!reader.readLine().isNullOrEmpty()) { }
|
||||
requests.incrementAndGet()
|
||||
requestBarrier?.let { barrier ->
|
||||
barrier.countDown()
|
||||
check(barrier.await(5, TimeUnit.SECONDS)) { "Requests did not overlap" }
|
||||
}
|
||||
val out = it.getOutputStream()
|
||||
val rejectedGuid = rejectGuid.get()
|
||||
if (reject.get() || (rejectedGuid != 0 && request.contains("_%08X".format(rejectedGuid)))) {
|
||||
out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
} else {
|
||||
val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN)
|
||||
.putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size)
|
||||
.put(ByteArray(24)).put(0).array()
|
||||
out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray())
|
||||
out.write(header)
|
||||
out.flush()
|
||||
deliveryGate?.await(5, TimeUnit.SECONDS)
|
||||
out.write(bytes)
|
||||
}
|
||||
out.flush()
|
||||
}
|
||||
} catch (_: java.io.IOException) {
|
||||
// Cancellation disconnects an owned request while this fixture serves it.
|
||||
} finally { sockets.remove(socket) }
|
||||
}
|
||||
}
|
||||
}.apply { start() }
|
||||
@@ -78,7 +103,15 @@ class EpicBoundedInstallTest {
|
||||
}
|
||||
}
|
||||
|
||||
@After fun tearDown() { server.close(); worker.join(6000); assertFalse(worker.isAlive) }
|
||||
@After fun tearDown() {
|
||||
deliveryGate?.countDown()
|
||||
server.close()
|
||||
worker.join(6000)
|
||||
sockets.forEach { it.close() }
|
||||
handlers.shutdownNow()
|
||||
assertTrue(handlers.awaitTermination(6, TimeUnit.SECONDS))
|
||||
assertFalse(worker.isAlive)
|
||||
}
|
||||
|
||||
private fun sha(data: ByteArray) = MessageDigest.getInstance("SHA-1").digest(data)
|
||||
|
||||
@@ -159,13 +192,9 @@ class EpicBoundedInstallTest {
|
||||
}
|
||||
manifest.uniqueChunks.add(secondChunk)
|
||||
manifest.files.last().parts.single().guid = secondChunk.guid
|
||||
val callback = object : EpicDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {}
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
if (stage == "stream" && items == 1) reject.set(true)
|
||||
}
|
||||
}
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install(callback = callback) }
|
||||
// Prefetch may fail early, but replacement is still ordered by the writer.
|
||||
rejectGuid.set(2)
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes())
|
||||
assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes())
|
||||
assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state)
|
||||
@@ -376,7 +405,7 @@ class EpicBoundedInstallTest {
|
||||
assertTrue(samples.all { it.second == bytes.size.toLong() })
|
||||
}
|
||||
|
||||
@Test fun evictedSharedChunksAdvanceWriteProgressOnEveryRefetchWithoutChangingStages() {
|
||||
@Test fun oversizedSharedChunkStaysPinnedAcrossPrefetchedFilesWithoutChangingStages() {
|
||||
bytes = ByteArray((EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 1024).toInt()) { (it % 251).toByte() }
|
||||
manifest.uniqueChunks.single().apply {
|
||||
windowSize = bytes.size
|
||||
@@ -407,7 +436,7 @@ class EpicBoundedInstallTest {
|
||||
}
|
||||
}
|
||||
assertNotNull(install(callback = callback))
|
||||
assertEquals(3, requests.get())
|
||||
assertEquals(1, requests.get())
|
||||
assertEquals(listOf(0L, 1024L, 2048L, 3072L), samples)
|
||||
assertEquals("install", stages.last())
|
||||
repeat(3) { assertArrayEquals(bytes.copyOfRange(it * 1024, (it + 1) * 1024), File(root, "part-$it.bin").readBytes()) }
|
||||
@@ -442,4 +471,175 @@ class EpicBoundedInstallTest {
|
||||
assertEquals(2 * half, EpicDownloadManager.estimateStagedFilePeak(manifest.files, emptyMap()))
|
||||
assertEquals(bytes.size.toLong(), EpicDownloadManager.estimatePeakAdditionalBytes(manifest.files, chunks, emptyMap(), false, old))
|
||||
}
|
||||
|
||||
private fun distinctFiles(count: Int) {
|
||||
val template = manifest.uniqueChunks.single()
|
||||
manifest.uniqueChunks = (1..count).map { id ->
|
||||
EpicDownloadManager.ChunkInfo().apply {
|
||||
guid = intArrayOf(id, 0, 0, 0)
|
||||
windowSize = template.windowSize
|
||||
fileSize = template.fileSize
|
||||
sha1 = template.sha1
|
||||
}
|
||||
}
|
||||
manifest.files = manifest.uniqueChunks.mapIndexed { index, chunk ->
|
||||
gameFile("file-$index.bin", 0, bytes.size).apply { parts.single().guid = chunk.guid }
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun prefetchOverlapsEightSinglePartFilesButNeverFetchesTheWholeManifest() {
|
||||
distinctFiles(20)
|
||||
requestBarrier = CountDownLatch(8)
|
||||
val cancel = AtomicBoolean()
|
||||
var sawWindow = false
|
||||
assertNull(install(cancel, EpicDownloadManager.Callback { message, _ ->
|
||||
if (message.startsWith("Downloading chunks (")) {
|
||||
assertEquals(8, requests.get())
|
||||
val cacheBytes = File(root, ".chunks").listFiles()!!.sumOf { it.length() }
|
||||
assertTrue(cacheBytes <= 8L * bytes.size)
|
||||
sawWindow = true
|
||||
cancel.set(true)
|
||||
}
|
||||
}))
|
||||
assertTrue("An eight-request barrier cannot pass with a sequential downloader", sawWindow)
|
||||
assertEquals(8, requests.get())
|
||||
assertTrue(root.walk().none { it.name.endsWith(".stage") || it.name.endsWith(".part") })
|
||||
}
|
||||
|
||||
@Test fun parallelWindowSpaceThresholdIsCheckedBeforeStartingAnyRequest() {
|
||||
distinctFiles(20)
|
||||
val cache = temp.newFolder("parallel-scratch")
|
||||
val peak = EpicDownloadManager.estimatePeakAdditionalBytes(
|
||||
manifest.files, manifest.uniqueChunks.associateBy { it.guidStr() }, emptyMap(), false,
|
||||
)
|
||||
assertEquals(8L * bytes.size, peak)
|
||||
ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096 - 1, peak.toInt() / 4096 - 1)
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache.path) }
|
||||
assertEquals(0, requests.get())
|
||||
ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096, peak.toInt() / 4096)
|
||||
requestBarrier = CountDownLatch(8)
|
||||
assertNotNull(install(cache = cache.path))
|
||||
assertEquals(20, requests.get())
|
||||
manifest.files.forEach { assertArrayEquals(bytes, File(root, it.filename).readBytes()) }
|
||||
}
|
||||
|
||||
@Test fun cancellationDuringParallelNetworkReadsJoinsWorkersBeforeReturning() {
|
||||
distinctFiles(20)
|
||||
requestBarrier = CountDownLatch(8)
|
||||
deliveryGate = CountDownLatch(1)
|
||||
val cancel = AtomicBoolean()
|
||||
val outcome = java.util.concurrent.atomic.AtomicReference<Any?>()
|
||||
val installer = Thread {
|
||||
try { outcome.set(install(cancel)) } catch (e: Throwable) { outcome.set(e) }
|
||||
}.apply { start() }
|
||||
try {
|
||||
assertTrue(requestBarrier!!.await(5, TimeUnit.SECONDS))
|
||||
cancel.set(true)
|
||||
installer.join(10_000)
|
||||
assertFalse("Installer must disconnect and join all download workers", installer.isAlive)
|
||||
assertNull(outcome.get())
|
||||
val cache = File(root, ".chunks")
|
||||
val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty()
|
||||
deliveryGate!!.countDown()
|
||||
Thread.sleep(150)
|
||||
assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty())
|
||||
assertTrue(snapshot.keys.none { it.endsWith(".part") })
|
||||
assertTrue(root.walk().none { it.name.endsWith(".stage") })
|
||||
assertEquals(8, requests.get())
|
||||
} finally {
|
||||
cancel.set(true)
|
||||
deliveryGate!!.countDown()
|
||||
installer.join(10_000)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun failedParallelFetchJoinsOtherNetworkReadersBeforeReturning() {
|
||||
distinctFiles(20)
|
||||
requestBarrier = CountDownLatch(8)
|
||||
deliveryGate = CountDownLatch(1)
|
||||
rejectGuid.set(1)
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertEquals(8, requests.get())
|
||||
val cache = File(root, ".chunks")
|
||||
val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty()
|
||||
deliveryGate!!.countDown()
|
||||
Thread.sleep(150)
|
||||
assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty())
|
||||
assertTrue(snapshot.keys.none { it.endsWith(".part") })
|
||||
assertTrue(root.walk().none { it.name.endsWith(".stage") })
|
||||
}
|
||||
|
||||
@Test fun ownedOrphanStagingIsRecoveredBeforeTheSpaceProbeWithoutTouchingManifestFiles() {
|
||||
val staging = EpicDownloadManager.recoverOwnedStaging(root, manifest.files)
|
||||
File(staging, "native-123-0.stage").writeBytes(ByteArray(bytes.size))
|
||||
val legitimate = File(root, ".droiddeck-epic-old.stage").apply { writeText("keep") }
|
||||
oldInstall()
|
||||
val peak = 2 * bytes.size
|
||||
ShadowStatFs.registerStats(root.path, 1024, peak / 4096, peak / 4096)
|
||||
assertNotNull(install())
|
||||
assertEquals("keep", legitimate.readText())
|
||||
assertEquals(listOf("owner"), staging.listFiles()!!.map { it.name })
|
||||
}
|
||||
|
||||
@Test fun stagingRecoveryRefusesUnownedAndManifestOwnedNamespaces() {
|
||||
val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY).apply { mkdir() }
|
||||
val untouched = File(staging, "legitimate.stage").apply { writeText("keep") }
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertEquals("keep", untouched.readText())
|
||||
File(staging, "owner").writeText("DroidDeck Epic staging v1\n")
|
||||
manifest.files.first().filename = "${EpicDownloadManager.STAGING_DIRECTORY}/legitimate.stage"
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertEquals("keep", untouched.readText())
|
||||
assertEquals(0, requests.get())
|
||||
}
|
||||
|
||||
@Test fun loadedNativeAbiFailureIsExplicitAndNeverFallsBackToJavaDownloads() {
|
||||
val field = com.droiddeck.launcher.stores.StoresNative::class.java.getDeclaredField("loaded")
|
||||
field.isAccessible = true
|
||||
field.set(null, true)
|
||||
try {
|
||||
oldInstall()
|
||||
val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertTrue(error.message.orEmpty().contains("Native Epic install failed"))
|
||||
assertEquals(0, requests.get())
|
||||
assertOldInstallIntact()
|
||||
} finally {
|
||||
com.droiddeck.launcher.stores.StoresNative.forget()
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun stagingRecoveryRejectsSymlinkNamespaceOwnerAndEntriesWithoutDeletingOutsideData() {
|
||||
val outside = temp.newFolder("outside-staging")
|
||||
val externalStage = File(outside, "keep.stage").apply { writeText("keep") }
|
||||
val externalOwner = File(outside, "owner").apply { writeText("DroidDeck Epic staging v1\n") }
|
||||
val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY)
|
||||
try {
|
||||
java.nio.file.Files.createSymbolicLink(staging.toPath(), outside.toPath())
|
||||
} catch (e: java.nio.file.FileSystemException) {
|
||||
org.junit.Assume.assumeNoException("Host does not permit symbolic links", e)
|
||||
}
|
||||
try {
|
||||
val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertTrue(error.message.orEmpty().contains("Unsafe Epic staging"))
|
||||
assertEquals("keep", externalStage.readText())
|
||||
java.nio.file.Files.delete(staging.toPath())
|
||||
staging.mkdir()
|
||||
java.nio.file.Files.createSymbolicLink(File(staging, "owner").toPath(), externalOwner.toPath())
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
java.nio.file.Files.delete(File(staging, "owner").toPath())
|
||||
File(staging, "owner").writeText("DroidDeck Epic staging v1\n")
|
||||
val localStage = File(staging, "local.stage").apply { writeText("local") }
|
||||
java.nio.file.Files.createSymbolicLink(File(staging, "linked.stage").toPath(), externalStage.toPath())
|
||||
assertThrows(EpicDownloadManager.InstallException::class.java) { install() }
|
||||
assertEquals("local", localStage.readText())
|
||||
assertEquals("keep", externalStage.readText())
|
||||
assertEquals(0, requests.get())
|
||||
} finally {
|
||||
if (java.nio.file.Files.isSymbolicLink(staging.toPath())) java.nio.file.Files.delete(staging.toPath())
|
||||
for (name in listOf("owner", "linked.stage")) {
|
||||
val entry = File(staging, name).toPath()
|
||||
if (java.nio.file.Files.isSymbolicLink(entry)) java.nio.file.Files.delete(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,14 +15,14 @@ import org.robolectric.annotation.Config
|
||||
@Config(sdk = [33])
|
||||
class EpicJsonManifestTest {
|
||||
@Test
|
||||
fun boundedInstallPeakCountsOnlyOneLiveChunkForSequentialChunks() {
|
||||
fun boundedInstallPeakReservesParallelIncomingChunksBeforeWriting() {
|
||||
val first = chunk(1)
|
||||
val second = chunk(2)
|
||||
val file = file(part(1, 0, 100), part(2, 0, 100))
|
||||
val chunks = mapOf(first.guidStr() to first, second.guidStr() to second)
|
||||
|
||||
assertEquals(300L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true))
|
||||
assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false))
|
||||
assertEquals(200L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false))
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -45,7 +45,7 @@ class EpicJsonManifestTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun distantSharedChunksAreEvictedAndRefetchedWithinTheCacheCeiling() {
|
||||
fun sharedChunksWithinTheWindowStayPinnedWithinTheCacheCeiling() {
|
||||
val first = chunk(1).apply { windowSize = 40 * 1024 * 1024 }
|
||||
val second = chunk(2).apply { windowSize = 40 * 1024 * 1024 }
|
||||
val files = listOf(
|
||||
@@ -63,6 +63,43 @@ class EpicJsonManifestTest {
|
||||
assertTrue(cachePeak <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 40L * 1024 * 1024)
|
||||
}
|
||||
|
||||
@Test fun scheduleDeduplicatesWindowAndEvictsOnlyUnpinnedReusableChunks() {
|
||||
val size = 40L * 1024 * 1024
|
||||
val chunks = (1..12).map { chunk(it).apply { windowSize = size.toInt() } }
|
||||
.associateBy { it.guidStr() }
|
||||
val files = (1..12).map { file(part(it, 0, 1)) } + (1..12).map { file(part(it, 0, 1)) }
|
||||
val schedule = EpicDownloadManager.PrefetchSchedule(files, chunks, emptyMap())
|
||||
assertEquals(chunks.keys.take(8), schedule.fill())
|
||||
assertEquals(emptyList<String>(), schedule.fill())
|
||||
assertEquals(8 * size, schedule.bytes())
|
||||
assertEquals(emptyList<String>(), schedule.consume())
|
||||
assertEquals(listOf(chunks.keys.elementAt(8)), schedule.fill())
|
||||
assertEquals(9 * size, schedule.bytes())
|
||||
assertEquals(listOf(chunks.keys.first()), schedule.consume())
|
||||
var refetchedFirst = false
|
||||
repeat(22) {
|
||||
val fetched = schedule.fill()
|
||||
if (chunks.keys.first() in fetched) refetchedFirst = true
|
||||
assertTrue(schedule.bytes() <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 8 * size)
|
||||
val released = schedule.consume()
|
||||
assertTrue(released.none { it in schedule.window() })
|
||||
}
|
||||
assertTrue(refetchedFirst)
|
||||
assertEquals(0L, schedule.bytes())
|
||||
assertEquals(9 * size, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false))
|
||||
}
|
||||
|
||||
@Test fun cachedAndOldOutputBytesAreCreditedOnlyWhenTheScheduleReleasesThem() {
|
||||
val chunks = (1..8).map { chunk(it) }.associateBy { it.guidStr() }
|
||||
val files = (1..8).map { id -> file(part(id, 0, 100)).apply { filename = "file-$id" } }
|
||||
val existing = chunks.keys.associateWith { 100L }
|
||||
val old = files.associate { it.filename to 100L }
|
||||
assertEquals(800L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false))
|
||||
assertEquals(0L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, false))
|
||||
assertEquals(900L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true, old))
|
||||
assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, true, old))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun resumeCacheDropsCorruptAndUnneededEntriesButKeepsVerifiedChunks() {
|
||||
val dir = java.io.File(System.getProperty("user.dir"), "build/epic-cache-test-${System.nanoTime()}")
|
||||
|
||||
+60
-15
@@ -45,13 +45,46 @@ with a placeholder job. Failures are kept in the Downloads UI, notification and
|
||||
duplicated as transient failure toasts. Active queue persistence and ordinary backend pause/retry
|
||||
are unchanged.
|
||||
|
||||
Epic installs use the same bounded Java writer whether the native engine is available or not.
|
||||
It fetches one chunk at a time, verifies it, and writes the required parts into the current
|
||||
game file. Chunks are deleted after their last use; reusable chunks are retained up to 64 MiB,
|
||||
with at most one additional incoming chunk before eviction. Evicted shared chunks may be
|
||||
downloaded again. This trades parallel-download throughput and potentially extra network
|
||||
traffic for bounded temporary disk usage. The native batch APIs remain available but are not
|
||||
used by this install path.
|
||||
Epic installs use **native adaptive fetch/assembly** when `libdroiddeckstores.so` loads.
|
||||
The Java delta pass sends a versioned snapshot of pending files, chunk metadata, verified cache
|
||||
lengths, old output lengths and both measured free-space probes to Rust. The native planner and
|
||||
execution share one ordered moving-window policy. Before any chunk request, Rust chooses the
|
||||
largest safe lookahead (128, 64, 32, 16, 8, 4, 2 or 1 part references) and reusable-cache budget
|
||||
(64 MiB, 16 MiB or zero) whose simulated staged-file and cache peaks fit.
|
||||
Native device IDs also detect an explicitly named cache on the game volume, so two separate
|
||||
paths cannot incorrectly turn one filesystem's requirement into two independent checks.
|
||||
|
||||
| Storage situation | Native policy |
|
||||
| --- | --- |
|
||||
| Plenty of room | Up to 128 references ahead, up to 32 network requests, 64 MiB reusable data |
|
||||
| Tight | Smaller lookahead and/or reusable budget; parallel fetching remains where space permits |
|
||||
| Barely fits | One-reference lookahead, one request, no reusable retention if needed |
|
||||
| Does not fit even the minimal schedule | Explicit preflight error before requests or replacement |
|
||||
| Free space unknown | Conservative bounded lookahead (at most eight), not an invented zero |
|
||||
|
||||
The native fetch coordinator reuses `fetch_core`'s host scheduling, retries, adaptive network
|
||||
window and decode pool. The staged writer runs concurrently, waiting only for its next required
|
||||
chunk, and reads parts through the native assembler's streaming copy primitive. GUIDs in the
|
||||
lookahead are reserved once before submission, at their full decoded size. Chunks are released
|
||||
at last use on **both** cache layouts; distant reusable data is oldest-first bounded and may
|
||||
be refetched. At most the selected reusable budget plus distinct lookahead decoded sizes is
|
||||
resident/reserved. New request admission re-probes both volumes and can shrink unsent lookahead
|
||||
or retention if headroom disappears; already in-flight reservations remain protected.
|
||||
If even existing reservations cannot fit, the run fails and joins its workers rather than
|
||||
waiting indefinitely or deleting old game files. An install to SD retains its internal scratch
|
||||
cache, so assembled output is the only game data written to the card.
|
||||
|
||||
Only failure to **load** the native library selects the Java fallback. An ABI mismatch, native
|
||||
planning/network/hash/I/O failure or unsupported atomic swap does not silently rerun Java.
|
||||
Eight fallback Java workers prefetch the next eight chunk-part references, across file boundaries,
|
||||
while one writer assembles and verifies files in manifest order. Shared GUIDs within that window
|
||||
are fetched only once. Chunks are deleted after their last use; other reusable chunks are retained
|
||||
up to 64 MiB. At most eight additional decoded chunks (complete or in flight) coexist with that
|
||||
retained cache. The size bound is 64 MiB plus the decoded sizes of the window's distinct GUIDs,
|
||||
not a fixed eight-MiB allowance: manifests can supply larger chunk windows. Evicted distant shared
|
||||
chunks may be downloaded again. No worker downloads outside the moving window; cancellation and
|
||||
failure disconnect and join all workers before returning or removing the cache.
|
||||
The legacy native batch APIs remain available but are not used by the production install path.
|
||||
|
||||
The earlier batch design intentionally prioritizes parallel-download throughput. Native
|
||||
`epic/assemble.rs::build_assemble_plan` enables `scratch_cache` only for a nonempty cache path.
|
||||
@@ -61,9 +94,9 @@ when that flag is set**; the default beside-game cache is removed at the end. Th
|
||||
the scratch-cache assembly tests cover that distinction. Extending last-use reclamation to
|
||||
internal native assembly could keep parallel batch throughput and reduce the overlapping
|
||||
cache/output peak, but batch fetching still fills
|
||||
the cache before assembly. That is not equivalent to the bounded sequential writer's fixed
|
||||
retained-cache ceiling. This change keeps the bounded path and its explicit throughput/refetch
|
||||
tradeoff; it does not redesign or benchmark native download throughput.
|
||||
the cache before assembly. Production native adaptive and Java fallback paths instead overlap
|
||||
network fetches with assembly without retaining a whole-game cache. No native or Java throughput
|
||||
benchmark is claimed.
|
||||
|
||||
The bounded Epic download bar, measured rate and ETA count pending file bytes written, not unique
|
||||
compressed chunks fetched. Shared chunks therefore cannot complete the bar before later files
|
||||
@@ -71,9 +104,12 @@ are written, and an evicted chunk's refetch is reflected in the time spent doing
|
||||
Streaming stays in the Download stage until all files are written and verified, then enters
|
||||
Install for registration; it does not bounce between stages for each file.
|
||||
|
||||
The space check follows the manifest delta pass and simulates file writes, chunk fetches,
|
||||
and evictions, rather than adding the entire cache to the entire game. Each file is written to a
|
||||
unique sibling staging file on its output volume (never the literal `<filename>.part`, which can
|
||||
The space check follows the manifest delta pass and uses the same pure moving-window policy as
|
||||
execution, reserving all incoming decoded chunk sizes before writing parts and simulating
|
||||
last-use releases and unpinned oldest-first evictions. Existing verified cache bytes count against
|
||||
current free space and are credited only at their scheduled release. This avoids adding the entire
|
||||
cache to the entire game without undercounting concurrent downloads. Each file is written to a
|
||||
unique file in an owned staging namespace on its output volume (never the literal `<filename>.part`, which can
|
||||
be a legitimate manifest file), size/hash verified, synced and atomically renamed over its output.
|
||||
The old file remains until that swap succeeds. The planner credits its bytes only after that
|
||||
file's complete staging write, not by subtracting all old outputs up front. Both volume checks
|
||||
@@ -84,8 +120,17 @@ verified cached chunks are accounted for on resume. On a card, game and cache re
|
||||
are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed
|
||||
or cancelled staging files are removed without deleting old outputs. A successful run removes the
|
||||
cache. A resume within a large incomplete file restarts that file, not its individual parts.
|
||||
Abrupt process termination can leave a unique staging file; unlike normal failure/cancellation,
|
||||
its cleanup is not guaranteed, and the next space probe includes its occupied bytes.
|
||||
Both writers use `<installDir>/.droiddeck-epic-staging/`, whose `owner` marker identifies this
|
||||
namespace. Before the delta pass and free-space probes, the manager removes orphan `.stage`
|
||||
entries only from this owned namespace. A manifest naming that directory or any descendant
|
||||
fails without deletion; an unowned directory or unexpected entry also fails conservatively.
|
||||
The namespace must be a real directory at the expected canonical child of the install root.
|
||||
Ownership markers and staging entries must be no-follow regular files; symlink directories,
|
||||
markers and entries are rejected before recovery deletes anything.
|
||||
Ordinary game files with `.stage` or `.part` suffixes elsewhere are never recovery targets.
|
||||
Abrupt process termination is therefore recoverable on the next run without inferring ownership
|
||||
from a loose filename prefix. Cancellation/failure waits for all native/Java fetch writers before
|
||||
returning; completed verified cache chunks remain usable on resume.
|
||||
|
||||
## The sidecar
|
||||
|
||||
|
||||
Reference in new issue
Block a user