From f3715aac325f2b8280535199faf170d3cb27bbcf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 4 Oct 2026 02:57:17 -0400 Subject: [PATCH] M13.30: web depth and polish - GPU in browser job workers: each job worker opens its own WebGPU device (deferred readbacks); the Render Queue export and the analyses' frame loops are futures (render::passes, offload::run_request_async) whose frames render in passes, awaiting the device between them. GPU particles and Advanced 3D read back under keys too. workers.jobs reports passes and readbacks; the smoke test measures them. - Layer buffers in the browser's OPFS disk cache: frame workers back their layer caches with a PrefetchStore; the page plans what to fetch before a request from recent misses (LayerPrefetch over the shared DiskIndex). - Templates embed footage (default on, size-capped with a warning) and creating from a template extracts it next to the new project / into browser storage. - ICC LUT profiles (lut8/lut16/lutAtoB/lutBtoA, from the public ICC spec) in Simulate Output > My Custom RGB, baked into a 3D LUT for the viewer. Co-Authored-By: Claude Opus 5.5 --- apps/effectcraft-web/js/host.js | 103 ++- apps/effectcraft-web/src/app.rs | 6 +- apps/effectcraft-web/src/diskcache.rs | 89 ++- apps/effectcraft-web/src/files.rs | 7 + apps/effectcraft-web/src/frames.rs | 63 +- apps/effectcraft-web/src/worker.rs | 91 ++- apps/effectcraft-web/tests/smoke.mjs | 79 ++- apps/effectcraft-web/web/worker.js | 6 +- crates/color/src/icc.rs | 741 ++++++++++++++++++++- crates/engine/src/camera_track.rs | 34 +- crates/engine/src/commands/viewer_cmds.rs | 24 +- crates/engine/src/lib.rs | 12 + crates/engine/src/mask_track.rs | 55 +- crates/engine/src/offload.rs | 215 ++++-- crates/engine/src/remote.rs | 181 ++++- crates/engine/src/render_queue.rs | 53 +- crates/engine/src/roto.rs | 79 ++- crates/engine/src/templates.rs | 357 +++++++++- crates/engine/src/tests_color_view.rs | 66 ++ crates/engine/src/tracking.rs | 37 +- crates/engine/src/viewer.rs | 195 +++++- crates/engine/src/warp.rs | 34 +- crates/export/src/encode.rs | 8 +- crates/export/src/lib.rs | 45 +- crates/export/src/pipeline.rs | 36 +- crates/export/src/webm.rs | 7 +- crates/export/src/webm_av1.rs | 5 +- crates/gpu/src/adv3d.rs | 149 ++++- crates/gpu/src/lib.rs | 14 +- crates/gpu/src/particles.rs | 63 +- crates/gpu/src/tests_adv3d.rs | 10 +- crates/gpu/src/tests_deferred.rs | 6 +- crates/gpu/src/tests_deferred_jobs.rs | 109 +++ crates/gpu/src/tests_particles.rs | 6 +- crates/host/src/lib.rs | 76 ++- crates/render/src/cache.rs | 177 ++++- crates/render/src/disk_cache.rs | 55 ++ crates/render/src/lib.rs | 16 +- crates/render/src/passes.rs | 104 +++ crates/ui-egui/src/panels/forms.rs | 2 + crates/ui-egui/src/panels/templates.rs | 3 +- crates/ui-egui/tests/ui_web_gpu_storage.rs | 2 + crates/ui-egui/tests/ui_web_job_gpu.rs | 123 ++++ crates/ui-egui/tests/ui_web_offload.rs | 11 +- docs/parity.md | 34 +- docs/web.md | 103 ++- 46 files changed, 3304 insertions(+), 387 deletions(-) create mode 100644 crates/gpu/src/tests_deferred_jobs.rs create mode 100644 crates/render/src/passes.rs create mode 100644 crates/ui-egui/tests/ui_web_job_gpu.rs diff --git a/apps/effectcraft-web/js/host.js b/apps/effectcraft-web/js/host.js index 5435aba0..ec89d4cb 100644 --- a/apps/effectcraft-web/js/host.js +++ b/apps/effectcraft-web/js/host.js @@ -282,15 +282,31 @@ export function audioStop() { // worker once. const idle = []; const running = new Map(); // job id → {worker, done} +// Job workers open a WebGPU device of their own (unless `?nogpuworkers`): their jobs' frames +// render on it. `jobGpu`: what the last started worker reported; `jobStats`: the last jobs' +// {kind, gpu, passes, readbacks, ms}. +let jobWorkerGpu = true; +let jobGpu = null; +const jobStats = []; + +export function setJobWorkerGpu(on) { + jobWorkerGpu = !!on; +} function spawn(base) { const worker = new Worker(new URL("worker.js", base), { type: "module", name: "effectcraft-worker" }); - const w = { worker, files: new Map(), ready: null }; + const w = { worker, files: new Map(), ready: null, gpu: null }; w.ready = new Promise((res, rej) => { - worker.onmessage = (e) => { if (e.data && e.data.type === "ready") res(); }; + worker.onmessage = (e) => { + if (e.data && e.data.type === "gpu") { + w.gpu = e.data.gpu ? { adapter: e.data.gpu } : { cpu: e.data.error || "no GPU" }; + jobGpu = w.gpu; + } + if (e.data && e.data.type === "ready") res(); + }; worker.onerror = (e) => rej(new Error(e.message || "worker failed to start")); }); - worker.postMessage({ type: "init", module: self.effectcraftModule ?? null, base }); + worker.postMessage({ type: "init", module: self.effectcraftModule ?? null, base, gpu: jobWorkerGpu }); return w; } @@ -310,6 +326,9 @@ export function workerRun(id, json, files, base, onMessage) { } } else if (m.type === "file") { onMessage("file", null, m.path, new Uint8Array(m.bytes)); + } else if (m.type === "stats") { + jobStats.push({ kind: m.kind, gpu: m.gpu, passes: m.passes, readbacks: m.readbacks, ms: m.ms }); + if (jobStats.length > 16) jobStats.shift(); } }; w.worker.onerror = (e) => { @@ -337,9 +356,9 @@ export function workerCancel(id) { w.worker.terminate(); } -/// {running, idle}. +/// {running, idle, gpu, jobs: [{kind, gpu, passes, readbacks, ms}]}. export function workerStats() { - return { running: running.size, idle: idle.length }; + return { running: running.size, idle: idle.length, gpu: jobGpu, jobs: jobStats.slice() }; } // ------------------------------------------------------------------ frame workers @@ -370,53 +389,65 @@ export function frameWorkerStart(base, onMessage, gpu) { // ------------------------------------------------------------------ disk cache (OPFS) -// The disk cache's frames (src/diskcache.rs): one file per frame under -// effectcraft-cache/v1/frames/<32 hex>.ecc in the Origin Private File System. Frame workers write -// them with synchronous access handles (a temporary file moved into place, so a reader never -// sees a partial entry; every entry also ends in a checksum); the page lists, reads and deletes. +// The disk cache (src/diskcache.rs): one file per frame under +// effectcraft-cache/v1/frames/<32 hex>.ecc and one per layer buffer under +// effectcraft-cache/v1/layers/<32 hex>.ecc in the Origin Private File System (entry names with a +// `layers/` prefix are layer buffers). Frame workers write them with synchronous access handles +// (a temporary file moved into place, so a reader never sees a partial entry; every entry also +// ends in a checksum) and read the layer buffers they prefetch; the page lists, reads and +// deletes. const CACHE_ROOT = "effectcraft-cache"; -const CACHE_PATH = [CACHE_ROOT, "v1", "frames"]; -let cacheDirHandle = null; +const CACHE_KINDS = ["frames", "layers"]; +const cacheDirHandles = {}; -async function cacheDir() { - if (cacheDirHandle) return cacheDirHandle; +async function cacheDir(kind = "frames") { + if (cacheDirHandles[kind]) return cacheDirHandles[kind]; if (!navigator.storage || !navigator.storage.getDirectory) throw new Error("no Origin Private File System"); let d = await navigator.storage.getDirectory(); - for (const n of CACHE_PATH) d = await d.getDirectoryHandle(n, { create: true }); - cacheDirHandle = d; + for (const n of [CACHE_ROOT, "v1", kind]) d = await d.getDirectoryHandle(n, { create: true }); + cacheDirHandles[kind] = d; return d; } -/// Every cached entry: [{name, size, modified}]. Temporaries older than ten minutes (a writer -/// that died) are removed. +/// The folder and file name of an entry name (`layers/.ecc` or `.ecc`). +async function entryDir(name) { + if (name.startsWith("layers/")) return [await cacheDir("layers"), name.slice(7)]; + return [await cacheDir("frames"), name]; +} + +/// Every cached entry: [{name, size, modified}] (layer buffers named `layers/…`). Temporaries +/// older than ten minutes (a writer that died) are removed. export async function cacheList() { - const dir = await cacheDir(); const out = []; - const stale = []; - for await (const [name, h] of dir.entries()) { - if (h.kind !== "file") continue; - const f = await h.getFile(); - if (name.includes(".tmp")) { - if (Date.now() - f.lastModified > 600000) stale.push(name); - continue; + for (const kind of CACHE_KINDS) { + const dir = await cacheDir(kind); + const prefix = kind === "frames" ? "" : `${kind}/`; + const stale = []; + for await (const [name, h] of dir.entries()) { + if (h.kind !== "file") continue; + const f = await h.getFile(); + if (name.includes(".tmp")) { + if (Date.now() - f.lastModified > 600000) stale.push(name); + continue; + } + out.push({ name: prefix + name, size: f.size, modified: f.lastModified }); } - out.push({ name, size: f.size, modified: f.lastModified }); + for (const n of stale) { try { await dir.removeEntry(n); } catch {} } } - for (const n of stale) { try { await dir.removeEntry(n); } catch {} } return out; } /// An entry's bytes. export async function cacheRead(name) { - const dir = await cacheDir(); - const f = await (await dir.getFileHandle(name)).getFile(); + const [dir, file] = await entryDir(name); + const f = await (await dir.getFileHandle(file)).getFile(); return new Uint8Array(await f.arrayBuffer()); } /// Write an entry (in a worker: a synchronous access handle; elsewhere a writable stream); /// resolves with its size. -export async function cacheWrite(name, bytes) { - const dir = await cacheDir(); +export async function cacheWrite(fullName, bytes) { + const [dir, name] = await entryDir(fullName); const canMove = typeof FileSystemHandle !== "undefined" && "move" in FileSystemHandle.prototype; const tmp = canMove ? `${name}.tmp.${Math.random().toString(36).slice(2)}` : name; const fh = await dir.getFileHandle(tmp, { create: true }); @@ -442,13 +473,17 @@ export async function cacheWrite(name, bytes) { /// Delete entries (names); missing ones are ignored. export async function cacheDelete(names) { - const dir = await cacheDir(); - for (const n of names) { try { await dir.removeEntry(n); } catch {} } + for (const n of names) { + try { + const [dir, file] = await entryDir(n); + await dir.removeEntry(file); + } catch {} + } } /// Delete the whole cache. export async function cacheClear() { - cacheDirHandle = null; + for (const k of CACHE_KINDS) delete cacheDirHandles[k]; try { const root = await navigator.storage.getDirectory(); await root.removeEntry(CACHE_ROOT, { recursive: true }); diff --git a/apps/effectcraft-web/src/app.rs b/apps/effectcraft-web/src/app.rs index bf2c5147..c4a5dbac 100644 --- a/apps/effectcraft-web/src/app.rs +++ b/apps/effectcraft-web/src/app.rs @@ -141,7 +141,11 @@ pub fn session() -> Session { script: Some(effectcraft_host::script::runner), exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), config: Some(Arc::new(persist::config())), - offload: (!query_flag("noworkers")).then(|| Arc::new(worker::WorkerOffload) as Arc), + offload: (!query_flag("noworkers")).then(|| { + // Job workers render on a WebGPU device of their own (`?nogpuworkers`: their CPU). + worker::set_job_worker_gpu(!query_flag("nogpuworkers")); + Arc::new(worker::WorkerOffload) as Arc + }), browser: Some(Arc::new(browse::WebBrowser)), storage: Some(Arc::new(storage::WebStorage)), ..Default::default() diff --git a/apps/effectcraft-web/src/diskcache.rs b/apps/effectcraft-web/src/diskcache.rs index 177ce8d8..d96e6f13 100644 --- a/apps/effectcraft-web/src/diskcache.rs +++ b/apps/effectcraft-web/src/diskcache.rs @@ -13,13 +13,18 @@ //! - **Reads** are asynchronous: a frame whose key is indexed is read and decoded here instead //! of rendering it (`crate::frames`); a damaged or missing file is a miss and is forgotten. //! -//! Only frames are cached on disk in the browser (the desktop also keeps slow layer buffers): -//! a layer lookup happens in the middle of a synchronous render, where the browser's file -//! system can't be read. +//! - **Layer buffers** too (`layers/<32 hex>.ecc`): a layer lookup happens in the middle of a +//! synchronous render, where the browser's file system can't be read, so the frame workers +//! back their layer caches with a `PrefetchStore`. They report each frame's layer-cache +//! misses; before the next request the page names the recent misses that are on disk +//! ([`layer_prefetch`], `disk_cache::LayerPrefetch`), and the worker reads those entries +//! into memory before rendering, so the lookups find them. Slow buffers are written by the +//! worker after its frame and indexed here like frames (one least-recently-used order for +//! both kinds). use std::cell::RefCell; -use effectcraft_engine::render::disk_cache::{self, DiskIndex, Kind}; +use effectcraft_engine::render::disk_cache::{self, DiskIndex, Kind, LayerPrefetch}; use serde_json::{Value, json}; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; @@ -29,7 +34,7 @@ extern "C" { #[wasm_bindgen(js_name = cacheList)] fn cache_list() -> js_sys::Promise; #[wasm_bindgen(js_name = cacheRead)] - fn cache_read(name: &str) -> js_sys::Promise; + pub(crate) fn cache_read(name: &str) -> js_sys::Promise; #[wasm_bindgen(js_name = cacheWrite)] pub(crate) fn cache_write(name: &str, bytes: js_sys::Uint8Array) -> js_sys::Promise; #[wasm_bindgen(js_name = cacheDelete)] @@ -53,6 +58,10 @@ struct State { writes: u64, evictions: u64, error: Option, + /// Recent layer-cache misses of the frame workers (what to prefetch). + prefetch: LayerPrefetch, + layer_hits: u64, + layer_writes: u64, } thread_local! { @@ -68,19 +77,19 @@ pub fn init() { wasm_bindgen_futures::spawn_local(async { match JsFuture::from(cache_list()).await { Ok(list) => { - let mut entries: Vec<(f64, u128, u64)> = js_sys::Array::from(&list) + let mut entries: Vec<(f64, Kind, u128, u64)> = js_sys::Array::from(&list) .iter() .filter_map(|e| { let get = |k: &str| js_sys::Reflect::get(&e, &k.into()).ok(); - let key = disk_cache::parse_entry_name(&get("name")?.as_string()?)?; - Some((get("modified")?.as_f64().unwrap_or(0.0), key, get("size")?.as_f64().unwrap_or(0.0) as u64)) + let (kind, key) = parse_name(&get("name")?.as_string()?)?; + Some((get("modified")?.as_f64().unwrap_or(0.0), kind, key, get("size")?.as_f64().unwrap_or(0.0) as u64)) }) .collect(); entries.sort_by(|a, b| a.0.total_cmp(&b.0)); ST.with(|s| { let mut s = s.borrow_mut(); - for (_, key, bytes) in entries { - s.index.insert(Kind::Frame, key, bytes); + for (_, kind, key, bytes) in entries { + s.index.insert(kind, key, bytes); } s.loaded = true; s.available = true; @@ -96,6 +105,46 @@ pub fn init() { }); } +/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers). +pub(crate) fn name_of(kind: Kind, key: u128) -> String { + match kind { + Kind::Frame => disk_cache::entry_name(key), + Kind::Layer => format!("layers/{}", disk_cache::entry_name(key)), + } +} + +fn parse_name(name: &str) -> Option<(Kind, u128)> { + match name.strip_prefix("layers/") { + Some(n) => Some((Kind::Layer, disk_cache::parse_entry_name(n)?)), + None => Some((Kind::Frame, disk_cache::parse_entry_name(name)?)), + } +} + +/// Layer entries (32 hex digits) a frame worker should read before its next frame: recent +/// misses that are on disk. +pub fn layer_prefetch(max: usize) -> Vec { + if !active() { + return vec![]; + } + ST.with(|s| { + let s = s.borrow(); + s.prefetch.plan(&s.index, max).into_iter().map(|k| format!("{k:032x}")).collect() + }) +} + +/// A frame worker's layer-cache misses and the buffers it served from what it fetched. +pub fn note_layers(misses: &[String], hits: &[String]) { + let parse = |v: &[String]| v.iter().filter_map(|k| u128::from_str_radix(k, 16).ok()).collect::>(); + ST.with(|s| { + let mut s = s.borrow_mut(); + s.prefetch.note_misses(parse(misses)); + for k in parse(hits) { + s.index.touch(Kind::Layer, k); + s.layer_hits += 1; + } + }); +} + /// Apply the settings: on/off and the limit (`max_gb`, at most half the origin's quota when it /// is known). pub fn configure(enabled: bool, max_gb: u32, quota: Option) { @@ -155,12 +204,15 @@ pub async fn read(key: u128) -> Option { frame } -/// A worker stored the frame of `key` (`bytes` on disk). -pub fn stored(key: u128, bytes: u64) { +/// A worker stored the frame (or, with `layer`, the layer buffer) of `key` (`bytes` on disk). +pub fn stored(key: u128, bytes: u64, layer: bool) { ST.with(|s| { let mut s = s.borrow_mut(); - s.index.insert(Kind::Frame, key, bytes); + s.index.insert(if layer { Kind::Layer } else { Kind::Frame }, key, bytes); s.writes += 1; + if layer { + s.layer_writes += 1; + } }); evict(); } @@ -174,7 +226,7 @@ fn evict() { v }); if !victims.is_empty() { - delete(victims.into_iter().map(|(_, k)| disk_cache::entry_name(k)).collect()); + delete(victims.into_iter().map(|(kind, k)| name_of(kind, k)).collect()); } } @@ -194,6 +246,7 @@ pub fn clear() -> (u64, u64) { let mut s = s.borrow_mut(); let r = (s.index.len() as u64, s.index.total()); s.index.clear(); + s.prefetch.clear(); r }); wasm_bindgen_futures::spawn_local(async { @@ -212,10 +265,12 @@ pub fn stats() -> Value { "enabled": s.enabled && s.available, "available": s.available, "loaded": s.loaded, - "folder": "Origin Private File System: effectcraft-cache/v1/frames", + "folder": "Origin Private File System: effectcraft-cache/v1 (frames, layers)", "entries": s.index.len(), - "frames": s.index.len(), - "layers": 0, + "frames": s.index.keys(Kind::Frame).len(), + "layers": s.index.keys(Kind::Layer).len(), + "layerHits": s.layer_hits, + "layerWrites": s.layer_writes, "bytes": s.index.total(), "maxBytes": s.max_bytes, "hits": s.hits, diff --git a/apps/effectcraft-web/src/files.rs b/apps/effectcraft-web/src/files.rs index 78700e28..05f86d42 100644 --- a/apps/effectcraft-web/src/files.rs +++ b/apps/effectcraft-web/src/files.rs @@ -124,6 +124,10 @@ impl Services for WebServices { } download(file_name(path), data).map_err(|e| std::io::Error::other(format!("download failed: {e:?}"))) } + fn store_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { + put(path, data.into()); + Ok(()) + } } /// Media import from the table (the bytes are also handed to the media pool, which decodes them). @@ -137,6 +141,9 @@ impl effectcraft_engine::Importer for WebImporter { self.pool.add_bytes(path, bytes.clone()); effectcraft_media::probe_bytes(path, bytes).map_err(|e| e.to_string()) } + fn register(&self, path: &str, data: &[u8]) { + self.pool.add_bytes(path, get(path).unwrap_or_else(|| data.into())); + } } /// Render Queue output: files land in the table and are downloaded after the render diff --git a/apps/effectcraft-web/src/frames.rs b/apps/effectcraft-web/src/frames.rs index d22f1ef1..fc703eba 100644 --- a/apps/effectcraft-web/src/frames.rs +++ b/apps/effectcraft-web/src/frames.rs @@ -38,7 +38,7 @@ extern "C" { #[wasm_bindgen(js_name = frameWorkerPost)] fn frame_worker_post(index: u32, msg: &JsValue, transfer: &js_sys::Array); #[wasm_bindgen(js_name = hasWebGpu)] - fn has_web_gpu() -> bool; + pub(crate) fn has_web_gpu() -> bool; } /// One frame worker as the page sees it. @@ -134,7 +134,10 @@ impl WebFrames { frame_worker_post(index, &obj(&[("type", "frame".into()), ("json", t.into())]), &js_sys::Array::new()); } let disk = job.disk_key.map(|k| format!("{k:032x}")); - let req = FrameMsg::Render { id, revision: job.revision, comp: job.comp, time: job.t, opts: Box::new(job.opts), disk }; + // Layer buffers from the disk cache: what to read before rendering. + let layers = crate::diskcache::active(); + let prefetch = crate::diskcache::layer_prefetch(64); + let req = FrameMsg::Render { id, revision: job.revision, comp: job.comp, time: job.t, opts: Box::new(job.opts), disk, layers, prefetch }; frame_worker_post( index, &obj(&[("type", "frame".into()), ("json", serde_json::to_string(&req).unwrap_or_default().into())]), @@ -210,7 +213,8 @@ fn on_message(index: u32, json: JsValue, bytes: JsValue, error: JsValue) { } let reply: FrameReply = serde_json::from_str(&json.as_string()?).ok()?; match reply { - FrameReply::Frame { id, width, height, ms, passes, gpu } => { + FrameReply::Frame { id, width, height, ms, passes, gpu, layer_misses, layer_hits } => { + crate::diskcache::note_layers(&layer_misses, &layer_hits); let (_, d) = w.busy.take_if(|b| b.0 == id)?; let rgba = bytes.dyn_into::().map(|b| b.to_vec()).unwrap_or_default(); st.rendered += 1; @@ -231,9 +235,9 @@ fn on_message(index: u32, json: JsValue, bytes: JsValue, error: JsValue) { w.mirror.reset(); None } - FrameReply::Stored { key, bytes } => { + FrameReply::Stored { key, bytes, layer } => { if let Ok(k) = u128::from_str_radix(&key, 16) { - crate::diskcache::stored(k, bytes); + crate::diskcache::stored(k, bytes, layer); } None } @@ -410,7 +414,9 @@ fn post_reply(r: &FrameReply, px: Option<&[u8]>) { } fn new_server() -> FrameServer { - FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions))) + let mut s = FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions))); + s.set_layer_store(Some(Arc::new(effectcraft_engine::render::PrefetchStore::default()))); + s } /// Set up a frame worker (called once by `web/worker.js` after `workerInit`): its frame @@ -450,9 +456,14 @@ pub async fn worker_frame(json: String) { return; } }; - let disk = match &msg { - FrameMsg::Render { disk, .. } => disk.clone(), - _ => None, + let (disk, layers) = match &msg { + FrameMsg::Render { disk, layers, prefetch, .. } => { + if *layers { + prefetch_layers(prefetch).await; + } + (disk.clone(), *layers) + } + _ => (None, false), }; let mut replies = SERVER.with(|c| c.borrow_mut().get_or_insert_with(new_server).handle(msg)); while SERVER.with(|c| c.borrow().as_ref().is_some_and(|s| s.waiting())) { @@ -466,9 +477,41 @@ pub async fn worker_frame(json: String) { let entry = effectcraft_engine::render::disk_cache::frame_entry(*width, *height, px); let name = format!("{key}.ecc"); match JsFuture::from(crate::diskcache::cache_write(&name, js_sys::Uint8Array::from(&entry[..]))).await { - Ok(n) => post_reply(&FrameReply::Stored { key: key.clone(), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64 }, None), + Ok(n) => post_reply(&FrameReply::Stored { key: key.clone(), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64, layer: false }, None), Err(e) => log::warn!("frame worker: disk cache write failed: {e:?}"), } } } + if layers { + store_layers().await; + } +} + +/// Read the layer entries the page named into the server's store (those it lacks). +async fn prefetch_layers(keys: &[String]) { + let Some(store) = SERVER.with(|c| c.borrow().as_ref().and_then(|s| s.layer_store().cloned())) else { return }; + for k in keys { + let Ok(key) = u128::from_str_radix(k, 16) else { continue }; + if store.has(key) { + continue; + } + let name = crate::diskcache::name_of(effectcraft_engine::render::disk_cache::Kind::Layer, key); + if let Ok(b) = JsFuture::from(crate::diskcache::cache_read(&name)).await + && let Ok(b) = b.dyn_into::() + { + store.provide(key, &b.to_vec()); + } + } +} + +/// Write the slow layer buffers of the frame just rendered and report them. +async fn store_layers() { + let Some(store) = SERVER.with(|c| c.borrow().as_ref().and_then(|s| s.layer_store().cloned())) else { return }; + for (key, entry) in store.take_writes() { + let name = crate::diskcache::name_of(effectcraft_engine::render::disk_cache::Kind::Layer, key); + match JsFuture::from(crate::diskcache::cache_write(&name, js_sys::Uint8Array::from(&entry[..]))).await { + Ok(n) => post_reply(&FrameReply::Stored { key: format!("{key:032x}"), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64, layer: true }, None), + Err(e) => log::warn!("frame worker: layer cache write failed: {e:?}"), + } + } } diff --git a/apps/effectcraft-web/src/worker.rs b/apps/effectcraft-web/src/worker.rs index 07c9aadc..dbba7fff 100644 --- a/apps/effectcraft-web/src/worker.rs +++ b/apps/effectcraft-web/src/worker.rs @@ -9,10 +9,14 @@ //! segmentations a propagation job sends back are relayed to them. //! //! **Worker side** — `web/worker.js` instantiates the same wasm module (the page's compiled -//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`], then +//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`] and +//! [`worker_job_init`] (the worker's own WebGPU device, where workers have WebGPU), then //! [`worker_file`] per footage file and [`worker_job`] per job: a plain engine session (footage -//! from memory, expressions, the exporter) runs it blocking and posts replies as it goes. Each -//! worker has its own memory: no shared-memory threads, so this works on stable Rust. +//! from memory, expressions, the exporter) runs it and posts replies as it goes. With a device, +//! the job's frames (Render Queue frames, analysis input frames, particles, Advanced 3D) render +//! on it in passes, the job awaiting the GPU's readbacks between them +//! ([`effectcraft_engine::offload::run_request_async`]). Each worker has its own memory: no +//! shared-memory threads, so this works on stable Rust. use std::cell::RefCell; use std::collections::HashMap; @@ -32,6 +36,8 @@ extern "C" { fn worker_cancel(id: f64); #[wasm_bindgen(js_name = workerStats)] pub fn stats() -> JsValue; + #[wasm_bindgen(js_name = setJobWorkerGpu)] + pub fn set_job_worker_gpu(on: bool); } type OnMessage = Closure; @@ -42,6 +48,8 @@ thread_local! { /// The worker's session and its media pool. static SESSION: RefCell> = const { RefCell::new(None) }; static POOL: RefCell>> = const { RefCell::new(None) }; + /// The job worker's GPU device (deferred readbacks), when it has one. + static GPU: RefCell> = const { RefCell::new(None) }; } /// The page's offload: one worker per running job (idle workers are reused). @@ -169,9 +177,42 @@ pub fn worker_file(path: String, bytes: js_sys::Uint8Array) { }); } -/// Run one job (a JSON [`WorkerRequest`]) and post its replies. +/// A job worker's GPU (called once by `web/worker.js` after `workerInit`): with `gpu` and +/// WebGPU in workers, its own device with deferred readbacks becomes the session's +/// accelerator. Posts `{type: "gpu", gpu?, error?}`. +#[wasm_bindgen(js_name = workerJobInit)] +pub async fn worker_job_init(gpu: bool) { + let r = if !gpu { + Err("GPU job workers are off (?nogpuworkers)".to_string()) + } else if !crate::frames::has_web_gpu() { + Err("no WebGPU in workers".to_string()) + } else { + effectcraft_gpu::Gpu::request_deferred().await + }; + let msg = match r { + Ok(g) => { + let name = effectcraft_engine::render::Accelerator::name(&g); + SESSION.with(|c| { + if let Some(s) = c.borrow_mut().as_mut() { + s.accel = Some(Arc::new(g.clone())); + } + }); + GPU.with(|c| *c.borrow_mut() = Some(g)); + log::info!("job worker: GPU {name}"); + obj(&[("type", "gpu".into()), ("gpu", name.into())]) + } + Err(e) => { + log::info!("job worker: CPU only ({e})"); + obj(&[("type", "gpu".into()), ("error", e.into())]) + } + }; + post(&msg, None); +} + +/// Run one job (a JSON [`WorkerRequest`]) and post its replies, then `{type: "stats", gpu, +/// passes, readbacks, ms}` (the device's render passes and readbacks during the job). #[wasm_bindgen(js_name = workerJob)] -pub fn worker_job(json: String) { +pub async fn worker_job(json: String) { let send: effectcraft_engine::offload::Post = Rc::new(|r: WorkerReply| { let done = matches!(r, WorkerReply::Done); let t = serde_json::to_string(&r).unwrap_or_default(); @@ -185,13 +226,37 @@ pub fn worker_job(json: String) { return; } }; - SESSION.with(|c| { - let mut c = c.borrow_mut(); - let Some(s) = c.as_mut() else { - send(WorkerReply::Failed { error: "worker not initialised".into() }); - send(WorkerReply::Done); - return; - }; - effectcraft_engine::offload::run_request(s, req, &send); + // The session leaves its slot while the job runs (the job awaits the GPU in between; + // messages are handled one at a time, so nothing else needs it meanwhile). + let Some(mut s) = SESSION.with(|c| c.borrow_mut().take()) else { + send(WorkerReply::Failed { error: "worker not initialised".into() }); + send(WorkerReply::Done); + return; + }; + let gpu = GPU.with(|c| c.borrow().clone()); + let before = gpu.as_ref().and_then(|g| g.deferred_stats()).unwrap_or_default(); + let t0 = web_time::Instant::now(); + let kind = format!("{:?}", req.job.kind()); + // `Done` is posted last by the job; the stats go out just before it. + let stats_sent = Rc::new(std::cell::Cell::new(false)); + let (g2, s2) = (gpu.clone(), stats_sent.clone()); + let send2: effectcraft_engine::offload::Post = Rc::new(move |r: WorkerReply| { + if matches!(r, WorkerReply::Done) && !s2.replace(true) { + let after = g2.as_ref().and_then(|g| g.deferred_stats()).unwrap_or_default(); + post( + &obj(&[ + ("type", "stats".into()), + ("kind", kind.as_str().into()), + ("gpu", JsValue::from_bool(g2.is_some())), + ("passes", ((after.0 - before.0) as f64).into()), + ("readbacks", ((after.1 - before.1) as f64).into()), + ("ms", (t0.elapsed().as_secs_f64() * 1000.0).into()), + ]), + None, + ); + } + send(r); }); + effectcraft_engine::offload::run_request_async(&mut s, req, &send2).await; + SESSION.with(|c| *c.borrow_mut() = Some(s)); } diff --git a/apps/effectcraft-web/tests/smoke.mjs b/apps/effectcraft-web/tests/smoke.mjs index 47ae22ad..4bae1ada 100644 --- a/apps/effectcraft-web/tests/smoke.mjs +++ b/apps/effectcraft-web/tests/smoke.mjs @@ -11,7 +11,10 @@ // AudioContext starts on a user gesture; preview playback feeds it), File ▸ Save As (download), // re-opening the saved project through `effectcraft.addFile`, persistence across a reload (the // session, imported media, settings, a project saved to browser storage, Open Recent) and an -// offline reload through the service worker. Prints a JSON report; exits non-zero on failure. +// offline reload through the service worker; M13.24: GPU frame workers, the storage manager and +// the disk cache; M13.30: GPU job workers (Render Queue frames and an analysis's input frames on +// the worker's WebGPU device, in passes) and layer buffers in the disk cache. Prints a JSON +// report; exits non-zero on failure. // `--headed` shows the browser. import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync, readdirSync, statSync, mkdtempSync } from "node:fs"; @@ -310,6 +313,55 @@ try { check(bg.maxGapMs < 400, `event loop blocked ${bg.maxGapMs} ms during the render`); check(bg.progressFrames > 0, "progress reported while rendering"); + // GPU in the job workers (M13.30): a Render Queue job and a Warp Stabilizer analysis whose + // frames have GPU effects render on the job worker's own WebGPU device, in passes (the job + // awaits the device's readbacks between them), while the page's event loop stays free. + if (report.steps.gpuWorkers.effects) { + await js(`effectcraft.execute("renderQueue.setRender", {index: 3, render: false})`); + report.steps.gpuJobs = await js(`(async () => { + let maxGapMs = 0, last = performance.now(); + const timer = setInterval(() => { const now = performance.now(); maxGapMs = Math.max(maxGapMs, now - last); last = now; }, 10); + await effectcraft.execute("layer.newSolid", {name: "GPU Job Solid", color: "#3366cc", width: 320, height: 180}); + await effectcraft.execute("effect.apply", {effect: "Fractal Noise"}); + await effectcraft.execute("prop.addKey", {path: "effects/#1/transform/offset", time: 0, value: [0.5, 0.5]}); + await effectcraft.execute("prop.addKey", {path: "effects/#1/transform/offset", time: 1, value: [0.65, 0.58]}); + await effectcraft.execute("effect.apply", {effect: "Gaussian Blur"}); + await effectcraft.execute("prop.set", {path: "effects/#2/blurriness", value: 12}); + // (A resolution no earlier render used: Backend Auto warms this comp and scale up afresh, + // GPU first.) + await effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "png", output: "gpujob_[####].png", resolution: 0.2, timeSpan: "custom", start: 0, end: 0.4}}); + const t0 = performance.now(); + const r = await effectcraft.execute("renderQueue.render", {}); + const renderMs = performance.now() - t0; + // Applying Warp Stabilizer starts its analysis in a job worker (as in After Effects). + await effectcraft.execute("effect.apply", {effect: "Warp Stabilizer"}); + const t1 = performance.now(); + let warp; + do { + await new Promise((res) => setTimeout(res, 200)); + warp = await effectcraft.execute("warp.status", {}); + } while (!warp.analyzed && performance.now() - t1 < 180000); + const warpMs = performance.now() - t1; + clearInterval(timer); + const workers = effectcraft.info().workers; + for (let k = 0; k < 7; k++) await effectcraft.execute("edit.undo", {}); + return {status: r.items.map((i) => i.status), renderMs, warp, warpMs, workers, maxGapMs}; + })()`); + const gj = report.steps.gpuJobs; + gj.download = await waitDownload((n) => n.startsWith("gpujob")); + const jobs = gj.workers.jobs || []; + const rj = jobs.filter((j) => j.kind === "Render").pop(); + const wj = jobs.filter((j) => j.kind === "Warp").pop(); + check(gj.workers.gpu && gj.workers.gpu.adapter, `job workers have a GPU: ${JSON.stringify(gj.workers)}`); + check(gj.status.every((st) => st === "Done"), `GPU job render: ${JSON.stringify(gj.status)}`); + check(rj && rj.gpu && rj.readbacks >= 3 && rj.passes >= 6, `Render Queue frames on the job worker's GPU, in passes: ${JSON.stringify(rj)}`); + check(gj.warp.analyzed, `Warp Stabilizer analysed in a job worker: ${JSON.stringify(gj.warp)}`); + check(wj && wj.gpu && wj.readbacks > 0, `Warp Stabilizer input frames on the job worker's GPU: ${JSON.stringify(wj)}`); + check(gj.maxGapMs < 400, `event loop blocked ${gj.maxGapMs} ms during the GPU jobs`); + } else { + report.steps.gpuJobs = { note: "no WebGPU in workers here: job workers render on their CPU" }; + } + // Audio: the AudioContext starts on a user gesture. report.steps.audio = { before: (await js("effectcraft.info()")).audio }; check(report.steps.audio.before.state !== "running", "audio context not running before a gesture"); @@ -367,6 +419,13 @@ try { await js(`effectcraft.execute("prefs.set", {key: "general.recentItems", value: 7})`); const saved = await js(`effectcraft.saveToBrowser("/persist-test.ecproj")`); await js(`effectcraft.execute("layer.newSolid", {name: "Unsaved Solid"})`); + // A layer slow enough on the CPU for its buffer to go to the disk cache too (M13.30). + await js(`(async () => { + await effectcraft.execute("effect.apply", {effect: "Fractal Noise"}); + await effectcraft.execute("prop.set", {path: "effects/#1/complexity", value: 10}); + await effectcraft.execute("effect.apply", {effect: "Gaussian Blur"}); + await effectcraft.execute("prop.set", {path: "effects/#2/blurriness", value: 60}); + })()`); // Disk cache: frames the workers render now are stored in the Origin Private File System // (the CPU renderer sends every frame to the workers). await js(`effectcraft.execute("render.backend", {backend: "cpu"})`); @@ -386,6 +445,7 @@ try { return {writesBefore: w0, after: st}; })()`); check(stored.after.writes >= stored.writesBefore + diskTimes.length, `frames written to the disk cache: ${JSON.stringify(stored)}`); + check(stored.after.layers > 0 && stored.after.layerWrites > 0, `layer buffers written to the disk cache: ${JSON.stringify(stored.after)}`); await sleep(1500); // the session snapshot is taken at most every second await js("effectcraft.flush()"); const before = await js("effectcraft.listStored()"); @@ -424,6 +484,23 @@ try { })()`); const dcr = report.steps.diskCache; dcr.stored = stored; + // Layer buffers from the disk cache (M13.30): after a purge (the workers' memory is empty), a + // frame not seen yet renders and reports its layer misses; the page then names the ones on + // disk, and the worker of the next frame reads them before rendering: its lookups hit. + dcr.layers = await js(`(async () => { + const before = await effectcraft.execute("cache.diskStats", {}); + const hits = []; + for (const t of [2.05, 2.15, 2.25]) { + await effectcraft.execute("edit.purge", {what: "memory"}); + await effectcraft.execute("time.set", {time: t}); + const r0 = effectcraft.info().frameWorkers.rendered, t0 = performance.now(); + while (effectcraft.info().frameWorkers.rendered <= r0 && performance.now() - t0 < 30000) await new Promise((r) => setTimeout(r, 100)); + hits.push((await effectcraft.execute("cache.diskStats", {})).layerHits); + } + return {before, after: await effectcraft.execute("cache.diskStats", {}), hits}; + })()`); + check(dcr.layers.before.layers > 0, `layer entries after reload: ${JSON.stringify(dcr.layers.before)}`); + check(dcr.layers.after.layerHits > dcr.layers.before.layerHits, `layer buffers served from the disk cache: ${JSON.stringify(dcr.layers)}`); check(dcr.listed >= diskTimes.length, `disk cache entries after reload: ${JSON.stringify(dcr)}`); check(dcr.frameWorkers.diskHits >= diskTimes.length, `cache hits after reload: ${JSON.stringify(dcr.frameWorkers)}`); const cleared = await js(`effectcraft.execute("storage.clear", {what: "diskCache"})`); diff --git a/apps/effectcraft-web/web/worker.js b/apps/effectcraft-web/web/worker.js index 4bd1ab54..7c6752e5 100644 --- a/apps/effectcraft-web/web/worker.js +++ b/apps/effectcraft-web/web/worker.js @@ -1,7 +1,8 @@ // An EffectCraft engine instance in a Web Worker (src/worker.rs, src/frames.rs, // crates/engine/src/offload.rs and remote.rs). The page sends the compiled module, then: // - job workers: footage files and jobs (Render Queue, analyses, Roto Brush); the worker replies -// with JSON messages and rendered files; +// with JSON messages and rendered files. With `gpu: true` it opens its own WebGPU device and +// the job's frames render on it in passes (the job awaits the device's readbacks); // - frame workers (`frames: true`): footage files and frame messages (project syncs as diffs, // render requests, Roto Brush segmentations); the worker replies with frames (pixels // transferred). With `gpu: true` it opens its own WebGPU device and renders GPU effects on @@ -15,7 +16,7 @@ let queue = Promise.resolve(); async function handle(m) { if (m.type === "file") ec.workerFile(m.path, m.bytes); - else if (m.type === "job") ec.workerJob(m.json); + else if (m.type === "job") await ec.workerJob(m.json); else if (m.type === "frame") await ec.workerFrame(m.json); } @@ -26,6 +27,7 @@ self.onmessage = (e) => { await init({ module_or_path: m.module ?? new URL("effectcraft_web_bg.wasm", m.base) }); ec.workerInit(); if (m.frames) await ec.workerFrameInit(!!m.gpu); + else await ec.workerJobInit(!!m.gpu); self.postMessage({ type: "ready" }); })(); return; diff --git a/crates/color/src/icc.rs b/crates/color/src/icc.rs index b740c123..e85927a8 100644 --- a/crates/color/src/icc.rs +++ b/crates/color/src/icc.rs @@ -3,8 +3,15 @@ //! `bXYZ`), the media white point (`wtpt`), the chromatic adaptation (`chad`), the tone curves //! (`rTRC`, `gTRC`, `bTRC` as `curv` or `para`) and the description (`desc` / `mluc`). //! -//! Used by View ▸ Simulate Output ▸ My Custom RGB: the profile becomes primaries, a white point -//! and a curve. LUT-based (`A2B0`) and non-RGB profiles are rejected. +//! LUT-based profiles ([`parse_profile`], [`LutProfile`]): `A2B0` / `B2A0` tables of type +//! `lut8Type`, `lut16Type`, `lutAToBType` and `lutBToAType` (ICC.1:2010 §10.10–10.12): +//! curves (`curv`, `para`), matrices and multilinear CLUT interpolation, with the PCS in XYZ or +//! Lab (the legacy 16-bit Lab encoding for `lut16Type`). A profile without `B2A0` is inverted +//! numerically. Implemented from the public specification only. +//! +//! Used by View ▸ Simulate Output ▸ My Custom RGB: a matrix/TRC profile becomes primaries, a white +//! point and a curve; a LUT-based profile is baked into a 3D LUT for the viewer ([`Lut3d`]). +//! Non-RGB profiles are rejected. use crate::space::{bradford, invert, mul_vec}; @@ -296,6 +303,645 @@ pub fn write_matrix_profile(desc: &str, primaries: [[f64; 2]; 3], white: [f64; 2 out } +// ---------------------------------------------------------------- LUT-based profiles + +/// One processing element of a LUT transform. Values between elements are normalised to 0..1 +/// (ICC.1:2010 §10.10–10.12: `lut8Type`, `lut16Type`, `lutAToBType`, `lutBToAType`). +#[derive(Clone, Debug, PartialEq)] +pub enum Stage { + /// One curve per channel. + Curves(Vec), + /// A 3×3 matrix and offsets (`lutAToBType` / `lutBToAType`; `lut8/16Type` without offsets). + Matrix([[f64; 3]; 3], [f64; 3]), + Clut(Clut), +} + +/// A colour lookup table: `grid[i]` points along input `i`, `outputs` values per point +/// (normalised 0..1), the first input varying slowest (ICC order). +#[derive(Clone, Debug, PartialEq)] +pub struct Clut { + pub grid: Vec, + pub outputs: usize, + pub data: Vec, +} + +impl Clut { + /// Multilinear interpolation at `x` (0..1 per input). + pub fn eval(&self, x: &[f64]) -> Vec { + let n = self.grid.len(); + let mut base = 0usize; + let mut stride = vec![0usize; n]; + let mut s = self.outputs; + for i in (0..n).rev() { + stride[i] = s; + s *= self.grid[i].max(1); + } + let mut frac = vec![0.0; n]; + let mut step = vec![0usize; n]; + for i in 0..n { + let g = self.grid[i].max(1); + let f = x.get(i).copied().unwrap_or(0.0).clamp(0.0, 1.0) * (g - 1) as f64; + let k = (f.floor() as usize).min(g.saturating_sub(2)); + frac[i] = if g > 1 { f - k as f64 } else { 0.0 }; + step[i] = if g > 1 { stride[i] } else { 0 }; + base += k * stride[i]; + } + let mut out = vec![0.0; self.outputs]; + for corner in 0..(1usize << n) { + let mut w = 1.0; + let mut off = base; + for i in 0..n { + if corner >> i & 1 == 1 { + w *= frac[i]; + off += step[i]; + } else { + w *= 1.0 - frac[i]; + } + } + if w == 0.0 { + continue; + } + for (o, v) in out.iter_mut().enumerate() { + *v += w * self.data.get(off + o).copied().unwrap_or(0.0) as f64; + } + } + out + } +} + +/// A LUT transform (`A2B0` device → PCS, or `B2A0` PCS → device). +#[derive(Clone, Debug, PartialEq)] +pub struct Lut { + pub inputs: usize, + pub outputs: usize, + pub stages: Vec, + /// `lut16Type`: PCS Lab uses the legacy 16-bit encoding (0xFF00 = L 100). + pub legacy_lab: bool, +} + +impl Lut { + /// Run the transform on normalised values. + pub fn eval(&self, x: &[f64]) -> Vec { + let mut v: Vec = x.iter().map(|v| v.clamp(0.0, 1.0)).collect(); + for s in &self.stages { + v = match s { + Stage::Curves(c) => v.iter().enumerate().map(|(i, x)| c.get(i).map_or(*x, |t| t.eval(*x))).collect(), + Stage::Matrix(m, o) if v.len() == 3 => (0..3).map(|i| (m[i][0] * v[0] + m[i][1] * v[1] + m[i][2] * v[2] + o[i]).clamp(0.0, 1.0)).collect(), + Stage::Matrix(..) => v, + Stage::Clut(c) => c.eval(&v), + }; + } + v + } +} + +/// The profile connection space of a profile. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Pcs { + Xyz, + Lab, +} + +/// D50 white (X, Y, Z) of the PCS. +const D50_XYZ: [f64; 3] = [0.9642, 1.0, 0.8249]; + +fn lab_to_xyz(l: [f64; 3]) -> [f64; 3] { + let fy = (l[0] + 16.0) / 116.0; + let fx = fy + l[1] / 500.0; + let fz = fy - l[2] / 200.0; + let inv = |t: f64| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f64 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + [D50_XYZ[0] * inv(fx), D50_XYZ[1] * inv(fy), D50_XYZ[2] * inv(fz)] +} + +fn xyz_to_lab(x: [f64; 3]) -> [f64; 3] { + let f = |t: f64| if t > (6.0f64 / 29.0).powi(3) { t.cbrt() } else { t / (3.0 * (6.0f64 / 29.0).powi(2)) + 4.0 / 29.0 }; + let (fx, fy, fz) = (f(x[0] / D50_XYZ[0]), f(x[1] / D50_XYZ[1]), f(x[2] / D50_XYZ[2])); + [116.0 * fy - 16.0, 500.0 * (fx - fy), 200.0 * (fy - fz)] +} + +/// Normalised PCS values (a LUT's output or input) → CIE XYZ (D50). +pub fn pcs_to_xyz(pcs: Pcs, legacy: bool, n: &[f64]) -> [f64; 3] { + let g = |i: usize| n.get(i).copied().unwrap_or(0.0); + match pcs { + Pcs::Xyz => [0, 1, 2].map(|i| g(i) * 65535.0 / 32768.0), + Pcs::Lab => { + let k = if legacy { 65535.0 / 65280.0 } else { 1.0 }; + lab_to_xyz([g(0) * k * 100.0, g(1) * k * 255.0 - 128.0, g(2) * k * 255.0 - 128.0]) + } + } +} + +/// CIE XYZ (D50) → normalised PCS values. +pub fn xyz_to_pcs(pcs: Pcs, legacy: bool, xyz: [f64; 3]) -> [f64; 3] { + match pcs { + Pcs::Xyz => xyz.map(|v| (v * 32768.0 / 65535.0).clamp(0.0, 1.0)), + Pcs::Lab => { + let l = xyz_to_lab(xyz); + let k = if legacy { 65280.0 / 65535.0 } else { 1.0 }; + [(l[0] / 100.0 * k).clamp(0.0, 1.0), ((l[1] + 128.0) / 255.0 * k).clamp(0.0, 1.0), ((l[2] + 128.0) / 255.0 * k).clamp(0.0, 1.0)] + } + } +} + +/// What a LUT-based RGB profile says: its `A2B0` (device → PCS) and, when present, its `B2A0` +/// (PCS → device). +#[derive(Clone, Debug, PartialEq)] +pub struct LutProfile { + pub description: String, + pub pcs: Pcs, + pub a2b: Lut, + pub b2a: Option, + pub version: u8, +} + +impl LutProfile { + /// Device RGB (0..1) → CIE XYZ (D50, relative to the PCS white). + pub fn to_xyz(&self, rgb: [f64; 3]) -> [f64; 3] { + pcs_to_xyz(self.pcs, self.a2b.legacy_lab, &self.a2b.eval(&rgb)) + } + + /// CIE XYZ (D50) → device RGB (0..1): through `B2A0`, or by inverting `A2B0` (Gauss-Newton + /// from `guess`, clipped to the device cube) when the profile has none. + pub fn from_xyz(&self, xyz: [f64; 3], guess: [f64; 3]) -> [f64; 3] { + if let Some(b) = &self.b2a { + let v = b.eval(&xyz_to_pcs(self.pcs, b.legacy_lab, xyz)); + return [0, 1, 2].map(|i| v.get(i).copied().unwrap_or(0.0).clamp(0.0, 1.0)); + } + self.invert(xyz, guess) + } + + /// Device RGB whose `A2B0` result is closest to `xyz` (least squares in CIE Lab). + pub fn invert(&self, xyz: [f64; 3], guess: [f64; 3]) -> [f64; 3] { + let target = xyz_to_lab(xyz); + let err = |rgb: [f64; 3]| { + let l = xyz_to_lab(self.to_xyz(rgb)); + [l[0] - target[0], l[1] - target[1], l[2] - target[2]] + }; + let norm = |e: [f64; 3]| e[0] * e[0] + e[1] * e[1] + e[2] * e[2]; + let mut x = guess.map(|v| v.clamp(0.0, 1.0)); + let mut e = err(x); + for _ in 0..24 { + if norm(e) < 1e-6 { + break; + } + // Finite-difference Jacobian (one-sided, inward at the cube's faces). + let h = 1e-3; + let mut j = [[0.0; 3]; 3]; + for c in 0..3 { + let mut y = x; + let d = if x[c] + h <= 1.0 { h } else { -h }; + y[c] += d; + let ey = err(y); + for r in 0..3 { + j[r][c] = (ey[r] - e[r]) / d; + } + } + let det = j[0][0] * (j[1][1] * j[2][2] - j[1][2] * j[2][1]) - j[0][1] * (j[1][0] * j[2][2] - j[1][2] * j[2][0]) + + j[0][2] * (j[1][0] * j[2][1] - j[1][1] * j[2][0]); + if det.abs() < 1e-12 { + break; + } + let ji = crate::space::invert(&j); + let dx = crate::space::mul_vec(&ji, e); + // Damped step: halve until the error drops. + let mut t = 1.0; + let mut improved = false; + for _ in 0..8 { + let y = [0, 1, 2].map(|i| (x[i] - t * dx[i]).clamp(0.0, 1.0)); + let ey = err(y); + if norm(ey) < norm(e) { + x = y; + e = ey; + improved = true; + break; + } + t *= 0.5; + } + if !improved { + break; + } + } + x + } + + /// Chromaticities and white the profile's device approximately has (what the dialog shows): + /// the primaries' and white's `A2B0` results (CIE xy, D50-relative), and the gamma that + /// best fits its neutral ramp. + pub fn approximate(&self) -> ([[f64; 2]; 3], [f64; 2], f64) { + let prim = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]].map(|c| xy(self.to_xyz(c))); + let wy = self.to_xyz([1.0; 3])[1].max(1e-9); + let ramp = Trc::Table((0..=64).map(|i| ((self.to_xyz([i as f64 / 64.0; 3])[1] / wy).clamp(0.0, 1.0) * 65535.0).round() as u16).collect()); + (prim, D50, ramp.fit_gamma()) + } +} + +/// A parsed RGB profile: matrix/TRC, or LUT-based (`A2B0`, which takes precedence when both are +/// present, as the ICC specification says). +#[derive(Clone, Debug, PartialEq)] +pub enum Profile { + Matrix(RgbProfile), + Lut(LutProfile), +} + +fn tag_table(b: &[u8]) -> Result, String> { + let n = be32(b, 128).ok_or("truncated tag table")? as usize; + let mut tags = std::collections::HashMap::new(); + for i in 0..n.min(200) { + let o = 132 + i * 12; + let sig = b.get(o..o + 4).ok_or("truncated tag table")?; + let off = be32(b, o + 4).ok_or("truncated tag table")? as usize; + let size = be32(b, o + 8).ok_or("truncated tag table")? as usize; + if off.checked_add(size).is_some_and(|e| e <= b.len()) { + tags.insert(String::from_utf8_lossy(sig).into_owned(), &b[off..off + size]); + } + } + Ok(tags) +} + +/// Parse an RGB ICC profile, LUT-based or matrix/TRC. +pub fn parse_profile(b: &[u8]) -> Result { + if b.len() < 132 || b.get(36..40) != Some(b"acsp") { + return Err("not an ICC profile".into()); + } + if b.get(16..20) != Some(b"RGB ") { + return Err("not an RGB profile (only RGB display profiles can be simulated)".into()); + } + let pcs = match b.get(20..24) { + Some(b"XYZ ") => Pcs::Xyz, + Some(b"Lab ") => Pcs::Lab, + _ => return Err("unknown profile connection space".into()), + }; + let tags = tag_table(b)?; + let Some(a2b) = tags.get("A2B0") else { return parse(b).map(Profile::Matrix) }; + let a2b = parse_lut(a2b, false).map_err(|e| format!("A2B0: {e}"))?; + if a2b.inputs != 3 || a2b.outputs != 3 { + return Err("A2B0 must map 3 device channels to 3 PCS values".into()); + } + let b2a = match tags.get("B2A0").map(|t| parse_lut(t, true)) { + Some(Ok(l)) if l.inputs == 3 && l.outputs == 3 => Some(l), + _ => None, + }; + Ok(Profile::Lut(LutProfile { description: description(tags.get("desc").copied()).unwrap_or_default(), pcs, a2b, b2a, version: b[8] })) +} + +/// A `curv` or `para` element at `o` of `t`; returns it and its padded length. +fn curve_at(t: &[u8], o: usize) -> Result<(Trc, usize), String> { + match t.get(o..o + 4) { + Some(b"curv") => { + let count = be32(t, o + 8).ok_or("bad curv")? as usize; + let trc = match count { + 0 => Trc::Gamma(1.0), + 1 => Trc::Gamma(be16(t, o + 12).ok_or("bad curv")? as f64 / 256.0), + _ => Trc::Table((0..count).map(|i| be16(t, o + 12 + i * 2)).collect::>>().ok_or("truncated curv")?), + }; + Ok((trc, (12 + count * 2).div_ceil(4) * 4)) + } + Some(b"para") => { + let ty = be16(t, o + 8).ok_or("bad para")?; + let np = [1, 3, 4, 5, 7].get(ty as usize).copied().ok_or("unknown para function")?; + let mut p = [0.0; 7]; + for (k, v) in p.iter_mut().enumerate().take(np) { + *v = s15f16(t, o + 12 + k * 4).ok_or("truncated para")?; + } + Ok((if ty == 0 { Trc::Gamma(p[0]) } else { Trc::Parametric(ty, p) }, 12 + np * 4)) + } + _ => Err("unsupported curve element".into()), + } +} + +fn curves_at(t: &[u8], o: usize, n: usize) -> Result, String> { + let mut v = vec![]; + let mut o = o; + for _ in 0..n { + let (c, len) = curve_at(t, o)?; + v.push(c); + o += len; + } + Ok(v) +} + +fn clut_len(grid: &[usize], outputs: usize) -> Option { + grid.iter().try_fold(outputs, |a, g| a.checked_mul(*g)).filter(|n| *n <= 1 << 26) +} + +/// `lut8Type` / `lut16Type` / `lutAToBType` / `lutBToAType`. `b2a`: the element order of a +/// PCS → device table. +pub fn parse_lut(t: &[u8], b2a: bool) -> Result { + let sig = t.get(0..4).ok_or("empty tag")?; + match sig { + b"mft1" | b"mft2" => { + let wide = sig == b"mft2"; + let (i, o, g) = (*t.get(8).ok_or("truncated")? as usize, *t.get(9).ok_or("truncated")? as usize, *t.get(10).ok_or("truncated")? as usize); + if i == 0 || o == 0 || i > 8 || o > 15 || g < 2 { + return Err("bad table dimensions".into()); + } + let mut m = [[0.0; 3]; 3]; + for (k, v) in m.iter_mut().flatten().enumerate() { + *v = s15f16(t, 12 + k * 4).ok_or("truncated matrix")?; + } + let (n_in, n_out, mut p) = + if wide { (be16(t, 48).ok_or("truncated")? as usize, be16(t, 50).ok_or("truncated")? as usize, 52) } else { (256, 256, 48) }; + if n_in < 2 || n_out < 2 { + return Err("bad table sizes".into()); + } + let w = if wide { 2 } else { 1 }; + let read = |p: usize| -> Option { if wide { be16(t, p) } else { t.get(p).map(|v| *v as u16 * 257) } }; + let table = |n: usize, p: &mut usize| -> Result { + let v = (0..n).map(|k| read(*p + k * w)).collect::>>().ok_or("truncated table")?; + *p += n * w; + Ok(Trc::Table(v)) + }; + let ins = (0..i).map(|_| table(n_in, &mut p)).collect::, _>>()?; + let grid = vec![g; i]; + let len = clut_len(&grid, o).ok_or("table too large")?; + let data = (0..len).map(|k| read(p + k * w).map(|v| v as f32 / 65535.0)).collect::>>().ok_or("truncated table")?; + p += len * w; + let outs = (0..o).map(|_| table(n_out, &mut p)).collect::, _>>()?; + let mut stages = vec![]; + // The matrix applies only to XYZ input (a PCS → device table); identity is skipped. + let ident = m == [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + if b2a && i == 3 && !ident { + stages.push(Stage::Matrix(m, [0.0; 3])); + } + stages.push(Stage::Curves(ins)); + stages.push(Stage::Clut(Clut { grid, outputs: o, data })); + stages.push(Stage::Curves(outs)); + Ok(Lut { inputs: i, outputs: o, stages, legacy_lab: wide }) + } + b"mAB " | b"mBA " => { + let (i, o) = (*t.get(8).ok_or("truncated")? as usize, *t.get(9).ok_or("truncated")? as usize); + if i == 0 || o == 0 || i > 15 || o > 15 { + return Err("bad channel counts".into()); + } + let off = |k: usize| be32(t, 12 + k * 4).map(|v| v as usize).ok_or("truncated offsets"); + let (ob, om, oc, oclut, oa) = (off(0)?, off(1)?, off(2)?, off(3)?, off(4)?); + let a_n = if sig == b"mAB " { i } else { o }; + let b_n = if sig == b"mAB " { o } else { i }; + let bc = if ob > 0 { Some(Stage::Curves(curves_at(t, ob, b_n)?)) } else { None }; + let ac = if oa > 0 { Some(Stage::Curves(curves_at(t, oa, a_n)?)) } else { None }; + let mtx = if om > 0 { + let v = (0..12).map(|k| s15f16(t, om + k * 4)).collect::>>().ok_or("truncated matrix")?; + Some(Stage::Matrix([[v[0], v[1], v[2]], [v[3], v[4], v[5]], [v[6], v[7], v[8]]], [v[9], v[10], v[11]])) + } else { + None + }; + let mc = if oc > 0 { Some(Stage::Curves(curves_at(t, oc, 3)?)) } else { None }; + let clut = if oclut > 0 { + let (ci, co) = (i, o); + let grid: Vec = (0..ci).map(|k| t.get(oclut + k).map(|v| *v as usize)).collect::>().ok_or("truncated clut")?; + if grid.iter().any(|g| *g < 2) { + return Err("bad grid".into()); + } + let prec = *t.get(oclut + 16).ok_or("truncated clut")? as usize; + let len = clut_len(&grid, co).ok_or("table too large")?; + let d = oclut + 20; + let data = match prec { + 1 => (0..len).map(|k| t.get(d + k).map(|v| *v as f32 / 255.0)).collect::>>(), + 2 => (0..len).map(|k| be16(t, d + k * 2).map(|v| v as f32 / 65535.0)).collect::>>(), + _ => return Err("bad clut precision".into()), + } + .ok_or("truncated clut")?; + Some(Stage::Clut(Clut { grid, outputs: co, data })) + } else { + None + }; + if bc.is_none() { + return Err("the B curves are required".into()); + } + // A → CLUT → M → matrix → B (A2B); B → matrix → M → CLUT → A (B2A). + let order = if sig == b"mAB " { [ac, clut, mc, mtx, bc] } else { [bc, mtx, mc, clut, ac] }; + Ok(Lut { inputs: i, outputs: o, stages: order.into_iter().flatten().collect(), legacy_lab: false }) + } + _ => Err(format!("unsupported table type `{}`", String::from_utf8_lossy(sig))), + } +} + +/// The kind of table [`write_lut_profile`] writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LutKind { + /// `lut8Type` (PCS Lab). + Lut8, + /// `lut16Type` (PCS XYZ). + Lut16, + /// `lutAToBType` / `lutBToAType` (PCS Lab, v4) with power-curve A curves and a matrix element. + AToB, +} + +/// Build a LUT-based RGB display profile (tests, generated profiles): `A2B0` samples `to_xyz` +/// (device RGB → XYZ D50) on a `grid`³ table; with `from_xyz`, a `B2A0` samples it on the PCS. +pub fn write_lut_profile( + desc: &str, + kind: LutKind, + grid: usize, + to_xyz: &dyn Fn([f64; 3]) -> [f64; 3], + from_xyz: Option<&dyn Fn([f64; 3]) -> [f64; 3]>, +) -> Vec { + let (pcs, legacy) = match kind { + LutKind::Lut8 => (Pcs::Lab, false), + LutKind::Lut16 => (Pcs::Xyz, true), + LutKind::AToB => (Pcs::Lab, false), + }; + // The A curves of an AToB table (a power curve) spread the grid evenly over lightness. + let a_gamma = 0.75; + let pt = |k: usize| k as f64 / (grid - 1) as f64; + let mut a2b_vals = vec![]; + let mut b2a_vals = vec![]; + for r in 0..grid { + for g in 0..grid { + for b in 0..grid { + let mut rgb = [pt(r), pt(g), pt(b)]; + if kind == LutKind::AToB { + rgb = rgb.map(|v| v.powf(1.0 / a_gamma)); + } + a2b_vals.extend(xyz_to_pcs(pcs, legacy, to_xyz(rgb))); + if let Some(f) = from_xyz { + // A lut16 B2A0 over XYZ gets cube-root input curves (even steps in lightness). + let node = [pt(r), pt(g), pt(b)]; + let x = pcs_to_xyz(pcs, legacy, &if kind == LutKind::Lut16 { node.map(|v| v * v * v) } else { node }); + let d = f(x).map(|v| v.clamp(0.0, 1.0)); + b2a_vals.extend(d); + } + } + } + } + let s15 = |v: f64| ((v * 65536.0).round() as i32).to_be_bytes(); + let pad4 = |t: &mut Vec| { + while !t.len().is_multiple_of(4) { + t.push(0); + } + }; + let mft = |vals: &[f64], wide: bool, cube_in: bool| -> Vec { + let mut t = if wide { b"mft2\0\0\0\0".to_vec() } else { b"mft1\0\0\0\0".to_vec() }; + t.extend([3, 3, grid as u8, 0]); + for k in 0..9 { + t.extend(s15(if k % 4 == 0 { 1.0 } else { 0.0 })); + } + let n = 256usize; + if wide { + t.extend((n as u16).to_be_bytes()); + t.extend((n as u16).to_be_bytes()); + } + let put = |t: &mut Vec, v: f64| { + if wide { t.extend(((v.clamp(0.0, 1.0) * 65535.0).round() as u16).to_be_bytes()) } else { t.push((v.clamp(0.0, 1.0) * 255.0).round() as u8) } + }; + for _ in 0..3 { + for k in 0..n { + let x = k as f64 / (n - 1) as f64; + put(&mut t, if cube_in { x.cbrt() } else { x }); + } + } + for v in vals { + put(&mut t, *v); + } + for _ in 0..3 { + for k in 0..n { + put(&mut t, k as f64 / (n - 1) as f64); + } + } + pad4(&mut t); + t + }; + let curv_identity = || b"curv\0\0\0\0\0\0\0\0".to_vec(); + let para_gamma = |g: f64| { + let mut t = b"para\0\0\0\0\0\0\0\0".to_vec(); + t.extend(s15(g)); + t + }; + let mab = |vals: &[f64], a2b: bool| -> Vec { + let mut t = if a2b { b"mAB \0\0\0\0".to_vec() } else { b"mBA \0\0\0\0".to_vec() }; + t.extend([3, 3, 0, 0]); + let offsets_at = t.len(); + t.extend([0u8; 20]); + // B curves (identity). + let ob = t.len(); + for _ in 0..3 { + t.extend(curv_identity()); + } + // Matrix: identity, zero offsets (exercises the element). + let om = t.len(); + for k in 0..9 { + t.extend(s15(if k % 4 == 0 { 1.0 } else { 0.0 })); + } + for _ in 0..3 { + t.extend(s15(0.0)); + } + // M curves (identity). + let oc = t.len(); + for _ in 0..3 { + t.extend(curv_identity()); + } + let oclut = t.len(); + let mut g = [0u8; 16]; + g[..3].fill(grid as u8); + t.extend(g); + t.extend([2, 0, 0, 0]); + for v in vals { + t.extend(((v.clamp(0.0, 1.0) * 65535.0).round() as u16).to_be_bytes()); + } + pad4(&mut t); + let oa = t.len(); + for _ in 0..3 { + t.extend(para_gamma(if a2b { a_gamma } else { 1.0 })); + } + for (k, o) in [ob, om, oc, oclut, oa].into_iter().enumerate() { + t[offsets_at + k * 4..offsets_at + k * 4 + 4].copy_from_slice(&(o as u32).to_be_bytes()); + } + t + }; + let table = |vals: &[f64], a2b: bool| match kind { + LutKind::Lut8 => mft(vals, false, false), + LutKind::Lut16 => mft(vals, true, !a2b), + LutKind::AToB => mab(vals, a2b), + }; + let dsc = { + let mut t = b"desc\0\0\0\0".to_vec(); + t.extend((desc.len() as u32 + 1).to_be_bytes()); + t.extend(desc.as_bytes()); + t.push(0); + t.extend([0u8; 12 + 67]); + pad4(&mut t); + t + }; + let mut tags: Vec<(&[u8; 4], Vec)> = vec![(b"desc", dsc), (b"A2B0", table(&a2b_vals, true))]; + if from_xyz.is_some() { + tags.push((b"B2A0", table(&b2a_vals, false))); + } + let mut tab = vec![]; + let mut data = vec![]; + let base = 128 + 4 + tags.len() * 12; + for (sig, t) in &tags { + tab.extend(*sig); + tab.extend(((base + data.len()) as u32).to_be_bytes()); + tab.extend((t.len() as u32).to_be_bytes()); + data.extend(t); + pad4(&mut data); + } + let total = base + data.len(); + let mut h = vec![0u8; 128]; + h[0..4].copy_from_slice(&(total as u32).to_be_bytes()); + h[8] = if kind == LutKind::AToB { 4 } else { 2 }; + h[12..16].copy_from_slice(b"mntr"); + h[16..20].copy_from_slice(b"RGB "); + h[20..24].copy_from_slice(if pcs == Pcs::Lab { b"Lab " } else { b"XYZ " }); + h[36..40].copy_from_slice(b"acsp"); + for (k, v) in D50_XYZ.into_iter().enumerate() { + h[68 + k * 4..72 + k * 4].copy_from_slice(&s15(v)); + } + let mut out = h; + out.extend((tags.len() as u32).to_be_bytes()); + out.extend(tab); + out.extend(data); + out +} + +/// A baked 3D lookup table: `size`³ RGB points over the unit cube (red varying slowest), +/// evaluated trilinearly. +#[derive(Clone, Debug, PartialEq)] +pub struct Lut3d { + pub size: usize, + pub data: Vec<[f32; 3]>, +} + +impl Lut3d { + /// Sample `f` on a `size`³ grid. + pub fn bake(size: usize, f: impl Fn([f64; 3]) -> [f64; 3]) -> Lut3d { + let size = size.max(2); + let pt = |k: usize| k as f64 / (size - 1) as f64; + let data = (0..size * size * size) + .map(|i| { + let (r, g, b) = (i / (size * size), i / size % size, i % size); + f([pt(r), pt(g), pt(b)]).map(|v| v as f32) + }) + .collect(); + Lut3d { size, data } + } + + pub fn eval(&self, c: [f32; 3]) -> [f32; 3] { + let n = self.size; + let mut k = [0usize; 3]; + let mut f = [0f32; 3]; + for i in 0..3 { + let x = c[i].clamp(0.0, 1.0) * (n - 1) as f32; + k[i] = (x.floor() as usize).min(n - 2); + f[i] = x - k[i] as f32; + } + let at = |r: usize, g: usize, b: usize| self.data[(r * n + g) * n + b]; + let mut out = [0f32; 3]; + for corner in 0..8 { + let (dr, dg, db) = (corner >> 2 & 1, corner >> 1 & 1, corner & 1); + let w = (if dr == 1 { f[0] } else { 1.0 - f[0] }) * (if dg == 1 { f[1] } else { 1.0 - f[1] }) * (if db == 1 { f[2] } else { 1.0 - f[2] }); + if w == 0.0 { + continue; + } + let v = at(k[0] + dr, k[1] + dg, k[2] + db); + for i in 0..3 { + out[i] += w * v[i]; + } + } + out + } +} + #[cfg(test)] mod tests { use super::*; @@ -334,4 +980,95 @@ mod tests { b[16..20].copy_from_slice(b"CMYK"); assert!(parse(&b).unwrap_err().contains("RGB")); } + + /// A display: P3 primaries, D65 white, gamma 2.2 → XYZ D50 (and back). + fn device() -> (impl Fn([f64; 3]) -> [f64; 3], impl Fn([f64; 3]) -> [f64; 3]) { + let p3 = [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]]; + let m = crate::space::mul(&bradford(D65, D50), &crate::space::rgb_to_xyz(p3, D65)); + let mi = invert(&m); + (move |c: [f64; 3]| mul_vec(&m, c.map(|v| v.powf(2.2))), move |x: [f64; 3]| mul_vec(&mi, x).map(|v| v.clamp(0.0, 1.0).powf(1.0 / 2.2))) + } + + fn lab_de(a: [f64; 3], b: [f64; 3]) -> f64 { + let (a, b) = (xyz_to_lab(a), xyz_to_lab(b)); + ((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2) + (a[2] - b[2]).powi(2)).sqrt() + } + + #[test] + fn lut_profiles_evaluate_their_a2b0() { + let (to, from) = device(); + for (kind, grid, tol) in [(LutKind::Lut16, 17, 2.5), (LutKind::Lut8, 17, 3.0), (LutKind::AToB, 17, 1.5)] { + let b = write_lut_profile("LUT Monitor", kind, grid, &to, Some(&from)); + let Profile::Lut(p) = parse_profile(&b).unwrap() else { panic!("{kind:?}: not a LUT profile") }; + assert_eq!(p.description, "LUT Monitor"); + assert!(p.b2a.is_some()); + let mut worst = 0.0f64; + for k in 0..125 { + let c = [(k / 25) as f64 / 4.0, (k / 5 % 5) as f64 / 4.0, (k % 5) as f64 / 4.0]; + worst = worst.max(lab_de(p.to_xyz(c), to(c))); + // PCS → device through B2A0 lands back on mid-gamut colours (a coarse grid + // over linear XYZ can't follow the gamma curve: checked at its nodes below). + if p.pcs == Pcs::Lab && c.iter().all(|v| (0.25..=0.75).contains(v)) { + let back = p.from_xyz(to(c), [0.5; 3]); + assert!(lab_de(to(back), to(c)) < 6.0, "{kind:?} {c:?} → {back:?}"); + } + } + assert!(worst < tol, "{kind:?}: ΔE {worst}"); + // At B2A0's grid nodes the table gives what was sampled. + let b2a = p.b2a.as_ref().unwrap(); + let q = match kind { + LutKind::Lut8 => 2.0 / 255.0, + LutKind::Lut16 => 3e-3, + LutKind::AToB => 1e-3, + }; + for node in [[0.5, 0.5, 0.5], [0.25, 0.5, 0.75], [1.0, 0.0, 0.5]] { + // (lut16: through its cube-root input curves) + let node = if kind == LutKind::Lut16 { node.map(|v: f64| v * v * v) } else { node }; + let want = from(pcs_to_xyz(p.pcs, b2a.legacy_lab, &node)); + let got = b2a.eval(&node); + assert!((0..3).all(|i| (got[i] - want[i]).abs() < q), "{kind:?} {node:?}: {got:?} vs {want:?}"); + } + // The matrix reader rejects it; the approximation finds the primaries. + assert!(parse(&b).is_err()); + let (prim, _, gamma) = p.approximate(); + let red = xy(to([1.0, 0.0, 0.0])); + assert!((prim[0][0] - red[0]).abs() < 0.01 && (prim[0][1] - red[1]).abs() < 0.01, "{kind:?} {prim:?}"); + assert!((gamma - 2.2).abs() < 0.1, "{kind:?} gamma {gamma}"); + } + } + + #[test] + fn a2b0_without_b2a0_is_inverted() { + let (to, _) = device(); + let b = write_lut_profile("A2B only", LutKind::Lut16, 17, &to, None); + let Profile::Lut(p) = parse_profile(&b).unwrap() else { panic!() }; + assert!(p.b2a.is_none()); + for c in [[0.2, 0.5, 0.8], [0.9, 0.1, 0.3], [0.5, 0.5, 0.5], [1.0, 1.0, 1.0], [0.0, 0.0, 0.0]] { + let x = to(c); + let back = p.from_xyz(x, [0.5; 3]); + assert!(lab_de(p.to_xyz(back), x) < 0.5, "{c:?} → {back:?}"); + } + // A matrix/TRC profile still parses as one. + assert!(matches!(parse_profile(&write_matrix_profile("m", P709, D65, 2.2)).unwrap(), Profile::Matrix(_))); + } + + #[test] + fn clut_and_lut3d_interpolate() { + // A 2-point identity CLUT on 3 inputs is exact for every point. + let mut data = vec![]; + for r in 0..2 { + for g in 0..2 { + for b in 0..2 { + data.extend([r as f32, g as f32, b as f32]); + } + } + } + let c = Clut { grid: vec![2; 3], outputs: 3, data }; + let v = c.eval(&[0.25, 0.5, 0.75]); + assert!((v[0] - 0.25).abs() < 1e-6 && (v[1] - 0.5).abs() < 1e-6 && (v[2] - 0.75).abs() < 1e-6); + let l = Lut3d::bake(5, |c| [c[0] * c[0], c[1], 1.0 - c[2]]); + let o = l.eval([0.5, 0.3, 0.2]); + assert!((o[0] - 0.25).abs() < 1e-6 && (o[1] - 0.3).abs() < 1e-6 && (o[2] - 0.8).abs() < 1e-6, "{o:?}"); + assert!(parse_lut(b"xxxx\0\0\0\0", false).is_err()); + } } diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs index 0b823135..79a45884 100644 --- a/crates/engine/src/camera_track.rs +++ b/crates/engine/src/camera_track.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, Mutex}; use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; use effectcraft_keyframe::{Keyframe, Value}; use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; -use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer}; use effectcraft_time::Tick; use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; @@ -196,8 +196,11 @@ fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) Some((Arc::new(buf.img.clone()), buf.offset)) } -fn run_work(w: CamWork, shared: &CameraShared) { +async fn run_work(w: CamWork, shared: &CameraShared) { let t0 = web_time::Instant::now(); + // A browser job worker's GPU (deferred readbacks): frames render in passes. + let accel = crate::offload::job_accel(); + let accel = accel.as_deref(); let fail = |msg: &str| { let mut s = lock(&shared.state); s.error = Some(msg.to_string()); @@ -215,29 +218,32 @@ fn run_work(w: CamWork, shared: &CameraShared) { let (tracks, fresh) = match w.tracks.clone() { Some(t) => (t, false), None => { - let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel)); r.expr = w.expr.as_deref(); r.cache = Some(&w.cache); let r = &r; let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; - let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + let ctx0 = &ctx0; + let frame_at = |t: Tick| crate::offload::in_passes(accel, move |a| input_frame(&r.with_accel(a), &ctx0.at(t), layer, w.effect_index)); { let mut s = lock(&shared.state); s.step = 1; s.total = w.times.len() as u64; } let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); - let mut next = frame_at(w.times[0]); + let mut next = frame_at(w.times[0]).await; for k in 0..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { return cancelled(); } let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; - let ((), nf) = rayon::join( + let ((), nf) = crate::offload::join_fetch( + accel.is_some(), || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), - || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, - ); + || async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } }, + ) + .await; next = nf; let mut s = lock(&shared.state); s.done = k as u64 + 1; @@ -350,11 +356,19 @@ impl Session { st.total = n as u64; } let thread = if wait { - run_work(work, &shared); + crate::offload::run_or_queue({ + let sh = shared.clone(); + async move { run_work(work, &sh).await } + }); None } else { let sh = shared.clone(); - Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + Some( + std::thread::Builder::new() + .name("camera-track".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh))) + .map_err(|e| e.to_string())?, + ) }; self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); self.events.push(Event::ProjectChanged { revision: self.revision }); diff --git a/crates/engine/src/commands/viewer_cmds.rs b/crates/engine/src/commands/viewer_cmds.rs index 30dc6e4a..cdb43adb 100644 --- a/crates/engine/src/commands/viewer_cmds.rs +++ b/crates/engine/src/commands/viewer_cmds.rs @@ -346,7 +346,13 @@ fn simulate_output(s: &mut Session, p: &Value) -> Result { } fn custom_rgb_json(c: &CustomRgb) -> Value { - serde_json::to_value(c).unwrap_or_default() + let mut v = serde_json::to_value(c).unwrap_or_default(); + // A LUT-based profile's tables stay in Settings; the reply says it is one. + if let Some(o) = v.as_object_mut() { + o.remove("iccLut"); + o.insert("lutProfile".into(), json!(!c.icc_lut.is_empty())); + } + v } /// View ▸ Simulate Output ▸ My Custom RGB…: define the custom output device (kept in Settings) @@ -367,6 +373,7 @@ fn custom_rgb(s: &mut Session, p: &Value) -> Result { _ => (1.0, false), }; c.icc.clear(); + c.icc_lut = Default::default(); c.name = format!("Custom ({})", prof.label()); } } @@ -380,22 +387,33 @@ fn custom_rgb(s: &mut Session, p: &Value) -> Result { } } }; + let mut typed = false; for (k, slot) in [("red", &mut c.red), ("green", &mut c.green), ("blue", &mut c.blue), ("white", &mut c.white)] { if let Some(v) = pt(k)? { + typed |= *slot != v; *slot = v; } } if let Some(g) = f_p(p, "gamma") { + typed |= c.gamma != g; c.gamma = g; } if let Some(b) = b_p(p, "srgbCurve") { + typed |= c.srgb_curve != b; c.srgb_curve = b; } + // Typed-in numbers replace a LUT-based profile's tables (they only approximated it). + if typed { + c.icc_lut = Default::default(); + } if let Some(n) = str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()) { c.name = n.to_string(); } match str_p(p, "icc").map(str::trim) { - Some("") => c.icc.clear(), + Some("") => { + c.icc.clear(); + c.icc_lut = Default::default(); + } Some(path) if path != c.icc || b_p(p, "reload") == Some(true) => { let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; let mut from = CustomRgb::from_icc(&bytes, path).map_err(|e| bad(cmd, format!("{path}: {e}")))?; @@ -475,7 +493,7 @@ pub fn specs() -> Vec { "My Custom RGB...", [], None, - "{name?, red?/green?/blue?/white?: [x, y] (CIE xy), gamma?, srgbCurve?: bool, icc?: path (RGB ICC profile; read when it changes, \"\" clears), reload?, from?: a Simulate Output profile to start from, reset?, preserveRgb?, apply?: bool (default true: simulate it)} → the definition (kept in Settings)", + "{name?, red?/green?/blue?/white?: [x, y] (CIE xy), gamma?, srgbCurve?: bool, icc?: path (RGB ICC profile, matrix/TRC or LUT-based A2B0: read when it changes, \"\" clears; a LUT profile simulates through its tables, `lutProfile` in the reply; typed-in numbers replace it), reload?, from?: a Simulate Output profile to start from, reset?, preserveRgb?, apply?: bool (default true: simulate it)} → the definition (kept in Settings)", always, custom_rgb ), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ac929328..26dc819a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -92,6 +92,15 @@ pub trait Services: Send + Sync { fn exists(&self, path: &str) -> bool { std::path::Path::new(path).exists() } + /// Keep a file the app made for itself (footage extracted from a template): like + /// [`Services::write_file`], creating its folder, but never offered as a download (web: it + /// goes to browser storage only). + fn store_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { + if let Some(d) = std::path::Path::new(path).parent().filter(|d| !d.as_os_str().is_empty()) { + let _ = std::fs::create_dir_all(d); + } + self.write_file(path, data) + } } /// Native filesystem. @@ -109,6 +118,9 @@ impl Services for FsServices { pub trait Importer: Send + Sync { /// Probe a file and return footage metadata. fn probe(&self, path: &str) -> std::result::Result; + /// A footage file's bytes are now at `path` (extracted from a template): hosts whose media + /// layer reads from memory (the browser) register them; file-based ones need nothing. + fn register(&self, _path: &str, _data: &[u8]) {} } /// A keyframe reference: layer, property uid, key time (layer time). diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 9174258e..56bb0fa1 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -18,7 +18,7 @@ use std::sync::{Arc, Mutex}; use effectcraft_keyframe::{Keyframe, ShapePath, Value}; use effectcraft_project::{ItemId, LayerId, Project, Uid}; -use effectcraft_render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_render::{ExprHost, FootageSource, LayerCache, Renderer}; use effectcraft_time::Tick; use effectcraft_track as trk; use effectcraft_track::fit::Model; @@ -200,8 +200,11 @@ pub(crate) struct MaskWork { pub times: Vec, } -fn run_work(w: MaskWork, shared: &MaskTrackShared) { +async fn run_work(w: MaskWork, shared: &MaskTrackShared) { let t0 = web_time::Instant::now(); + // A browser job worker's GPU (deferred readbacks): frames render in passes. + let accel = crate::offload::job_accel(); + let accel = accel.as_deref(); let fail = |msg: &str| { let mut s = lock(&shared.state); s.error = Some(msg.to_string()); @@ -209,20 +212,22 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { }; let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; - let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel)); r.expr = w.expr.as_deref(); r.cache = Some(&w.cache); let r = &r; - let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); + let wr = &w; + let frame_at = + |t: Tick| crate::offload::in_passes(accel, move |a| source_frame(&r.with_accel(a), &wr.project, wr.comp, comp, layer, t, wr.expr.as_deref())); lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; - let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + let Some(first) = frame_at(w.times[0]).await else { return fail("the layer has no pixels to track") }; if w.method.is_face() { - return run_face(&w, shared, first, &frame_at, layer, t0); + return run_face(&w, shared, first, &frame_at, accel.is_some(), layer, t0).await; } let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); let mut path = w.path.clone(); lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None)); - let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { lock(&shared.state).cancelled = true; @@ -232,8 +237,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); break; }; - let (res, nf) = - rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + let (res, nf) = crate::offload::join_fetch( + accel.is_some(), + || tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), + || async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } }, + ) + .await; next = nf; let Some(step) = res else { lock(&shared.state).error = Some(format!("lost the mask's pixels at {:.3} s (too little texture inside the mask)", w.times[k].seconds())); @@ -254,11 +263,13 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { } /// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks. -fn run_face( +/// `deferred`: frames render in passes (awaited one after the other). +async fn run_face, [f64; 2])>>>( w: &MaskWork, shared: &MaskTrackShared, first: (Arc, [f64; 2]), - frame_at: &(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync), + frame_at: &(impl Fn(Tick) -> F + Sync), + deferred: bool, layer: &effectcraft_project::Layer, t0: web_time::Instant, ) { @@ -273,7 +284,7 @@ fn run_face( let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks)); let (path, pts) = key(&fit); lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts)); - let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { lock(&shared.state).cancelled = true; @@ -283,8 +294,12 @@ fn run_face( lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); break; }; - let (res, nf) = - rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + let (res, nf) = crate::offload::join_fetch( + deferred, + || tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), + || async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } }, + ) + .await; next = nf; let Some(fit) = res else { lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds())); @@ -389,11 +404,19 @@ impl Session { let shared = Arc::new(MaskTrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { - run_work(work, &shared); + crate::offload::run_or_queue({ + let sh = shared.clone(); + async move { run_work(work, &sh).await } + }); None } else { let sh = shared.clone(); - Some(std::thread::Builder::new().name("mask-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + Some( + std::thread::Builder::new() + .name("mask-track".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh))) + .map_err(|e| e.to_string())?, + ) }; self.mask_job = Some(MaskTrackJob { shared, thread, comp, layer, mask, direction }); self.poll_mask_track(); diff --git a/crates/engine/src/offload.rs b/crates/engine/src/offload.rs index 712549bd..8be8922c 100644 --- a/crates/engine/src/offload.rs +++ b/crates/engine/src/offload.rs @@ -6,8 +6,10 @@ //! [`Offload`] (set as [`Session::offload`]) ships a [`WorkerRequest`] — the serialized project, //! the footage files it reads and the job — and feeds the worker's [`WorkerReply`]s back into an //! [`Inbox`] that the session drains every frame ([`Session::poll_render`], -//! [`Session::poll_offload`]). The worker side is [`run_request`]: a plain session fed the -//! project runs the job blocking and reports through a callback. +//! [`Session::poll_offload`]). The worker side is [`run_request_async`]: a plain session fed the +//! project runs the job and reports through a callback. With a WebGPU device of the worker's +//! own (deferred readbacks, `effectcraft_render::passes`) the job's frames render on the GPU in +//! passes, the job awaiting the device between them; [`run_request`] runs it blocking. //! //! | Job | Worker runs | Reply applied on the UI thread | //! |---|---|---| @@ -26,6 +28,8 @@ //! Queue item being rendered becomes "User Stopped". use std::cell::{Cell, RefCell}; +use std::future::Future; +use std::pin::Pin; use std::rc::Rc; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -34,6 +38,7 @@ use effectcraft_keyframe::ShapePath; use effectcraft_project::render_queue::{RenderQueueItem, RenderStatus}; use effectcraft_project::tracking::TrackerSettings; use effectcraft_project::{ItemId, ItemKind, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{Accelerator, Backend, RenderOpts}; use effectcraft_time::Tick; use serde::{Deserialize, Serialize}; @@ -288,13 +293,92 @@ pub fn report(done: u64, total: u64) { }); } -fn with_hook(hook: Box, f: impl FnOnce() -> R) -> R { +/// Run `f` (a future: an analysis awaited to its end) with `hook` receiving its progress. +async fn with_hook(hook: Box, f: impl Future) -> R { HOOK.with(|h| *h.borrow_mut() = Some(hook)); - let r = f(); + let r = f.await; HOOK.with(|h| *h.borrow_mut() = None); r } +// ---------------------------------------------------------------- GPU in job workers + +/// A boxed analysis future (see [`run_or_queue`]). +type Queued = Pin>>; + +thread_local! { + /// The job worker's accelerator while it runs a request: a WebGPU device with deferred + /// readbacks (see [`job_accel`]). + static JOB_ACCEL: RefCell>> = const { RefCell::new(None) }; + /// Analyses started while [`run_request_async`] runs a request on this thread: awaited by it + /// (`None`: not inside one, analyses run to completion when started). + static QUEUED: RefCell>> = const { RefCell::new(None) }; +} + +/// The accelerator analyses render their input frames with: the job worker's GPU while it runs +/// a request ([`run_request_async`] with a session whose accelerator defers its readbacks). +/// Elsewhere (the desktop's analysis threads) analyses render on the CPU, as before. +pub(crate) fn job_accel() -> Option> { + JOB_ACCEL.with(|a| a.borrow().clone()) +} + +/// Render options of an analysis's input frames: 100 %; with an accelerator, the project's +/// renderer (Mercury GPU Acceleration: its GPU effects on the device; Software Only: the CPU). +pub(crate) fn analysis_opts(accel: Option<&dyn Accelerator>) -> RenderOpts { + RenderOpts { scale: 1.0, backend: if accel.is_some() { Backend::Auto } else { Backend::Cpu }, ..Default::default() } +} + +/// A render in passes ([`effectcraft_render::passes::in_passes`]): its value. +pub(crate) async fn in_passes<'a, T>(accel: Option<&'a dyn Accelerator>, render: impl FnMut(Option<&'a dyn Accelerator>) -> T) -> T { + effectcraft_render::passes::in_passes(accel, render).await.value +} + +/// `work()` while the next frame renders (`fetch`): side by side on the thread pool, or — with +/// `deferred` readbacks, where a frame renders in passes awaiting the GPU — one after the other. +pub(crate) async fn join_fetch>( + deferred: bool, + work: impl FnOnce() -> R + Send, + fetch: impl FnOnce() -> Fut + Send, +) -> (R, F) { + if deferred { + let r = work(); + return (r, fetch().await); + } + rayon::join(work, || effectcraft_render::passes::block_on(fetch())) +} + +/// Run an analysis started with `wait`: inside [`run_request_async`] it is queued and awaited +/// there (its frames may render in passes); anywhere else it runs to completion now. +pub(crate) fn run_or_queue(f: impl Future + 'static) { + let f: Queued = Box::pin(f); + let now = QUEUED.with(|q| match q.borrow_mut().as_mut() { + Some(v) => { + v.push(f); + None + } + None => Some(f), + }); + if let Some(f) = now { + effectcraft_render::passes::block_on(f); + } +} + +/// Await the analyses [`run_or_queue`] queued, then let the session apply their results. +async fn drain(s: &mut Session) { + loop { + let next = QUEUED.with(|q| q.borrow_mut().as_mut().and_then(|v| (!v.is_empty()).then(|| v.remove(0)))); + match next { + Some(f) => f.await, + None => break, + } + } + s.poll_track(); + s.poll_mask_track(); + s.poll_warp(false); + s.poll_camera(false); + s.poll_roto(false); +} + // ---------------------------------------------------------------- worker side /// Where a worker sends its replies. @@ -303,19 +387,36 @@ pub type Post = Rc; /// Run `req` on `s` (a session with footage, expressions and an exporter, but no project yet), /// blocking, reporting through `post`. Always ends with [`WorkerReply::Done`]. pub fn run_request(s: &mut Session, req: WorkerRequest, post: &Post) { + effectcraft_render::passes::block_on(run_request_async(s, req, post)); +} + +/// [`run_request`] as a future: with an accelerator whose readbacks are deferred +/// ([`Session::accel`]: a browser job worker's own WebGPU device) the Render Queue's frames +/// and the analyses' input frames render on it in passes, awaiting the device between them +/// (the browser delivers readbacks only while the worker is back in its event loop). +pub async fn run_request_async(s: &mut Session, req: WorkerRequest, post: &Post) { + let accel = s.accel.clone().filter(|a| a.miss_gate().is_some()); + if let Some(a) = &accel { + // Buffers computed from placeholders of a pass that missed are not kept. + s.layer_cache.set_gate(a.miss_gate()); + } + JOB_ACCEL.with(|a| *a.borrow_mut() = accel); + QUEUED.with(|q| *q.borrow_mut() = Some(vec![])); match Project::from_json(&req.project) { Ok(p) => { s.replace_project(p, None); - if let Err(e) = run_job(s, req.job, post.clone()) { + if let Err(e) = run_job(s, req.job, post.clone()).await { post(WorkerReply::Failed { error: e }); } } Err(e) => post(WorkerReply::Failed { error: format!("project: {e}") }), } + QUEUED.with(|q| *q.borrow_mut() = None); + JOB_ACCEL.with(|a| *a.borrow_mut() = None); post(WorkerReply::Done); } -fn run_job(s: &mut Session, job: WorkerJob, post: Post) -> Result<(), String> { +async fn run_job(s: &mut Session, job: WorkerJob, post: Post) -> Result<(), String> { let kind = job.kind(); let target = job.target(); match job { @@ -334,50 +435,59 @@ fn run_job(s: &mut Session, job: WorkerJob, post: Post) -> Result<(), String> { post(WorkerReply::Render { state: sh.snapshot() }); } }; - crate::render_queue::run_items(s, exporter, items, &shared, &mut |sh| notify(sh, false)); + crate::render_queue::run_items(s, exporter, items, &shared, &mut |sh| notify(sh, false)).await; notify(&shared, true); Ok(()) } - WorkerJob::Warp { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_warp(comp, layer, effect, true).map(|_| ())), - WorkerJob::Camera { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_camera(comp, layer, effect, true).map(|_| ())), - WorkerJob::RotoFreeze { comp, layer, effect } => analysis(s, kind, target, post, |s| { - s.start_roto(comp, layer, effect, crate::roto::RotoTask::Freeze, crate::roto::Direction::Both, true).map(|_| ()) - }), - WorkerJob::RotoPropagate { comp, layer, effect, direction } => propagate(s, comp, layer, effect, direction, post), - WorkerJob::Track { comp, layer, tracker, direction, settings, points, times } => analysis(s, kind, target, post, |s| { - let work = crate::tracking::Work { - project: s.project.clone(), - footage: s.footage.clone(), - expr: s.expr.clone(), - cache: s.layer_cache.clone(), - comp, - layer, - settings, - points, - times, - }; - s.start_track(work, tracker, direction, true) - }), - WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times } => analysis(s, kind, target, post, |s| { - let work = crate::mask_track::MaskWork { - project: s.project.clone(), - footage: s.footage.clone(), - expr: s.expr.clone(), - cache: s.layer_cache.clone(), - comp, - layer, - path, - method, - times, - }; - s.start_mask_track(work, mask, direction, true) - }), + WorkerJob::Warp { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_warp(comp, layer, effect, true).map(|_| ())).await, + WorkerJob::Camera { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_camera(comp, layer, effect, true).map(|_| ())).await, + WorkerJob::RotoFreeze { comp, layer, effect } => { + analysis(s, kind, target, post, |s| { + s.start_roto(comp, layer, effect, crate::roto::RotoTask::Freeze, crate::roto::Direction::Both, true).map(|_| ()) + }) + .await + } + WorkerJob::RotoPropagate { comp, layer, effect, direction } => propagate(s, comp, layer, effect, direction, post).await, + WorkerJob::Track { comp, layer, tracker, direction, settings, points, times } => { + analysis(s, kind, target, post, |s| { + let work = crate::tracking::Work { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + cache: s.layer_cache.clone(), + comp, + layer, + settings, + points, + times, + }; + s.start_track(work, tracker, direction, true) + }) + .await + } + WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times } => { + analysis(s, kind, target, post, |s| { + let work = crate::mask_track::MaskWork { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + cache: s.layer_cache.clone(), + comp, + layer, + path, + method, + times, + }; + s.start_mask_track(work, mask, direction, true) + }) + .await + } } } /// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms) /// as they are computed, so stopping the job keeps what was propagated. -fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> { +async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> { let keys: Vec = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?; let sent = Rc::new(RefCell::new(std::collections::HashSet::new())); let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone()); @@ -393,7 +503,12 @@ fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: cr } }); s.events.clear(); - with_hook(hook, || s.start_roto(comp, layer, effect, crate::roto::RotoTask::Propagate, dir, true))?; + with_hook(hook, async { + let r = s.start_roto(comp, layer, effect, crate::roto::RotoTask::Propagate, dir, true); + drain(s).await; + r + }) + .await?; let segs = segs_to_send(&keys, &mut sent.borrow_mut()); if !segs.is_empty() { post(WorkerReply::Segs { segs }); @@ -407,7 +522,7 @@ fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: cr Ok(()) } -fn analysis( +async fn analysis( s: &mut Session, kind: JobKind, target: Option<(ItemId, LayerId, Uid)>, @@ -425,7 +540,13 @@ fn analysis( p(WorkerReply::Progress { done, total }); } }); - with_hook(hook, || run(s))?; + with_hook(hook, async { + let r = run(s); + // The analysis was queued: await it, then the session writes its result. + drain(s).await; + r + }) + .await?; let message = s .events .iter() @@ -856,10 +977,10 @@ mod tests { report(1, 2); // no hook: nothing happens let seen = Rc::new(RefCell::new(vec![])); let s2 = seen.clone(); - with_hook(Box::new(move |d, t| s2.borrow_mut().push((d, t))), || { + effectcraft_render::passes::block_on(with_hook(Box::new(move |d, t| s2.borrow_mut().push((d, t))), async { report(1, 3); report(2, 3); - }); + })); report(3, 3); assert_eq!(*seen.borrow(), vec![(1, 3), (2, 3)]); } diff --git a/crates/engine/src/remote.rs b/crates/engine/src/remote.rs index fb3e33e9..207367b0 100644 --- a/crates/engine/src/remote.rs +++ b/crates/engine/src/remote.rs @@ -22,6 +22,14 @@ //! - Disk cache: a render request may carry the frame's disk-cache key ([`FrameMsg::Render`] //! `disk`); the transport stores the finished frame under it (the browser: in the Origin //! Private File System) and reports [`FrameReply::Stored`]. +//! - Layer buffers on disk: with `layers`, the server's layer cache is backed by a +//! [`PrefetchStore`] ([`FrameServer::set_layer_store`]): before the request the transport +//! fetches the entries named in `prefetch` (recent misses that are on disk, chosen by the +//! page, `disk_cache::LayerPrefetch`), so the lookups in the middle of the render find them; +//! the reply lists the layer keys that missed and those served from the store +//! ([`FrameReply::Frame`] `layer_misses` / `layer_hits`), and slow buffers queue up for the +//! transport to write after the frame ([`PrefetchStore::take_writes`], reported as +//! [`FrameReply::Stored`] with `layer`). //! //! The transport (structured-clone messages, transferable pixel buffers) is the web crate's; //! everything here is portable and tested natively. @@ -29,7 +37,7 @@ use std::sync::Arc; use effectcraft_project::{ItemId, Project}; -use effectcraft_render::{Accelerator, AutoKey, Backend, ExprHost, FootageSource, Image, LayerCache, RenderOpts, Renderer}; +use effectcraft_render::{Accelerator, AutoKey, Backend, ExprHost, FootageSource, Image, LayerCache, LayerStore, PrefetchStore, RenderOpts, Renderer}; use effectcraft_time::Tick; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -140,6 +148,12 @@ pub enum FrameMsg { opts: Box, #[serde(default, skip_serializing_if = "Option::is_none")] disk: Option, + /// Back the layer cache with the disk cache's layer buffers (see the module docs). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + layers: bool, + /// Layer entries (32 hex digits) the transport fetches before rendering. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + prefetch: Vec, }, /// Roto Brush segmentations computed elsewhere (a propagation worker). Segs { segs: Vec }, @@ -163,9 +177,21 @@ pub enum FrameReply { passes: u32, #[serde(default)] gpu: bool, + /// Layer-cache lookups that missed the store (salted keys, 32 hex digits). + #[serde(default, rename = "layerMisses", skip_serializing_if = "Vec::is_empty")] + layer_misses: Vec, + /// Layer buffers served from the store (fetched from disk). + #[serde(default, rename = "layerHits", skip_serializing_if = "Vec::is_empty")] + layer_hits: Vec, + }, + /// The frame (or, with `layer`, a layer buffer) was stored in the disk cache under `key` + /// (`bytes` on disk). + Stored { + key: String, + bytes: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + layer: bool, }, - /// The frame was stored in the disk cache under `key` (`bytes` on disk). - Stored { key: String, bytes: u64 }, /// The worker's renderer: its GPU adapter, or why it has none. Info { #[serde(default)] @@ -234,6 +260,9 @@ pub struct FrameServer { revision: Option, /// The frame being rendered in passes. pending: Option, + /// The disk cache's layer buffers (see the module docs) and the project's salt. + layer_store: Option>, + salt: u64, } /// A frame whose readbacks are in flight. @@ -254,7 +283,33 @@ pub const MAX_PASSES: u32 = 24; impl FrameServer { pub fn new(footage: Arc, expr: Option>) -> FrameServer { - FrameServer { footage, expr, cache: Arc::new(LayerCache::default()), accel: None, doc: Value::Null, project: None, revision: None, pending: None } + FrameServer { + footage, + expr, + cache: Arc::new(LayerCache::default()), + accel: None, + doc: Value::Null, + project: None, + revision: None, + pending: None, + layer_store: None, + salt: 0, + } + } + + /// The store behind the layer cache when a request asks for `layers` (the browser: the + /// disk cache's layer buffers, fetched before the request). + pub fn set_layer_store(&mut self, store: Option>) { + self.layer_store = store; + } + + pub fn layer_store(&self) -> Option<&Arc> { + self.layer_store.as_ref() + } + + /// The salt of the replica's layer keys on disk (keys are `(salt, key)`). + pub fn salt(&self) -> u64 { + self.salt } /// Render with `accel` when the request's backend asks for it (the worker's GPU). Its miss @@ -283,6 +338,8 @@ impl FrameServer { fn load(&mut self, revision: u64) -> Result<(), String> { let p = Project::from_json(&self.doc.to_string()).map_err(|e| e.to_string())?; + self.salt = effectcraft_render::disk_cache::footage_salt(&p); + self.cache.set_store_salt(self.salt); self.project = Some(Arc::new(p)); self.revision = Some(revision); Ok(()) @@ -319,9 +376,14 @@ impl FrameServer { } FrameMsg::Purge => { self.cache.clear(); + if let Some(st) = &self.layer_store { + st.clear(); + } vec![] } - FrameMsg::Render { id, revision, comp, time, opts, .. } => { + FrameMsg::Render { id, revision, comp, time, opts, layers, .. } => { + let store = self.layer_store.clone().filter(|_| layers); + self.cache.set_store(store.map(|s| (s as Arc, self.salt))); let fail = |error: String, stale: bool| vec![(FrameReply::Failed { id, error, stale }, None)]; if self.revision != Some(revision) { return fail(format!("the replica is at {:?}, not {revision}", self.revision), true); @@ -376,12 +438,20 @@ impl FrameServer { } let ms = pr.t0.elapsed().as_secs_f64() * 1000.0; let gpu = r.accel.is_some() && r.active_accel().is_some(); + let hex = |v: Vec| v.into_iter().map(|k| format!("{k:032x}")).collect::>(); + let (layer_misses, layer_hits) = match &self.layer_store { + Some(st) => (hex(st.take_misses()), hex(st.take_hits())), + None => (vec![], vec![]), + }; if let (Some((key, chose)), Some(a)) = (pr.auto, &self.accel) && let Some(pick) = a.auto_pick() { pick.record(key, chose, ms); } - vec![(FrameReply::Frame { id: pr.id, width: img.width, height: img.height, ms, passes: pr.passes, gpu }, Some(rgba8_premultiplied(&img)))] + vec![( + FrameReply::Frame { id: pr.id, width: img.width, height: img.height, ms, passes: pr.passes, gpu, layer_misses, layer_hits }, + Some(rgba8_premultiplied(&img)), + )] } fn lost(&mut self, error: String) -> Vec<(FrameReply, Option>)> { @@ -435,6 +505,77 @@ mod tests { rgba8_premultiplied(&r.comp_frame(comp, t)) } + /// Layer buffers through the disk store: a worker's slow buffers queue up as entries; the + /// page plans what another worker fetches before its frame (recent misses on disk); that + /// worker's mid-render lookups find them and the pixels match. + #[test] + fn layer_buffers_go_through_the_disk_store() { + use effectcraft_render::disk_cache::{DiskIndex, Kind, LayerPrefetch}; + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let comp = s.active_comp_id().unwrap(); + let opts = RenderOpts { scale: 0.25, ..Default::default() }; + let t = Tick::from_seconds_f64(1.0); + let req = |id: u64, layers: bool, prefetch: Vec| { + round(&FrameMsg::Render { id, revision: s.revision, comp, time: t, opts: Box::new(opts), disk: None, layers, prefetch }) + }; + let worker = || { + let mut w = server(&s); + let store = Arc::new(PrefetchStore::new(64 << 20).with_min_ms(0)); + w.set_layer_store(Some(store.clone())); + let mut m = Mirror::default(); + assert!(w.handle(round(&m.sync(s.revision, &s.project).unwrap())).is_empty()); + (w, store) + }; + let unhex = |v: &[String]| v.iter().map(|k| u128::from_str_radix(k, 16).unwrap()).collect::>(); + // Worker A: everything misses; its buffers queue up for the disk. + let (mut a, store_a) = worker(); + let out = a.handle(req(1, true, vec![])); + let (FrameReply::Frame { layer_misses, layer_hits, .. }, Some(px_a)) = &out[0] else { panic!("{:?}", out[0].0) }; + assert!(!layer_misses.is_empty() && layer_hits.is_empty()); + let writes = store_a.take_writes(); + assert!(!writes.is_empty()); + assert!(writes.iter().all(|(k, _)| (*k >> 64) as u64 == a.salt()), "keys are salted per project"); + let mut index = DiskIndex::default(); + let mut disk = std::collections::HashMap::new(); + for (k, e) in writes { + index.insert(Kind::Layer, k, e.len() as u64); + disk.insert(k, e); + } + index.insert(Kind::Frame, 1, 10); + // The page: recent misses that are on disk, most recent first. + let mut plan = LayerPrefetch::default(); + plan.note_misses(unhex(layer_misses)); + let keys = plan.plan(&index, 64); + assert!(!keys.is_empty() && keys.len() <= index.len()); + // Worker B fetches them before its first frame: its lookups hit. + let (mut b, store_b) = worker(); + for k in &keys { + assert!(store_b.provide(*k, &disk[k])); + } + assert!(!store_b.provide(1, b"damaged")); + let out = b.handle(req(1, true, keys.iter().map(|k| format!("{k:032x}")).collect())); + let (FrameReply::Frame { layer_misses: mb, layer_hits: hb, .. }, Some(px_b)) = &out[0] else { panic!("{:?}", out[0].0) }; + assert_eq!(unhex(hb).len(), keys.len(), "every fetched buffer was used"); + assert!(mb.len() < layer_misses.len()); + assert_eq!(px_a, px_b); + assert_eq!(*px_b, local(&s, comp, t, opts)); + // Nothing new to write: what B rendered itself was rendered by A too. + assert!(store_b.take_writes().iter().all(|(k, _)| disk.contains_key(k))); + // Without `layers` the store is not consulted. + let (mut c, store_c) = worker(); + c.handle(req(1, false, vec![])); + assert!(store_c.take_misses().is_empty() && store_c.take_writes().is_empty()); + // Purge empties the store. + b.handle(round(&FrameMsg::Purge)); + assert!(!store_b.has(keys[0])); + // LRU across frames and layers in the shared index: a touched layer outlives the frame. + index.touch(Kind::Layer, keys[0]); + let order = index.lru_order(); + assert_eq!(order.last(), Some(&(Kind::Layer, keys[0]))); + assert!(order.iter().position(|e| *e == (Kind::Frame, 1)).unwrap() < order.len() - 1); + } + #[test] fn replica_follows_edits_through_diffs_and_renders_the_same_frames() { let mut s = Session::default(); @@ -450,8 +591,9 @@ mod tests { assert!(send(first, &mut w).is_empty()); assert!(m.sync(s.revision, &s.project).is_none(), "nothing new"); let t = Tick::from_seconds_f64(1.0); - let render = - |w: &mut FrameServer, rev: u64, id: u64| w.handle(round(&FrameMsg::Render { id, revision: rev, comp, time: t, opts: Box::new(opts), disk: None })); + let render = |w: &mut FrameServer, rev: u64, id: u64| { + w.handle(round(&FrameMsg::Render { id, revision: rev, comp, time: t, opts: Box::new(opts), disk: None, layers: false, prefetch: vec![] })) + }; let out = render(&mut w, s.revision, 1); let (FrameReply::Frame { id, width, height, .. }, Some(px)) = &out[0] else { panic!("{:?}", out[0].0) }; assert_eq!((*id, *width as usize * *height as usize * 4), (1, px.len())); @@ -487,6 +629,8 @@ mod tests { time: Tick::from_seconds_f64(0.5), opts: Box::new(RenderOpts { scale: 0.5, draft: true, roi: Some([1.0, 2.0, 3.0, 4.0]), ..Default::default() }), disk: Some(format!("{:032x}", 0xabcdu128)), + layers: true, + prefetch: vec![format!("{:032x}", 7u128)], }, ] { let a = serde_json::to_string(&m).unwrap(); @@ -494,8 +638,9 @@ mod tests { assert!(a.starts_with("{\"type\":"), "{a}"); } for r in [ - FrameReply::Frame { id: 1, width: 2, height: 3, ms: 4.5, passes: 2, gpu: true }, - FrameReply::Stored { key: "00ff".into(), bytes: 9 }, + FrameReply::Frame { id: 1, width: 2, height: 3, ms: 4.5, passes: 2, gpu: true, layer_misses: vec!["0a".into()], layer_hits: vec![] }, + FrameReply::Stored { key: "00ff".into(), bytes: 9, layer: false }, + FrameReply::Stored { key: "00fe".into(), bytes: 8, layer: true }, FrameReply::Info { gpu: Some("Adapter (BrowserWebGpu)".into()), error: None }, FrameReply::Failed { id: 1, error: "x".into(), stale: true }, FrameReply::Resync { error: "y".into() }, @@ -596,7 +741,7 @@ mod tests { w.set_accel(Some(acc.clone())); let mut m = Mirror::default(); w.handle(round(&m.sync(s.revision, &s.project).unwrap())); - let req = |id| round(&FrameMsg::Render { id, revision: s.revision, comp, time: t, opts: Box::new(opts), disk: None }); + let req = |id| round(&FrameMsg::Render { id, revision: s.revision, comp, time: t, opts: Box::new(opts), disk: None, layers: false, prefetch: vec![] }); assert!(w.handle(req(1)).is_empty() && w.waiting(), "the first pass misses"); let mut out = vec![]; for _ in 0..10 { @@ -621,11 +766,21 @@ mod tests { assert_eq!(out[0].1.as_deref(), Some(&px[..])); // CPU requests never touch the accelerator; Auto times whole frames. let cpu = RenderOpts { backend: Backend::Cpu, ..opts }; - let out = w.handle(round(&FrameMsg::Render { id: 3, revision: s.revision, comp, time: t, opts: Box::new(cpu), disk: None })); + let out = + w.handle(round(&FrameMsg::Render { id: 3, revision: s.revision, comp, time: t, opts: Box::new(cpu), disk: None, layers: false, prefetch: vec![] })); assert!(matches!(out[0].0, FrameReply::Frame { id: 3, passes: 1, gpu: false, .. })); for id in 4..12 { let auto = RenderOpts { backend: Backend::Auto, ..opts }; - let mut out = w.handle(round(&FrameMsg::Render { id, revision: s.revision, comp, time: t, opts: Box::new(auto), disk: None })); + let mut out = w.handle(round(&FrameMsg::Render { + id, + revision: s.revision, + comp, + time: t, + opts: Box::new(auto), + disk: None, + layers: false, + prefetch: vec![], + })); while w.waiting() { acc.settle(); out = w.resume(); diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index 9756b183..c8d26937 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -60,8 +60,17 @@ pub trait Exporter: Send + Sync { /// Render and write one item (blocking). `progress(done, total)` returns `false` to cancel; /// a cancelled export returns `Err("cancelled")`. fn export(&self, job: &ExportJob, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result; + /// [`Exporter::export`] as a future: renders whose GPU readbacks are deferred (a browser + /// job worker's WebGPU device) await the device between a frame's passes. The default runs + /// the blocking export. + fn export_async<'a>(&'a self, job: &'a ExportJob<'a>, progress: &'a mut dyn FnMut(u64, u64) -> bool) -> LocalFuture<'a, Result> { + Box::pin(async move { self.export(job, progress) }) + } } +/// A boxed future that stays on its thread. +pub type LocalFuture<'a, T> = std::pin::Pin + 'a>>; + /// The error string an [`Exporter`] returns when cancelled. pub const CANCELLED: &str = "cancelled"; @@ -196,8 +205,10 @@ fn unix_now() -> u64 { web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0) } -/// Render `items` with the session's footage, expressions and GPU (a worker's render). -pub(crate) fn run_items( +/// Render `items` with the session's footage, expressions and GPU (a worker's render). A +/// future: with deferred GPU readbacks (a browser job worker) each frame awaits the device +/// between its passes. +pub(crate) async fn run_items( s: &Session, exporter: Arc, items: Vec>, @@ -214,10 +225,10 @@ pub(crate) fn run_items( items, nested_switches: s.prefs.general.switches_affect_nested_comps, }; - run_work(work, shared, notify); + run_work(work, shared, notify).await; } -fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { +async fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { let t0 = web_time::Instant::now(); { let mut s = lock(&shared.state); @@ -257,18 +268,21 @@ fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { label: format!("#{index} {name} (output module {})", mi + 1), nested_switches: w.nested_switches, }; - let rr = w.exporter.export(&job, &mut |done, total| { - { - let mut s = lock(&shared.state); - s.done = done; - s.total = total; - s.elapsed = t0.elapsed().as_secs_f64(); - s.item_elapsed = t1.elapsed().as_secs_f64(); - s.remaining = (done > 0).then(|| s.item_elapsed / done as f64 * total.saturating_sub(done) as f64); - } - notify(shared); - !shared.cancel.load(Ordering::Relaxed) - }); + let rr = w + .exporter + .export_async(&job, &mut |done, total| { + { + let mut s = lock(&shared.state); + s.done = done; + s.total = total; + s.elapsed = t0.elapsed().as_secs_f64(); + s.item_elapsed = t1.elapsed().as_secs_f64(); + s.remaining = (done > 0).then(|| s.item_elapsed / done as f64 * total.saturating_sub(done) as f64); + } + notify(shared); + !shared.cancel.load(Ordering::Relaxed) + }) + .await; match rr { Ok(res) if mi == 0 => r = Ok(res), Ok(_) => {} @@ -474,11 +488,14 @@ impl Session { }; let shared = Arc::new(JobShared::default()); if wait { - run_work(work, &shared, &mut |_| {}); + effectcraft_render::passes::block_on(run_work(work, &shared, &mut |_| {})); self.render_job = Some(RenderJob { shared, thread: None, remote: None }); } else { let sh = shared.clone(); - let thread = std::thread::Builder::new().name("render-queue".into()).spawn(move || run_work(work, &sh, &mut |_| {})).map_err(|e| e.to_string())?; + let thread = std::thread::Builder::new() + .name("render-queue".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh, &mut |_| {}))) + .map_err(|e| e.to_string())?; self.render_job = Some(RenderJob { shared, thread: Some(thread), remote: None }); } self.poll_render(); diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index 2c1828f8..ad922ed5 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -277,9 +277,14 @@ struct Work { task: RotoTask, } -/// Renders the effect's input frames for a worker, keeping the last few. +/// Renders the effect's input frames for a worker, keeping the last few. With a browser job +/// worker's GPU (deferred readbacks), [`FrameSource::prefetch`] renders the frames a step +/// will read in passes beforehand; [`FrameSource::get`] renders on the CPU what it lacks. pub(crate) struct FrameSource<'a> { r: Renderer<'a>, + accel: Option<&'a dyn effectcraft_render::Accelerator>, + /// Frames in `memo`, oldest first. + order: std::collections::VecDeque, ctx0: EvalCtx<'a>, comp: &'a effectcraft_project::Comp, layer: &'a Layer, @@ -305,15 +310,55 @@ impl<'a> FrameSource<'a> { r.expr = expr; r.cache = Some(cache); let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) }; - FrameSource { r, ctx0, comp, layer, index, size, memo: HashMap::new() } + FrameSource { r, accel: None, order: Default::default(), ctx0, comp, layer, index, size, memo: HashMap::new() } + } + + /// Render with `accel` (deferred readbacks: see [`FrameSource::prefetch`]). + pub(crate) fn with_accel(mut self, accel: Option<&'a dyn effectcraft_render::Accelerator>) -> Self { + self.accel = accel; + self.r.opts = crate::offload::analysis_opts(accel); + self + } + + /// Render frames `fs` (those not kept yet) in passes on the GPU, so the step that reads + /// them finds them ([`FrameSource::get`] can't wait for the GPU). + pub(crate) async fn prefetch(&mut self, fs: &[i64]) { + let Some(a) = self.accel else { return }; + for &f in fs { + if self.memo.contains_key(&f) { + continue; + } + let this = &*self; + let img = crate::offload::in_passes(Some(a), |acc| this.render(f, &this.r.with_accel(acc))).await; + if let Some(img) = img { + self.keep(f, img); + } + } + } + + fn keep(&mut self, f: i64, img: Arc) { + while self.order.len() >= 8 { + if let Some(old) = self.order.pop_front() { + self.memo.remove(&old); + } + } + self.order.push_back(f); + self.memo.insert(f, img); } pub(crate) fn get(&mut self, f: i64) -> Option> { if let Some(i) = self.memo.get(&f) { return Some(i.clone()); } + // Not prefetched: the CPU (a pass of the GPU could only return placeholders here). + let img = self.render(f, &self.r.with_accel(None))?; + self.keep(f, img.clone()); + Some(img) + } + + fn render(&self, f: i64, r: &Renderer) -> Option> { let t = comp_time_of(self.comp, self.layer, f); - let buf = self.r.layer_input(&self.ctx0.at(t), self.layer, self.index)?; + let buf = r.layer_input(&self.ctx0.at(t), self.layer, self.index)?; let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; @@ -325,17 +370,14 @@ impl<'a> FrameSource<'a> { } else { (*buf).clone() }; - let img = Arc::new(fx::layer_grid(&buf, self.size)); - if self.memo.len() > 4 { - self.memo.clear(); - } - self.memo.insert(f, img.clone()); - Some(img) + Some(Arc::new(fx::layer_grid(&buf, self.size))) } } -fn run_work(w: Work, shared: &RotoShared) { +async fn run_work(w: Work, shared: &RotoShared) { let t0 = web_time::Instant::now(); + // A browser job worker's GPU (deferred readbacks): input frames render in passes. + let accel = crate::offload::job_accel(); let fail = |msg: &str| { let mut s = lock(&shared.state); s.error = Some(msg.to_string()); @@ -343,7 +385,8 @@ fn run_work(w: Work, shared: &RotoShared) { }; let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; - let mut src = FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size); + let mut src = + FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size).with_accel(accel.as_deref()); { let mut s = lock(&shared.state); s.task = Some(w.task); @@ -359,6 +402,8 @@ fn run_work(w: Work, shared: &RotoShared) { s.finished = true; return; } + // The frame and its neighbours (what a step reads), on the GPU. + src.prefetch(&[*f, f - 1, f + 1]).await; let mut get = |g: i64| src.get(g); let ok = match w.task { RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), @@ -516,11 +561,19 @@ impl Session { let wait = wait || cfg!(target_arch = "wasm32"); let shared = Arc::new(RotoShared::default()); let thread = if wait { - run_work(work, &shared); + crate::offload::run_or_queue({ + let sh = shared.clone(); + async move { run_work(work, &sh).await } + }); None } else { let sh = shared.clone(); - Some(std::thread::Builder::new().name("roto".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + Some( + std::thread::Builder::new() + .name("roto".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh))) + .map_err(|e| e.to_string())?, + ) }; self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); self.events.push(Event::ProjectChanged { revision: self.revision }); diff --git a/crates/engine/src/templates.rs b/crates/engine/src/templates.rs index afa9b146..00d17a98 100644 --- a/crates/engine/src/templates.rs +++ b/crates/engine/src/templates.rs @@ -8,6 +8,13 @@ //! (the open template container of `essential.exportTemplate`: a stored ZIP with //! `manifest.json` (`kind: "project"`), `project.ecproj`, `poster.png` and `thumb.txt`) into //! the config store's `Templates` folder. +//! - **Embedded footage**: saving embeds the footage files the project reads (default on, +//! `embedFootage`), as `media//` entries like `essential.exportTemplate`, up to +//! [`EMBED_LIMIT_MB`] in all (`embedLimitMB`); footage beyond the cap stays referenced by its +//! path and the save warns. Creating from the template extracts the files next to the new +//! project (`projectPath`: `/ Footage/`), into `footageDir`, or into the +//! settings folder's `Template Footage//` (the browser: its storage), and points the +//! footage at them. //! - Opening a template (`templates.create`) makes an untitled copy: the template itself is never //! changed. //! @@ -31,6 +38,10 @@ use crate::{EngineError, Result, Session, cmd, query}; pub const TEMPLATES_DIR: &str = "Templates"; /// Template file extension. pub const EXT: &str = "ectemplate"; +/// Default cap on the footage a saved template embeds (MB, all files together). +pub const EMBED_LIMIT_MB: f64 = 256.0; +/// Folder (under the settings folder) that footage extracted from templates goes to. +pub const FOOTAGE_DIR: &str = "Template Footage"; /// Gallery thumbnail size (RGB, letterboxed). pub const THUMB_W: usize = 256; pub const THUMB_H: usize = 144; @@ -611,20 +622,148 @@ pub fn list(s: &Session) -> Vec { v } -/// The project of a template id (or of an `.ectemplate` file). -fn project_of(s: &Session, id: Option<&str>, path: Option<&str>) -> Result<(Project, String)> { +/// A template's project, its name and its archive entries (embedded footage), by id or by +/// `.ectemplate` file. +type Opened = (Project, String, Vec<(String, Vec)>); + +fn project_of(s: &Session, id: Option<&str>, path: Option<&str>) -> Result { if let Some(path) = path { let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; - let (m, p, _) = read_template(&bytes).map_err(|e| bad("templates.create", e))?; - return Ok((p, str_of(&m, "name"))); + let (m, p, e) = read_template(&bytes).map_err(|e| bad("templates.create", e))?; + return Ok((p, str_of(&m, "name"), e)); } let id = id.ok_or_else(|| bad("templates.create", "missing `id` (see templates.list) or `path`"))?; if let Some(f) = id.strip_prefix("user/") { - let (m, p, _) = read_template(&read_user(s, f)?).map_err(|e| bad("templates.create", e))?; - return Ok((p, str_of(&m, "name"))); + let (m, p, e) = read_template(&read_user(s, f)?).map_err(|e| bad("templates.create", e))?; + return Ok((p, str_of(&m, "name"), e)); } let b = builtin(id).ok_or_else(|| bad("templates.create", format!("unknown template `{id}` (see templates.list)")))?; - Ok((cached_builtin(id)?, b.name.to_string())) + Ok((cached_builtin(id)?, b.name.to_string(), vec![])) +} + +fn base_name(path: &str) -> String { + path.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("footage").to_string() +} + +/// What [`embed_footage`] did. +#[derive(Default)] +struct Embedded { + /// `{item, files, bytes}` per embedded footage item (the manifest's `media`). + media: Vec, + bytes: u64, + /// Footage left referenced: `{item, name, bytes?, reason}`. + skipped: Vec, +} + +/// Embed the footage files `proj` reads as `media//` entries (paths rewritten to +/// the entries), until `limit` bytes; the rest stays referenced. +fn embed_footage(s: &Session, proj: &mut Project, entries: &mut Vec<(String, Vec)>, limit: u64) -> Embedded { + let mut out = Embedded::default(); + let mut ids: Vec = proj.items.keys().copied().collect(); + ids.sort_by_key(|i| i.0); + for id in ids { + let Some(it) = proj.items.get_mut(&id) else { continue }; + let name = it.name.clone(); + let effectcraft_project::ItemKind::Footage(f) = &mut it.kind else { continue }; + if f.data.is_some() || f.path.is_empty() { + continue; + } + let files: Vec = if f.sequence.is_empty() { vec![f.path.clone()] } else { f.sequence.clone() }; + let mut data = vec![]; + let mut missing = None; + for file in &files { + match s.services.read_file(file) { + Ok(b) => data.push((file.clone(), b)), + Err(e) => { + missing = Some(format!("{file}: {e}")); + break; + } + } + } + if let Some(reason) = missing { + out.skipped.push(json!({"item": id.0, "name": name, "reason": reason})); + continue; + } + let size: u64 = data.iter().map(|(_, b)| b.len() as u64).sum(); + if out.bytes + size > limit { + out.skipped.push(json!({"item": id.0, "name": name, "bytes": size, "reason": "over the size limit"})); + continue; + } + let mut stored = vec![]; + for (file, bytes) in data { + let entry = format!("media/{}/{}", id.0, base_name(&file)); + entries.push((entry.clone(), bytes)); + stored.push(entry); + } + f.path = stored[0].clone(); + if !f.sequence.is_empty() { + f.sequence = stored.clone(); + } + out.bytes += size; + out.media.push(json!({"item": id.0, "files": stored, "bytes": size})); + } + out +} + +/// Where footage extracted from template `name` goes (see the module docs). +fn footage_dir(s: &Session, name: &str, footage_dir: Option<&str>, project_path: Option<&str>) -> String { + if let Some(d) = footage_dir.filter(|d| !d.is_empty()) { + return d.trim_end_matches(['/', '\\']).to_string(); + } + if let Some(pp) = project_path.map(std::path::Path::new) { + let stem = pp.file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Project".into()); + return pp.parent().unwrap_or(std::path::Path::new("")).join(format!("{stem} Footage")).to_string_lossy().into_owned(); + } + let safe: String = name.chars().map(|c| if c.is_alphanumeric() || " -_()".contains(c) { c } else { '_' }).collect(); + let safe = if safe.trim().is_empty() { "Template".to_string() } else { safe.trim().to_string() }; + let base = match s.config.as_ref().and_then(|c| c.dir()) { + Some(d) => d.join(FOOTAGE_DIR), + None => PathBuf::from(format!("/{FOOTAGE_DIR}")), + }; + // A fresh folder per created project. + let mut k = 1; + loop { + let d = base.join(if k == 1 { safe.clone() } else { format!("{safe} {k}") }); + let ds = d.to_string_lossy().into_owned(); + if !s.services.exists(&ds) || k > 999 { + return ds; + } + k += 1; + } +} + +/// Write a template's embedded footage (`media/…` entries) into `dir` and point the project's +/// footage at the files. Returns the files written. +fn extract_footage(s: &Session, proj: &mut Project, entries: &[(String, Vec)], dir: &str) -> Result> { + let mut written = std::collections::HashMap::new(); + let mut used = std::collections::HashSet::new(); + for (k, v) in entries.iter().filter(|(k, _)| k.starts_with("media/")) { + // `media//` → one folder; a name another item took gets the item's id. + let file = base_name(k); + let item = k.trim_start_matches("media/").split('/').next().unwrap_or("0"); + let name = if used.insert(file.clone()) { file } else { format!("{item}-{file}") }; + let out = format!("{dir}/{name}"); + s.services.store_file(&out, v).map_err(|e| EngineError::Other(format!("cannot write {out}: {e}")))?; + if let Some(imp) = &s.importer { + imp.register(&out, v); + } + written.insert(k.clone(), out); + } + for it in proj.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + if let Some(w) = written.get(&f.path) { + f.path = w.clone(); + } + for x in &mut f.sequence { + if let Some(w) = written.get(x) { + *x = w.clone(); + } + } + } + } + let mut v: Vec = written.into_values().collect(); + v.sort(); + Ok(v) } // ------------------------------------------------------------------------- commands @@ -648,7 +787,13 @@ fn thumb_cmd(s: &mut Session, p: &Value) -> Result { } fn create(s: &mut Session, p: &Value) -> Result { - let (proj, name) = project_of(s, str_p(p, "id"), str_p(p, "path"))?; + let (mut proj, name, entries) = project_of(s, str_p(p, "id"), str_p(p, "path"))?; + let project_path = str_p(p, "projectPath").filter(|p| !p.is_empty()); + let mut footage = vec![]; + if entries.iter().any(|(k, _)| k.starts_with("media/")) { + let dir = footage_dir(s, &name, str_p(p, "footageDir"), project_path); + footage = extract_footage(s, &mut proj, &entries, &dir)?; + } let cid = main_comp(&proj); let poster = cid.and_then(|c| proj.comp(c)).map(|c| c.poster_time).unwrap_or(Tick::ZERO); s.replace_project(proj, None); @@ -657,8 +802,13 @@ fn create(s: &mut Session, p: &Value) -> Result { s.open_comp(c); s.set_time(poster); } + let mut untitled = true; + if let Some(pp) = project_path { + s.execute("file.saveAs", json!({"path": pp}))?; + untitled = false; + } s.toast(format!("New project from template “{name}”")); - Ok(json!({"template": name, "comp": cid.map(|c| c.0), "untitled": true})) + Ok(json!({"template": name, "comp": cid.map(|c| c.0), "untitled": untitled, "footage": footage})) } fn save_as(s: &mut Session, p: &Value) -> Result { @@ -700,9 +850,26 @@ fn save_as(s: &mut Session, p: &Value) -> Result { "poster": "poster.png", "thumbnail": "thumb.txt", }); + let mut manifest = manifest; + // Embedded footage (default on, capped): the rest stays referenced, with a warning. + let mut media_entries = vec![]; + let mut warning = None; + if p.get("embedFootage").and_then(Value::as_bool).unwrap_or(true) { + let limit_mb = p.get("embedLimitMB").and_then(Value::as_f64).unwrap_or(EMBED_LIMIT_MB).max(0.0); + let e = embed_footage(s, &mut proj, &mut media_entries, (limit_mb * 1024.0 * 1024.0) as u64); + let over: Vec<&Value> = e.skipped.iter().filter(|v| v["reason"] == "over the size limit").collect(); + if !over.is_empty() { + let mb = over.iter().filter_map(|v| v["bytes"].as_u64()).sum::() as f64 / 1048576.0; + warning = Some(format!("{} footage item(s) ({mb:.1} MB) are over the {limit_mb:.0} MB embedding limit and stay linked to their files", over.len())); + } + manifest["media"] = json!(e.media); + manifest["embeddedBytes"] = json!(e.bytes); + manifest["linkedFootage"] = json!(e.skipped); + } let t = s.time_of(cid); let mut entries: Vec<(String, Vec)> = vec![("manifest.json".into(), serde_json::to_vec_pretty(&manifest).unwrap_or_default()), ("project.ecproj".into(), proj.to_json().into_bytes())]; + entries.extend(media_entries); if let Ok((w, h, rgba)) = s.render_rgba8(cid, t, 320) { if let Ok(png) = crate::commands::comp_more::encode_png(&rgba, w, h) { entries.push(("poster.png".into(), png)); @@ -711,8 +878,20 @@ fn save_as(s: &mut Session, p: &Value) -> Result { } let bytes = effectcraft_lottie::zip::store(&entries); let at = write_user(s, &file, &bytes)?; - s.toast(format!("Saved template “{name}”")); - Ok(json!({"id": format!("user/{file}"), "name": name, "path": at, "bytes": bytes.len()})) + match &warning { + Some(w) => s.events.push(crate::Event::Toast { message: format!("Saved template “{name}”: {w}"), error: false }), + None => s.toast(format!("Saved template “{name}”")), + } + Ok(json!({ + "id": format!("user/{file}"), + "name": name, + "path": at, + "bytes": bytes.len(), + "embedded": manifest["media"].as_array().map_or(0, |m| m.len()), + "embeddedBytes": manifest["embeddedBytes"], + "linked": manifest["linkedFootage"], + "warning": warning, + })) } fn delete(s: &mut Session, p: &Value) -> Result { @@ -735,13 +914,21 @@ pub fn specs() -> Vec { list_cmd ), query!("templates.thumbnail", "Template Thumbnail", "{id} → {id, width, height, rgb (hex)}", thumb_cmd), - cmd!("templates.create", "New Project from Template", [], None, "{id (templates.list) | path (.ectemplate)} → an untitled copy", always, create), + cmd!( + "templates.create", + "New Project from Template", + [], + None, + "{id (templates.list) | path (.ectemplate), projectPath? (save the new project there; embedded footage goes to ` Footage/` next to it), footageDir?} → an untitled copy (embedded footage extracted: `footage` lists the files)", + always, + create + ), cmd!( "templates.saveAs", "Save as Template...", ["File"], None, - "{name, description?, category?, overwrite?: bool (default true)} → {id, path}", + "{name, description?, category?, overwrite?: bool (default true), embedFootage?: bool (default true: the footage files go into the template), embedLimitMB? (default 256: footage beyond it stays linked, with a warning)} → {id, path, embedded, embeddedBytes, linked: [{item, name, reason}], warning?}", has_comps, save_as ), @@ -864,6 +1051,150 @@ mod tests { } } + fn add_footage(s: &mut Session, name: &str, path: &str, sequence: Vec) -> ItemId { + let f = effectcraft_project::Footage { path: path.into(), width: 4, height: 4, has_video: true, sequence, ..Default::default() }; + Arc::make_mut(&mut s.project).add_item(name, effectcraft_color::Label::Aqua, None, effectcraft_project::ItemKind::Footage(f)) + } + + fn footage_paths(s: &Session) -> Vec<(String, String, Vec)> { + let mut v: Vec<_> = s + .project + .items + .values() + .filter_map(|i| match &i.kind { + effectcraft_project::ItemKind::Footage(f) => Some((i.name.clone(), f.path.clone(), f.sequence.clone())), + _ => None, + }) + .collect(); + v.sort(); + v + } + + /// Saving embeds the footage (capped, with a warning); creating extracts it next to the + /// new project, or into the settings folder, and points the footage at the copies. + #[test] + fn templates_embed_and_extract_footage() { + let root = std::env::temp_dir().join(format!("ec-tpl-footage-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let media = root.join("media"); + std::fs::create_dir_all(&media).unwrap(); + let file = |n: &str, len: usize, b: u8| { + let p = media.join(n); + std::fs::write(&p, vec![b; len]).unwrap(); + p.to_string_lossy().into_owned() + }; + let still = file("still.png", 1000, 1); + let seq: Vec = (0..3).map(|k| file(&format!("shot_{k:03}.png"), 500, 10 + k as u8)).collect(); + let big = file("big.mov", 2 * 1024 * 1024, 7); + let mut s = Session { config: Some(Arc::new(DirConfig::new(root.join("config")))), ..Default::default() }; + s.execute("templates.create", json!({"id": "lower-third"})).unwrap(); + add_footage(&mut s, "Still", &still, vec![]); + add_footage(&mut s, "Shot", &seq[0], seq.clone()); + add_footage(&mut s, "Big", &big, vec![]); + add_footage(&mut s, "Gone", &media.join("gone.png").to_string_lossy(), vec![]); + let r = s.execute("templates.saveAs", json!({"name": "With Footage", "embedLimitMB": 1})).unwrap(); + assert_eq!(r["embedded"], 2, "{r}"); + assert_eq!(r["embeddedBytes"], 2500); + let linked = r["linked"].as_array().unwrap(); + assert_eq!(linked.len(), 2, "{r}"); + let warning = r["warning"].as_str().unwrap(); + assert!(warning.contains("1 footage item") && warning.contains("1 MB"), "{warning}"); + assert!(s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.contains("embedding limit")))); + // The archive holds the files under media//. + let bytes = read_user(&s, "With Footage.ectemplate").unwrap(); + let entries = effectcraft_lottie::zip::read_stored(&bytes); + let names: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).filter(|k| k.starts_with("media/")).collect(); + assert_eq!(names.len(), 4, "{names:?}"); + assert!(names.iter().any(|n| n.ends_with("/still.png")) && names.iter().any(|n| n.ends_with("/shot_002.png"))); + // The originals go away: the template still has them. + std::fs::remove_file(&still).unwrap(); + for f in &seq { + std::fs::remove_file(f).unwrap(); + } + let proj_path = root.join("work").join("Promo.ecproj"); + std::fs::create_dir_all(proj_path.parent().unwrap()).unwrap(); + let r = s.execute("templates.create", json!({"id": "user/With Footage.ectemplate", "projectPath": proj_path.to_string_lossy()})).unwrap(); + assert_eq!(r["untitled"], false); + assert_eq!(r["footage"].as_array().unwrap().len(), 4, "{r}"); + assert_eq!(s.path.as_deref(), Some(&*proj_path.to_string_lossy())); + assert!(proj_path.exists()); + let dir = root.join("work").join("Promo Footage"); + let fp = footage_paths(&s); + let get = |n: &str| fp.iter().find(|f| f.0 == n).unwrap().clone(); + assert_eq!(get("Still").1, dir.join("still.png").to_string_lossy()); + assert_eq!(std::fs::read(dir.join("still.png")).unwrap(), vec![1u8; 1000]); + let shot = get("Shot"); + assert_eq!(shot.2.len(), 3); + assert_eq!(shot.1, shot.2[0]); + assert_eq!(std::fs::read(&shot.2[2]).unwrap(), vec![12u8; 500]); + // Over the cap: still linked to the original file. + assert_eq!(get("Big").1, big); + // Without a project path: a fresh folder under the settings folder each time. + s.execute("templates.create", json!({"id": "user/With Footage.ectemplate"})).unwrap(); + let base = root.join("config").join(FOOTAGE_DIR); + assert!(base.join("With Footage").join("still.png").exists()); + assert!(s.path.is_none()); + s.execute("templates.create", json!({"id": "user/With Footage.ectemplate"})).unwrap(); + assert!(base.join("With Footage 2").join("still.png").exists()); + // Not embedding: no media entries. + let r = s.execute("templates.saveAs", json!({"name": "Linked", "embedFootage": false})).unwrap(); + assert_eq!(r["embedded"], 0); + let entries = effectcraft_lottie::zip::read_stored(&read_user(&s, "Linked.ectemplate").unwrap()); + assert!(entries.iter().all(|(k, _)| !k.starts_with("media/"))); + let _ = std::fs::remove_dir_all(&root); + } + + /// The browser: no settings folder; the files go to browser storage quietly + /// (`Services::store_file`) and the media layer is told about them (`Importer::register`). + #[test] + fn template_footage_into_browser_storage() { + #[derive(Default)] + struct Mem(Mutex>>, Mutex>); + struct Svc(Arc); + impl crate::Services for Svc { + fn read_file(&self, path: &str) -> std::io::Result> { + self.0.0.lock().unwrap().get(path).cloned().ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, path.to_string())) + } + fn write_file(&self, _: &str, _: &[u8]) -> std::io::Result<()> { + Err(std::io::Error::other("downloads are not for extracted footage")) + } + fn store_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { + self.0.0.lock().unwrap().insert(path.into(), data.to_vec()); + Ok(()) + } + fn exists(&self, path: &str) -> bool { + self.0.0.lock().unwrap().keys().any(|k| k == path || k.starts_with(&format!("{path}/"))) + } + } + struct Imp(Arc); + impl crate::Importer for Imp { + fn probe(&self, path: &str) -> std::result::Result { + Err(path.into()) + } + fn register(&self, path: &str, _: &[u8]) { + self.0.1.lock().unwrap().push(path.into()); + } + } + let mem = Arc::new(Mem::default()); + mem.0.lock().unwrap().insert("/clip.png".into(), vec![3; 64]); + let mut s = Session { + config: Some(Arc::new(MemoryConfig::default())), + services: Arc::new(Svc(mem.clone())), + importer: Some(Arc::new(Imp(mem.clone()))), + ..Default::default() + }; + s.execute("templates.create", json!({"id": "title-card"})).unwrap(); + add_footage(&mut s, "Clip", "/clip.png", vec![]); + s.execute("templates.saveAs", json!({"name": "Web"})).unwrap(); + s.execute("templates.create", json!({"id": "user/Web.ectemplate"})).unwrap(); + let want = format!("/{FOOTAGE_DIR}/Web/clip.png"); + assert_eq!(mem.0.lock().unwrap().get(&want), Some(&vec![3; 64])); + assert_eq!(*mem.1.lock().unwrap(), vec![want.clone()]); + assert!(footage_paths(&s).iter().any(|f| f.1 == want)); + s.execute("templates.create", json!({"id": "user/Web.ectemplate"})).unwrap(); + assert!(mem.0.lock().unwrap().contains_key(&format!("/{FOOTAGE_DIR}/Web 2/clip.png"))); + } + #[test] fn file_names_are_safe() { assert_eq!(file_name_for("A/B: C"), "A_B_ C.ectemplate"); diff --git a/crates/engine/src/tests_color_view.rs b/crates/engine/src/tests_color_view.rs index 7d42f739..7b999394 100644 --- a/crates/engine/src/tests_color_view.rs +++ b/crates/engine/src/tests_color_view.rs @@ -286,3 +286,69 @@ fn my_custom_rgb_simulation() { assert_eq!(s.prefs.custom_rgb, crate::viewer::CustomRgb::default()); assert_eq!(s.execute("view.displayColor", json!({})).unwrap()["customRgb"]["gamma"], json!(2.2)); } + +/// My Custom RGB from a LUT-based (`A2B0`) ICC profile: the viewer runs the profile's tables +/// (baked into a 3D LUT) and matches the same device described as a matrix/TRC profile. +#[test] +fn my_custom_rgb_lut_profiles() { + use effectcraft_color::icc::{D50, LutKind, write_lut_profile}; + use effectcraft_color::space; + let mut s = session(); + s.config = Some(std::sync::Arc::new(crate::config::MemoryConfig::default())); + s.execute("file.projectSettings", json!({"workingSpace": "srgb"})).unwrap(); + // The device: P3 primaries, D65 white, gamma 2.4. + let p3 = [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]]; + let d65 = [0.3127, 0.3290]; + let m = space::mul(&space::bradford(d65, D50), &space::rgb_to_xyz(p3, d65)); + let mi = space::invert(&m); + let to = move |c: [f64; 3]| space::mul_vec(&m, c.map(|v| v.powf(2.4))); + let from = move |x: [f64; 3]| space::mul_vec(&mi, x).map(|v| v.clamp(0.0, 1.0).powf(1.0 / 2.4)); + s.execute( + "view.customRgb", + json!({"name": "Matrix", "red": p3[0], "green": p3[1], "blue": p3[2], "white": d65, "gamma": 2.4, "srgbCurve": false, "preserveRgb": false}), + ) + .unwrap(); + // (Inside both gamuts, away from the edges where a coarse B2A0 grid bends.) + let colors = [px(128, 90, 200), px(90, 170, 110), px(225, 225, 225), px(60, 60, 60), px(200, 120, 100)]; + let reference: Vec> = colors + .iter() + .map(|c| { + let mut c = c.clone(); + DisplayColor::of(&s).unwrap().apply(&mut c); + c + }) + .collect(); + let dir = std::env::temp_dir(); + for (k, (kind, b2a)) in [(LutKind::Lut16, true), (LutKind::AToB, true), (LutKind::Lut8, true), (LutKind::Lut16, false)].into_iter().enumerate() { + let icc = write_lut_profile("LUT Display", kind, 33, &to, if b2a { Some(&from) } else { None }); + let path = dir.join(format!("ec-lut-{}-{k}.icc", std::process::id())); + std::fs::write(&path, &icc).unwrap(); + let r = s.execute("view.customRgb", json!({"icc": path.to_string_lossy(), "preserveRgb": false})).unwrap(); + let _ = std::fs::remove_file(&path); + assert_eq!((r["name"].as_str(), r["lutProfile"].as_bool()), (Some("LUT Display"), Some(true)), "{r}"); + assert!(r.get("iccLut").is_none()); + // The numbers approximate the device. + let c = s.prefs.custom_rgb.clone(); + assert!((c.red[0] - 0.680).abs() < 0.01 && (c.green[1] - 0.690).abs() < 0.01, "{kind:?} {c:?}"); + assert!((c.gamma - 2.4).abs() < 0.1, "{kind:?} {}", c.gamma); + let dc = DisplayColor::of(&s).unwrap(); + assert!(dc.is_lut()); + let tol = if kind == LutKind::Lut8 { 6 } else { 4 }; + for (c, want) in colors.iter().zip(&reference) { + let mut got = c.clone(); + dc.apply(&mut got); + let d = (0..3).map(|i| (got[0][i] as i32 - want[0][i] as i32).abs()).max().unwrap(); + assert!(d <= tol, "{kind:?} b2a {b2a}: {c:?} → {got:?}, matrix {want:?}"); + } + } + // Kept in Settings (base64) and restored with the preferences. + let saved = s.config.as_ref().unwrap().read(crate::prefs::PREFS_FILE).unwrap(); + assert!(saved.contains("\"iccLut\"")); + let back: crate::viewer::CustomRgb = serde_json::from_value(serde_json::from_str::(&saved).unwrap()["customRgb"].clone()).unwrap(); + assert_eq!(back, s.prefs.custom_rgb); + assert!(format!("{back:?}").len() < 400); + // Typed-in numbers replace the tables. + s.execute("view.customRgb", json!({"gamma": 2.2})).unwrap(); + assert!(s.prefs.custom_rgb.icc_lut.is_empty()); + assert!(!DisplayColor::of(&s).unwrap().is_lut()); +} diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index a4386199..1a03cb4c 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -12,7 +12,7 @@ use effectcraft_geom::{Mat3, vec2}; use effectcraft_project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions, TrackerSettings}; use effectcraft_project::{Comp, ItemId, Keyframe, Layer, LayerId, Project, PropGroup, Uid, Value}; use effectcraft_raster::Image; -use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer}; use effectcraft_time::Tick; use effectcraft_track as trk; use serde::{Deserialize, Serialize}; @@ -228,8 +228,11 @@ pub(crate) fn source_frame( Some((Arc::new(buf.img), buf.offset)) } -pub(crate) fn run_work(w: Work, shared: &TrackShared) { +pub(crate) async fn run_work(w: Work, shared: &TrackShared) { let t0 = web_time::Instant::now(); + // A browser job worker's GPU (deferred readbacks): frames render in passes. + let accel = crate::offload::job_accel(); + let accel = accel.as_deref(); let fail = |msg: &str| { let mut s = lock(&shared.state); s.error = Some(msg.to_string()); @@ -237,16 +240,18 @@ pub(crate) fn run_work(w: Work, shared: &TrackShared) { }; let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; - let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel)); r.expr = w.expr.as_deref(); r.cache = Some(&w.cache); let r = &r; - let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); + let wr = &w; + let frame_at = + |t: Tick| crate::offload::in_passes(accel, move |a| source_frame(&r.with_accel(a), &wr.project, wr.comp, comp, layer, t, wr.expr.as_deref())); { let mut s = lock(&shared.state); s.total = w.times.len().saturating_sub(1) as u64; } - let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + let Some(first) = frame_at(w.times[0]).await else { return fail("the layer has no pixels to track") }; let specs: Vec = w.points.iter().map(|p| p.spec).collect(); let mut tracker = trk::Tracker::new(track_options(&w.settings.options), &specs, &trk::Frame { img: &first.0, offset: first.1 }); let ref_c: Vec<[f64; 2]> = specs.iter().map(|s| s.center).collect(); @@ -263,7 +268,7 @@ pub(crate) fn run_work(w: Work, shared: &TrackShared) { .map(|(p, a)| (p.uid, trk::PointResult { center: p.spec.center, confidence: 100.0, rotation: 0.0, scale: 1.0, status: trk::Status::Tracked }, *a)) .collect(), }); - let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { lock(&shared.state).cancelled = true; @@ -274,8 +279,12 @@ pub(crate) fn run_work(w: Work, shared: &TrackShared) { break; }; // Track this frame while the next one renders. - let (res, nf) = - rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + let (res, nf) = crate::offload::join_fetch( + accel.is_some(), + || tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), + || async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } }, + ) + .await; next = nf; if res.iter().any(|p| p.status == trk::Status::Stopped) { lock(&shared.state).low_confidence_stop = true; @@ -363,11 +372,19 @@ impl Session { let shared = Arc::new(TrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { - run_work(work, &shared); + crate::offload::run_or_queue({ + let sh = shared.clone(); + async move { run_work(work, &sh).await } + }); None } else { let sh = shared.clone(); - Some(std::thread::Builder::new().name("track-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + Some( + std::thread::Builder::new() + .name("track-analyze".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh))) + .map_err(|e| e.to_string())?, + ) }; self.track_job = Some(TrackJob { shared, thread, comp, layer, tracker, direction }); self.poll_track(); diff --git a/crates/engine/src/viewer.rs b/crates/engine/src/viewer.rs index 22385995..45551ce6 100644 --- a/crates/engine/src/viewer.rs +++ b/crates/engine/src/viewer.rs @@ -522,11 +522,72 @@ pub struct CustomRgb { pub srgb_curve: bool, /// The ICC profile the numbers were read from (empty: typed in). pub icc: String, + /// A LUT-based (`A2B0`) profile's bytes: the simulation runs the profile's tables (baked + /// into a 3D LUT) instead of the numbers above, which then only approximate it. + #[serde(skip_serializing_if = "IccLut::is_empty")] + pub icc_lut: IccLut, +} + +/// The bytes of a LUT-based ICC profile, kept in Settings as base64. +#[derive(Clone, Default, PartialEq)] +pub struct IccLut(pub Option>>); + +impl IccLut { + pub fn is_empty(&self) -> bool { + self.0.is_none() + } + + /// The parsed profile. + pub fn profile(&self) -> Option { + match effectcraft_color::icc::parse_profile(self.0.as_deref()?) { + Ok(effectcraft_color::icc::Profile::Lut(p)) => Some(p), + _ => None, + } + } + + fn hash(&self) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + self.0.as_deref().hash(&mut h); + h.finish() + } +} + +impl std::fmt::Debug for IccLut { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.0 { + Some(b) => write!(f, "IccLut({} bytes, {:016x})", b.len(), self.hash()), + None => write!(f, "IccLut(none)"), + } + } +} + +impl Serialize for IccLut { + fn serialize(&self, s: S) -> std::result::Result { + s.serialize_str(&self.0.as_deref().map(|b| effectcraft_track::roto::rle::base64_encode(b)).unwrap_or_default()) + } +} + +impl<'de> Deserialize<'de> for IccLut { + fn deserialize>(d: D) -> std::result::Result { + let s = String::deserialize(d)?; + Ok(IccLut(effectcraft_track::roto::rle::base64_decode(&s).filter(|b| !b.is_empty()).map(Arc::new))) + } } impl Default for CustomRgb { fn default() -> Self { - CustomRgb { name: "My Custom RGB".into(), red: P709[0], green: P709[1], blue: P709[2], white: D65, gamma: 2.2, srgb_curve: false, icc: String::new() } + CustomRgb { + name: "My Custom RGB".into(), + red: P709[0], + green: P709[1], + blue: P709[2], + white: D65, + gamma: 2.2, + srgb_curve: false, + icc: String::new(), + icc_lut: IccLut::default(), + } } } @@ -564,26 +625,54 @@ impl CustomRgb { Ok(()) } - /// Take primaries, white and curve from an RGB ICC profile. + /// Take primaries, white and curve from an RGB ICC profile. A LUT-based profile is kept + /// whole ([`CustomRgb::icc_lut`]); its numbers are the primaries' and white's colours + /// (adapted to D65) and the gamma that fits its neutral ramp. pub fn from_icc(bytes: &[u8], path: &str) -> std::result::Result { - let p = effectcraft_color::icc::parse(bytes)?; - let name = if p.description.trim().is_empty() { - std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "ICC Profile".into()) - } else { - p.description.trim().to_string() - }; - let gamma = (p.trc.iter().map(|t| t.fit_gamma()).sum::() / 3.0 * 1000.0).round() / 1000.0; + use effectcraft_color::icc::{Profile, parse_profile}; + let stem = || std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "ICC Profile".into()); + let name = |d: &str| if d.trim().is_empty() { stem() } else { d.trim().to_string() }; let r = |c: [f64; 2]| c.map(|v| (v * 1e5).round() / 1e5); - Ok(CustomRgb { - name, - red: r(p.primaries[0]), - green: r(p.primaries[1]), - blue: r(p.primaries[2]), - white: r(p.white), - gamma, - srgb_curve: p.trc[0].is_srgb(), - icc: path.to_string(), - }) + match parse_profile(bytes)? { + Profile::Matrix(p) => { + let gamma = (p.trc.iter().map(|t| t.fit_gamma()).sum::() / 3.0 * 1000.0).round() / 1000.0; + Ok(CustomRgb { + name: name(&p.description), + red: r(p.primaries[0]), + green: r(p.primaries[1]), + blue: r(p.primaries[2]), + white: r(p.white), + gamma, + srgb_curve: p.trc[0].is_srgb(), + icc: path.to_string(), + icc_lut: IccLut::default(), + }) + } + Profile::Lut(p) => { + let (prims, _, gamma) = p.approximate(); + // The PCS is D50-relative: the device white shows as the viewer's D65. + let a = space::bradford(effectcraft_color::icc::D50, D65); + let adapt = |c: [f64; 2]| { + let v = space::mul_vec(&a, [c[0] / c[1], 1.0, (1.0 - c[0] - c[1]) / c[1]]); + let sum = v[0] + v[1] + v[2]; + let c = [(v[0] / sum).clamp(1e-4, 0.9998), (v[1] / sum).clamp(1e-4, 0.9998)]; + // Table rounding can put a primary a hair outside the spectrum locus. + let k = (0.9999 / (c[0] + c[1])).min(1.0); + [c[0] * k, c[1] * k] + }; + Ok(CustomRgb { + name: name(&p.description), + red: r(adapt(prims[0])), + green: r(adapt(prims[1])), + blue: r(adapt(prims[2])), + white: D65, + gamma: (gamma * 1000.0).round() / 1000.0, + srgb_curve: false, + icc: path.to_string(), + icc_lut: IccLut(Some(Arc::new(bytes.to_vec()))), + }) + } + } } fn prof(&self) -> Prof { @@ -643,10 +732,12 @@ const D65: [f64; 2] = [0.3127, 0.3290]; /// The viewer's colour conversion from the rendered frame (sRGB-encoded display pixels) to /// what the monitor shows (View ▸ Use Display Color Management, View ▸ Simulate Output and /// the display profile in Settings ▸ Previews). `None` when nothing changes. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] pub struct DisplayColor { sim: Option<(Conv, Option)>, display: Option, + /// A LUT-based My Custom RGB profile: the whole simulation baked into a 3D LUT. + lut: Option>, } impl DisplayColor { @@ -669,7 +760,12 @@ impl DisplayColor { if !dcm { // Without display colour management the working space's numbers go to the screen. let raw = Conv::new(src, Prof::of(ws, linear)); - return Some(DisplayColor { sim: None, display: Some(raw) }); + return Some(DisplayColor { sim: None, display: Some(raw), lut: None }); + } + if sim.profile == SimProfile::MyCustom + && let Some(lut) = lut_simulation(custom, src, display, sim.preserve_rgb) + { + return Some(DisplayColor { sim: None, display: None, lut: Some(lut) }); } let out = match sim.profile { SimProfile::MyCustom => Some(custom.prof()), @@ -686,7 +782,7 @@ impl DisplayColor { ) }); let display = (display != source).then(|| Conv::new(src, Prof::of(display, false))); - (sim.is_some() || display.is_some()).then_some(DisplayColor { sim, display }) + (sim.is_some() || display.is_some()).then_some(DisplayColor { sim, display, lut: None }) } /// The conversion a session's viewer uses. @@ -706,8 +802,16 @@ impl DisplayColor { ) } + /// Whether the simulation runs a LUT-based ICC profile (baked into a 3D LUT). + pub fn is_lut(&self) -> bool { + self.lut.is_some() + } + /// One straight colour (0..1 floats). pub fn apply_straight(&self, c: [f32; 3]) -> [f32; 3] { + if let Some(l) = &self.lut { + return l.eval(c.map(|v| v.clamp(0.0, 1.0))).map(|v| v.clamp(0.0, 1.0)); + } let c = match &self.sim { // Through the output's 8-bit encoding (clipped and quantised like the real output), // then shown on the display: converted from the output profile, or — Preserve RGB — @@ -739,6 +843,53 @@ impl DisplayColor { } } +/// Grid points per axis of a baked LUT simulation (a profile without `B2A0` is inverted +/// numerically per point, so its grid is coarser). +const SIM_LUT_SIZE: usize = 33; +const SIM_LUT_SIZE_INVERTED: usize = 17; + +/// My Custom RGB with a LUT-based profile: rendered colours (`src`) → the device's numbers +/// (`B2A0`, or `A2B0` inverted), clipped and quantised to 8 bits like the real output → what the +/// device shows (`A2B0`; Preserve RGB: the numbers taken as sRGB) → the display. Baked once per +/// profile and conversion (the last one is cached). +fn lut_simulation(custom: &CustomRgb, src: Prof, display: ColorSpace, preserve: bool) -> Option> { + type Cache = Option<(u64, Arc)>; + static CACHE: std::sync::Mutex = std::sync::Mutex::new(None); + custom.icc_lut.0.as_ref()?; + let key = { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + (custom.icc_lut.hash(), format!("{src:?}{display:?}{preserve}")).hash(&mut h); + h.finish() + }; + if let Some((k, l)) = CACHE.lock().ok()?.as_ref() + && *k == key + { + return Some(l.clone()); + } + let prof = custom.icc_lut.profile()?; + let to_d50 = space::mul(&space::bradford(D65, effectcraft_color::icc::D50), &src.xyz); + let from_d50 = space::bradford(effectcraft_color::icc::D50, D65); + let disp = Prof::of(display, false); + let disp_inv = space::invert(&disp.xyz); + let srgb_to_display = Conv::new(Prof::new(P709, Curve::Srgb), disp); + let size = if prof.b2a.is_some() { SIM_LUT_SIZE } else { SIM_LUT_SIZE_INVERTED }; + let lut = effectcraft_color::icc::Lut3d::bake(size, |c| { + let lin = c.map(|v| src.curve.decode(v as f32) as f64); + let dev = prof.from_xyz(space::mul_vec(&to_d50, lin), c).map(|v| (v.clamp(0.0, 1.0) * 255.0).round() / 255.0); + if preserve { + return srgb_to_display.apply(dev.map(|v| v as f32)).map(|v| v as f64); + } + let xyz = space::mul_vec(&from_d50, prof.to_xyz(dev)); + space::mul_vec(&disp_inv, xyz).map(|v| disp.curve.encode(v as f32) as f64) + }); + let lut = Arc::new(lut); + if let Ok(mut c) = CACHE.lock() { + *c = Some((key, lut.clone())); + } + Some(lut) +} + // ---------------------------------------------------------------- snapshot /// Take Snapshot (Shift+F5): a rendered frame kept for comparison (Show Snapshot, F5). diff --git a/crates/engine/src/warp.rs b/crates/engine/src/warp.rs index ae956a28..45007bc6 100644 --- a/crates/engine/src/warp.rs +++ b/crates/engine/src/warp.rs @@ -19,7 +19,7 @@ use effectcraft_effects::warp_stab::{self, ANALYSIS, ANALYSIS_KEY}; use effectcraft_keyframe::Value; use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Project, PropGroup, Uid}; use effectcraft_raster::Image; -use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer}; use effectcraft_time::Tick; use effectcraft_track::stabilize::{AnalyzeOpts, Analyzer, StabSettings, WarpAnalysis}; use serde::Serialize; @@ -193,8 +193,11 @@ fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) Some((Arc::new(buf.img.clone()), buf.offset)) } -fn run_work(w: WarpWork, shared: &WarpShared) { +async fn run_work(w: WarpWork, shared: &WarpShared) { let t0 = web_time::Instant::now(); + // A browser job worker's GPU (deferred readbacks): frames render in passes. + let accel = crate::offload::job_accel(); + let accel = accel.as_deref(); let fail = |msg: &str| { let mut s = lock(&shared.state); s.error = Some(msg.to_string()); @@ -202,20 +205,21 @@ fn run_work(w: WarpWork, shared: &WarpShared) { }; let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; - let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel)); r.expr = w.expr.as_deref(); r.cache = Some(&w.cache); let r = &r; let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; - let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + let ctx0 = &ctx0; + let frame_at = |t: Tick| crate::offload::in_passes(accel, move |a| input_frame(&r.with_accel(a), &ctx0.at(t), layer, w.effect_index)); { let mut s = lock(&shared.state); s.step = 1; s.total = w.times.len() as u64; } let mut an = Analyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); - let mut next = frame_at(w.times[0]); + let mut next = frame_at(w.times[0]).await; for k in 0..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { let mut s = lock(&shared.state); @@ -224,10 +228,12 @@ fn run_work(w: WarpWork, shared: &WarpShared) { return; } let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; - let ((), nf) = rayon::join( + let ((), nf) = crate::offload::join_fetch( + accel.is_some(), || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), - || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, - ); + || async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } }, + ) + .await; next = nf; let mut s = lock(&shared.state); s.done = k as u64 + 1; @@ -343,11 +349,19 @@ impl Session { let wait = wait || cfg!(target_arch = "wasm32"); let shared = Arc::new(WarpShared::default()); let thread = if wait { - run_work(work, &shared); + crate::offload::run_or_queue({ + let sh = shared.clone(); + async move { run_work(work, &sh).await } + }); None } else { let sh = shared.clone(); - Some(std::thread::Builder::new().name("warp-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + Some( + std::thread::Builder::new() + .name("warp-analyze".into()) + .spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh))) + .map_err(|e| e.to_string())?, + ) }; self.warp_job = Some(WarpJob { shared, thread, comp, layer, effect, key }); self.events.push(Event::ProjectChanged { revision: self.revision }); diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index 167b5f33..777c4dd7 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -249,7 +249,7 @@ pub(crate) fn pcm_bytes(buf: &[f32], fmt: AudioFormat, big_endian: bool) -> Vec< type Writer<'f, 's> = Mp4Writer<&'f mut crate::out::Out<'s>>; -pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) async fn movie(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { let rate = job.settings.rate(comp); let fmt = job.output.format; let channels = match job.output.channels { @@ -264,11 +264,11 @@ pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re }; let brand = if fmt == OutputFormat::ProRes { Brand::Mov } else { Brand::Mp4 }; let batch = batch_size(); - let render = |k: u64| -> Vec { job.pixels(&job.frame(comp, k), comp, channels, w, h) }; + let render = |_: u64, img: effectcraft_raster::Image| -> Vec { job.pixels(&img, comp, channels, w, h) }; // Encode the first batch before creating tracks: encoders finalise their config on frame 1. let first_end = batch.min(st.total); - let first: Vec> = (0..first_end).into_par_iter().map(render).collect(); + let first: Vec> = job.frames(comp, (0..first_end).collect(), render).await; let mut pending = Vec::new(); for (k, px) in first.iter().enumerate() { pending.extend(venc.encode(px, k as u64)?); @@ -356,7 +356,7 @@ pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let mut i = first_end; while i < total { let end = (i + batch).min(total); - let frames: Vec> = (i..end).into_par_iter().map(render).collect(); + let frames: Vec> = job.frames(comp, (i..end).collect(), render).await; for (k, px) in frames.iter().enumerate() { let ps = venc.encode(px, i + k as u64)?; write_video(&mut mux, ps)?; diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index ae889742..84cf40c3 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -41,7 +41,6 @@ use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, Out use effectcraft_project::{Comp, ItemId, Project}; use effectcraft_raster::Image; use effectcraft_render::{ExprHost, FootageSource}; -use rayon::prelude::*; pub use effectcraft_project::render_queue; pub use effectcraft_project::render_queue::StorageQuota; @@ -159,9 +158,17 @@ pub fn wants_audio(job: &Job) -> bool { /// Run the export (blocking). `progress` is called after every batch; returning `false` cancels. pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result { + effectcraft_render::passes::block_on(export_async(job, progress)) +} + +/// [`export`] as a future: with an accelerator whose readbacks are deferred (WebGPU in a +/// browser worker) every frame renders in passes, awaiting the device in between +/// ([`effectcraft_render::passes`]); otherwise it never waits and frames render in parallel +/// batches as in [`export`]. +pub async fn export_async(job: &Job<'_>, progress: &mut dyn FnMut(&Progress) -> bool) -> Result { let t0 = Instant::now(); let cx = Cx::new(job); - let mut r = export_with(&cx, progress, t0); + let mut r = export_with(&cx, progress, t0).await; let log = cx.write_log(&r, t0.elapsed().as_secs_f64()); if let Ok(rep) = &mut r { rep.log = log; @@ -170,19 +177,19 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< r } -fn export_with(cx: &Cx, progress: &mut dyn FnMut(&Progress) -> bool, t0: Instant) -> Result { +async fn export_with(cx: &Cx<'_>, progress: &mut dyn FnMut(&Progress) -> bool, t0: Instant) -> Result { let comp = cx.comp().ok_or(ExportError::NoComp)?; let total = cx.settings.frame_count(comp); let mut st = State { t0, total, done: 0, progress }; st.advance(0)?; let (w, h) = output_size(comp, cx.settings, cx.output); let mut report = match cx.output.format { - OutputFormat::H264 | OutputFormat::Hevc | OutputFormat::Av1 | OutputFormat::ProRes => encode::movie(cx, comp, w, h, &mut st)?, - OutputFormat::Gif => gif_export(cx, comp, w, h, &mut st)?, - OutputFormat::WebM => webm::webm(cx, comp, w, h, &mut st)?, + OutputFormat::H264 | OutputFormat::Hevc | OutputFormat::Av1 | OutputFormat::ProRes => encode::movie(cx, comp, w, h, &mut st).await?, + OutputFormat::Gif => gif_export(cx, comp, w, h, &mut st).await?, + OutputFormat::WebM => webm::webm(cx, comp, w, h, &mut st).await?, OutputFormat::Wav => webm::audio_file(cx, comp, false, &mut st)?, OutputFormat::Aiff => webm::audio_file(cx, comp, true, &mut st)?, - f if f.is_sequence() => sequence(cx, comp, w, h, &mut st)?, + f if f.is_sequence() => sequence(cx, comp, w, h, &mut st).await?, f => return Err(ExportError::Unsupported(f.label().into())), }; report.seconds = t0.elapsed().as_secs_f64(); @@ -233,7 +240,7 @@ fn first_frame_number(job: &Job, comp: &Comp) -> i64 { job.settings.first_frame(comp) } -fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +async fn sequence(cx: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { let f0 = first_frame_number(cx, comp); let fmt = cx.output.format; let channels = match cx.output.channels { @@ -245,14 +252,10 @@ fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result> = (i..end) - .into_par_iter() - .map(|k| { + let ks: Vec = (i..end).filter(|k| !(cx.settings.skip_existing && out::exists(cx.sink, &sequence_path(cx.path, f0 + *k as i64)))).collect(); + let written: Vec> = cx + .frames(comp, ks, |k, img| { let path = sequence_path(cx.path, f0 + k as i64); - if cx.settings.skip_existing && out::exists(cx.sink, &path) { - return Ok(0); - } - let img = cx.frame(comp, k); // Encode in memory first: the storage hook decides where it fits. let mut buf = std::io::Cursor::new(Vec::new()); write_still(cx, &mut buf, fmt, &img, comp, channels, w, h, cx.output.quality)?; @@ -262,7 +265,7 @@ fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result( } } -fn gif_export(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +async fn gif_export(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { if w > u16::MAX as u32 || h > u16::MAX as u32 { return Err(ExportError::Unsupported("GIF frames are limited to 65535 px".into())); } @@ -357,14 +360,12 @@ fn gif_export(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result> = (i..end) - .into_par_iter() - .map(|k| { - let img = job.frame(comp, k); + let frames: Vec> = job + .frames(comp, (i..end).collect(), |_, img| { let mut px = job.pixels(&img, comp, job.output.channels, w, h); gif::Frame::from_rgba_speed(w as u16, h as u16, &mut px, 10) }) - .collect(); + .await; for mut f in frames { clock += cs_per_frame; let d = (clock.round() as u64).saturating_sub(emitted).max(1); diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index d1ad01bf..3b1e7161 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -75,7 +75,7 @@ impl<'a> Cx<'a> { self.project.comp(self.job.comp) } - fn render_at(&self, t: Tick) -> Image { + async fn render_at(&self, t: Tick) -> Image { let s = self.job.settings; let opts = RenderOpts { scale: s.resolution.clamp(0.01, 4.0), @@ -93,17 +93,43 @@ impl<'a> Cx<'a> { let mut r = Renderer::new(&self.project, self.job.footage, opts); r.expr = self.job.expr; r.accel = self.job.accel; - r.comp_frame(self.job.comp, t) + // With deferred GPU readbacks (a browser worker) the frame renders in passes. + effectcraft_render::passes::comp_frame(&r, self.job.comp, t).await + } + + /// Whether frames render in passes (an accelerator with deferred readbacks). + pub fn deferred(&self) -> bool { + effectcraft_render::passes::is_deferred(self.job.accel) + } + + /// Output frames `ks`, each mapped by `f` (encoding): rendered in parallel batches, or one + /// by one in passes when readbacks are deferred ([`Cx::deferred`]). + pub async fn frames(&self, comp: &Comp, ks: Vec, f: impl Fn(u64, Image) -> T + Sync + Send) -> Vec { + if self.deferred() { + let mut out = Vec::with_capacity(ks.len()); + for k in ks { + let img = self.frame_async(comp, k).await; + out.push(f(k, img)); + } + return out; + } + use rayon::prelude::*; + ks.into_par_iter().map(|k| f(k, self.frame(comp, k))).collect() } /// Output frame `i`: sampled (fields, pulldown), at the colour depth, cropped and resized. pub fn frame(&self, comp: &Comp, i: u64) -> Image { + effectcraft_render::passes::block_on(self.frame_async(comp, i)) + } + + /// [`Cx::frame`] as a future (deferred readbacks: the frame renders in passes). + pub async fn frame_async(&self, comp: &Comp, i: u64) -> Image { let t0 = Instant::now(); let img = match self.job.settings.sample(comp, i) { - FrameSample::Progressive(t) => self.render_at(t), + FrameSample::Progressive(t) => self.render_at(t).await, FrameSample::Fields { first, second, upper_first } => { - let a = self.render_at(first); - if second == first { a } else { interleave(&a, &self.render_at(second), upper_first) } + let a = self.render_at(first).await; + if second == first { a } else { interleave(&a, &self.render_at(second).await, upper_first) } } }; let mut img = img; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index de7c8684..98993b9e 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -17,7 +17,6 @@ use std::io::{Seek, SeekFrom, Write}; use effectcraft_project::Comp; use effectcraft_project::render_queue::{AudioFormat, Channels}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; -use rayon::prelude::*; use crate::encode::{mix, pcm_bytes}; use crate::{Cx, Report, Result, State, batch_size, io, wants_audio}; @@ -134,9 +133,9 @@ pub(crate) fn opus_encoder(job: &Cx) -> effectcraft_opusenc::OpusEncoder { effectcraft_opusenc::OpusEncoder::with_application(channels, job.output.opus_bitrate_kbps.clamp(6, 510) * 1000, app) } -pub(crate) fn webm(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) async fn webm(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { if job.output.webm_codec == effectcraft_project::render_queue::WebmVideoCodec::Av1 { - return crate::webm_av1::webm_av1(job, comp, w, h, st); + return crate::webm_av1::webm_av1(job, comp, w, h, st).await; } let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; @@ -260,7 +259,7 @@ pub(crate) fn webm(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Res // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and // alpha side by side. let pixels: Vec> = - (i..end).into_par_iter().map(|k| job.pixels(&job.frame(comp, k), comp, if alpha { Channels::Rgba } else { colour_channels }, w, h)).collect(); + job.frames(comp, (i..end).collect(), |_, img| job.pixels(&img, comp, if alpha { Channels::Rgba } else { colour_channels }, w, h)).await; let mut frames: Vec = Vec::with_capacity(pixels.len()); for (j, px) in pixels.iter().enumerate() { let (color, a) = rayon::join( diff --git a/crates/export/src/webm_av1.rs b/crates/export/src/webm_av1.rs index 69378e4f..f9b053b8 100644 --- a/crates/export/src/webm_av1.rs +++ b/crates/export/src/webm_av1.rs @@ -9,7 +9,6 @@ use std::io::{Seek, SeekFrom, Write}; use effectcraft_project::Comp; use effectcraft_project::render_queue::Channels; use effectcraft_time::{TICKS_PER_SECOND, Tick}; -use rayon::prelude::*; use crate::encode::VideoEncoder; use crate::encode::mix; @@ -41,7 +40,7 @@ fn cluster(blocks: &mut [Block]) -> Vec { out } -pub(crate) fn webm_av1(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { let rate = job.settings.rate(comp); let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output)?; let with_audio = wants_audio(job); @@ -149,7 +148,7 @@ pub(crate) fn webm_av1(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> let end = (i + batch).min(total); // RGB, or the alpha matte alone (Channels: Alpha); AV1 in WebM carries no alpha track. let channels = if job.output.channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }; - let frames: Vec> = (i..end).into_par_iter().map(|k| job.pixels(&job.frame(comp, k), comp, channels, w, h)).collect(); + let frames: Vec> = job.frames(comp, (i..end).collect(), |_, img| job.pixels(&img, comp, channels, w, h)).await; for (j, px) in frames.iter().enumerate() { let k = i + j as u64; for p in venc.encode(px, k)? { diff --git a/crates/gpu/src/adv3d.rs b/crates/gpu/src/adv3d.rs index e0836e9f..ebcfcca7 100644 --- a/crates/gpu/src/adv3d.rs +++ b/crates/gpu/src/adv3d.rs @@ -324,14 +324,44 @@ fn raster_into(e: &mut Enc, s: &Scene) -> Option<(wgpu::Texture, wgpu::Texture)> /// Rasterise a scene and read it back ([`effectcraft_render::Accelerator::raster_3d`]). `None` /// when the device can't (no readback, size or buffer limits). +/// +/// With deferred readbacks (a browser worker) both targets are read back under the scene's key +/// ([`crate::deferred`]): the first pass starts them and gets an empty target (the pass misses). pub(crate) fn render(g: &GpuContext, s: &Scene) -> Option { - if !g.can_wait() { + if !g.can_readback() { return None; } let mut e = Enc::new(g); - let (color, zout) = raster_into(&mut e, s)?; - let cb = e.read_texture(&color, s.width, s.height, 8)?; - let zb = e.read_texture(&zout, s.width, s.height, 4)?; + let (cb, zb) = match g.deferred.clone() { + Some(d) => { + use crate::deferred::Lookup::*; + let mut k = crate::deferred::Key::new(4); + key_scene(&mut k, s); + let kc = k.finish(); + let kz = kc ^ 1; + match (d.lookup(kc), d.lookup(kz)) { + (Ready(c), Ready(z)) => (c?.bytes, z?.bytes), + (Absent, Absent) => { + let (color, zout) = raster_into(&mut e, s)?; + e.read_texture_async(&color, s.width, s.height, 8, d.start(kc, s.width, s.height, [0.0; 2], 1.0)); + e.read_texture_async(&zout, s.width, s.height, 4, d.start(kz, s.width, s.height, [0.0; 2], 1.0)); + d.miss(); + return Some(empty_target(s.width, s.height)); + } + _ => { + d.miss(); + return Some(empty_target(s.width, s.height)); + } + } + } + None => { + let (color, zout) = raster_into(&mut e, s)?; + (e.read_texture(&color, s.width, s.height, 8)?, e.read_texture(&zout, s.width, s.height, 4)?) + } + }; + if cb.len() != (s.width * s.height * 8) as usize || zb.len() != (s.width * s.height * 4) as usize { + return None; + } let color = cb.chunks_exact(8).map(|c| [0, 1, 2, 3].map(|k| half_to_f32(u16::from_le_bytes([c[2 * k], c[2 * k + 1]])))).collect(); let depth = zb .chunks_exact(4) @@ -343,6 +373,34 @@ pub(crate) fn render(g: &GpuContext, s: &Scene) -> Option { Some(Target { width: s.width, height: s.height, color, depth }) } +/// A placeholder target (a pass whose readbacks are in flight). +fn empty_target(w: u32, h: u32) -> Target { + let n = (w * h) as usize; + Target { width: w, height: h, color: vec![[0.0; 4]; n], depth: vec![f32::INFINITY; n] } +} + +/// Everything that determines a scene's pixels, for deferred readback keys: the packed +/// buffers, the geometry, the target size. +fn key_scene(k: &mut crate::deferred::Key, s: &Scene) { + k.u64(s.width as u64); + k.u64(s.height as u64); + k.u64(s.ssaa as u64); + k.u64(s.opaque_count as u64); + for b in pack(s) { + k.u64(b.len() as u64); + k.bytes(&b); + } + let mut v = Vec::with_capacity(s.vertices.len() * VERTEX_SIZE as usize); + for x in &s.vertices { + for f in x.pos.iter().chain(&x.normal).chain(&x.uv).chain(&x.tangent) { + v.extend_from_slice(&f.to_le_bytes()); + } + v.extend_from_slice(&x.material.to_le_bytes()); + } + k.bytes(&v); + k.bytes(&s.indices.iter().flat_map(|i| i.to_le_bytes()).collect::>()); +} + // ---------------------------------------------------------------- after the rasteriser /// Compute kernels of `adv3d.wgsl` (resolve, motion-blur accumulation, depth of field, @@ -673,9 +731,12 @@ pub(crate) fn finish(e: &mut Enc, r: &Resolved, encode: bool, canvas: Option<&Gp /// [`effectcraft_render::Accelerator::render_3d`]: the whole prepared run on the GPU, read /// back. pub(crate) fn render_prepared(g: &GpuContext, prep: &Prepared) -> Option { - if !g.can_wait() { + if !g.can_readback() { return None; } + if let Some(d) = g.deferred.clone() { + return render_prepared_deferred(g, &d, prep); + } let mut e = Enc::new(g); let Some(r) = resolve_run(&mut e, prep)? else { return Some(None) }; let img = finish(&mut e, &r, !prep.linear, None); @@ -692,6 +753,84 @@ pub(crate) fn render_prepared(g: &GpuContext, prep: &Prepared) -> Option, prep: &Prepared) -> Option { + use crate::deferred::Lookup::*; + if prep.dof.is_some() { + return None; + } + let (w, h) = prep.out; + let mut k = crate::deferred::Key::new(5); + k.u64(w as u64); + k.u64(h as u64); + k.u64(prep.samples as u64); + k.f64(prep.scale); + k.u64(prep.linear as u64); + let mut any = false; + for i in 0..prep.samples.max(1) { + let s = prep.scene(i); + any |= !s.indices.is_empty(); + key_scene(&mut k, &s); + } + if !any { + return Some(None); + } + let ki = k.finish(); + let kz = ki ^ 1; + match (d.lookup(ki), d.lookup(kz)) { + (Ready(a), Ready(b)) => { + let (a, b) = (a?, b?); + let mut img = effectcraft_render::Image::new(a.width, a.height); + let dst: &mut [u8] = bytemuck::cast_slice_mut(&mut img.data); + if dst.len() != a.bytes.len() || b.bytes.len() != dst.len() / 4 { + return None; + } + dst.copy_from_slice(&a.bytes); + let depth = b + .bytes + .chunks_exact(4) + .map(|c| { + let z = f32::from_le_bytes([c[0], c[1], c[2], c[3]]); + if z >= BIG * 0.5 { f32::INFINITY } else { z } + }) + .collect(); + return Some(Some((img, depth))); + } + (Absent, Absent) => {} + _ => { + d.miss(); + return Some(None); + } + } + let mut e = Enc::new(g); + let Some(r) = resolve_run(&mut e, prep)? else { return Some(None) }; + let img = finish(&mut e, &r, !prep.linear, None); + let _ = e.read_keyed(&img, ki, [0.0; 2], 1.0); + let size = w as u64 * h as u64 * 4; + let rb = g.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("adv3d depth readback"), + size, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + e.encoder().copy_buffer_to_buffer(&r.depth, 0, &rb, 0, size); + e.submit(); + let done = d.start(kz, w, h, [0.0; 2], 1.0); + let b = rb.clone(); + rb.map_async(wgpu::MapMode::Read, .., move |res| { + let out = res.ok().and_then(|_| b.get_mapped_range(..).ok().map(|v| v.to_vec())); + b.unmap(); + done(out); + }); + #[cfg(not(target_arch = "wasm32"))] + let _ = g.device.poll(wgpu::PollType::Poll); + d.miss(); + Some(None) +} + /// Draw a prepared run over the GPU canvas (the compositor's walk; no readback unless the /// depth of field needs its radius range). `None` = render it on the CPU. pub(crate) fn draw_run(e: &mut Enc, prep: &Prepared, canvas: &GpuImage) -> Option { diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 2c23f08c..554decc1 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -269,7 +269,7 @@ impl Accelerator for Gpu { } fn particles(&self) -> Option<&dyn effectcraft_effects::psim::ParticleSim> { - self.ctx.can_wait().then_some(self as &dyn effectcraft_effects::psim::ParticleSim) + self.ctx.can_readback().then_some(self as &dyn effectcraft_effects::psim::ParticleSim) } fn auto_pick(&self) -> Option<&effectcraft_render::AutoPick> { @@ -295,6 +295,16 @@ impl Accelerator for Gpu { fn miss_gate(&self) -> Option> { self.ctx.deferred().map(|d| d.gate()) } + + fn settle(&self) -> Option + '_>>> { + let d = self.ctx.deferred()?.clone(); + // Natively the device delivers the readbacks when polled: wait for them here. + #[cfg(not(target_arch = "wasm32"))] + if d.in_flight() > 0 { + let _ = self.ctx.device.poll(wgpu::PollType::wait_indefinitely()); + } + Some(Box::pin(async move { d.settled().await })) + } } impl effectcraft_effects::psim::ParticleSim for Gpu { @@ -314,6 +324,8 @@ mod tests_adv3d; #[cfg(test)] mod tests_deferred; #[cfg(test)] +mod tests_deferred_jobs; +#[cfg(test)] mod tests_fx_color; #[cfg(test)] mod tests_fx_depth; diff --git a/crates/gpu/src/particles.rs b/crates/gpu/src/particles.rs index 487fcfbe..d06a52af 100644 --- a/crates/gpu/src/particles.rs +++ b/crates/gpu/src/particles.rs @@ -108,14 +108,30 @@ fn uniforms(req: &SimRequest, n: u32, s0: u64, prev_n: u32) -> Vec { } /// Simulate `req` (see [`effectcraft_effects::psim::ParticleSim`]). +/// +/// With deferred readbacks (a browser worker) the state's readback is keyed by the request +/// ([`crate::deferred`]): the first pass starts it and gets no particles (the pass misses and is +/// rendered again), later passes read the bytes. pub(crate) fn simulate(g: &GpuContext, req: &SimRequest) -> Option> { - if !g.can_wait() || req.steps > u32::MAX as u64 { + if !g.can_readback() || req.steps > u32::MAX as u64 { return None; } let n = req.births.len() as u32; if n == 0 { return Some(vec![]); } + let deferred = g.deferred.clone(); + let key = deferred.as_ref().map(|_| sim_key(req)); + if let (Some(d), Some(k)) = (&deferred, key) { + match d.lookup(k) { + crate::deferred::Lookup::Ready(r) => return r.map(|r| parse(&r.bytes)), + crate::deferred::Lookup::InFlight => { + d.miss(); + return Some(vec![]); + } + crate::deferred::Lookup::Absent => {} + } + } let size = n as u64 * STATE_BYTES; if size > g.device.limits().max_storage_buffer_binding_size { return None; @@ -176,6 +192,20 @@ pub(crate) fn simulate(g: &GpuContext, req: &SimRequest) -> Option Option = view.chunks_exact(4).map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect(); + let parts = parse(&view); drop(view); read.unmap(); - let parts = f - .chunks_exact(12) + keep_checkpoint(g, req, n, next); + Some(parts) +} + +/// The deferred readback key of a request: everything that determines the state. +fn sim_key(req: &SimRequest) -> u128 { + let mut k = crate::deferred::Key::new(3); + k.u64(req.key); + k.u64(req.steps); + k.u64(req.births.len() as u64); + k.bytes(&req.births.iter().flat_map(|b| b.to_le_bytes()).collect::>()); + k.debug(&req.system); + k.finish() +} + +/// Live particles of a state buffer's bytes. +fn parse(bytes: &[u8]) -> Vec { + let f: Vec = bytes.chunks_exact(4).map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect(); + f.chunks_exact(12) .enumerate() .filter(|(_, s)| s[9] != 0.0) .map(|(id, s)| SimParticle { p: [s[0], s[1], s[2]], v: [s[4], s[5], s[6]], age: s[3], life: s[7], rnd: s[8], id: id as u32 }) - .collect(); - // Keep the new state as a checkpoint. + .collect() +} + +/// Keep a simulated state on the GPU as a checkpoint (the oldest go beyond the limit). +fn keep_checkpoint(g: &GpuContext, req: &SimRequest, n: u32, next: wgpu::Buffer) { if let Ok(mut c) = g.particle_states.lock() { let now = c.clock; c.list.retain(|k| !(k.key == req.key && k.steps == req.steps)); @@ -202,7 +252,6 @@ pub(crate) fn simulate(g: &GpuContext, req: &SimRequest) -> Option; diff --git a/crates/gpu/src/tests_adv3d.rs b/crates/gpu/src/tests_adv3d.rs index 736931af..7f4511d9 100644 --- a/crates/gpu/src/tests_adv3d.rs +++ b/crates/gpu/src/tests_adv3d.rs @@ -35,7 +35,7 @@ fn set(l: &mut effectcraft_project::Layer, path: &str, v: Value) { /// Primitives with metal/dielectric materials, a shadow-casting spot light and a point light, /// an environment light, a semi-transparent textured card. -fn scene_project() -> (Project, ItemId) { +pub(crate) fn scene_project() -> (Project, ItemId) { let mut p = Project::default(); let mut c = Comp::new(240, 160, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); c.renderer = R3::Advanced3D; @@ -90,7 +90,7 @@ fn scene_project() -> (Project, ItemId) { (p, cid) } -fn scene(p: &Project, cid: ItemId) -> Scene { +pub(crate) fn scene(p: &Project, cid: ItemId) -> Scene { let r = Renderer::new(p, &NoFootage, RenderOpts::default()); let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); @@ -177,7 +177,7 @@ fn gpu_raster_perf_smoke() { /// The lit scene seen through a depth-of-field camera (hexagonal iris, diffraction fringe, /// highlight boost) with the sphere moving under motion blur. -fn dof_mb_project() -> (Project, ItemId) { +pub(crate) fn dof_mb_project() -> (Project, ItemId) { let (mut p, cid) = scene_project(); add(&mut p, cid, LayerSource::Camera, |l| { let zoom = match l.props.prop_mut("cameraOptions/zoom").unwrap().value { @@ -208,7 +208,7 @@ fn dof_mb_project() -> (Project, ItemId) { /// Share of pixels (0–1) differing by more than 1/255 on any channel, and the largest /// difference. -fn off_share(a: &[[f32; 4]], b: &[[f32; 4]]) -> (f64, f32) { +pub(crate) fn off_share(a: &[[f32; 4]], b: &[[f32; 4]]) -> (f64, f32) { let mut off = 0; let mut max = 0.0f32; for (p, q) in a.iter().zip(b) { @@ -222,7 +222,7 @@ fn off_share(a: &[[f32; 4]], b: &[[f32; 4]]) -> (f64, f32) { (off as f64 / a.len().max(1) as f64, max) } -fn run_of(p: &Project, cid: ItemId) -> (EvalCtx<'_>, Vec<&effectcraft_project::Layer>) { +pub(crate) fn run_of(p: &Project, cid: ItemId) -> (EvalCtx<'_>, Vec<&effectcraft_project::Layer>) { let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); (ctx, run) diff --git a/crates/gpu/src/tests_deferred.rs b/crates/gpu/src/tests_deferred.rs index c19153a1..e5eec605 100644 --- a/crates/gpu/src/tests_deferred.rs +++ b/crates/gpu/src/tests_deferred.rs @@ -16,7 +16,7 @@ use crate::tests::{Pattern, Scene, diff, n, set, tolerance, v3}; /// The device, held exclusively: deferred state is per device and a frame renders at a time /// (a frame worker renders one frame at a time too). -fn deferred_gpu() -> Option<(&'static Gpu, std::sync::MutexGuard<'static, ()>)> { +pub(crate) fn deferred_gpu() -> Option<(&'static Gpu, std::sync::MutexGuard<'static, ()>)> { static G: OnceLock> = OnceLock::new(); static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); let g = G.get_or_init(|| pollster::block_on(Gpu::request_deferred()).map_err(|e| eprintln!("deferred gpu tests skipped: {e}")).ok()).as_ref()?; @@ -25,7 +25,7 @@ fn deferred_gpu() -> Option<(&'static Gpu, std::sync::MutexGuard<'static, ()>)> /// Footage with GPU effects around a CPU-only one, a solid with Glow, and a precomp whose /// layer is blurred inside and outside (a chain whose input is another chain's result). -fn scene() -> Scene { +pub(crate) fn scene() -> Scene { let cpu_only = "ec.blur.ccradial"; assert!(!effectcraft_effects::GPU_EFFECTS.contains(&cpu_only)); let mut s = Scene::new(BitDepth::Bpc32); @@ -56,7 +56,7 @@ fn scene() -> Scene { } /// Render in passes with deferred readbacks; returns the frame and the number of passes. -fn render_passes(g: &Gpu, s: &Scene, opts: RenderOpts, cache: &LayerCache, t: Tick) -> (effectcraft_raster::Image, usize) { +pub(crate) fn render_passes(g: &Gpu, s: &Scene, opts: RenderOpts, cache: &LayerCache, t: Tick) -> (effectcraft_raster::Image, usize) { cache.set_gate(g.miss_gate()); let mut r = Renderer::new(&s.p, &Pattern, opts); r.accel = Some(g); diff --git a/crates/gpu/src/tests_deferred_jobs.rs b/crates/gpu/src/tests_deferred_jobs.rs new file mode 100644 index 00000000..7f10f53d --- /dev/null +++ b/crates/gpu/src/tests_deferred_jobs.rs @@ -0,0 +1,109 @@ +//! Deferred readbacks for what job workers and frame workers render besides effect chains: +//! GPU particles and Advanced 3D read back under keys too, so they render on the device in +//! passes; and the job loops' pass driver (`effectcraft_render::passes`) settles natively. + +use effectcraft_effects::EffectCtx; +use effectcraft_project::BitDepth; +use effectcraft_render::three_d::adv; +use effectcraft_render::{Accelerator, Backend, LayerCache, NoFootage, RenderOpts, Renderer}; +use effectcraft_time::Tick; + +use crate::tests::{Pattern, diff, n, tolerance}; +use crate::tests_adv3d::{off_share, run_of, scene, scene_project}; +use crate::tests_deferred::{deferred_gpu, render_passes}; +use crate::tests_particles::{Host, agree, params}; + +/// The device's readbacks have landed (the browser's event loop; natively a poll). +fn settle(g: &crate::Gpu) { + g.wait(); + pollster::block_on(g.settled()); +} + +#[test] +fn particles_resolve_in_passes() { + let Some((g, _lock)) = deferred_gpu() else { return }; + let host = Host(g); + for (id, vals) in [ + ("ec.sim.ccparticleworld", vec![("birthRate", n(2.0)), ("longevity", n(1.2)), ("extras/randomSeed", n(7.0))]), + ("ec.sim.particleplayground", vec![("grid/particlesAcross", n(0.0)), ("cannon/particlesPerSecond", n(90.0))]), + ] { + let p = params(id, &vals); + let ctx = |env| EffectCtx { params: &p, time: 1.5, layer_size: [320.0, 240.0], seed: 5, adjustment: false, env }; + let run = |c: &EffectCtx| { + if id == "ec.sim.particleplayground" { effectcraft_effects::playground_state(c) } else { effectcraft_effects::particle_state(id, c) } + }; + let cpu = run(&ctx(Default::default())).unwrap(); + let env = || effectcraft_effects::EffectEnv { host: Some(&host), ..Default::default() }; + g.frame_begin(); + g.pass_begin(); + // First pass: the simulation runs, its readback starts; no particles yet, the pass missed. + let first = run(&ctx(env())).unwrap(); + assert!(first.is_empty() && g.pass_missed(), "{id}"); + settle(g); + g.pass_begin(); + let gpu = run(&ctx(env())).unwrap(); + assert!(!g.pass_missed(), "{id}: served from the readback"); + agree(&format!("deferred {id}"), &cpu, &gpu, 0.01, cpu.len() / 100 + 1); + } +} + +#[test] +fn advanced_3d_resolves_in_passes() { + let Some((g, _lock)) = deferred_gpu() else { return }; + let (p, cid) = scene_project(); + // One scene rasterised (the CPU compositor's per-scene path). + let s = scene(&p, cid); + let cpu = adv::raster::render(&s); + g.frame_begin(); + g.pass_begin(); + let t = g.raster_3d(&s).expect("handled"); + assert!(g.pass_missed() && t.color.iter().all(|c| c[3] == 0.0), "a placeholder first"); + settle(g); + g.pass_begin(); + let t = g.raster_3d(&s).expect("handled"); + assert!(!g.pass_missed()); + let (share, _) = off_share(&cpu.color, &t.color); + assert!(share < 0.015, "raster: {:.3} % of pixels differ", share * 100.0); + // A whole prepared run (resolve, encoding) read back under its key. + let (ctx, run) = run_of(&p, cid); + let out = (ctx.comp.width, ctx.comp.height); + let r = Renderer::new(&p, &NoFootage, RenderOpts::default()); + let prep = adv::Prepared::new(&r, &ctx, &run, out); + let (cimg, _) = adv::render_prepared(&prep).expect("cpu drew"); + g.frame_begin(); + g.pass_begin(); + assert!(matches!(g.render_3d(&prep), Some(None)) && g.pass_missed(), "nothing drawn yet: a placeholder"); + settle(g); + g.pass_begin(); + let (gimg, depth) = g.render_3d(&prep).expect("handled").expect("drawn"); + assert!(!g.pass_missed()); + assert_eq!(depth.len(), (out.0 * out.1) as usize); + let (share, _) = off_share(&cimg.data, &gimg.data); + assert!(share <= 0.01, "run: {:.3} % of pixels differ", share * 100.0); +} + +/// The job loops' driver: `passes::comp_frame` awaits the device between passes (natively +/// `Accelerator::settle` polls it) and gives the frame `render_passes` gives. +#[test] +fn job_frames_render_in_passes() { + let Some((g, _lock)) = deferred_gpu() else { return }; + let s = crate::tests_deferred::scene(); + let t = Tick::from_seconds_f64(0.5); + let cache = LayerCache::default(); + cache.set_gate(g.miss_gate()); + let opts = RenderOpts { backend: Backend::Gpu, ..Default::default() }; + let (want, _) = render_passes(g, &s, opts, &LayerCache::default(), t); + let mut r = Renderer::new(&s.p, &Pattern, opts); + r.accel = Some(g); + r.cache = Some(&cache); + let before = g.deferred_stats().unwrap(); + let img = effectcraft_render::passes::block_on(effectcraft_render::passes::comp_frame(&r, s.cid, t)); + let after = g.deferred_stats().unwrap(); + assert!(after.0 - before.0 >= 2 && after.1 > before.1, "rendered in passes with readbacks: {before:?} → {after:?}"); + assert_eq!(img.data, want.data); + let cpu = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Cpu, ..Default::default() }).comp_frame_cpu(s.cid, t); + assert_eq!(diff(&cpu, &img, tolerance(BitDepth::Bpc32) * 2.0).over, 0); + // `in_passes` without a deferred accelerator: one pass, at once. + let one = effectcraft_render::passes::block_on(effectcraft_render::passes::in_passes(None, |_| 7)); + assert_eq!((one.value, one.passes, one.accelerated), (7, 1, false)); +} diff --git a/crates/gpu/src/tests_particles.rs b/crates/gpu/src/tests_particles.rs index e3ef97a8..d2a229b0 100644 --- a/crates/gpu/src/tests_particles.rs +++ b/crates/gpu/src/tests_particles.rs @@ -10,7 +10,7 @@ use effectcraft_keyframe::Value; use crate::Gpu; use crate::tests::{gpu, n}; -struct Host(&'static Gpu); +pub(crate) struct Host(pub(crate) &'static Gpu); impl EffectHost for Host { fn layer(&self, _id: u64, _masks_and_effects: bool) -> Option { @@ -24,7 +24,7 @@ impl EffectHost for Host { } } -fn params(id: &str, vals: &[(&str, Value)]) -> Params { +pub(crate) fn params(id: &str, vals: &[(&str, Value)]) -> Params { let spec = effectcraft_effects::find(id).unwrap(); let size = [320.0, 240.0]; let mut p = Params { values: spec.params.iter().map(|ps| (ps.id.to_string(), effectcraft_effects::default_value(ps, size))).collect() }; @@ -47,7 +47,7 @@ fn states(id: &str, p: &Params, t: f64, host: &Host) -> (Vec, Vec 20, "{label}: the scene has particles ({})", cpu.len()); let gi: std::collections::HashMap = gpu.iter().map(|q| (q.id, q)).collect(); let mut missing = 0; diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index f0115f6f..80d902e9 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -30,39 +30,49 @@ impl Exporter for FileExporter { effectcraft_export::available_formats() } fn export(&self, job: &ExportJob, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { - let j = effectcraft_export::Job { - project: job.project, - footage: job.footage, - expr: job.expr, - accel: job.accel, - comp: job.item.comp, - settings: &job.item.settings, - output: &job.item.output, - path: job.path, - sink: self.sink.as_deref(), - options: effectcraft_export::JobOptions { - log: job.item.log, - label: job.label.clone(), - storage: job.storage, - overflow: job.project.render_prefs.overflow_folders.clone(), - }, - nested_switches: job.nested_switches, - }; - match effectcraft_export::export(&j, &mut |p| progress(p.done, p.total)) { - Ok(r) => Ok(ExportResult { - path: r.path, - frames: r.frames, - width: r.width, - height: r.height, - bytes: r.bytes, - seconds: r.seconds, - audio: r.audio, - log: r.log, - overflow: r.overflow, - }), - Err(effectcraft_export::ExportError::Cancelled) => Err(effectcraft_engine::render_queue::CANCELLED.into()), - Err(e) => Err(e.to_string()), - } + effectcraft_engine::render::passes::block_on(self.export_async(job, progress)) + } + + fn export_async<'a>( + &'a self, + job: &'a ExportJob<'a>, + progress: &'a mut dyn FnMut(u64, u64) -> bool, + ) -> effectcraft_engine::render_queue::LocalFuture<'a, Result> { + Box::pin(async move { + let j = effectcraft_export::Job { + project: job.project, + footage: job.footage, + expr: job.expr, + accel: job.accel, + comp: job.item.comp, + settings: &job.item.settings, + output: &job.item.output, + path: job.path, + sink: self.sink.as_deref(), + options: effectcraft_export::JobOptions { + log: job.item.log, + label: job.label.clone(), + storage: job.storage, + overflow: job.project.render_prefs.overflow_folders.clone(), + }, + nested_switches: job.nested_switches, + }; + match effectcraft_export::export_async(&j, &mut |p| progress(p.done, p.total)).await { + Ok(r) => Ok(ExportResult { + path: r.path, + frames: r.frames, + width: r.width, + height: r.height, + bytes: r.bytes, + seconds: r.seconds, + audio: r.audio, + log: r.log, + overflow: r.overflow, + }), + Err(effectcraft_export::ExportError::Cancelled) => Err(effectcraft_engine::render_queue::CANCELLED.into()), + Err(e) => Err(e.to_string()), + } + }) } } diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index da2ee829..8c85490b 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -32,6 +32,8 @@ use crate::eval::EvalCtx; pub struct LayerCache { inner: Mutex, disk: Mutex, u64)>>, + /// Another persistent store when there is no disk cache (the browser's, [`PrefetchStore`]). + store: Mutex, u64)>>, /// While set, inserts are dropped (see [`LayerCache::set_gate`]). gate: Mutex>>, } @@ -74,6 +76,7 @@ impl LayerCache { LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }), disk: Mutex::new(None), + store: Mutex::new(None), gate: Mutex::new(None), } } @@ -113,10 +116,32 @@ impl LayerCache { self.disk.lock().ok()?.as_ref().map(|d| d.0.clone()) } - fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { - let d = self.disk.lock().ok()?; - let (dc, salt) = d.as_ref()?; - Some((dc.clone(), ((*salt as u128) << 64) | key as u128)) + /// Attach (or detach) a [`LayerStore`] used when no disk cache is attached (the browser's + /// Origin Private File System through a [`PrefetchStore`]); `salt` as in + /// [`LayerCache::set_disk`]. + pub fn set_store(&self, store: Option<(Arc, u64)>) { + if let Ok(mut s) = self.store.lock() { + *s = store; + } + } + + /// Change the store's salt (a new project or footage), keeping the store. + pub fn set_store_salt(&self, salt: u64) { + if let Ok(mut d) = self.store.lock() + && let Some((_, s)) = d.as_mut() + { + *s = salt; + } + } + + fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { + let salted = |salt: u64| ((salt as u128) << 64) | key as u128; + if let Some((dc, salt)) = self.disk.lock().ok()?.as_ref() { + return Some((dc.clone() as Arc, salted(*salt))); + } + let s = self.store.lock().ok()?; + let (st, salt) = s.as_ref()?; + Some((st.clone(), salted(*salt))) } pub fn get(&self, key: u64) -> Option> { @@ -139,7 +164,7 @@ impl LayerCache { drop(g); // Disk cache: a hit comes back into memory. let (dc, dk) = self.disk_key(key)?; - let buf = Arc::new(dc.get_layer(dk)?); + let buf = dc.get_layer(dk)?; self.insert_mem(key, buf.clone()); Some(buf) } @@ -223,6 +248,148 @@ impl LayerCache { } } +/// Persistent storage of layer buffers behind a [`LayerCache`] (keys salted per project). +pub trait LayerStore: Send + Sync { + /// A stored buffer (synchronous: mid-render). + fn get_layer(&self, key: u128) -> Option>; + /// Keep a buffer that was slow to render. + fn put_layer(&self, key: u128, buf: &Buf); + /// Buffers faster to render than this (ms) are not stored. + fn min_layer_ms(&self) -> u64 { + 0 + } +} + +/// Layers buffers rendered slower than this go to the browser's disk cache (ms). +pub const PREFETCH_MIN_LAYER_MS: u64 = 20; + +/// A [`LayerStore`] over storage that can't be read in the middle of a render (the browser's +/// Origin Private File System, read asynchronously). Lookups are served from buffers fetched +/// before the frame ([`PrefetchStore::provide`]); the keys a render looked up and didn't find +/// are recorded ([`PrefetchStore::take_misses`]) so the transport can fetch them before the next +/// frame; buffers to store queue up as sealed entries ([`PrefetchStore::take_writes`]) for the +/// transport to write after the frame. Fetched buffers are kept least recently used first, up +/// to a byte budget. +pub struct PrefetchStore { + st: Mutex, + budget: usize, + min_ms: u64, +} + +#[derive(Default)] +struct Prefetched { + ready: HashMap, u64)>, + bytes: usize, + clock: u64, + misses: Vec, + hits: Vec, + writes: Vec<(u128, Vec)>, + /// Keys stored or queued (not written twice). + known: std::collections::HashSet, +} + +impl Default for PrefetchStore { + fn default() -> Self { + PrefetchStore::new(256 << 20) + } +} + +impl PrefetchStore { + pub fn new(budget: usize) -> PrefetchStore { + PrefetchStore { st: Mutex::new(Prefetched::default()), budget, min_ms: PREFETCH_MIN_LAYER_MS } + } + + /// Store buffers that took at least `ms` to render (default [`PREFETCH_MIN_LAYER_MS`]). + pub fn with_min_ms(mut self, ms: u64) -> PrefetchStore { + self.min_ms = ms; + self + } + + /// Whether `key`'s buffer is here. + pub fn has(&self, key: u128) -> bool { + self.st.lock().is_ok_and(|s| s.ready.contains_key(&key)) + } + + /// A fetched entry (sealed bytes, [`crate::disk_cache::layer_entry`]) for `key`; `false` + /// when damaged. + pub fn provide(&self, key: u128, entry: &[u8]) -> bool { + let Some(buf) = crate::disk_cache::read_layer_entry(entry) else { return false }; + let bytes = buf.img.data.len() * std::mem::size_of::(); + let Ok(mut s) = self.st.lock() else { return false }; + s.clock += 1; + let now = s.clock; + if let Some((old, _)) = s.ready.insert(key, (Arc::new(buf), now)) { + s.bytes -= old.img.data.len() * std::mem::size_of::(); + } + s.bytes += bytes; + s.known.insert(key); + while s.bytes > self.budget { + let Some((&k, _)) = s.ready.iter().min_by_key(|(_, (_, t))| *t) else { break }; + if let Some((b, _)) = s.ready.remove(&k) { + s.bytes -= b.img.data.len() * std::mem::size_of::(); + } + } + true + } + + /// Keys looked up and not found since the last call (each once). + pub fn take_misses(&self) -> Vec { + self.st.lock().map(|mut s| std::mem::take(&mut s.misses)).unwrap_or_default() + } + + /// Keys served since the last call (each once). + pub fn take_hits(&self) -> Vec { + self.st.lock().map(|mut s| std::mem::take(&mut s.hits)).unwrap_or_default() + } + + /// Entries to write: (key, sealed bytes). + pub fn take_writes(&self) -> Vec<(u128, Vec)> { + self.st.lock().map(|mut s| std::mem::take(&mut s.writes)).unwrap_or_default() + } + + /// Forget everything (Empty Disk Cache, Purge). + pub fn clear(&self) { + if let Ok(mut s) = self.st.lock() { + *s = Prefetched::default(); + } + } +} + +impl LayerStore for PrefetchStore { + fn get_layer(&self, key: u128) -> Option> { + let mut s = self.st.lock().ok()?; + s.clock += 1; + let now = s.clock; + match s.ready.get_mut(&key) { + Some((b, t)) => { + *t = now; + let b = b.clone(); + if !s.hits.contains(&key) { + s.hits.push(key); + } + Some(b) + } + None => { + if !s.misses.contains(&key) && s.misses.len() < 1024 { + s.misses.push(key); + } + None + } + } + } + + fn put_layer(&self, key: u128, buf: &Buf) { + let Ok(mut s) = self.st.lock() else { return }; + if s.known.insert(key) { + s.writes.push((key, crate::disk_cache::layer_entry(buf))); + } + } + + fn min_layer_ms(&self) -> u64 { + self.min_ms + } +} + /// FNV-1a style 64-bit hasher fed by `Debug` output (exact float formatting) and raw values. struct KeyHasher(u64); diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index bd78442a..6204da45 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -173,6 +173,38 @@ impl DiskIndex { } } +/// Which layer buffers a browser frame worker should fetch from the disk cache before its next +/// frame (its lookups mid-render are synchronous, the Origin Private File System is not): the +/// layer-cache misses the workers reported for recent frames, most recent first, that are on +/// disk ([`DiskIndex`], kind [`Kind::Layer`]). +#[derive(Default)] +pub struct LayerPrefetch { + recent: std::collections::VecDeque, +} + +/// Misses [`LayerPrefetch`] remembers. +const PREFETCH_RECENT: usize = 512; + +impl LayerPrefetch { + /// A frame's layer-cache misses (salted keys). + pub fn note_misses(&mut self, keys: impl IntoIterator) { + for k in keys { + self.recent.retain(|x| *x != k); + self.recent.push_front(k); + } + self.recent.truncate(PREFETCH_RECENT); + } + + /// Up to `max` keys to fetch before the next frame. + pub fn plan(&self, index: &DiskIndex, max: usize) -> Vec { + self.recent.iter().copied().filter(|k| index.contains(Kind::Layer, *k)).take(max).collect() + } + + pub fn clear(&mut self) { + self.recent.clear(); + } +} + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] enum Job { Write(Kind, u128, Vec), @@ -584,6 +616,29 @@ pub fn read_frame_entry(data: &[u8]) -> Option { decode_frame(verify(data, Kind::Frame)?) } +/// A sealed layer-buffer entry: what the browser's frame workers write to the Origin Private +/// File System (`layers/<32 hex>.ecc`). +pub fn layer_entry(buf: &Buf) -> Vec { + seal(Kind::Layer, &encode_layer(buf)) +} + +/// A layer buffer from a sealed entry's bytes (`None` when damaged). +pub fn read_layer_entry(data: &[u8]) -> Option { + decode_layer(verify(data, Kind::Layer)?) +} + +impl crate::cache::LayerStore for DiskCache { + fn get_layer(&self, key: u128) -> Option> { + DiskCache::get_layer(self, key).map(Arc::new) + } + fn put_layer(&self, key: u128, buf: &Buf) { + DiskCache::put_layer(self, key, buf); + } + fn min_layer_ms(&self) -> u64 { + DiskCache::min_layer_ms(self) + } +} + /// An entry's file name in its kind's folder (`<32 hex>.ecc`). pub fn entry_name(key: u128) -> String { format!("{key:032x}.ecc") diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index fa261ade..4b6c7f3e 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -14,6 +14,7 @@ pub mod disk_cache; use color::Region; pub mod eval; pub mod masks; +pub mod passes; pub mod shapes; pub mod styles; pub mod text; @@ -22,7 +23,7 @@ pub mod three_d; use std::sync::Arc; pub use auto::{AutoKey, AutoPick}; -pub use cache::{CacheStats, LayerCache}; +pub use cache::{CacheStats, LayerCache, LayerStore, PrefetchStore}; use effectcraft_color::BlendMode; use effectcraft_effects::{Buf, EffectCtx, EffectEnv, EffectHost, LayerPixels, Params}; use effectcraft_geom::{Mat3, Mat4, vec2}; @@ -313,6 +314,13 @@ pub trait Accelerator: Send + Sync { fn miss_gate(&self) -> Option> { None } + /// With deferred readbacks: a future that resolves once the readbacks in flight have + /// landed (the browser delivers them from its event loop; natively the device is polled + /// before it returns, so the future is ready). `None` = nothing to wait for. See + /// [`passes::in_passes`]. + fn settle(&self) -> Option + '_>>> { + None + } } /// Where an effect stack runs (see [`Renderer::run_effects_on`]): a CPU buffer, or an image @@ -631,6 +639,12 @@ impl<'a> Renderer<'a> { EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr, footage: Some(self.footage) } } + /// This renderer with another accelerator (or none): the passes of + /// [`passes::in_passes`] render with the accelerator, its CPU fallback without. + pub fn with_accel(&self, accel: Option<&'a dyn Accelerator>) -> Renderer<'a> { + Renderer { accel, ..*self } + } + /// The accelerator to use for this render, if any (see [`Backend`]). pub fn active_accel(&self) -> Option<&'a dyn Accelerator> { let a = self.accel?; diff --git a/crates/render/src/passes.rs b/crates/render/src/passes.rs new file mode 100644 index 00000000..5f0373b8 --- /dev/null +++ b/crates/render/src/passes.rs @@ -0,0 +1,104 @@ +//! Rendering with deferred readbacks from loops that are not frame servers (the browser's +//! job workers: Render Queue renders and analyses). +//! +//! A frame worker renders one frame per request and resumes it pass by pass +//! (`FrameServer::resume`). Jobs render many frames from a loop; written as `async` functions, +//! the loop awaits [`in_passes`] for each frame: with an accelerator whose readbacks are +//! deferred ([`Accelerator::miss_gate`]), the render runs again after every pass that missed, +//! awaiting the device in between ([`Accelerator::settle`]), until a pass completes; with any +//! other accelerator (or none) it runs once and the future is ready at once. Desktop threads +//! drive the same futures with [`block_on`]. + +use std::future::Future; +use std::task::{Context, Poll, Waker}; + +use effectcraft_project::ItemId; +use effectcraft_raster::Image; +use effectcraft_time::Tick; + +use crate::{Accelerator, AutoKey, Backend, Renderer}; + +/// Passes before a frame gives up on deferred readbacks and renders on the CPU. +pub const MAX_PASSES: u32 = 24; + +/// What [`in_passes`] returns. +#[derive(Clone, Debug)] +pub struct Passed { + pub value: T, + /// Render passes (1 without deferred readbacks). + pub passes: u32, + /// Rendered with the accelerator (not the CPU fallback after [`MAX_PASSES`]). + pub accelerated: bool, +} + +/// Whether `accel` defers its readbacks (renders take passes). +pub fn is_deferred(accel: Option<&dyn Accelerator>) -> bool { + accel.is_some_and(|a| a.miss_gate().is_some()) +} + +/// Run `render` (one render with the accelerator it is given) until a pass doesn't miss: see +/// the module docs. After [`MAX_PASSES`] it renders once without the accelerator (the CPU). +/// Caches the render writes to must hold the accelerator's miss gate +/// ([`crate::LayerCache::set_gate`]). +pub async fn in_passes<'a, T>(accel: Option<&'a dyn Accelerator>, mut render: impl FnMut(Option<&'a dyn Accelerator>) -> T) -> Passed { + let Some(a) = accel.filter(|a| a.miss_gate().is_some()) else { + return Passed { value: render(accel), passes: 1, accelerated: accel.is_some() }; + }; + a.frame_begin(); + for pass in 1..=MAX_PASSES { + a.pass_begin(); + let value = render(Some(a)); + if !a.pass_missed() { + return Passed { value, passes: pass, accelerated: true }; + } + match a.settle() { + Some(s) => s.await, + None => break, + } + } + Passed { value: render(None), passes: MAX_PASSES + 1, accelerated: false } +} + +/// A top-level comp frame through [`in_passes`] ([`Renderer::comp_frame`] in passes). Backend +/// Auto picks CPU or GPU for the whole frame (all its passes) and is timed as a whole, as the +/// frame workers do. +pub async fn comp_frame(r: &Renderer<'_>, comp: ItemId, t: Tick) -> Image { + let Some(a) = r.accel.filter(|a| a.miss_gate().is_some()) else { return r.comp_frame(comp, t) }; + let mut opts = r.opts; + let mut auto = None; + if opts.backend == Backend::Auto { + if !r.project.settings.gpu_acceleration { + opts.backend = Backend::Cpu; + } else if opts.roi.is_none() + && let Some(pick) = a.auto_pick() + { + let key = AutoKey::new(comp, opts.scale, false); + let gpu = pick.choose(key); + opts.backend = if gpu { Backend::Gpu } else { Backend::Cpu }; + auto = Some((key, gpu)); + } + } + let t0 = web_time::Instant::now(); + let img = if opts.backend == Backend::Cpu { + Renderer { opts, accel: None, ..*r }.comp_frame(comp, t) + } else { + in_passes(Some(a), |acc| Renderer { opts, accel: acc, ..*r }.comp_frame(comp, t)).await.value + }; + if let (Some((key, chose)), Some(pick)) = (auto, a.auto_pick()) { + pick.record(key, chose, t0.elapsed().as_secs_f64() * 1e3); + } + img +} + +/// Drive `f` to completion on this thread. For futures that only wait on deferred readbacks, +/// whose [`Accelerator::settle`] blocks natively: they never stay pending there. +pub fn block_on(f: F) -> F::Output { + let mut f = std::pin::pin!(f); + let mut cx = Context::from_waker(Waker::noop()); + loop { + if let Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + std::thread::yield_now(); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 6eaf1081..a484572a 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -583,6 +583,8 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { Field::text("name", "Template name", &name), Field::text("description", "Description", ""), Field::text("category", "Category", "My Templates"), + Field::bool("embedFootage", "Embed footage files", true), + Field::num("embedLimitMB", "Embed up to (MB)", effectcraft_engine::templates::EMBED_LIMIT_MB), ], ) } diff --git a/crates/ui-egui/src/panels/templates.rs b/crates/ui-egui/src/panels/templates.rs index bae62928..26969a69 100644 --- a/crates/ui-egui/src/panels/templates.rs +++ b/crates/ui-egui/src/panels/templates.rs @@ -218,10 +218,11 @@ mod tests { assert_eq!(app.dialog_state.form.command, "templates.saveAs"); app.dialog_state.form.fields[0].kind = crate::panels::forms::FieldKind::Text("Brand Intro".into()); frame(&mut app, &ctx); - for id in ["form.field.name", "form.field.description", "form.ok", "form.cancel"] { + for id in ["form.field.name", "form.field.description", "form.field.embedFootage", "form.field.embedLimitMB", "form.ok", "form.cancel"] { assert!(app.auto.find(id).is_some(), "{id}"); } let params = app.dialog_state.form.params(); + assert_eq!((params["embedFootage"].as_bool(), params["embedLimitMB"].as_f64()), (Some(true), Some(256.0)), "{params}"); app.session.execute("templates.saveAs", params).unwrap(); app.dialog = None; app.ui.start_screen = true; diff --git a/crates/ui-egui/tests/ui_web_gpu_storage.rs b/crates/ui-egui/tests/ui_web_gpu_storage.rs index a25bd4a7..f322fac1 100644 --- a/crates/ui-egui/tests/ui_web_gpu_storage.rs +++ b/crates/ui-egui/tests/ui_web_gpu_storage.rs @@ -79,6 +79,8 @@ impl RemoteFrames for GpuWorker { time: job.t, opts: Box::new(job.opts), disk: disk.clone(), + layers: false, + prefetch: vec![], })); // The worker awaits its device between passes (`Gpu::settled`); natively a poll. while server.waiting() { diff --git a/crates/ui-egui/tests/ui_web_job_gpu.rs b/crates/ui-egui/tests/ui_web_job_gpu.rs new file mode 100644 index 00000000..e24829a8 --- /dev/null +++ b/crates/ui-egui/tests/ui_web_job_gpu.rs @@ -0,0 +1,123 @@ +//! Headless checks for M13.30 (GPU in the browser's job workers), run natively: a job worker +//! session whose accelerator is a device with deferred readbacks (the worker's own WebGPU +//! device) runs the Render Queue and an analysis with their frames rendered on the device in +//! passes (`offload::run_request_async`, `render::passes`), and gets what a CPU worker gets. + +use std::sync::{Arc, Mutex}; + +use effectcraft_engine::Session; +use effectcraft_engine::offload::{Inbox, Offload, Post, WorkerReply, WorkerRequest}; +use effectcraft_gpu::Gpu; +use serde_json::json; + +/// Holds offloaded jobs until the test runs them. +#[derive(Default)] +struct Manual(Mutex)>>); + +impl Offload for Manual { + fn start(&self, req: WorkerRequest, inbox: Arc) -> Result<(), String> { + self.0.lock().unwrap().push((req, inbox)); + Ok(()) + } + fn cancel(&self, _: u64) {} +} + +type Files = Arc)>>>; + +fn exporter(files: Files) -> Arc { + let sink: Arc) + Send + Sync> = Arc::new(move |p: &str, d: Vec| files.lock().unwrap().push((p.to_string(), d))); + Arc::new(effectcraft_host::FileExporter { sink: Some(sink) }) +} + +/// Run `req` on a worker session (`gpu`: its accelerator); its replies and the files it wrote. +fn run(req: WorkerRequest, gpu: Option<&Gpu>) -> (Vec, Vec<(String, Vec)>) { + let files: Files = Arc::default(); + let mut w = Session { exporter: Some(exporter(files.clone())), ..Default::default() }; + w.accel = gpu.map(|g| Arc::new(g.clone()) as Arc); + let replies = Arc::new(Mutex::new(vec![])); + let r2 = replies.clone(); + let post: Post = std::rc::Rc::new(move |r| r2.lock().unwrap().push(r)); + effectcraft_engine::offload::run_request(&mut w, req, &post); + let r = replies.lock().unwrap().clone(); + let f = files.lock().unwrap().clone(); + (r, f) +} + +fn page() -> (Session, Arc) { + let off = Arc::new(Manual::default()); + let mut s = Session { offload: Some(off.clone()), exporter: Some(exporter(Arc::default())), ..Default::default() }; + s.execute("comp.new", json!({"name": "Job", "width": 96, "height": 64, "frameRate": 10, "duration": 0.6})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": "#3070c0"})).unwrap()["layer"].as_u64().unwrap(); + // Moving texture (Fractal Noise, offset animated), then a blur: GPU effects. + s.execute("effect.apply", json!({"layers": [l], "effect": "ec.noise.fractal"})).unwrap(); + s.execute("prop.addKey", json!({"layer": l, "path": "effects/#1/transform/offset", "time": 0.0, "value": [0.5, 0.5]})).unwrap(); + s.execute("prop.addKey", json!({"layer": l, "path": "effects/#1/transform/offset", "time": 0.5, "value": [0.6, 0.56]})).unwrap(); + s.execute("effect.apply", json!({"layers": [l], "effect": "ec.blur.gaussian"})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#2/blurriness", "value": 3.0})).unwrap(); + (s, off) +} + +#[test] +fn render_queue_jobs_render_on_the_worker_gpu() { + let Some(g) = Gpu::headless_deferred() else { + eprintln!("no GPU adapter: skipped"); + return; + }; + let (mut s, off) = page(); + s.execute("renderQueue.add", json!({"format": "png", "output": "/job/f_[#####].png"})).unwrap(); + s.execute("renderQueue.render", json!({"wait": false})).unwrap(); + let (req, _) = off.0.lock().unwrap().pop().expect("the render went to the worker"); + let (cpu_replies, mut cpu_files) = run(req.clone(), None); + let before = g.deferred_stats().unwrap(); + let (gpu_replies, mut gpu_files) = run(req, Some(&g)); + // (the CPU worker writes a batch's files in parallel) + cpu_files.sort(); + gpu_files.sort(); + let after = g.deferred_stats().unwrap(); + let done = |r: &[WorkerReply]| r.iter().any(|r| matches!(r, WorkerReply::Item { update } if format!("{update:?}").contains("Done"))); + assert!(done(&cpu_replies) && done(&gpu_replies), "{gpu_replies:?}"); + assert!(matches!(gpu_replies.last(), Some(WorkerReply::Done))); + // Backend Auto (what renders use) warms up both sides, GPU first: the GPU frames rendered + // in passes on the device (their chains, then the frame read back). + let frames = gpu_files.len() as u64; + assert_eq!(frames, 6); + assert!(after.0 - before.0 >= 6, "passes {before:?} → {after:?}"); + assert!(after.1 - before.1 >= 6, "readbacks {before:?} → {after:?}"); + assert_eq!(after.2, 0, "nothing left in flight"); + // The same frames as the CPU worker's (within GPU rounding). + assert_eq!(cpu_files.iter().map(|f| &f.0).collect::>(), gpu_files.iter().map(|f| &f.0).collect::>()); + for ((name, a), (_, b)) in cpu_files.iter().zip(&gpu_files) { + let a = image::load_from_memory(a).unwrap().to_rgba8(); + let b = image::load_from_memory(b).unwrap().to_rgba8(); + assert_eq!(a.dimensions(), b.dimensions()); + let worst = a.as_raw().iter().zip(b.as_raw()).map(|(x, y)| x.abs_diff(*y)).max().unwrap(); + assert!(worst <= 3, "{name}: differs by {worst} levels"); + } +} + +#[test] +fn analyses_render_their_input_on_the_worker_gpu() { + let Some(g) = Gpu::headless_deferred() else { + eprintln!("no GPU adapter: skipped"); + return; + }; + let (mut s, off) = page(); + let l = s.active_comp().unwrap().layers[0].id; + s.execute("layer.select", json!({"layers": [l.0]})).unwrap(); + s.execute("effect.apply", json!({"layers": [l.0], "effect": "Warp Stabilizer"})).unwrap(); + s.execute("warp.analyze", json!({"layer": l.0, "wait": false})).unwrap(); + let (req, _) = off.0.lock().unwrap().pop().expect("the analysis went to the worker"); + let (cpu, _) = run(req.clone(), None); + let before = g.deferred_stats().unwrap(); + let (gpu, _) = run(req, Some(&g)); + let after = g.deferred_stats().unwrap(); + let group = |r: &[WorkerReply]| r.iter().find_map(|r| if let WorkerReply::Group { group, .. } = r { Some(group.clone()) } else { None }); + let failed = |r: &[WorkerReply]| r.iter().find_map(|r| if let WorkerReply::Failed { error } = r { Some(error.clone()) } else { None }); + assert_eq!(group(&cpu).is_some(), group(&gpu).is_some(), "cpu {:?} / gpu {:?}", failed(&cpu), failed(&gpu)); + assert!(group(&gpu).is_some(), "{:?}", failed(&gpu)); + // The input frames (the effects above Warp Stabilizer) rendered on the device in passes. + assert!(after.1 - before.1 >= 6, "readbacks {before:?} → {after:?}"); + assert!(after.0 - before.0 >= 12, "passes {before:?} → {after:?}"); + let progress = gpu.iter().filter(|r| matches!(r, WorkerReply::Progress { .. })).count(); + assert!(progress >= 1); +} diff --git a/crates/ui-egui/tests/ui_web_offload.rs b/crates/ui-egui/tests/ui_web_offload.rs index 12fe0d5b..b766c73c 100644 --- a/crates/ui-egui/tests/ui_web_offload.rs +++ b/crates/ui-egui/tests/ui_web_offload.rs @@ -69,7 +69,16 @@ impl RemoteFrames for FakeWorker { assert!(server.handle(rt(&m)).is_empty()); } *next += 1; - let out = server.handle(rt(&FrameMsg::Render { id: *next, revision: job.revision, comp: job.comp, time: job.t, opts: Box::new(job.opts), disk: None })); + let out = server.handle(rt(&FrameMsg::Render { + id: *next, + revision: job.revision, + comp: job.comp, + time: job.t, + opts: Box::new(job.opts), + disk: None, + layers: false, + prefetch: vec![], + })); match out.into_iter().next() { Some((FrameReply::Frame { width, height, ms, .. }, Some(rgba))) => pending.push((done, RemoteFrame { width, height, rgba, ms })), other => done(Err(format!("{:?}", other.map(|o| o.0)))), diff --git a/docs/parity.md b/docs/parity.md index b88dafda..53c266c4 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -20,7 +20,7 @@ Partial features: | id | Tier | Done | What is missing | |---|---|---|---| | EFF-5 GPU effects | P1 | 0.97 | GPU kernels for the audio-driven effects and the stepped particle systems' sprite pass (the CC light / transition families, Numbers, Timecode and Time effects landed in M13.22; 3D Channel, the VR family, OCIO / LUT and the simulations' render passes in M13.23); custom `.ocio` configs, Shatter's wireframe views and Foam's texture / environment / flow-map extras render on the CPU | -| WEB-1 Web app | P0 | 0.97 | GPU effects in Render Queue / analysis job workers (they render on their CPU); layer buffers in the browser's disk cache (frames only); threads inside one engine instance (the decided design is one engine instance per worker; a threaded build needs nightly `build-std`). M13.24 added GPU effects and the GPU compositor in the frame workers (their own WebGPU devices, deferred readbacks: frames render in passes; Backend Auto per comp), the disk cache in the Origin Private File System (written by the workers with sync access handles, LRU under the settings' limit, served after a reload), the storage manager (Settings ▸ Disk ▸ Browser Storage, `storage.*`) and fixed a WGSL constant Chrome rejected (it disabled every GPU kernel in Chrome) | +| WEB-1 Web app | P0 | 0.98 | Threads inside one engine instance (the decided design is one engine instance per worker; a threaded build needs nightly `build-std`); Content-Aware Fill runs on the page, not in a job worker. M13.30 added GPU rendering in the job workers (Render Queue frames, analysis input frames, GPU particles and Advanced 3D read back under keys and render in passes on each job worker's own WebGPU device) and layer buffers in the browser's disk cache (prefetched before a frame from the previous frames' misses). M13.24 added GPU effects and the GPU compositor in the frame workers (their own WebGPU devices, deferred readbacks: frames render in passes; Backend Auto per comp), the disk cache in the Origin Private File System (written by the workers with sync access handles, LRU under the settings' limit, served after a reload), the storage manager (Settings ▸ Disk ▸ Browser Storage, `storage.*`) and fixed a WGSL constant Chrome rejected (it disabled every GPU kernel in Chrome) | | UI-7 Home screen | P1 | 0.95 | A "start from a template" gallery | | MSK-4 Roto Brush | P2 | 0.85 | Segmentation is classical (graph cut + optical flow), not a learned model | | TRK-3 Face tracking | P2 | 0.9 | Classical fitter: weak on profile and occluded faces; Rolling Shutter Ripple is approximated | @@ -154,7 +154,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Compositions | ≈ 81% | 3.3 | Mocha-style planar tracks for templates (CMP-7: Essential Graphics mirrored and linked properties landed in M13.11; Font and uniform Scale controls, Composition ▸ Open in Essential Graphics, Save Frame As ▸ Photoshop Layers / ProEXR and the VR Comp Editor landed in M13.7; the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | | Animation | 70% | 9.0 | puppet depth beyond pins, recording, rigging and follow-through (puppet pin recording with Record Options landed in M13.1; pin selection, rotate/scale handles, nulls for pins and Follow-Through in M13.14–M13.16), Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 96% | 0.3 | no extruded strokes (M13.12: the Variable Font Axes animator re-spaces the text — advances follow the animated axes; M13.6: variable font axes in the character style — `layer.setText variations`, the Character panel's Variable Font Axes fields — shape with HVAR / gvar advances and draw at that design-space position; OpenType features — stylistic sets, discretionary ligatures, contextual / stylistic alternates, swash, titling, ordinals, fractions, figure styles, true small caps / all small caps and superior / inferior glyphs with faux fallback — per character with the Character panel's OpenType popup and `text.fontFeatures` landed in M13.2; vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | -| Web | 97% | 0.2 | GPU effects in Render Queue / analysis workers (job workers render on their CPU), layer buffers in the browser's disk cache (frames only), no shared-memory threads inside one engine instance (the decided design is one engine instance per worker; a threaded build needs nightly `build-std`) (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2; viewer frames in frame workers fed by project diffs, Roto Brush propagation in a worker, non-blocking `wait: true` jobs and a browser Media Browser (File System Access folders, browser storage) in M13.10; WEB-1 in M13.24: GPU effects and the GPU compositor in the frame workers on their own WebGPU devices with deferred readbacks (frames render in passes; Backend Auto per comp), the disk cache in the Origin Private File System (written by the workers with sync access handles, LRU under the settings' limit, served after a reload), the storage manager (Settings ▸ Disk ▸ Browser Storage, `storage.info` / `storage.persist` / `storage.clear`), and a WGSL constant Chrome rejected (it disabled every GPU kernel in Chrome) fixed) | +| Web | 98% | 0.1 | No shared-memory threads inside one engine instance (the decided design is one engine instance per worker; a threaded build needs nightly `build-std`); Content-Aware Fill runs on the page (M13.30: Render Queue and analyses render on each job worker's own WebGPU device in passes, particles and Advanced 3D read back under keys, layer buffers in the browser's disk cache; browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2; viewer frames in frame workers fed by project diffs, Roto Brush propagation in a worker, non-blocking `wait: true` jobs and a browser Media Browser (File System Access folders, browser storage) in M13.10; WEB-1 in M13.24: GPU effects and the GPU compositor in the frame workers on their own WebGPU devices with deferred readbacks (frames render in passes; Backend Auto per comp), the disk cache in the Origin Private File System (written by the workers with sync access handles, LRU under the settings' limit, served after a reload), the storage manager (Settings ▸ Disk ▸ Browser Storage, `storage.info` / `storage.persist` / `storage.clear`), and a WGSL constant Chrome rejected (it disabled every GPU kernel in Chrome) fixed) | | 3D | 88% | 5.5 | multi-view layouts, the Extended Viewer for Advanced 3D comps (Classic 3D Extended Viewer landed in M13.5 UI completion); collapsed precomps of another size seen through the parent's camera render (fixed in M13.2); stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D depth of field with the iris and highlight options, collapsed precomps as real Advanced 3D geometry and extruded text/shape strokes landed in M13.8; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6; Advanced 3D end to end on the GPU (motion blur, iris depth of field, compositing) in M13.11 | | Effects | ≈ 85% | 3.0 | GPU versions of the remaining effects (236 run on the GPU since M13.23, EFF-5: M13.23 added the 3D Channel and Immersive Video families, Apply Color LUT, the OCIO effects, Color Profile Converter and the simulations' render passes (Card Wipe included); M13.22 added the CC light family, the CC transitions, Block Dissolve, Radial Shadow, CC Bender / Blobbylize / Cylinder / Sphere / Spotlight / Environment, 3D Glasses, Numbers, Timecode and the time effects joined the 166 of M13.13; audio-driven effects and the stepped particle systems still render on the CPU) and the missing controls listed as partial in [effects.md](effects.md) (every After Effects effect exists since M9.11, M12.5 and M12.6; parameter names, order, twirl-downs, popups, units and defaults were aligned in M9.12) | | Interface | 75% | 5.0 | more Learn tutorials and pixel-level fidelity of dialogs (the Home ▸ Templates gallery (eight original built-in templates, user templates from File ▸ Save as Template…) and View ▸ Simulate Output ▸ My Custom RGB… landed in M13.25; Timeline layer reordering by drag, a non-snapping viewer pan, a working rename field and twirl arrows, and the full set of property reveal shortcuts — double presses, Alt+Shift keyframes, Ctrl+` — landed in M13.17–M13.20; the Home ▸ Learn tab with interactive tutorials and a UI fidelity pass landed in M13.5 UI completion; visual editors for Lumetri RGB / hue-saturation curves, Colorama's output cycle wheel, Glow's colour map and Reshape's correspondence points (viewer handles), a scrolling Preview panel and AE-style Composition / Timeline tabs (close, label swatch, viewer lock) landed in M13.10; Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | @@ -399,8 +399,9 @@ folder (in the browser: the settings store); the gallery lists built-in and user user cards can be deleted, and opening any template makes an untitled copy. Commands: `templates.list`, `templates.thumbnail`, `templates.create`, `templates.saveAs`, `templates.delete`, `file.newFromTemplate`; automation ids `home.tab.templates`, -`home.templates.` / `.open` / `.delete`, `home.templates.saveCurrent`. Not yet: footage -used by a saved template is referenced by path, not embedded. +`home.templates.` / `.open` / `.delete`, `home.templates.saveCurrent`. M13.30: saving embeds +the footage files (default on, capped at 256 MB with a warning; the rest stays linked) and +creating from the template extracts them next to the new project (or into browser storage). **View ▸ Simulate Output ▸ My Custom RGB…**: a dialog defines a custom output device by primaries and white point (CIE xy) and a gamma or the sRGB curve, or reads them from an RGB @@ -408,8 +409,9 @@ matrix/TRC ICC profile (v2/v4: colorants, `wtpt`, `chad`, `curv`/`para` tone cur parametric curves are fitted to a gamma unless they are the sRGB curve); the definition is kept in Settings (`customRgb`) and simulated like the built-in profiles, with Preserve RGB (`view.customRgb`, `view.simulateOutput {profile: "myCustom"}`). Generic dialog forms now register -`form.field.`, `form.ok` and `form.cancel` automation ids. Not yet: LUT-based (`A2B0`) -ICC profiles. +`form.field.`, `form.ok` and `form.cancel` automation ids. M13.30: LUT-based (`A2B0`) +ICC profiles too (`lut8Type`, `lut16Type`, `lutAToBType` / `lutBToAType`, evaluated from the +public ICC specification and baked into a 3D LUT for the viewer). ### M13.26: Premiere Pro interop via timeline interchange @@ -460,6 +462,26 @@ composited at 8 and 32 bpc: default to the layer). Shatter's wireframe views and Foam's User Defined texture, Environment Map and flow-map preview render on the CPU. +## Update: M13.30 web depth and polish + +- **GPU in the browser's job workers** (WEB-1): each job worker opens its own WebGPU device + (deferred readbacks) like the frame workers. The Render Queue's export and the analyses' + frame loops are futures (`effectcraft_render::passes`, `offload::run_request_async`): every + frame renders in passes on the device and the job awaits the GPU between them, so Render + Queue renders and the input frames of Warp Stabilizer, 3D Camera Tracker, Track Motion, mask + tracking and Roto Brush (prefetched before each step) use the GPU. GPU particles and + Advanced 3D (`raster_3d`, `render_3d`) read back under keys, so they run on the device in + frame and job workers too (a depth-of-field run still resolves on the CPU, its scenes on + the GPU). `effectcraft.info().workers` reports the job workers' adapter and each job's passes + and readbacks; the headless-Chrome smoke test measures them. +- **Layer buffers in the browser's disk cache**: frame workers back their layer caches with a + `PrefetchStore`; the page plans which layer entries to read before a request from the + previous frames' misses (`LayerPrefetch` over the shared `DiskIndex`, one LRU order for + frames and layers), and slow buffers are written to `effectcraft-cache/v1/layers`. +- **Templates embed footage**: `templates.saveAs {embedFootage (default true), embedLimitMB + (default 256)}`; `templates.create {projectPath?, footageDir?}` extracts it. +- **ICC A2B0 profiles** in View ▸ Simulate Output ▸ My Custom RGB (see above). + ## Highest-value gaps, in order 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10). diff --git a/docs/web.md b/docs/web.md index 73ff79f7..2c49e9a1 100644 --- a/docs/web.md +++ b/docs/web.md @@ -43,15 +43,15 @@ URL flags: `?empty` (a blank project; the session is not restored or recorded), project instead of the last session), `?home` (show the start screen), `?storage=indexeddb` / `?storage=memory` (force a storage backend), `?noworkers` (render, analyse and draw viewer frames on the page's thread), `?frameworkers=N` (frame workers, default 2; 0 = viewer frames on -the page's thread), `?nogpuworkers` (frame workers render on their CPU, without a WebGPU device of -their own), `?nosw` (don't register the service worker). +the page's thread), `?nogpuworkers` (frame and job workers render on their CPU, without a WebGPU +device of their own), `?nosw` (don't register the service worker). ## How the desktop pieces map to the browser | Desktop | Web | |---|---| | frame render thread pool (`ui-egui/src/frames.rs`) | frame workers (below), each with its own WebGPU device: `Frames::pump` hands queued frames to them, most important first (the viewer's frame, then prefetch); frames without effects that the page's GPU compositor can keep on the GPU render on the page. Without workers, queued frames render on the UI thread after each egui frame within 40 ms (24 ms while playing) | -| Render Queue on a background thread | a Web Worker running a second engine instance (below); progress streams back, the page never waits | +| Render Queue on a background thread | a Web Worker running a second engine instance with its own WebGPU device (below); progress streams back, the page never waits | | tracker / mask tracker / Warp Stabilizer / 3D Camera Tracker / Roto Brush Freeze threads | the same workers; the analysed property group comes back as one undo step | | `rayon` parallel loops (raster, effects, export batches) | rayon's global pool falls back to the calling thread when threads are unavailable: same code, serial | | `std::time::Instant` / `SystemTime` (panic on wasm32) | `web-time` (re-exports `std::time` on native) | @@ -66,7 +66,7 @@ their own), `?nosw` (don't register the service worker). | TCP control channel / MCP | `window.effectcraft` (below) | | cpal audio output | Web Audio (below) | | GPU compositor on the desktop's wgpu device | the same compositor on eframe's WebGPU device, and GPU effects on each frame worker's own WebGPU device (below) | -| disk cache folder (Settings ▸ Disk ▸ Disk Cache) | viewer frames in the Origin Private File System, written by the frame workers (below) | +| disk cache folder (Settings ▸ Disk ▸ Disk Cache) | viewer frames and slow layer buffers in the Origin Private File System, written by the frame workers (below) | | (the file system's own tools) | the storage manager: Settings ▸ Disk ▸ Browser Storage and `storage.*` (below) | ### Persistence @@ -106,9 +106,11 @@ compiled twice) and its own memory (`crates/engine/src/offload.rs`, `src/worker. `track`, `maskTrack`, `rotoFreeze` with the target and analysis parameters). 2. `js/host.js` takes an idle worker from its pool (or starts one), sends it the footage bytes it doesn't have yet, then the request. -3. The worker runs the job on a plain engine session, blocking, and posts `WorkerReply`s: render - progress (at most every 100 ms), item started/finished, rendered files (transferred), analysis - progress, the analysed property group, `failed`, `done`. +3. The worker runs the job on a plain engine session and posts `WorkerReply`s: render progress + (at most every 100 ms), item started/finished, rendered files (transferred), analysis + progress, the analysed property group, `failed`, `done`; then `{type: "stats"}` with the + job's GPU passes and readbacks. With a WebGPU device of its own the job's frames render on + it in passes (below, **GPU in job workers**). 4. The page applies them every frame: `Session::poll_render` updates the queue items and the progress bar, `Session::poll_offload` writes an analysis result as one undo step (the effect / tracker / mask group is replaced; other edits made meanwhile are kept) and the downloads start. @@ -154,6 +156,38 @@ thread as before. `effectcraft.info().frameWorkers` reports `{workers, alive, bu failed, lastMs, syncs, syncBytes, gpu, gpuFrames, lastPasses, diskHits}` (`gpu`: each worker's adapter, `{cpu: why}` without one, `null` until it reported). +**GPU in job workers (M13.30).** Each job worker opens its own WebGPU device too +(`workerJobInit`; `?nogpuworkers`: their CPU) and makes it the session's accelerator. A job +can't block on a readback either, so the job loops are futures rather than blocking loops: +`effectcraft_engine::offload::run_request_async` runs the request, and every frame it renders +goes through `effectcraft_render::passes` (`in_passes`, `comp_frame`): render a pass, and while +it missed, await the device (`Accelerator::settle`, the browser's event loop delivers the +readbacks) and render again, as `FrameServer::resume` does for viewer frames (the desktop drives +the same futures with `passes::block_on`; they never wait there). + +- **Render Queue**: `effectcraft_export::export_async` renders each frame in passes (frames + with deferred readbacks render one by one instead of in parallel batches) and encodes as + before (`Exporter::export_async`). Renders use Backend Auto, as on the desktop: per comp + the worker warms up both sides (GPU first) and then renders on the faster one, each frame's + time measured across all its passes. +- **Analyses** (Warp Stabilizer, 3D Camera Tracker, Track Motion, mask and face tracking, Roto + Brush freeze and propagation): their work functions are `async`; started with `wait` inside + a job they are queued and awaited (`offload::run_or_queue`), and each input frame renders in + passes with the GPU backend (`offload::job_accel`). Roto Brush's segmentation reads frames + through a callback, so its frame source prefetches the frame and its neighbours in passes + before each step. On the desktop analyses render their input on the CPU, as before. +- **Particles and Advanced 3D** read back under keys like effect chains (`crate::deferred`): + a particle simulation by its request (system, steps, births), a rasterised scene by + everything it uploads, a whole prepared run by its scenes and output; the first pass starts + the readback and gets a placeholder (no particles, an empty target), later passes read the + bytes. A run with depth of field resolves on the CPU (its blur needs a readback mid-run), + each of its scenes still rasterised on the GPU. So frames with particles or Advanced 3D + render on the device in frame workers and job workers alike. + +`effectcraft.info().workers` reports `{running, idle, gpu: {adapter} | {cpu: why}, jobs: +[{kind, gpu, passes, readbacks, ms}]}` (the last jobs). Content-Aware Fill runs on the page +(it is not a worker job). + **Threading design: one engine instance per worker (decided, M13.24).** Shared-memory threads inside one instance (SharedArrayBuffer + atomics, a `wasm-bindgen-rayon`-style pool) need the standard library rebuilt with atomics (`-Z build-std`, nightly Rust) and every crate's statics @@ -211,7 +245,8 @@ readback, not even in a worker: a buffer maps only once the thread returns to it The CPU renderer, which runs GPU effect chains and reads their results back mid-frame, therefore can't use the page's device. Instead each frame worker opens its own WebGPU device (`navigator.gpu` in dedicated workers: Chromium, Safari Technology Preview; `Gpu::request_deferred`) -and renders with **deferred readbacks** (`crates/gpu/src/deferred.rs`): +and renders with **deferred readbacks** (`crates/gpu/src/deferred.rs`; job workers do the same, +above): 1. Every readback the renderer needs is keyed by what determines its pixels: a GPU effect chain by its input buffer's pixels and the chain's parameters, the top-level frame by comp, time and @@ -235,8 +270,7 @@ chains still on the GPU. Backend **Auto** works in the workers: it picks CPU or the frames' total times across their passes (`AutoPick`), as on the desktop. Without WebGPU in workers (Firefox, older Safari) or with `?nogpuworkers`, the workers render on their CPU, as before. A frame with effects goes to a worker when the workers have a GPU, so the page never runs -an effect on its thread. Particles and Advanced 3D scenes (which read back without a key) simulate -and rasterise on the worker's CPU. +an effect on its thread. Particles and Advanced 3D read back under keys too (M13.30, above). `effectcraft.workerFrameCheck({time?, scale?, backend?})` renders the active comp's frame in a worker (its GPU by default) and on the page's CPU and reports the differences (`{maxDiff, @@ -262,13 +296,27 @@ scale, view, render options). - A frame whose key is indexed is read and decoded on the page (asynchronously) instead of being rendered; a damaged or missing file is a miss and is forgotten. The timeline's blue cache bar shows the frames on disk. -- `cache.diskStats` reports the browser's cache (`{enabled, available, loaded, entries, bytes, - maxBytes, hits, misses, writes, evictions}`); Empty Disk Cache / `edit.purge {what: "disk"}` - clears it. `effectcraft.info().diskCache` has the same. +- `cache.diskStats` reports the browser's cache (`{enabled, available, loaded, entries, frames, + layers, bytes, maxBytes, hits, misses, writes, evictions, layerHits, layerWrites}`); Empty Disk + Cache / `edit.purge {what: "disk"}` clears it. `effectcraft.info().diskCache` has the same. -Only frames are cached on disk in the browser: the desktop also keeps slow layer buffers, but a -layer lookup happens in the middle of a synchronous render, where the browser's file system can't -be read. +**Layer buffers (M13.30).** Slow layer buffers (≥ 20 ms to render) go to the disk cache too +(`effectcraft-cache/v1/layers/<32 hex>.ecc`, `disk_cache::layer_entry`, keys salted per project +and footage as on the desktop). A layer lookup happens in the middle of a synchronous render, +where the browser's file system can't be read, so it is made synchronous by fetching first: + +1. Each frame worker backs its layer cache with a `PrefetchStore` (`effectcraft_render::cache`): + lookups are served from buffers fetched before the request; a lookup that finds nothing is + recorded, and buffers to keep queue up as sealed entries. +2. The reply lists the frame's layer misses and the buffers served from the store + (`FrameReply::Frame` `layerMisses` / `layerHits`); after posting the frame the worker writes + the queued entries (sync access handles) and reports each (`FrameReply::Stored` `layer`). +3. The page keeps one index for frames and layers (`DiskIndex`: one least-recently-used order; + hits touch entries) and remembers the workers' recent misses (`disk_cache::LayerPrefetch`). + A render request names the recent misses that are on disk (`FrameMsg::Render` `prefetch`, + at most 64); the worker reads those it lacks into its store before rendering, so their + lookups hit (another worker's buffers, buffers from an earlier visit, buffers dropped by a + purge). ### Storage manager @@ -350,7 +398,11 @@ the M13.24 ones: GPU effects (Gaussian Blur, Glow) rendered on the frame workers devices in passes while the page's event loop stays free, the worker's GPU frame matching the page's CPU render (`workerFrameCheck`), the storage manager (`storage.info`, `storage.persist`, the Settings ▸ Disk widgets), and the disk cache: frames written to the Origin Private File -System, served from it after a reload without rendering, and cleared with `storage.clear`. +System, served from it after a reload without rendering, and cleared with `storage.clear`; and +the M13.30 ones: a Render Queue job and a Warp Stabilizer analysis on the job worker's own +WebGPU device (`workers.jobs`: GPU passes and readbacks per job) while the page stays +responsive, and layer buffers written to the disk cache and served from it after a reload and a +purge (`layerHits`). Native unit tests cover the storage model (`apps/effectcraft-web/src/store.rs`: write coalescing, file table, `ConfigStore` / `FileOps`, auto-save and crash recovery through the @@ -369,15 +421,20 @@ the page's GPU keeps frames without effects, frames stored under their disk keys "reload" without rendering, the blue cache bar sees them; the storage manager's widgets and commands), the disk index (`disk_cache.rs`), the store's usage by kind and clearing (`store.rs`), `storage.*` (`storage.rs`), and every WGSL float literal fitting f32 (browsers reject the whole -module otherwise; `tests.rs`). +module otherwise; `tests.rs`). M13.30: job workers on a deferred device +(`crates/ui-egui/tests/ui_web_job_gpu.rs`: a Render Queue job and a Warp Stabilizer analysis +render their frames in passes on the device, with the CPU worker's pixels / result), +particles, Advanced 3D and the job pass driver in passes (`crates/gpu/src/tests_deferred_jobs.rs`), +layer buffers through the disk store and the prefetch plan (`remote.rs` +`layer_buffers_go_through_the_disk_store`). ## Gaps - No shared-memory threads: each worker is a separate engine instance (memory per worker: the project replica and the footage it reads); see the threading design above. -- GPU effects run in the frame workers (viewer frames, RAM preview). Render Queue renders and - analyses in job workers still run on their CPU (their loops are synchronous); particles and - Advanced 3D scenes in frame workers too. Browsers without WebGPU in workers render frames on - the workers' CPU. -- The browser's disk cache holds frames, not layer buffers. +- Browsers without WebGPU in workers render frames, Render Queue jobs and analyses on the + workers' CPU. A frame renders at most 24 passes on the GPU, then on the CPU. +- Content-Aware Fill runs on the page's thread (it is not offloaded to a job worker). +- A layer buffer is fetched from disk only after some worker missed it in a recent frame (the + first frame after a reload renders its layers). - Cancelling an analysis drops its partial result (the worker is terminated).