From 5e440a6607c87aa111618b97921dd2049443c6ed Mon Sep 17 00:00:00 2001 From: Davi Andrei <67552001+SaymonMaster14@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:59:57 -0300 Subject: [PATCH 1/2] Windows: Media Foundation / Direct3D 11 hardware H.264 + HEVC decode (#30) New backend in crates/platform (media_foundation/): a Direct3D-aware decoder MFT is driven at the level of single access units (Annex B), given the process's D3D11 video device through an IMFDXGIDeviceManager so it decodes with DXVA, and its NV12 / P010 textures are read back through a staging texture into planar Yuv8 / Yuv16. H.264 Baseline/Main/High and HEVC Main / Main 10, 4:2:0. - Hardware is verified: non-D3D-aware or async MFTs, streams the GPU's DXVA decoder does not list, and system-memory output (Microsoft's MFT software fallback) are declined or fail the stream so HybridDecoder continues with our decoder. - mfplat.dll is loaded at run time so Windows N editions still start. - register() / registered() / hardware_decoder_for() cover Windows; perf.stats reports the backend. - Tests (Windows): bit-exact parity with the software decoders at 640x360, 1080p and 2160p (resets, seeks, flush), forced mid-stream failure, damaged input, concurrency, repeated init. - Bench: runs on Windows (ffmpeg on PATH, process/thread CPU clocks), HEVC Main 10 2160p row, hardware counters per row. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MMorBHwct72udS2LiC3ZHp --- Cargo.lock | 12 + apps/filmcraft-cli/src/main.rs | 17 +- apps/filmcraft/src/main.rs | 2 +- crates/codecs/src/hw.rs | 13 + crates/engine/src/perf.rs | 3 + crates/platform/Cargo.toml | 20 +- crates/platform/README.md | 37 +- crates/platform/examples/mfprobe.rs | 64 +++ crates/platform/src/annexb.rs | 82 ++++ crates/platform/src/biplanar.rs | 139 +++++++ crates/platform/src/lib.rs | 44 +- crates/platform/src/media_foundation/gpu.rs | 284 +++++++++++++ crates/platform/src/media_foundation/mft.rs | 383 ++++++++++++++++++ crates/platform/src/media_foundation/mod.rs | 410 +++++++++++++++++++ crates/platform/tests/media_foundation.rs | 425 ++++++++++++++++++++ crates/platform/tests/setting.rs | 5 +- crates/ui-egui/Cargo.toml | 3 + crates/ui-egui/examples/bench/fixtures.rs | 21 +- crates/ui-egui/examples/bench/main.rs | 7 + crates/ui-egui/examples/bench/sections.rs | 14 +- crates/ui-egui/src/frames.rs | 12 +- 21 files changed, 1980 insertions(+), 17 deletions(-) create mode 100644 crates/platform/examples/mfprobe.rs create mode 100644 crates/platform/src/annexb.rs create mode 100644 crates/platform/src/biplanar.rs create mode 100644 crates/platform/src/media_foundation/gpu.rs create mode 100644 crates/platform/src/media_foundation/mft.rs create mode 100644 crates/platform/src/media_foundation/mod.rs create mode 100644 crates/platform/tests/media_foundation.rs diff --git a/Cargo.lock b/Cargo.lock index 63e28933..7505dec2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,6 +992,16 @@ dependencies = [ "windows 0.54.0", ] +[[package]] +name = "cpu-time" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e393a7668fe1fad3075085b86c781883000b4ede868f43627b34a87c8b7ded" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2099,6 +2109,7 @@ dependencies = [ "objc2-core-media", "objc2-core-video", "objc2-video-toolbox", + "windows 0.62.2", ] [[package]] @@ -2202,6 +2213,7 @@ dependencies = [ name = "filmcraft-ui-egui" version = "0.2.1" dependencies = [ + "cpu-time", "eframe", "egui", "egui_kittest", diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs index f45af27e..ee46691c 100644 --- a/apps/filmcraft-cli/src/main.rs +++ b/apps/filmcraft-cli/src/main.rs @@ -149,14 +149,19 @@ fn format_for(path: &str) -> Option<&'static str> { }) } +/// OS hardware video decoders (VideoToolbox on macOS, Media Foundation on Windows) in front of our +/// own, as in the desktop app. Also what `mcp` and the headless commands decode with. +fn register_hardware_decoders() -> filmcraft_platform::Availability { + filmcraft_platform::register() +} + #[tokio::main] async fn main() { if matches!(std::env::args().nth(1).as_deref(), Some("--version" | "-V")) { println!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")); return; } - // OS hardware video decoders (VideoToolbox on macOS) in front of our own, as in the desktop app. - let _ = filmcraft_platform::register(); + register_hardware_decoders(); let a = Args::parse(std::env::args().skip(1)); let Some(cmd) = a.pos(0) else { usage("missing subcommand") }; match cmd { @@ -373,6 +378,14 @@ async fn main() { #[cfg(test)] mod format_tests { + /// The CLI (and MCP / headless runs, which share its entry point) registers the hardware + /// decoders at start-up. + #[test] + fn startup_registers_the_hardware_decoders() { + let hardware = super::register_hardware_decoders(); + assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}"); + } + #[test] fn export_format_from_extension() { assert_eq!(super::format_for("out/clip.MXF"), Some("mxf-op1a")); diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index 2e465902..52454830 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -275,6 +275,6 @@ mod tests { fn startup_registers_the_hardware_decoders_without_a_logger() { assert!(!log::log_enabled!(log::Level::Info)); let hardware = super::register_hardware_decoders(); - assert_eq!(filmcraft_platform::registered(), cfg!(target_os = "macos"), "{hardware:?}"); + assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}"); } } diff --git a/crates/codecs/src/hw.rs b/crates/codecs/src/hw.rs index e16664bd..2ea255f6 100644 --- a/crates/codecs/src/hw.rs +++ b/crates/codecs/src/hw.rs @@ -29,6 +29,19 @@ pub fn hardware_decoding() -> bool { ENABLED.load(Ordering::Relaxed) } +static BACKEND: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// Record the OS decoder backend `filmcraft_platform::register` put in front of our decoders +/// ("VideoToolbox", "Media Foundation"), for `perf.stats`. +pub fn set_hw_backend(name: &'static str) { + *BACKEND.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(name); +} + +/// The registered hardware decoder backend, if any ([`set_hw_backend`]). +pub fn hw_backend() -> Option<&'static str> { + *BACKEND.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// Process-wide hardware decoding counters (they only grow; subtract two snapshots to measure an /// interval). Frames decoded in software are `GopStats::frames - frames`. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/crates/engine/src/perf.rs b/crates/engine/src/perf.rs index f1b420fa..947246b2 100644 --- a/crates/engine/src/perf.rs +++ b/crates/engine/src/perf.rs @@ -37,6 +37,8 @@ pub fn decode_json() -> Value { // hardware decoders created, streams handed to software up front, mid-stream fallbacks. "hardware": { "enabled": filmcraft_codecs::hw::hardware_decoding(), + // the OS decoder backend registered at startup (null where there is none) + "backend": filmcraft_codecs::hw::hw_backend(), "frames": hw.frames, "softwareFrames": g.frames.saturating_sub(hw.frames), "sessions": hw.sessions, @@ -92,6 +94,7 @@ mod tests { assert!(v["decode"]["hardware"][k].is_number(), "decode.hardware.{k} in {v}"); } assert!(v["decode"]["hardware"]["enabled"].is_boolean()); + assert!(v["decode"]["hardware"]["backend"].is_null() || v["decode"]["hardware"]["backend"].is_string()); assert!(v["media"]["openSources"].is_number()); assert_eq!(v["jobs"]["running"], json!(0)); assert_eq!(s.history.undo.len(), undo, "a query adds no undo step"); diff --git a/crates/platform/Cargo.toml b/crates/platform/Cargo.toml index 152110eb..128b1b0c 100644 --- a/crates/platform/Cargo.toml +++ b/crates/platform/Cargo.toml @@ -22,18 +22,32 @@ undocumented_unsafe_blocks = "deny" [dependencies] filmcraft-codecs = { workspace = true } +filmcraft-color = { workspace = true } +filmcraft-frame = { workspace = true } filmcraft-isobmff = { workspace = true } +filmcraft-time = { workspace = true } log = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] -filmcraft-color = { workspace = true } -filmcraft-frame = { workspace = true } -filmcraft-time = { workspace = true } objc2-core-foundation = "0.3.2" objc2-core-media = "0.3.2" objc2-core-video = "0.3.2" objc2-video-toolbox = "0.3.2" +# Windows: Media Foundation decoder MFTs + Direct3D 11 / DXVA through the official `windows` bindings +# (MIT OR Apache-2.0; already in the lockfile through wgpu / winit). +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.62", features = [ + "Win32_Foundation", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", + "Win32_Graphics_Dxgi_Common", + "Win32_Media_MediaFoundation", + "Win32_System_Com", + "Win32_System_LibraryLoader", +] } + [dev-dependencies] filmcraft-testkit = { workspace = true } filmcraft-media = { workspace = true } diff --git a/crates/platform/README.md b/crates/platform/README.md index e725a75d..fed10dab 100644 --- a/crates/platform/README.md +++ b/crates/platform/README.md @@ -7,7 +7,7 @@ It is the one crate of the workspace allowed to contain `unsafe`, under the rule ```rust // at startup (the desktop app, filmcraft-cli, the bench) -let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS +let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS, Available("Media Foundation") on Windows ``` ## What it does @@ -21,6 +21,35 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox") reorder buffer of the stream's own depth (`max_num_reorder_frames` / `sps_max_num_reorder_pics`) restores presentation order; a run starting at an HEVC CRA leaves out its RASL pictures, as our decoder does. Each seek (`reset`) starts a fresh session. +- **Windows: Media Foundation + Direct3D 11 / DXVA, H.264 (`avcC`) and HEVC (`hvcC`)**, 8-bit 4:2:0 + (H.264 Baseline / Main / High, HEVC Main) and 10-bit 4:2:0 (HEVC Main 10) + (`media_foundation/`). A Direct3D-aware decoder MFT (Microsoft's H.264 decoder, the HEVC Video + Extensions' decoder, or a vendor's synchronous hardware MFT) is driven at the level of single + access units, so there is no Source Reader and no second demuxer: the container samples FilmCraft + already read go in as Annex B (`annexb.rs`; parameter sets are put in front of the first sample of + a run). The process-wide Direct3D 11 device (`D3D11_CREATE_DEVICE_VIDEO_SUPPORT`, multithread + protected) is handed to the MFT through an `IMFDXGIDeviceManager`, which makes it decode with DXVA + on the GPU's video engine and return NV12 / P010 Direct3D 11 textures. Each picture is then read + back through a staging texture (**the one GPU to CPU copy**, `gpu.rs` `Readback`) and turned into + planar `Yuv8` / `Yuv16` (`biplanar.rs`: chroma deinterleaved, P010 shifted down, cropped to the + conformance window). The MFT returns pictures in presentation order; a run starting at an HEVC CRA + leaves out its RASL pictures, as our decoder does; `reset` (a seek) flushes the MFT. + - *Hardware is verified, not assumed.* An MFT that is not Direct3D-aware, asynchronous, or does not + provide its own output samples is not used; the GPU's DXVA decoder must list the profile, format + and size (`ID3D11VideoDevice::CheckVideoDecoderFormat` / `GetVideoDecoderConfigCount`); and a + picture that is not a Direct3D 11 texture (what Microsoft's decoders return when they fall back + to software inside the MFT) fails the stream, so `HybridDecoder` continues with our decoder. + Windows' own software decoding is never used in place of ours. + - *Declined up front* (our decoder is used): field-coded H.264, H.264 profiles other than + Baseline / Main / High, 10-bit H.264, 4:2:2 / 4:4:4 / monochrome, HEVC profiles other than + Main / Main Still Picture / Main 10, larger than 8192×8192, no Direct3D 11 video device, no + DXVA decoder for the stream on this GPU, no decoder MFT (HEVC needs the *HEVC Video Extensions* + from the Microsoft Store, which the Windows "N" editions and some installs lack). + - `mfplat.dll` is loaded at run time (`mft.rs`), not linked: Windows "N" editions without the Media + Feature Pack still start FilmCraft, which then decodes in software. + - After a `flush` (which drains the MFT) the MFT only restarts at an IDR picture; the GOP cache + always seeks after a flush, and a caller that continues from the middle of a GOP gets an error + that `HybridDecoder` answers by replaying the run in software. - **Other systems:** `register()` does nothing and returns `Availability::Unavailable`. - **`HybridDecoder`** (`hybrid.rs`, safe code): the hardware decoder plus the means to build our software decoder for the same `SampleEntry` (`filmcraft_codecs::software_video_decoder`). On a @@ -36,14 +65,16 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox") - **Never undecodable:** the factory declines (returns `None`, so the software decoder is used) when Settings ▸ Playback ▸ Hardware decoding is Off, for formats it does not take (field-coded H.264, bit depths other than 8 / 10, 4:4:4 or monochrome, luma / chroma depth mismatch, larger - than 8192×8192) and when VideoToolbox cannot create a hardware session. + than 8192×8192; on Windows also 4:2:2 and the profiles listed above) and when the OS cannot + create a hardware session (VideoToolbox) or a GPU-backed decoder (Media Foundation). - **Interchangeable:** colour, pixel aspect, pts, presentation order, `is_random_access` and `is_disposable` come from the software decoders' own helpers (`filmcraft_codecs::hw`, `video::vui_color`, `sar_par`). - **Never crash:** no `unwrap` / `expect` / `panic!` outside tests; the output callback runs under `catch_unwind`; every `unsafe` block has a `// SAFETY:` comment; the public API is safe. - **Counted:** `perf.stats` `decode.hardware` (frames, software frames, sessions, declined, - fallbacks; `filmcraft_codecs::hw::hw_stats`). + fallbacks; `filmcraft_codecs::hw::hw_stats`) and `backend` (the registered backend's name, + `filmcraft_codecs::hw::hw_backend`). ## Tests diff --git a/crates/platform/examples/mfprobe.rs b/crates/platform/examples/mfprobe.rs new file mode 100644 index 00000000..83564481 --- /dev/null +++ b/crates/platform/examples/mfprobe.rs @@ -0,0 +1,64 @@ +//! Diagnostics for the Windows backend: why does Media Foundation take or decline a stream, and +//! where does a decode spend its time? `mfprobe [--time] ...` (`--time` decodes every +//! sample and prints the GPU decode, GPU to CPU readback and plane conversion costs per picture). +#[cfg(target_os = "windows")] +fn main() { + use filmcraft_codecs::VideoDecoder; + println!("register: {:?}", filmcraft_platform::register()); + let time = std::env::args().any(|a| a == "--time"); + for path in std::env::args().skip(1).filter(|a| !a.starts_with("--")) { + let Ok(bytes) = std::fs::read(&path) else { + println!("{path}: unreadable"); + continue; + }; + let Ok(file) = filmcraft_isobmff::open(bytes.clone()) else { + println!("{path}: not an MP4"); + continue; + }; + let Some(t) = file.track_of_kind(filmcraft_isobmff::TrackKind::Video) else { + println!("{path}: no video"); + continue; + }; + let Some(entry) = file.tracks[t].entries.first() else { continue }; + let Some(Ok(info)) = filmcraft_codecs::hw::NalStreamInfo::from_entry(entry) else { + println!("{path}: not avcC/hvcC"); + continue; + }; + let mut d = match filmcraft_platform::media_foundation::MfDecoder::new(info) { + Ok(d) => d, + Err(e) => { + println!("{path}: declined: {e}"); + continue; + } + }; + println!("{path}: hardware ok ({} via {})", d.name(), d.mft_name()); + if !time { + continue; + } + let (mut n, t0) = (0usize, std::time::Instant::now()); + for i in 0..file.tracks[t].samples.len() { + let Ok(s) = file.read_sample(&bytes, t, i) else { break }; + match d.decode(&s, file.tracks[t].samples[i].pts) { + Ok(f) => n += f.len(), + Err(e) => { + println!(" error at sample {i}: {e}"); + break; + } + } + } + n += d.flush().len(); + let wall = t0.elapsed(); + let t = d.timings(); + let per = |x: std::time::Duration| x.as_secs_f64() * 1000.0 / t.frames.max(1) as f64; + println!( + " {n} pictures in {:.0} ms ({:.1} fps): DXVA decode {:.2} ms/picture, GPU to CPU readback {:.2} ms, plane conversion {:.2} ms", + wall.as_secs_f64() * 1000.0, + n as f64 / wall.as_secs_f64(), + per(t.decode), + per(t.readback), + per(t.convert) + ); + } +} +#[cfg(not(target_os = "windows"))] +fn main() {} diff --git a/crates/platform/src/annexb.rs b/crates/platform/src/annexb.rs new file mode 100644 index 00000000..837732d8 --- /dev/null +++ b/crates/platform/src/annexb.rs @@ -0,0 +1,82 @@ +//! Length-prefixed (`avcC` / `hvcC`) access units to Annex B byte streams, the input format of +//! Windows' H.264 and HEVC decoder MFTs. Safe code, tested on every target. + +use filmcraft_codecs::hw::NalStreamInfo; + +const START_CODE: [u8; 4] = [0, 0, 0, 1]; + +/// `sample` (NAL units behind `info.length_size`-byte lengths) as an Annex B access unit. With +/// `headers` the stream's parameter sets (from the sample entry) go first, unless the sample +/// carries parameter sets of its own: a decoder starting (or restarting after a flush) at this +/// sample then has what it needs. Errors on a malformed sample: lengths that run past its end, +/// empty NAL units or bytes after the last unit. +pub fn to_annex_b(info: &NalStreamInfo, sample: &[u8], headers: bool) -> Result, String> { + let nals = info.nals(sample); + let used = nals.iter().try_fold(0usize, |a, n| a.checked_add(n.len())?.checked_add(info.length_size)).ok_or("sample size overflows")?; + if nals.is_empty() || used != sample.len() || nals.iter().any(|n| n.is_empty()) { + return Err("malformed length-prefixed sample".into()); + } + let in_band = nals.iter().any(|n| n.first().is_some_and(|&h| info.is_parameter_set(info.nal_type(h)))); + let add_headers = headers && !in_band; + let prefix: usize = if add_headers { info.parameter_sets.iter().map(|p| p.len() + START_CODE.len()).sum() } else { 0 }; + let mut out = Vec::with_capacity(prefix + sample.len() + nals.len() * START_CODE.len()); + if add_headers { + for p in &info.parameter_sets { + out.extend_from_slice(&START_CODE); + out.extend_from_slice(p); + } + } + for n in nals { + out.extend_from_slice(&START_CODE); + out.extend_from_slice(n); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use filmcraft_codecs::hw::NalCodec; + + fn info(length_size: usize, codec: NalCodec) -> NalStreamInfo { + NalStreamInfo { + codec, + length_size, + highest_tid: None, + parameter_sets: vec![vec![0x67, 1, 2], vec![0x68, 3]], + coded: (16, 16), + crop: (0, 0, 16, 16), + chroma_format_idc: 1, + bit_depth_luma: 8, + bit_depth_chroma: 8, + interlaced: false, + profile_idc: 100, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + reorder: 0, + } + } + + #[test] + fn converts_lengths_to_start_codes() { + let i = info(4, NalCodec::H264); + let s = [0, 0, 0, 2, 0x65, 9, 0, 0, 0, 3, 0x41, 7, 7]; + assert_eq!(to_annex_b(&i, &s, false).unwrap(), [0, 0, 0, 1, 0x65, 9, 0, 0, 0, 1, 0x41, 7, 7]); + // headers go first, unless the sample brings its own + let with = to_annex_b(&i, &s, true).unwrap(); + assert_eq!(&with[..11], [0, 0, 0, 1, 0x67, 1, 2, 0, 0, 0, 1]); + assert_eq!(with.len(), 4 + 3 + 4 + 2 + 13); + let own = [0, 0, 0, 2, 0x67, 5, 0, 0, 0, 2, 0x65, 9]; + assert_eq!(to_annex_b(&i, &own, true).unwrap(), [0, 0, 0, 1, 0x67, 5, 0, 0, 0, 1, 0x65, 9]); + // two-byte lengths + assert_eq!(to_annex_b(&info(2, NalCodec::H264), &[0, 2, 0x65, 9], false).unwrap(), [0, 0, 0, 1, 0x65, 9]); + } + + #[test] + fn malformed_samples_are_errors() { + let i = info(4, NalCodec::H264); + for s in [&[][..], &[0, 0, 0], &[0, 0, 0, 9, 1], &[0, 0, 0, 1, 0x65, 0xff], &[0, 0, 0, 0], &[0xff, 0xff, 0xff, 0xff, 1]] { + assert!(to_annex_b(&i, s, true).is_err(), "{s:?}"); + } + } +} diff --git a/crates/platform/src/biplanar.rs b/crates/platform/src/biplanar.rs new file mode 100644 index 00000000..9615d75f --- /dev/null +++ b/crates/platform/src/biplanar.rs @@ -0,0 +1,139 @@ +//! Biplanar 4:2:0 pictures (NV12, and P010 with its 10-bit samples in the high bits of 16-bit +//! words) as read back from a decoder surface, into the planar [`PixelData::Yuv8`] / +//! [`PixelData::Yuv16`] frames the rest of FilmCraft uses. Safe code, tested on every target. +//! +//! This is the one place the Windows backend copies pixels (surface to staging texture to these +//! planes). A zero-copy path would replace it, not the decoder around it. + +use std::sync::Arc; + +use filmcraft_frame::{Chroma, PixelData, VideoFrame, pool}; + +/// A mapped biplanar surface: a luma plane followed by interleaved chroma, `stride` bytes per row. +pub struct Biplanar<'a> { + pub luma: &'a [u8], + pub chroma: &'a [u8], + pub stride: usize, + /// Rows in the luma plane (the surface height, which may exceed the picture's). + pub rows: usize, +} + +/// Where the picture is in the surface and how its samples are stored. +#[derive(Clone, Copy, Debug)] +pub struct Geometry { + /// Output rectangle in luma samples: x, y, width, height. + pub crop: (u32, u32, u32, u32), + /// Significant bits per sample: 8 (NV12) or 10 (P010). + pub bits: u32, + pub color: filmcraft_color::ColorInfo, + pub par: (u32, u32), +} + +fn row(plane: &[u8], stride: usize, y: usize, x: usize, len: usize) -> Result<&[u8], String> { + let start = y.checked_mul(stride).and_then(|s| s.checked_add(x)).ok_or("plane offset overflows")?; + plane.get(start..start.checked_add(len).ok_or("plane offset overflows")?).ok_or_else(|| "plane row out of range".to_string()) +} + +/// Copy the picture at `g.crop` out of `src` into a planar 4:2:0 frame (deinterleaving chroma, +/// shifting 10-bit samples down from the high bits). Errors when the surface is smaller than the +/// rectangle needs. +pub fn to_frame(src: &Biplanar, g: &Geometry) -> Result { + let (cx, cy, w, h) = g.crop; + let (ox, oy, w, h) = (cx as usize, cy as usize, w as usize, h as usize); + if w == 0 || h == 0 || !matches!(g.bits, 8 | 10) { + return Err(format!("cannot read a {w}x{h} picture of {} bits", g.bits)); + } + let bps = if g.bits > 8 { 2 } else { 1 }; + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let (cox, coy) = (ox / 2, oy / 2); + let needs_rows = oy.checked_add(h).ok_or("picture size overflows")?; + let needs_cols = ox.checked_add(w).and_then(|c| c.checked_mul(bps)).ok_or("picture size overflows")?; + if src.rows < needs_rows || src.stride < needs_cols || src.stride < (cox + cw) * 2 * bps { + return Err(format!("decoded surface ({} rows, {} byte rows) is smaller than the {w}x{h} picture at {ox},{oy}", src.rows, src.stride)); + } + let data = if bps == 1 { + let mut yp = pool::take_u8(w * h); + for y in 0..h { + yp.extend_from_slice(row(src.luma, src.stride, oy + y, ox, w)?); + } + let (mut u, mut v) = (pool::take_u8(cw * ch), pool::take_u8(cw * ch)); + for y in 0..ch { + // two passes over the row: each is a simple strided copy the compiler vectorises + let pairs = row(src.chroma, src.stride, coy + y, cox * 2, cw * 2)?.as_chunks::<2>().0; + u.extend(pairs.iter().map(|p| p[0])); + v.extend(pairs.iter().map(|p| p[1])); + } + PixelData::Yuv8 { planes: [Arc::new(yp), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, alpha: None } + } else { + let shift = 16 - g.bits; + let mut yp = pool::take_u16(w * h); + for y in 0..h { + yp.extend(row(src.luma, src.stride, oy + y, ox * 2, w * 2)?.as_chunks::<2>().0.iter().map(|b| u16::from_le_bytes([b[0], b[1]]) >> shift)); + } + let (mut u, mut v) = (pool::take_u16(cw * ch), pool::take_u16(cw * ch)); + for y in 0..ch { + let quads = row(src.chroma, src.stride, coy + y, cox * 4, cw * 4)?.as_chunks::<4>().0; + u.extend(quads.iter().map(|q| u16::from_le_bytes([q[0], q[1]]) >> shift)); + v.extend(quads.iter().map(|q| u16::from_le_bytes([q[2], q[3]]) >> shift)); + } + PixelData::Yuv16 { planes: [Arc::new(yp), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, bits: g.bits, alpha: None } + }; + Ok(VideoFrame { width: w as u32, height: h as u32, data, color: g.color, par: g.par, pts: filmcraft_time::Tick::ZERO }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn geometry(crop: (u32, u32, u32, u32), bits: u32) -> Geometry { + Geometry { crop, bits, color: filmcraft_color::ColorInfo::REC709, par: (1, 1) } + } + + #[test] + fn nv12_is_deinterleaved_and_cropped() { + // 8x6 surface, stride 10; the picture is the 4x4 at (2, 2) + let (stride, rows) = (10usize, 6usize); + let luma: Vec = (0..stride * rows).map(|i| i as u8).collect(); + let chroma: Vec = (0..stride * rows / 2).map(|i| 100 + i as u8).collect(); + let f = to_frame(&Biplanar { luma: &luma, chroma: &chroma, stride, rows }, &geometry((2, 2, 4, 4), 8)).unwrap(); + let PixelData::Yuv8 { planes, chroma: c, .. } = &f.data else { panic!("8-bit") }; + assert_eq!(*c, Chroma::C420); + assert_eq!(&planes[0][..], &[22, 23, 24, 25, 32, 33, 34, 35, 42, 43, 44, 45, 52, 53, 54, 55]); + // chroma rows 1..3, pairs from byte 2: row 1 starts at 10, row 2 at 20 + assert_eq!(&planes[1][..], &[112, 114, 122, 124]); + assert_eq!(&planes[2][..], &[113, 115, 123, 125]); + } + + #[test] + fn p010_shifts_down_from_the_high_bits() { + let (stride, rows) = (8usize, 2usize); + let mut luma = vec![0u8; stride * rows]; + for (i, s) in luma.as_chunks_mut::<2>().0.iter_mut().enumerate() { + *s = (((i as u16) + 1) << 6).to_le_bytes(); + } + let mut chroma = vec![0u8; stride]; + for (i, s) in chroma.as_chunks_mut::<2>().0.iter_mut().enumerate() { + *s = ((500 + i as u16) << 6).to_le_bytes(); + } + let f = to_frame(&Biplanar { luma: &luma, chroma: &chroma, stride, rows }, &geometry((0, 0, 4, 2), 10)).unwrap(); + let PixelData::Yuv16 { planes, bits, .. } = &f.data else { panic!("16-bit") }; + assert_eq!(*bits, 10); + assert_eq!(&planes[0][..], &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!((&planes[1][..], &planes[2][..]), (&[500, 502][..], &[501, 503][..])); + } + + #[test] + fn surfaces_smaller_than_the_picture_are_errors() { + let luma = vec![0u8; 64]; + let chroma = vec![0u8; 32]; + let src = Biplanar { luma: &luma, chroma: &chroma, stride: 8, rows: 8 }; + for crop in [(0, 0, 9, 8), (0, 0, 8, 9), (4, 0, 8, 8), (0, 0, 0, 8), (u32::MAX, 0, 8, 8), (0, 0, 8, 8)] { + let r = to_frame(&src, &geometry(crop, 8)); + assert_eq!(r.is_ok(), crop == (0, 0, 8, 8), "{crop:?}"); + } + assert!(to_frame(&src, &geometry((0, 0, 4, 4), 12)).is_err()); + // a short chroma plane is an error, not a panic + let short = Biplanar { luma: &luma, chroma: &chroma[..8], stride: 8, rows: 8 }; + assert!(to_frame(&short, &geometry((0, 0, 8, 8), 8)).is_err()); + } +} diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index 5a2a6802..e89f4fed 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -22,7 +22,11 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +mod annexb; +mod biplanar; pub mod hybrid; +#[cfg(target_os = "windows")] +pub mod media_foundation; #[cfg(target_os = "macos")] #[allow(unsafe_code)] pub mod videotoolbox; @@ -45,9 +49,16 @@ pub fn register() -> Availability { #[cfg(target_os = "macos")] { filmcraft_codecs::register_video_decoder(videotoolbox_factory); + filmcraft_codecs::hw::set_hw_backend("VideoToolbox"); Availability::Available("VideoToolbox") } - #[cfg(not(target_os = "macos"))] + #[cfg(target_os = "windows")] + { + filmcraft_codecs::register_video_decoder(media_foundation_factory); + filmcraft_codecs::hw::set_hw_backend("Media Foundation"); + Availability::Available("Media Foundation") + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] { Availability::Unavailable("no hardware video decoder for this system yet") } @@ -59,7 +70,11 @@ pub fn registered() -> bool { { filmcraft_codecs::video_decoder_registered(videotoolbox_factory) } - #[cfg(not(target_os = "macos"))] + #[cfg(target_os = "windows")] + { + filmcraft_codecs::video_decoder_registered(media_foundation_factory) + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] { false } @@ -72,7 +87,11 @@ pub fn hardware_decoder_for(entry: &filmcraft_isobmff::SampleEntry) -> bool { { filmcraft_codecs::hw::NalStreamInfo::from_entry(entry).and_then(|r| r.ok()).is_some_and(|info| videotoolbox::VtDecoder::new(info).is_ok()) } - #[cfg(not(target_os = "macos"))] + #[cfg(target_os = "windows")] + { + filmcraft_codecs::hw::NalStreamInfo::from_entry(entry).and_then(|r| r.ok()).is_some_and(|info| media_foundation::MfDecoder::new(info).is_ok()) + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] { let _ = entry; false @@ -96,3 +115,22 @@ pub fn videotoolbox_factory(entry: &filmcraft_isobmff::SampleEntry) -> Option Option>> { + if !filmcraft_codecs::hw::hardware_decoding() { + return None; + } + let info = filmcraft_codecs::hw::NalStreamInfo::from_entry(entry)?.ok()?; + match media_foundation::MfDecoder::new(info.clone()) { + Ok(mf) => Some(Ok(Box::new(HybridDecoder::new(Box::new(mf), entry.clone(), info)))), + Err(why) => { + log::info!("hardware decoding declined for {} video: {why}", entry.codec.name()); + filmcraft_codecs::hw::note_hw_declined(); + None + } + } +} diff --git a/crates/platform/src/media_foundation/gpu.rs b/crates/platform/src/media_foundation/gpu.rs new file mode 100644 index 00000000..8286e255 --- /dev/null +++ b/crates/platform/src/media_foundation/gpu.rs @@ -0,0 +1,284 @@ +//! The Direct3D 11 side of the Windows decoder: the video-capable device shared by every decoder +//! (with the `IMFDXGIDeviceManager` the decoder MFTs get it through), what DXVA can decode on it, +//! and the readback of a decoded surface into system memory (GPU to staging texture to bytes). +//! +//! FFI module (docs/adr/0001-platform-ffi.md): every `unsafe` block has a `// SAFETY:` comment, +//! the public items are safe and return `Result<_, String>`, no COM pointer leaves this module +//! except as an opaque handle for the MFT wrapper. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use windows::Win32::Foundation::{HMODULE, RPC_E_CHANGED_MODE}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_STAGING, D3D11_VIDEO_DECODER_DESC, D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread, ID3D11Texture2D, ID3D11VideoDevice, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_P010}; +use windows::Win32::Graphics::Dxgi::{IDXGIAdapter, IDXGIDevice}; +use windows::Win32::Media::MediaFoundation::{IMFDXGIBuffer, IMFDXGIDeviceManager, IMFSample}; +use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx}; +use windows::core::{GUID, Interface}; + +use super::mft::MfApi; +use crate::biplanar::Biplanar; + +/// What a surface format holds: 8-bit NV12 or 10-bit P010 (both 4:2:0, chroma interleaved). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SurfaceFormat { + Nv12, + P010, +} + +impl SurfaceFormat { + pub fn bits(self) -> u32 { + match self { + Self::Nv12 => 8, + Self::P010 => 10, + } + } + + fn dxgi(self) -> DXGI_FORMAT { + match self { + Self::Nv12 => DXGI_FORMAT_NV12, + Self::P010 => DXGI_FORMAT_P010, + } + } +} + +thread_local! { + /// COM is initialised (multithreaded) once per thread that touches a decoder; decoders move + /// between frame-worker threads. Never uninitialised: the threads end with the process. + static COM: Result<(), String> = init_com(); +} + +fn init_com() -> Result<(), String> { + // SAFETY: CoInitializeEx has no pointer arguments; calling it again on a thread that already + // initialised COM (in either apartment model) is allowed and reported through the HRESULT. + let hr = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + // A thread already in a single-threaded apartment (the UI thread) keeps it: MF decoders and + // D3D11 are free-threaded and work from there too. + if hr.is_ok() || hr == RPC_E_CHANGED_MODE { Ok(()) } else { Err(format!("COM initialisation failed ({hr:?})")) } +} + +/// Make sure the calling thread can use COM objects. +pub fn ensure_com() -> Result<(), String> { + COM.with(Clone::clone) +} + +/// A Direct3D 11 device created for video decoding, with its DXGI device manager. +pub struct Gpu { + device: ID3D11Device, + context: ID3D11DeviceContext, + video: ID3D11VideoDevice, + manager: IMFDXGIDeviceManager, + name: String, + /// Set when the device was removed or reset: new decoders get a new device. + lost: AtomicBool, + /// Serialises our use of the immediate context (the device is also multithread protected, + /// which covers the decoder MFTs' own use of it). + gate: Mutex<()>, +} + +// SAFETY: the device and its immediate context are used from several threads on purpose: the +// device is created multithread-protected (`ID3D11Multithread::SetMultithreadProtected`), the +// DXGI device manager is a free-threaded object, and our own use of the context is behind `gate`. +unsafe impl Send for Gpu {} +// SAFETY: see above. +unsafe impl Sync for Gpu {} + +static SHARED: Mutex>> = Mutex::new(None); + +impl Gpu { + /// The process-wide device (created on first use, replaced after a device loss). + pub fn shared(api: &MfApi) -> Result, String> { + ensure_com()?; + let mut slot = SHARED.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(g) = slot.as_ref().filter(|g| g.usable()) { + return Ok(g.clone()); + } + let g = Arc::new(Gpu::create(api)?); + log::info!("hardware decoding: Direct3D 11 device on {}", g.name); + *slot = Some(g.clone()); + Ok(g) + } + + fn create(api: &MfApi) -> Result { + let levels = [D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]; + let (mut device, mut context) = (None, None); + // SAFETY: the out-pointers refer to live locals; no adapter is passed (the default one), + // and the feature level slice outlives the call. + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&levels), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .map_err(|e| format!("no Direct3D 11 video device: {e}"))?; + let (device, context) = device.zip(context).ok_or("Direct3D 11 returned no device")?; + // Decoder MFTs on other threads use this device while we copy surfaces out of it. + let mt: ID3D11Multithread = device.cast().map_err(|e| format!("no ID3D11Multithread: {e}"))?; + // SAFETY: a plain COM call on a live interface (it returns the previous setting). + let _ = unsafe { mt.SetMultithreadProtected(true) }; + let video: ID3D11VideoDevice = device.cast().map_err(|e| format!("the device has no video decoding: {e}"))?; + let manager = api.create_device_manager(&device)?; + let name = adapter_name(&device).unwrap_or_else(|| "an unknown adapter".into()); + Ok(Gpu { device, context, video, manager, name, lost: AtomicBool::new(false), gate: Mutex::new(()) }) + } + + fn usable(&self) -> bool { + // SAFETY: a plain COM call on a live interface. + !self.lost.load(Ordering::Relaxed) && unsafe { self.device.GetDeviceRemovedReason() }.is_ok() + } + + /// The adapter's description (for logs). + pub fn name(&self) -> &str { + &self.name + } + + /// The device manager decoder MFTs are given (`MFT_MESSAGE_SET_D3D_MANAGER`), as the integer + /// that message carries. Valid while this `Gpu` lives. + pub fn manager_param(&self) -> usize { + self.manager.as_raw() as usize + } + + /// A failed call may mean the device is gone: if so, mark it so new decoders get a fresh one. + pub fn note_failure(&self) { + if !self.usable() { + log::warn!("Direct3D 11 device lost ({}); new decoders will create another", self.name); + self.lost.store(true, Ordering::Relaxed); + } + } + + /// Whether DXVA on this device decodes `profile` into `format` at `size` (so a stream is + /// declined up front instead of failing on its first sample). + pub fn supports(&self, profile: GUID, format: SurfaceFormat, size: (u32, u32)) -> Result<(), String> { + // SAFETY: plain COM calls on a live interface with a valid descriptor. + unsafe { + let ok = self.video.CheckVideoDecoderFormat(&profile, format.dxgi()).map_err(|e| format!("decoder format check failed: {e}"))?; + if !ok.as_bool() { + return Err(format!("the GPU has no DXVA decoder for this profile in {format:?}")); + } + let desc = D3D11_VIDEO_DECODER_DESC { Guid: profile, SampleWidth: size.0, SampleHeight: size.1, OutputFormat: format.dxgi() }; + let configs = self.video.GetVideoDecoderConfigCount(&desc).map_err(|e| format!("decoder configuration query failed: {e}"))?; + if configs == 0 { + return Err(format!("the GPU's DXVA decoder does not take {}x{} pictures", size.0, size.1)); + } + } + Ok(()) + } +} + +fn adapter_name(device: &ID3D11Device) -> Option { + let dxgi: IDXGIDevice = device.cast().ok()?; + // SAFETY: plain COM calls on live interfaces; the description is a plain struct filled by DXGI. + let desc = unsafe { dxgi.GetAdapter().ok()?.GetDesc().ok()? }; + let len = desc.Description.iter().position(|&c| c == 0).unwrap_or(desc.Description.len()); + Some(String::from_utf16_lossy(desc.Description.get(..len)?)) +} + +/// Reads decoded surfaces back into system memory through a staging texture it keeps between +/// pictures (recreated when the surface format or size changes). This is the GPU to CPU copy of +/// the backend: a zero-copy path would share the surface with the renderer instead. +#[derive(Default)] +pub struct Readback { + staging: Option<(ID3D11Texture2D, D3D11_TEXTURE2D_DESC)>, +} + +// SAFETY: the staging texture belongs to the device `Gpu` makes thread-safe; a `Readback` is used +// by one thread at a time (`&mut self`) and every use of the context is behind the `Gpu` gate. +unsafe impl Send for Readback {} + +impl Readback { + /// Copy the picture in `sample` (a decoder output sample backed by a Direct3D 11 texture) into + /// system memory and hand it to `f` as a mapped biplanar surface. Errors when the sample is + /// not texture-backed, which means the decoder did not use the GPU. + pub fn read(&mut self, gpu: &Gpu, sample: &IMFSample, format: SurfaceFormat, f: impl FnOnce(&Biplanar) -> Result) -> Result { + let r = self.read_inner(gpu, sample, format, f); + if r.is_err() { + gpu.note_failure(); + } + r + } + + fn read_inner(&mut self, gpu: &Gpu, sample: &IMFSample, format: SurfaceFormat, f: impl FnOnce(&Biplanar) -> Result) -> Result { + // SAFETY: plain COM calls on live interfaces; `GetResource` writes one interface pointer + // (a new reference) into `tex`, which `Option` owns and releases. + let (tex, index) = unsafe { + let buffer = sample.GetBufferByIndex(0).map_err(|e| format!("decoder output has no buffer: {e}"))?; + let dxgi: IMFDXGIBuffer = buffer.cast().map_err(|_| "decoder output is in system memory: the decoder is not using the GPU".to_string())?; + let mut tex: Option = None; + dxgi.GetResource(&ID3D11Texture2D::IID, &mut tex as *mut Option as *mut *mut c_void) + .map_err(|e| format!("decoder output is not a 2D texture: {e}"))?; + (tex.ok_or("decoder output has no texture")?, dxgi.GetSubresourceIndex().map_err(|e| format!("no subresource index: {e}"))?) + }; + let mut desc = D3D11_TEXTURE2D_DESC::default(); + // SAFETY: a plain COM call writing a descriptor struct. + unsafe { tex.GetDesc(&mut desc) }; + if desc.Format != format.dxgi() { + return Err(format!("decoded surface has DXGI format {} instead of {format:?}", desc.Format.0)); + } + if desc.Width == 0 || desc.Height == 0 || desc.Width > 16384 || desc.Height > 16384 { + return Err(format!("decoded surface is {}x{}", desc.Width, desc.Height)); + } + let staging = self.staging_for(gpu, &desc)?; + let _gate = gpu.gate.lock().unwrap_or_else(PoisonError::into_inner); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + // SAFETY: both textures belong to this device; the copy and the mapping happen under the + // context gate. After a successful `Map` the memory at `pData` holds `RowPitch` bytes per + // row for `Height` luma rows followed by `Height / 2` chroma rows (the NV12 / P010 staging + // layout), valid until `Unmap`, which runs before this block ends. + unsafe { + gpu.context.CopySubresourceRegion(staging, 0, 0, 0, 0, &tex, index, None); + gpu.context.Map(staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)).map_err(|e| format!("cannot map the decoded surface: {e}"))?; + } + let result = (|| { + let stride = mapped.RowPitch as usize; + let rows = desc.Height as usize; + let luma_len = stride.checked_mul(rows).ok_or("surface size overflows")?; + let chroma_len = stride.checked_mul(rows.div_ceil(2)).ok_or("surface size overflows")?; + if mapped.pData.is_null() || stride == 0 { + return Err("the decoded surface is not mapped".to_string()); + } + // SAFETY: see the `Map` call: `luma_len + chroma_len` bytes are mapped at `pData`. + let all = unsafe { std::slice::from_raw_parts(mapped.pData as *const u8, luma_len + chroma_len) }; + let (luma, chroma) = all.split_at(luma_len); + f(&Biplanar { luma, chroma, stride, rows }) + })(); + // SAFETY: the subresource was mapped above and nothing borrowed from it outlives `result`. + unsafe { gpu.context.Unmap(staging, 0) }; + result + } + + fn staging_for(&mut self, gpu: &Gpu, src: &D3D11_TEXTURE2D_DESC) -> Result<&ID3D11Texture2D, String> { + let fits = self.staging.as_ref().is_some_and(|(_, d)| d.Width == src.Width && d.Height == src.Height && d.Format == src.Format); + if !fits { + let desc = D3D11_TEXTURE2D_DESC { + Width: src.Width, + Height: src.Height, + MipLevels: 1, + ArraySize: 1, + Format: src.Format, + SampleDesc: windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + MiscFlags: 0, + }; + let mut tex: Option = None; + // SAFETY: a valid descriptor and out-pointer; no initial data. + unsafe { gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)) }.map_err(|e| format!("cannot create the readback texture: {e}"))?; + self.staging = Some((tex.ok_or("no readback texture")?, desc)); + } + self.staging.as_ref().map(|(t, _)| t).ok_or_else(|| "no readback texture".to_string()) + } +} diff --git a/crates/platform/src/media_foundation/mft.rs b/crates/platform/src/media_foundation/mft.rs new file mode 100644 index 00000000..a0c829fe --- /dev/null +++ b/crates/platform/src/media_foundation/mft.rs @@ -0,0 +1,383 @@ +//! The Media Foundation side of the Windows decoder: finding a Direct3D-aware H.264 / HEVC decoder +//! MFT, handing it the shared device manager so it decodes with DXVA, and driving it +//! (`ProcessInput` / `ProcessOutput`, drain, flush) at the level of single access units. +//! +//! FFI module (docs/adr/0001-platform-ffi.md): every `unsafe` block has a `// SAFETY:` comment, +//! the public items are safe and return `Result<_, String>`. +//! +//! `mfplat.dll` is loaded at run time instead of linked: Windows "N" editions ship without the +//! Media Feature Pack, and FilmCraft must still start (and decode in software) on them. + +use std::ffi::c_void; +use std::mem::ManuallyDrop; +use std::sync::OnceLock; + +use filmcraft_codecs::hw::NalCodec; +use windows::Win32::Foundation::HMODULE; +use windows::Win32::Graphics::Direct3D11::ID3D11Device; +use windows::Win32::Media::MediaFoundation::{ + IMFActivate, IMFDXGIDeviceManager, IMFMediaBuffer, IMFMediaType, IMFSample, IMFTransform, MF_E_NO_MORE_TYPES, MF_E_NOTACCEPTING, + MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_MPEG2_PROFILE, MF_MT_SUBTYPE, + MF_SA_D3D11_AWARE, MF_TRANSFORM_ASYNC, MF_VERSION, MFMediaType_Video, MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_FLAG, MFT_ENUM_FLAG_ASYNCMFT, + MFT_ENUM_FLAG_HARDWARE, MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_SYNCMFT, MFT_FRIENDLY_NAME_Attribute, MFT_MESSAGE_COMMAND_DRAIN, + MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, + MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoFormat_P010, + MFVideoInterlace_Progressive, +}; +use windows::Win32::System::Com::CoTaskMemFree; +use windows::Win32::System::LibraryLoader::{GetProcAddress, LOAD_LIBRARY_SEARCH_SYSTEM32, LoadLibraryExW}; +use windows::core::{GUID, HRESULT, Interface, PCSTR, w}; + +use super::gpu::{Gpu, SurfaceFormat, ensure_com}; + +type StartupFn = unsafe extern "system" fn(u32, u32) -> HRESULT; +type CreateFn = unsafe extern "system" fn(*mut *mut c_void) -> HRESULT; +type CreateBufferFn = unsafe extern "system" fn(u32, *mut *mut c_void) -> HRESULT; +type CreateManagerFn = unsafe extern "system" fn(*mut u32, *mut *mut c_void) -> HRESULT; +type EnumFn = unsafe extern "system" fn(GUID, u32, *const MFT_REGISTER_TYPE_INFO, *const MFT_REGISTER_TYPE_INFO, *mut *mut *mut c_void, *mut u32) -> HRESULT; + +/// The few `mfplat.dll` entry points the backend uses, resolved at run time. +pub struct MfApi { + create_media_type: CreateFn, + create_sample: CreateFn, + create_memory_buffer: CreateBufferFn, + create_manager: CreateManagerFn, + enum_transforms: EnumFn, +} + +/// The exported function `name` of `module` as a function pointer of type `F`. +/// +/// # Safety +/// `F` must be the `extern "system" fn` type that `name` really has. +unsafe fn symbol(module: HMODULE, name: PCSTR) -> Result { + if std::mem::size_of::() != std::mem::size_of::() { + return Err("not a function pointer type".into()); + } + // SAFETY: `module` is a loaded module handle and `name` a NUL-terminated string literal. + let p = unsafe { GetProcAddress(module, name) }.ok_or_else(|| "mfplat.dll lacks an expected function".to_string())?; + // SAFETY: `F` is pointer-sized (checked above) and, by this function's contract, the type of + // the exported function `p` points to. + Ok(unsafe { std::mem::transmute_copy::<_, F>(&p) }) +} + +/// Media Foundation, loaded and started once for the process (never shut down: decoders come and +/// go, and the threads of its worker queues end with the process). +pub fn api() -> Result<&'static MfApi, String> { + static API: OnceLock> = OnceLock::new(); + API.get_or_init(load).as_ref().map_err(Clone::clone) +} + +fn load() -> Result { + ensure_com()?; + // SAFETY: loads a system DLL by name from the system directory only. + let module = + unsafe { LoadLibraryExW(w!("mfplat.dll"), None, LOAD_LIBRARY_SEARCH_SYSTEM32) }.map_err(|_| "Media Foundation is not installed".to_string())?; + // SAFETY: each symbol is looked up with the signature Microsoft documents for it + // (MFStartup, MFCreateMediaType, MFCreateSample, MFCreateMemoryBuffer, + // MFCreateDXGIDeviceManager, MFTEnumEx). + let (startup, api) = unsafe { + ( + symbol::(module, PCSTR(c"MFStartup".as_ptr().cast()))?, + MfApi { + create_media_type: symbol(module, PCSTR(c"MFCreateMediaType".as_ptr().cast()))?, + create_sample: symbol(module, PCSTR(c"MFCreateSample".as_ptr().cast()))?, + create_memory_buffer: symbol(module, PCSTR(c"MFCreateMemoryBuffer".as_ptr().cast()))?, + create_manager: symbol(module, PCSTR(c"MFCreateDXGIDeviceManager".as_ptr().cast()))?, + enum_transforms: symbol(module, PCSTR(c"MFTEnumEx".as_ptr().cast()))?, + }, + ) + }; + // SAFETY: MFSTARTUP_FULL (0) with the version this binding was built for. + let hr = unsafe { startup(MF_VERSION, 0) }; + if hr.is_err() { + return Err(format!("MFStartup failed ({hr:?})")); + } + Ok(api) +} + +/// An out-pointer COM creation function's result as an interface. +fn made(hr: HRESULT, raw: *mut c_void, what: &str) -> Result { + if hr.is_err() || raw.is_null() { + return Err(format!("{what} failed ({hr:?})")); + } + // SAFETY: the creation function succeeded, so `raw` is a new reference to an object of type + // `T` that this takes ownership of. + Ok(unsafe { T::from_raw(raw) }) +} + +impl MfApi { + pub(super) fn create_device_manager(&self, device: &ID3D11Device) -> Result { + let (mut token, mut raw) = (0u32, std::ptr::null_mut()); + // SAFETY: both out-pointers refer to live locals. + let hr = unsafe { (self.create_manager)(&mut token, &mut raw) }; + let manager: IMFDXGIDeviceManager = made(hr, raw, "MFCreateDXGIDeviceManager")?; + // SAFETY: a plain COM call with the token the manager just returned. + unsafe { manager.ResetDevice(device, token) }.map_err(|e| format!("IMFDXGIDeviceManager::ResetDevice failed: {e}"))?; + Ok(manager) + } + + fn media_type(&self) -> Result { + let mut raw = std::ptr::null_mut(); + // SAFETY: the out-pointer refers to a live local. + let hr = unsafe { (self.create_media_type)(&mut raw) }; + made(hr, raw, "MFCreateMediaType") + } + + /// An input sample holding a copy of `data`, timestamped `time`. + fn sample_of(&self, data: &[u8], time: i64) -> Result { + let len = u32::try_from(data.len()).map_err(|_| "sample is too large".to_string())?; + let mut raw = std::ptr::null_mut(); + // SAFETY: the out-pointer refers to a live local. + let hr = unsafe { (self.create_memory_buffer)(len, &mut raw) }; + let buffer: IMFMediaBuffer = made(hr, raw, "MFCreateMemoryBuffer")?; + let mut raw = std::ptr::null_mut(); + // SAFETY: the out-pointer refers to a live local. + let hr = unsafe { (self.create_sample)(&mut raw) }; + let sample: IMFSample = made(hr, raw, "MFCreateSample")?; + // SAFETY: `Lock` gives a writable block of at least `max` bytes valid until `Unlock`; the + // copy stays within `len <= max` bytes and the buffer is unlocked before it is used. + unsafe { + let (mut ptr, mut max) = (std::ptr::null_mut::(), 0u32); + buffer.Lock(&mut ptr, Some(&mut max), None).map_err(|e| format!("cannot lock the input buffer: {e}"))?; + if ptr.is_null() || max < len { + let _ = buffer.Unlock(); + return Err("input buffer is too small".into()); + } + std::ptr::copy_nonoverlapping(data.as_ptr(), ptr, data.len()); + buffer.Unlock().map_err(|e| format!("cannot unlock the input buffer: {e}"))?; + buffer.SetCurrentLength(len).map_err(|e| format!("cannot size the input buffer: {e}"))?; + sample.AddBuffer(&buffer).map_err(|e| format!("cannot add the input buffer: {e}"))?; + sample.SetSampleTime(time).map_err(|e| format!("cannot timestamp the input: {e}"))?; + } + Ok(sample) + } + + /// The decoder MFTs for `codec`, hardware ones first, as activation objects. + fn decoders(&self, codec: NalCodec) -> Result, String> { + let input = MFT_REGISTER_TYPE_INFO { + guidMajorType: MFMediaType_Video, + guidSubtype: match codec { + NalCodec::H264 => MFVideoFormat_H264, + NalCodec::Hevc => MFVideoFormat_HEVC, + }, + }; + let flags = MFT_ENUM_FLAG(MFT_ENUM_FLAG_SYNCMFT.0 | MFT_ENUM_FLAG_ASYNCMFT.0 | MFT_ENUM_FLAG_HARDWARE.0 | MFT_ENUM_FLAG_SORTANDFILTER.0); + let (mut list, mut count) = (std::ptr::null_mut::<*mut c_void>(), 0u32); + // SAFETY: the type info and the out-pointers refer to live locals. + let hr = unsafe { (self.enum_transforms)(MFT_CATEGORY_VIDEO_DECODER, flags.0 as u32, &input, std::ptr::null(), &mut list, &mut count) }; + if hr.is_err() { + return Err(format!("MFTEnumEx failed ({hr:?})")); + } + let mut out = Vec::new(); + if !list.is_null() { + for i in 0..count as usize { + // SAFETY: `list` holds `count` pointers in memory allocated with CoTaskMemAlloc, each + // null or a reference to an `IMFActivate` that this takes over. + unsafe { + let p = *list.add(i); + if !p.is_null() { + out.push(IMFActivate::from_raw(p)); + } + } + } + // SAFETY: the array itself was allocated by MFTEnumEx with CoTaskMemAlloc. + unsafe { CoTaskMemFree(Some(list as *const c_void)) }; + } + Ok(out) + } +} + +/// What polling the decoder for output produced. +pub enum Poll { + /// A decoded picture and its timestamp. + Frame(IMFSample, i64), + /// The decoder needs more input (or has finished draining). + NeedInput, + /// The output format changed and was renegotiated; poll again. + FormatChanged, +} + +/// A Direct3D-aware decoder MFT, configured for one stream. +pub struct Mft { + transform: IMFTransform, + format: SurfaceFormat, + name: String, +} + +// SAFETY: the decoder MFTs are free-threaded COM objects (they are used from the frame workers' +// threads, one at a time: an `Mft` is only used through `&mut` / `&` of its owning decoder). +unsafe impl Send for Mft {} + +impl Mft { + /// Create the decoder for `codec` with output in `format`, for pictures of `size`, decoding + /// on the device of `gpu`. Errors (and so the stream is declined) when no Direct3D-aware + /// synchronous decoder MFT takes the stream. + pub fn new(api: &MfApi, gpu: &Gpu, codec: NalCodec, size: (u32, u32), format: SurfaceFormat) -> Result { + ensure_com()?; + let candidates = api.decoders(codec)?; + if candidates.is_empty() { + return Err(match codec { + NalCodec::H264 => "no H.264 decoder MFT".into(), + NalCodec::Hevc => "no HEVC decoder MFT (the HEVC Video Extensions are not installed)".into(), + }); + } + let mut reasons = Vec::new(); + for a in candidates { + match Self::configure(api, gpu, &a, codec, size, format) { + Ok(m) => return Ok(m), + Err(e) => reasons.push(e), + } + } + Err(reasons.join("; ")) + } + + fn configure(api: &MfApi, gpu: &Gpu, activate: &IMFActivate, codec: NalCodec, size: (u32, u32), format: SurfaceFormat) -> Result { + let name = friendly_name(activate).unwrap_or_else(|| "a decoder MFT".into()); + // SAFETY: plain COM calls on live interfaces. + let transform: IMFTransform = unsafe { activate.ActivateObject() }.map_err(|e| format!("{name}: cannot be created: {e}"))?; + // SAFETY: plain COM calls on live interfaces; the D3D manager parameter is a pointer to + // the device manager, kept alive by `gpu` (which every decoder holds an `Arc` of). + unsafe { + let attrs = transform.GetAttributes().map_err(|e| format!("{name}: no attributes: {e}"))?; + if attrs.GetUINT32(&MF_TRANSFORM_ASYNC).unwrap_or(0) != 0 { + return Err(format!("{name}: asynchronous decoder MFTs are not supported")); + } + if attrs.GetUINT32(&MF_SA_D3D11_AWARE).unwrap_or(0) == 0 { + return Err(format!("{name}: does not decode on Direct3D 11")); + } + transform + .ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, gpu.manager_param()) + .map_err(|e| format!("{name}: refused the Direct3D device manager: {e}"))?; + } + let mft = Mft { transform, format, name }; + let input = api.media_type()?; + // SAFETY: plain COM calls on live interfaces. + unsafe { + input.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video).map_err(|e| e.to_string())?; + input + .SetGUID( + &MF_MT_SUBTYPE, + &match codec { + NalCodec::H264 => MFVideoFormat_H264, + NalCodec::Hevc => MFVideoFormat_HEVC, + }, + ) + .map_err(|e| e.to_string())?; + input.SetUINT64(&MF_MT_FRAME_SIZE, (u64::from(size.0) << 32) | u64::from(size.1)).map_err(|e| e.to_string())?; + input.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32).map_err(|e| e.to_string())?; + if codec == NalCodec::Hevc { + // the HEVC decoder lists P010 output only when told the stream is Main 10 + // (eAVEncH265VProfile_Main_420_8 = 1, eAVEncH265VProfile_Main_420_10 = 2) + let profile = if format == SurfaceFormat::P010 { 2 } else { 1 }; + input.SetUINT32(&MF_MT_MPEG2_PROFILE, profile).map_err(|e| e.to_string())?; + } + mft.transform.SetInputType(0, &input, 0).map_err(|e| format!("{}: does not take this stream: {e}", mft.name))?; + } + mft.select_output()?; + // SAFETY: plain COM calls on live interfaces. + unsafe { + let info = mft.transform.GetOutputStreamInfo(0).map_err(|e| format!("{}: no output stream info: {e}", mft.name))?; + if info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 as u32 == 0 { + return Err(format!("{}: does not provide Direct3D output samples", mft.name)); + } + mft.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0).map_err(|e| e.to_string())?; + mft.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0).map_err(|e| e.to_string())?; + } + Ok(mft) + } + + /// Pick the decoder's NV12 / P010 output type. + fn select_output(&self) -> Result<(), String> { + let want = match self.format { + SurfaceFormat::Nv12 => MFVideoFormat_NV12, + SurfaceFormat::P010 => MFVideoFormat_P010, + }; + for i in 0..64 { + // SAFETY: plain COM calls on live interfaces. + unsafe { + match self.transform.GetOutputAvailableType(0, i) { + Ok(t) => { + if t.GetGUID(&MF_MT_SUBTYPE).ok() == Some(want) { + return self.transform.SetOutputType(0, &t, 0).map_err(|e| format!("{}: refused {:?} output: {e}", self.name, self.format)); + } + } + Err(e) if e.code() == MF_E_NO_MORE_TYPES => break, + Err(e) => return Err(format!("{}: no output types: {e}", self.name)), + } + } + } + Err(format!("{}: cannot output {:?}", self.name, self.format)) + } + + /// The MFT's name, for logs. + pub fn name(&self) -> &str { + &self.name + } + + /// Feed one Annex B access unit. `Ok(false)`: the decoder holds enough input already; take + /// its output ([`Mft::poll`]) and offer the sample again. + pub fn input(&self, api: &MfApi, annex_b: &[u8], time: i64) -> Result { + let sample = api.sample_of(annex_b, time)?; + // SAFETY: plain COM call on live interfaces. + match unsafe { self.transform.ProcessInput(0, &sample, 0) } { + Ok(()) => Ok(true), + Err(e) if e.code() == MF_E_NOTACCEPTING => Ok(false), + Err(e) => Err(format!("{}: ProcessInput failed: {e}", self.name)), + } + } + + /// Ask the decoder for its next picture. + pub fn poll(&self) -> Result { + let mut out = [MFT_OUTPUT_DATA_BUFFER { dwStreamID: 0, pSample: ManuallyDrop::new(None), dwStatus: 0, pEvents: ManuallyDrop::new(None) }]; + let mut status = 0u32; + // SAFETY: one output buffer descriptor with no sample (the decoder provides its own, as + // checked at creation) and a live status out-pointer. + let r = unsafe { self.transform.ProcessOutput(0, &mut out, &mut status) }; + // SAFETY: the descriptor's fields were initialised above and are only taken once; the + // decoder may have filled them in even on failure, and dropping them releases the + // references it handed over. + let (sample, _events) = unsafe { (ManuallyDrop::take(&mut out[0].pSample), ManuallyDrop::take(&mut out[0].pEvents)) }; + match r { + Ok(()) => { + let sample = sample.ok_or_else(|| format!("{}: produced no sample", self.name))?; + // SAFETY: a plain COM call on a live interface. + let time = unsafe { sample.GetSampleTime() }.unwrap_or(0); + Ok(Poll::Frame(sample, time)) + } + Err(e) if e.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => Ok(Poll::NeedInput), + Err(e) if e.code() == MF_E_TRANSFORM_STREAM_CHANGE => { + self.select_output()?; + Ok(Poll::FormatChanged) + } + Err(e) => Err(format!("{}: ProcessOutput failed: {e}", self.name)), + } + } + + /// Start draining: [`Mft::poll`] returns the pictures held back, then `NeedInput`. + pub fn drain(&self) -> Result<(), String> { + // SAFETY: plain COM call on a live interface. + unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) }.map_err(|e| format!("{}: drain failed: {e}", self.name)) + } + + /// Discard all queued input, output and reference pictures (a seek). + pub fn flush(&self) -> Result<(), String> { + // SAFETY: plain COM calls on a live interface. + unsafe { + self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0).map_err(|e| format!("{}: flush failed: {e}", self.name))?; + self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0).map_err(|e| format!("{}: restart failed: {e}", self.name)) + } + } +} + +fn friendly_name(a: &IMFActivate) -> Option { + // SAFETY: a plain COM call; `GetAllocatedString` returns a CoTaskMem string that the + // returned `PWSTR` wrapper does not free, so it is freed here after copying. + unsafe { + let mut s = windows::core::PWSTR::null(); + let mut len = 0u32; + a.GetAllocatedString(&MFT_FRIENDLY_NAME_Attribute, &mut s, &mut len).ok()?; + let name = s.to_string().ok(); + CoTaskMemFree(Some(s.0 as *const c_void)); + name + } +} diff --git a/crates/platform/src/media_foundation/mod.rs b/crates/platform/src/media_foundation/mod.rs new file mode 100644 index 00000000..b61652e7 --- /dev/null +++ b/crates/platform/src/media_foundation/mod.rs @@ -0,0 +1,410 @@ +//! Media Foundation / Direct3D 11 (Windows) hardware H.264 / HEVC decoding. +//! +//! The decoder is a Direct3D-aware decoder MFT (Microsoft's H.264 decoder, the HEVC Video +//! Extensions' decoder, or a vendor's synchronous hardware MFT) driven at the level of single +//! access units: the container demuxer FilmCraft already has supplies the samples, so there is no +//! Source Reader and no second media pipeline. The MFT is given the process's Direct3D 11 video +//! device through an `IMFDXGIDeviceManager`, which makes it decode with DXVA on the GPU's video +//! engine and return NV12 (8-bit) / P010 (10-bit) surfaces as Direct3D 11 textures. +//! +//! ```text +//! avcC / hvcC sample ──► Annex B ──► decoder MFT ──DXVA──► NV12 / P010 texture +//! │ copy to a staging texture (GPU → CPU) +//! ▼ +//! planar Yuv8 / Yuv16 VideoFrame ──► HybridDecoder +//! ``` +//! +//! "Hardware" is verified, not assumed: a decoder MFT that is not Direct3D-aware, or that hands +//! back system-memory samples (Microsoft's decoders do that when DXVA is unavailable), is +//! declined or fails the stream, so [`crate::HybridDecoder`] continues with FilmCraft's own +//! decoder and Windows' software decoding is never used in its place. Streams the GPU's DXVA +//! decoder does not list (profile, bit depth, chroma format, size) are declined up front. +//! +//! The readback (`gpu::Readback`, [`crate::biplanar`]) is the one GPU to CPU copy; a later +//! zero-copy path would hand the texture to the renderer instead and keep everything else. +//! +//! The modules that call into Windows (`gpu`, `mft`) are the FFI modules of this backend +//! (docs/adr/0001-platform-ffi.md); this one is safe code. + +#[allow(unsafe_code)] +mod gpu; +#[allow(unsafe_code)] +mod mft; + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use filmcraft_codecs::hw::{NalCodec, NalStreamInfo}; +use filmcraft_codecs::{CodecError, DecodedFrame, Result, VideoDecoder}; +use windows::Win32::Graphics::Direct3D11::{D3D11_DECODER_PROFILE_H264_VLD_NOFGT, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10}; +use windows::core::GUID; + +use self::gpu::{Gpu, Readback, SurfaceFormat}; +use self::mft::{MfApi, Mft, Poll}; +use crate::annexb::to_annex_b; +use crate::biplanar::{self, Geometry}; + +/// Largest picture the backend takes (as the VideoToolbox one). +const MAX_SIDE: u32 = 8192; +/// Pictures the decoder may hold back (a DPB is at most 16 frames; more means it lost track). +const MAX_IN_FLIGHT: usize = 64; +/// Upper bound on the outputs taken per call: a misbehaving MFT cannot make a decode call loop. +const MAX_OUTPUTS_PER_CALL: usize = 4096; + +/// Whether Media Foundation and a Direct3D 11 video device exist on this system. +pub fn available() -> bool { + mft::api().and_then(Gpu::shared).is_ok() +} + +/// What the decoder needs for a stream, or why it is declined. +fn plan(info: &NalStreamInfo) -> std::result::Result<(SurfaceFormat, GUID), String> { + if info.interlaced { + return Err("field-coded H.264".into()); + } + if info.chroma_format_idc != 1 { + return Err(format!("chroma_format_idc {}", info.chroma_format_idc)); + } + if info.bit_depth_luma != info.bit_depth_chroma || !matches!(info.bit_depth_luma, 8 | 10) { + return Err(format!("{}-bit luma / {}-bit chroma", info.bit_depth_luma, info.bit_depth_chroma)); + } + let (_, _, w, h) = info.crop; + let (cx, cy) = (info.crop.0, info.crop.1); + if w == 0 || h == 0 || w > MAX_SIDE || h > MAX_SIDE || cx.saturating_add(w) > info.coded.0 || cy.saturating_add(h) > info.coded.1 { + return Err(format!("picture size {w}x{h}")); + } + let ten = info.bit_depth_luma == 10; + match (info.codec, info.profile_idc, ten) { + // Baseline, Main and High (the profiles DXVA H.264 decoders list) + (NalCodec::H264, 66 | 77 | 100, false) => Ok((SurfaceFormat::Nv12, D3D11_DECODER_PROFILE_H264_VLD_NOFGT)), + (NalCodec::H264, p, _) => Err(format!("H.264 profile {p} at {} bits", info.bit_depth_luma)), + // Main and Main Still Picture + (NalCodec::Hevc, 1 | 3, false) => Ok((SurfaceFormat::Nv12, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN)), + (NalCodec::Hevc, 2, true) => Ok((SurfaceFormat::P010, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10)), + (NalCodec::Hevc, p, _) => Err(format!("HEVC profile {p} at {} bits", info.bit_depth_luma)), + } +} + +/// Where a decoder's time went (wall clock), for diagnostics (`examples/mfprobe.rs --time`). +#[derive(Clone, Copy, Debug, Default)] +pub struct Timings { + /// Pictures returned. + pub frames: u64, + /// Feeding the MFT and waiting for its pictures (the DXVA decode itself). + pub decode: Duration, + /// GPU to CPU: copy of the picture to the staging texture and its mapping. + pub readback: Duration, + /// CPU: staging memory to planar `Yuv8` / `Yuv16`. + pub convert: Duration, +} + +/// The Media Foundation decoder for one `avcC` / `hvcC` stream. +pub struct MfDecoder { + info: NalStreamInfo, + format: SurfaceFormat, + geometry: Geometry, + api: &'static MfApi, + gpu: Arc, + /// `None` only after a failed reset: made again on the next sample. + mft: Option, + readback: Readback, + /// Presentation times of submitted access units whose picture has not come out yet. + in_flight: Vec, + /// The next sample must carry the parameter sets (first of a run, after a reset or flush). + need_headers: bool, + /// The run started at a CRA / BLA picture: its RASL pictures are not decodable and are left + /// out (as the software decoder does). + skip_rasl: bool, + first: bool, + /// [`VideoDecoder::flush`] drained the MFT, which then behaves as at the end of a stream: it + /// only restarts at an IDR picture, the references of the run are gone. + drained: bool, + /// Test hook: fail every decode once this many samples were fed. + fail_after: Option, + fed: u64, + name: &'static str, + timings: Timings, +} + +impl MfDecoder { + /// A decoder for the stream, or why Media Foundation does not take it (unsupported format, no + /// DXVA decoder for it on the GPU, no Direct3D-aware decoder MFT). + pub fn new(info: NalStreamInfo) -> std::result::Result { + let (format, profile) = plan(&info)?; + let api = mft::api()?; + let gpu = Gpu::shared(api)?; + gpu.supports(profile, format, info.coded)?; + let (_, _, w, h) = info.crop; + let mft = Mft::new(api, &gpu, info.codec, (w, h), format)?; + log::info!("hardware decoding: {} on {}", mft.name(), gpu.name()); + let name = match info.codec { + NalCodec::H264 => "Media Foundation H.264", + NalCodec::Hevc => "Media Foundation HEVC", + }; + Ok(Self { + geometry: Geometry { crop: info.crop, bits: format.bits(), color: info.color, par: info.par }, + info, + format, + api, + gpu, + mft: Some(mft), + readback: Readback::default(), + in_flight: Vec::new(), + need_headers: true, + skip_rasl: false, + first: true, + drained: false, + fail_after: None, + fed: 0, + name, + timings: Timings::default(), + }) + } + + /// Test hook: from the `n`-th sample on, every decode fails as a lost device would. + pub fn fail_after(&mut self, n: u64) { + self.fail_after = Some(n); + } + + /// Where the time went so far. + pub fn timings(&self) -> Timings { + self.timings + } + + /// Name of the decoder MFT in use (diagnostics and tests). + pub fn mft_name(&self) -> &str { + self.mft.as_ref().map_or("", Mft::name) + } + + /// The submitted presentation time an output stamped `t` belongs to. + fn take_pts(&mut self, t: i64) -> i64 { + if let Some(i) = self.in_flight.iter().position(|&p| p == t) { + return self.in_flight.swap_remove(i); + } + // the MFT restamped it: pictures come out in presentation order, so the earliest + match self.in_flight.iter().enumerate().min_by_key(|(_, p)| **p).map(|(i, _)| i) { + Some(i) => self.in_flight.swap_remove(i), + None => t, + } + } + + /// Whether `sample` holds an IDR picture (H.264 IDR slice; HEVC IDR_W_RADL / IDR_N_LP). + fn is_idr(&self, sample: &[u8]) -> bool { + self.info.nal_types(sample).iter().any(|&t| match self.info.codec { + NalCodec::H264 => t == 5, + NalCodec::Hevc => matches!(t, 19 | 20), + }) + } + + /// Take every picture the decoder has ready (presentation order). + fn collect(&mut self, out: &mut Vec) -> std::result::Result<(), String> { + for _ in 0..MAX_OUTPUTS_PER_CALL { + let Some(mft) = self.mft.as_ref() else { return Ok(()) }; + match mft.poll()? { + Poll::NeedInput => return Ok(()), + Poll::FormatChanged => {} + Poll::Frame(sample, t) => { + let geometry = self.geometry; + let (t0, mut convert) = (Instant::now(), Duration::ZERO); + let frame = self.readback.read(&self.gpu, &sample, self.format, |b| { + let c0 = Instant::now(); + let f = biplanar::to_frame(b, &geometry); + convert = c0.elapsed(); + f + })?; + self.timings.frames += 1; + self.timings.convert += convert; + self.timings.readback += t0.elapsed().saturating_sub(convert); + let pts = self.take_pts(t); + out.push(DecodedFrame { pts, frame, draft: false }); + } + } + } + Err("the decoder keeps producing output".into()) + } + + fn feed(&mut self, annex_b: &[u8], pts: i64) -> std::result::Result, String> { + let mut out = Vec::new(); + for attempt in 0..2 { + let Some(mft) = self.mft.as_ref() else { return Err("no decoder".into()) }; + if mft.input(self.api, annex_b, pts)? { + self.in_flight.push(pts); + self.need_headers = false; + self.collect(&mut out)?; + // a decoder holds back a DPB's worth of pictures at most + if self.in_flight.len() > MAX_IN_FLIGHT { + return Err(format!("the decoder holds {} pictures back", self.in_flight.len())); + } + return Ok(out); + } + // full: take what it has ready, then offer the sample again + self.collect(&mut out)?; + if attempt == 1 { + break; + } + } + Err("the decoder does not accept input".into()) + } +} + +impl VideoDecoder for MfDecoder { + fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + self.fed += 1; + if self.fail_after.is_some_and(|n| self.fed > n) { + return Err(CodecError::Decode("Media Foundation decoder failed (test hook)".into())); + } + gpu::ensure_com().map_err(CodecError::Decode)?; + if self.drained { + // A drained MFT cannot go on from the middle of a GOP (software decoders can). The + // GOP cache seeks (`reset`) after every flush, so this is an error only for callers + // that continue after one: the hybrid decoder then replays the run in software. + if !self.is_idr(sample) { + return Err(CodecError::Decode("the hardware decoder was drained and restarts only at an IDR picture".into())); + } + self.drained = false; + self.first = true; + } + if self.info.codec == NalCodec::Hevc { + let types = self.info.nal_types(sample); + if types.iter().any(|t| (16..=23).contains(t)) { + // CRA / BLA starting a run: its RASL pictures reference pictures we never decoded. + self.skip_rasl = self.first && types.iter().any(|t| (16..=21).contains(t) && !(19..=20).contains(t)); + } else if self.skip_rasl && types.iter().any(|t| matches!(t, 8 | 9)) { + self.first = false; + return Ok(Vec::new()); + } + } + self.first = false; + let annex_b = to_annex_b(&self.info, sample, self.need_headers).map_err(CodecError::Decode)?; + if self.mft.is_none() { + let (_, _, w, h) = self.info.crop; + let mft = Mft::new(self.api, &self.gpu, self.info.codec, (w, h), self.format).map_err(CodecError::Decode)?; + self.mft = Some(mft); + self.need_headers = true; + } + let (t0, spent) = (Instant::now(), self.timings.readback + self.timings.convert); + let r = self.feed(&annex_b, pts).map_err(|e| { + self.gpu.note_failure(); + CodecError::Decode(e) + }); + let copying = (self.timings.readback + self.timings.convert).saturating_sub(spent); + self.timings.decode += t0.elapsed().saturating_sub(copying); + r + } + + fn flush(&mut self) -> Vec { + let mut out = Vec::new(); + if gpu::ensure_com().is_err() { + return out; + } + let drained = self.mft.as_ref().map(Mft::drain).transpose().and_then(|_| self.collect(&mut out)); + if let Err(e) = drained { + log::warn!("{}: {e} while flushing", self.name); + } + self.in_flight.clear(); + // what follows a drain starts like a new run: the decoder is given the headers again + self.need_headers = true; + self.drained = true; + out + } + + fn reset(&mut self) { + // after a seek no reference picture survives + if gpu::ensure_com().is_ok() && self.mft.as_ref().map(Mft::flush).transpose().is_err() { + self.mft = None; + } + self.in_flight.clear(); + self.need_headers = true; + self.skip_rasl = false; + self.first = true; + self.drained = false; + } + + fn name(&self) -> &str { + self.name + } + + fn is_random_access(&self, sample: &[u8]) -> Option { + self.info.is_random_access(sample) + } + + fn is_disposable(&self, sample: &[u8]) -> bool { + self.info.is_disposable(sample) + } +} + +impl Drop for MfDecoder { + fn drop(&mut self) { + // the decoder MFT is released on whichever thread drops the decoder: it needs COM + let _ = gpu::ensure_com(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(codec: NalCodec, profile_idc: u8, bits: u32, chroma_format_idc: u32) -> NalStreamInfo { + NalStreamInfo { + codec, + length_size: 4, + highest_tid: None, + parameter_sets: Vec::new(), + coded: (1920, 1088), + crop: (0, 0, 1920, 1080), + chroma_format_idc, + bit_depth_luma: bits, + bit_depth_chroma: bits, + interlaced: false, + profile_idc, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + reorder: 2, + } + } + + #[test] + fn takes_h264_baseline_main_high_and_hevc_main_main10() { + for p in [66, 77, 100] { + assert_eq!(plan(&info(NalCodec::H264, p, 8, 1)).unwrap().0, SurfaceFormat::Nv12, "H.264 profile {p}"); + } + assert_eq!(plan(&info(NalCodec::Hevc, 1, 8, 1)).unwrap(), (SurfaceFormat::Nv12, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN)); + assert_eq!(plan(&info(NalCodec::Hevc, 2, 10, 1)).unwrap(), (SurfaceFormat::P010, D3D11_DECODER_PROFILE_HEVC_VLD_MAIN10)); + } + + #[test] + fn declines_what_dxva_does_not_decode() { + // Hi10, High 4:2:2 / 4:4:4, Extended; 10-bit under an 8-bit profile; 4:2:2 / 4:4:4 / mono HEVC + for (codec, p, bits, chroma) in [ + (NalCodec::H264, 110, 10, 1), + (NalCodec::H264, 122, 10, 2), + (NalCodec::H264, 244, 8, 3), + (NalCodec::H264, 88, 8, 1), + (NalCodec::H264, 100, 10, 1), + (NalCodec::H264, 100, 8, 2), + (NalCodec::Hevc, 4, 10, 2), + (NalCodec::Hevc, 4, 12, 1), + (NalCodec::Hevc, 1, 10, 1), + (NalCodec::Hevc, 2, 8, 1), + (NalCodec::Hevc, 1, 8, 0), + (NalCodec::Hevc, 1, 8, 3), + ] { + assert!(plan(&info(codec, p, bits, chroma)).is_err(), "{codec:?} profile {p} {bits}-bit chroma {chroma}"); + } + let mut field_coded = info(NalCodec::H264, 100, 8, 1); + field_coded.interlaced = true; + assert!(plan(&field_coded).is_err()); + // luma and chroma depths that differ + let mut mixed = info(NalCodec::Hevc, 2, 10, 1); + mixed.bit_depth_chroma = 8; + assert!(plan(&mixed).is_err()); + } + + #[test] + fn declines_absurd_pictures() { + for crop in [(0, 0, 0, 1080), (0, 0, 1920, 0), (0, 0, 9000, 1080), (0, 0, 1080, 9000), (8, 0, 1920, 1080), (0, 16, 1920, 1080), (u32::MAX, 0, 16, 16)] { + let mut i = info(NalCodec::H264, 100, 8, 1); + i.crop = crop; + assert!(plan(&i).is_err(), "{crop:?}"); + } + } +} diff --git a/crates/platform/tests/media_foundation.rs b/crates/platform/tests/media_foundation.rs new file mode 100644 index 00000000..52d0b010 --- /dev/null +++ b/crates/platform/tests/media_foundation.rs @@ -0,0 +1,425 @@ +//! Media Foundation / Direct3D 11 decoding against our software decoders (Windows): every picture of H.264 High, +//! HEVC Main and HEVC Main 10 fixtures with B-frames must be identical (bit-exact planes, colour, +//! pixel aspect, pts and presentation order), also after `reset` + reseek and a mid-stream +//! `flush`; a forced mid-stream failure must continue in software with the software decoder's +//! exact output; damaged samples and parameter sets must give errors or fall back, never crash or +//! hang. Skips without ffmpeg (fixture generator) or without a hardware decoder. +#![cfg(target_os = "windows")] + +mod common; + +use std::time::Duration; + +use common::*; +use filmcraft_codecs::VideoDecoder; +use filmcraft_codecs::hw::NalStreamInfo; +use filmcraft_platform::HybridDecoder; +use filmcraft_platform::media_foundation::MfDecoder; + +/// The hardware decoder for a stream, or `None` (skip) when this machine has none for it. +fn hardware(s: &Stream) -> Option> { + match filmcraft_platform::media_foundation_factory(&s.entry) { + Some(Ok(d)) => { + assert!(d.name().starts_with("Media Foundation"), "{}", d.name()); + Some(d) + } + Some(Err(e)) => panic!("factory error: {e}"), + None => { + eprintln!("SKIPPED: no Media Foundation hardware decoder for {}", s.entry.codec.name()); + None + } + } +} + +fn software(s: &Stream) -> Box { + filmcraft_codecs::software_video_decoder(&s.entry).unwrap() +} + +/// Every picture of `s` from the hardware decoder equals the software decoder's: whole stream, +/// after `reset` + reseek to each later sync sample, after a mid-stream `flush`, and a full pass +/// after all the resets. +fn parity(name: &str, s: &Stream) { + let Some(mut hw) = hardware(s) else { return }; + let mut sw = software(s); + let a = decode_all(hw.as_mut(), &s.samples); + let b = decode_all(sw.as_mut(), &s.samples); + assert!(a.len() >= 40, "{name}: {} pictures", a.len()); + assert_same(name, &a, &b); + assert!(hw.name().starts_with("Media Foundation"), "{name}: stayed in hardware ({})", hw.name()); + + // reset + reseek to each later sync sample, decode a stretch, flush mid-stream + let syncs: Vec = (1..s.samples.len()).filter(|&i| s.sync[i]).collect(); + assert!(!syncs.is_empty(), "{name}: more than one GOP"); + for &k in syncs.iter().rev() { + let end = (k + 17).min(s.samples.len()); + for d in [&mut hw, &mut sw] { + d.reset(); + } + let a = decode_all(hw.as_mut(), &s.samples[k..end]); + let b = decode_all(sw.as_mut(), &s.samples[k..end]); + assert!(!a.is_empty(), "{name}: pictures after seeking to {k}"); + assert_same(&format!("{name} from sample {k}"), &a, &b); + } + // back to the start after seeking around: the whole stream again + hw.reset(); + sw.reset(); + assert_same(&format!("{name} after resets"), &decode_all(hw.as_mut(), &s.samples), &decode_all(sw.as_mut(), &s.samples)); + drained_decoder_restarts_only_at_idr(name, s); + // random-access / disposable answers match the software decoder's + for (smp, _) in &s.samples { + assert_eq!(hw.is_random_access(smp), sw.is_random_access(smp), "{name}"); + assert_eq!(hw.is_disposable(smp), sw.is_disposable(smp), "{name}"); + } +} + +/// After a `flush` (which drains the MFT) the run can only go on at an IDR picture; from the middle +/// of a GOP the decoder reports an error (the GOP cache seeks after every flush, and the hybrid +/// decoder would replay the run in software). After an IDR the pictures are the software decoder's. +fn drained_decoder_restarts_only_at_idr(name: &str, s: &Stream) { + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + let is_idr = |i: usize| { + info.nal_types(&s.samples[i].0).iter().any(|&t| if info.codec == filmcraft_codecs::hw::NalCodec::H264 { t == 5 } else { matches!(t, 19 | 20) }) + }; + let Ok(mut mf) = MfDecoder::new(info.clone()) else { return }; + let mid = s.samples.len() / 2; + let Some(idr) = (mid..s.samples.len()).find(|&i| s.sync[i] && is_idr(i)) else { return }; + let mut sw = software(s); + let before: Vec<_> = s.samples[..mid].iter().flat_map(|(d, p)| mf.decode(d, *p).unwrap()).collect(); + let drained = mf.flush(); + assert!(before.len() + drained.len() >= mid.saturating_sub(1), "{name}: flush returns the held pictures"); + if mid > 0 && !s.sync[mid] { + assert!(mf.decode(&s.samples[mid].0, s.samples[mid].1).is_err(), "{name}: continuing mid-GOP after a flush is an error"); + } + // flush, then an IDR: same pictures as software for the run that follows + mf.reset(); + let _ = decode_all(&mut mf, &s.samples[..idr.min(mid + 2)]); + let a = decode_all(&mut mf, &s.samples[idr..]); + let b = decode_all(sw.as_mut(), &s.samples[idr..]); + assert_same(&format!("{name} flush then IDR {idr}"), &a, &b); +} + +#[test] +fn bit_exact_with_the_software_decoders() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, _) in FIXTURES { + let Some(path) = named(&ff, name) else { continue }; + parity(name, &read_stream(&path)); + } +} + +/// 1920x1080 and 3840x2160 with B-frames and two GOPs: H.264 High, HEVC Main (open GOP) and +/// HEVC Main 10. +fn large_fixture(ff: &std::path::Path, codec: &str, size: &str) -> Option { + let (w, h) = size.split_once('x')?; + let src = format!("testsrc2=s={size}:r=24:d=2,noise=alls=12:allf=t"); + let (name, args): (String, Vec<&str>) = match codec { + "h264" => ( + format!("h264_{h}.mp4"), + vec![ + "-f", + "lavfi", + "-i", + &src, + "-c:v", + "libx264", + "-profile:v", + "high", + "-preset", + "veryfast", + "-x264-params", + "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0", + "-pix_fmt", + "yuv420p", + ], + ), + "hevc" => ( + format!("hevc_main_{h}.mp4"), + vec![ + "-f", + "lavfi", + "-i", + &src, + "-c:v", + "libx265", + "-preset", + "ultrafast", + "-tag:v", + "hvc1", + "-x265-params", + "log-level=error:bframes=4:keyint=24:min-keyint=24:scenecut=0:open-gop=1", + "-pix_fmt", + "yuv420p", + ], + ), + _ => ( + format!("hevc_main10_{h}.mp4"), + vec![ + "-f", + "lavfi", + "-i", + &src, + "-c:v", + "libx265", + "-preset", + "ultrafast", + "-profile:v", + "main10", + "-tag:v", + "hvc1", + "-x265-params", + "log-level=error:bframes=4:keyint=24:min-keyint=24:scenecut=0", + "-pix_fmt", + "yuv420p10le", + ], + ), + }; + let _ = w; + fixture(ff, &name, &args) +} + +fn large(codec: &str, size: &str) { + let ff = filmcraft_testkit::require_ffmpeg!(); + let Some(path) = large_fixture(&ff, codec, size) else { return }; + parity(&format!("{codec} {size}"), &read_stream(&path)); +} + +#[test] +fn h264_1080p_and_2160p_are_bit_exact() { + large("h264", "1920x1080"); + large("h264", "3840x2160"); +} + +#[test] +fn hevc_main_1080p_and_2160p_are_bit_exact() { + large("hevc", "1920x1080"); + large("hevc", "3840x2160"); +} + +#[test] +fn hevc_main10_1080p_and_2160p_are_bit_exact() { + large("hevc10", "1920x1080"); + large("hevc10", "3840x2160"); +} + +#[test] +fn mid_stream_failure_continues_in_software() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, _) in FIXTURES { + let Some(path) = named(&ff, name) else { continue }; + let s = read_stream(&path); + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + let Ok(_) = MfDecoder::new(info.clone()) else { + eprintln!("SKIPPED: no Media Foundation hardware decoder for {name}"); + continue; + }; + let reference = decode_all(software(&s).as_mut(), &s.samples); + // fail at the first sample, inside the first GOP, at a sync sample and just after one + let k = (1..s.samples.len()).find(|&i| s.sync[i]).unwrap(); + for fail_at in [0, 5, k, k + 1, k + 9] { + let mut vt = MfDecoder::new(info.clone()).unwrap(); + vt.fail_after(fail_at as u64); + let mut d = HybridDecoder::new(Box::new(vt), s.entry.clone(), info.clone()); + let before = filmcraft_codecs::hw::hw_stats().fallbacks; + let out = decode_all(&mut d, &s.samples); + assert!(!d.is_hardware(), "{name}: switched to software"); + assert!(filmcraft_codecs::hw::hw_stats().fallbacks > before, "{name}: fallback counted"); + assert_same(&format!("{name} failing at sample {fail_at}"), &out, &reference); + // the instance stays in software across seeks + d.reset(); + assert_same(&format!("{name} after the fallback and a reset"), &decode_all(&mut d, &s.samples), &reference); + } + } +} + +/// Run `f` on a thread; fail if it panics or takes longer than `limit`. +fn bounded(what: &str, limit: Duration, f: impl FnOnce() + Send + 'static) { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + let _ = tx.send(r.is_ok()); + }); + match rx.recv_timeout(limit) { + Ok(true) => {} + Ok(false) => panic!("{what}: panicked"), + Err(_) => panic!("{what}: hung"), + } +} + +#[test] +fn damaged_samples_and_parameter_sets_never_crash_or_hang() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, _) in FIXTURES { + let Some(path) = named(&ff, name) else { continue }; + let s = read_stream(&path); + if hardware(&s).is_none() { + continue; + } + // damaged samples (bit flips, truncation, corrupt length prefixes) through the hybrid + for seed in 1..=6u64 { + let samples = s.samples.clone(); + let entry = s.entry.clone(); + bounded(&format!("{name} seed {seed}"), Duration::from_secs(60), move || { + let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut d = filmcraft_platform::media_foundation_factory(&entry).unwrap().unwrap(); + for (i, (smp, pts)) in samples.iter().enumerate().take(40) { + let mut m = smp.clone(); + match xorshift(&mut seed) % 4 { + 0 => { + for _ in 0..1 + xorshift(&mut seed) % 8 { + let at = (xorshift(&mut seed) as usize) % m.len().max(1); + if let Some(b) = m.get_mut(at) { + *b ^= 1 << (xorshift(&mut seed) % 8); + } + } + } + 1 => m.truncate((xorshift(&mut seed) as usize) % m.len().max(1)), + 2 if m.len() > 4 => m[..4].copy_from_slice(&(xorshift(&mut seed) as u32).to_be_bytes()), + _ => {} + } + let _ = d.decode(&m, *pts); + if i % 13 == 12 { + d.reset(); + } + } + let _ = d.flush(); + }); + } + // damaged parameter sets: the factory declines or the decoder errors / falls back + let rec = match &s.entry.codec { + filmcraft_isobmff::CodecConfig::Avc(a) => a.to_bytes(), + filmcraft_isobmff::CodecConfig::Hevc(c) => c.to_bytes(), + _ => continue, + }; + let hevc = matches!(s.entry.codec, filmcraft_isobmff::CodecConfig::Hevc(_)); + let mut seed = 0x1234_5678u64; + for round in 0..24 { + let mut r = rec.clone(); + if round % 3 == 2 { + r.truncate(r.len() * (round + 1) / 30); + } else { + for _ in 0..1 + round % 4 { + let at = 6 + (xorshift(&mut seed) as usize) % r.len().saturating_sub(6).max(1); + if let Some(b) = r.get_mut(at) { + *b ^= 1 << (xorshift(&mut seed) % 8); + } + } + } + let samples: Vec<_> = s.samples.iter().take(12).cloned().collect(); + bounded(&format!("{name} record round {round}"), Duration::from_secs(60), move || { + let Some(info) = (if hevc { NalStreamInfo::from_hvcc(&r) } else { NalStreamInfo::from_avcc(&r) }).ok() else { return }; + let Ok(mut vt) = MfDecoder::new(info) else { return }; + for (smp, pts) in &samples { + if vt.decode(smp, *pts).is_err() { + break; + } + } + let _ = vt.flush(); + }); + } + } +} + +/// 4:2:2 (which our HEVC decoder does not decode): when Media Foundation takes it, its pictures must +/// match ffmpeg's decode exactly; when it does not, the factory declines. +#[test] +fn hevc_422_10bit_matches_ffmpeg_when_the_hardware_takes_it() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let args = [ + "-f", + "lavfi", + "-i", + "testsrc2=s=640x360:r=24:d=1,noise=alls=12:allf=t", + "-c:v", + "libx265", + "-preset", + "fast", + "-tag:v", + "hvc1", + "-x265-params", + "log-level=error:bframes=3:keyint=12:min-keyint=12:scenecut=0", + "-pix_fmt", + "yuv422p10le", + ]; + let Some(path) = fixture(&ff, "hevc_422_10.mp4", &args) else { return }; + let s = read_stream(&path); + let Some(mut hw) = hardware(&s) else { return }; + let out = decode_all(hw.as_mut(), &s.samples); + let o = + std::process::Command::new(&ff).args(["-v", "error", "-i", path.to_str().unwrap(), "-f", "rawvideo", "-pix_fmt", "yuv422p10le", "-"]).output().unwrap(); + let (w, h) = (640usize, 360usize); + let fsize = (w * h + 2 * (w / 2) * h) * 2; + assert_eq!(out.len(), o.stdout.len() / fsize, "frame count"); + for (i, f) in out.iter().enumerate() { + let raw: Vec = o.stdout[i * fsize..(i + 1) * fsize].as_chunks::<2>().0.iter().map(|b| u16::from_le_bytes([b[0], b[1]])).collect(); + let filmcraft_frame::PixelData::Yuv16 { planes, chroma, bits, .. } = &f.frame.data else { panic!("16-bit planes") }; + assert_eq!((*chroma, *bits), (filmcraft_frame::Chroma::C422, 10)); + let (y, c) = (w * h, (w / 2) * h); + assert!(planes[0][..] == raw[..y] && planes[1][..] == raw[y..y + c] && planes[2][..] == raw[y + c..], "frame {i} differs from ffmpeg"); + } +} + +/// Creating and dropping decoders again and again (every clip on a timeline makes one) neither +/// fails nor leaks Direct3D / Media Foundation objects into the next one. +#[test] +fn repeated_initialisation() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let Some(path) = named(&ff, "h264_high.mp4") else { return }; + let s = read_stream(&path); + let reference = decode_all(software(&s).as_mut(), &s.samples[..13]); + for round in 0..40 { + let Some(mut hw) = hardware(&s) else { return }; + if round % 8 == 0 { + assert_same(&format!("round {round}"), &decode_all(hw.as_mut(), &s.samples[..13]), &reference); + } + } +} + +/// Several decoders at once on different threads (the frame workers) share the device and give the +/// software decoder's pictures. +#[test] +fn concurrent_decoders_share_the_device() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let jobs: Vec<_> = ["h264_high.mp4", "hevc_main.mp4", "hevc_main10.mp4", "h264_high.mp4", "hevc_main.mp4"] + .into_iter() + .filter_map(|n| named(&ff, n)) + .map(|path| { + std::thread::spawn(move || { + let s = read_stream(&path); + let Some(mut hw) = hardware(&s) else { return }; + let n = s.samples.len().min(40); + let a = decode_all(hw.as_mut(), &s.samples[..n]); + hw.reset(); + let b = decode_all(hw.as_mut(), &s.samples[..n]); + let c = decode_all(software(&s).as_mut(), &s.samples[..n]); + assert_same("concurrent", &a, &c); + assert_same("concurrent, after a reset", &b, &c); + }) + }) + .collect(); + for j in jobs { + j.join().unwrap(); + } +} + +/// A decoder dropped, used and moved between threads (decoders migrate between frame workers). +#[test] +fn decoder_moves_between_threads() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let Some(path) = named(&ff, "hevc_main.mp4") else { return }; + let s = read_stream(&path); + let Some(mut hw) = hardware(&s) else { return }; + let reference = decode_all(software(&s).as_mut(), &s.samples[..30]); + let samples = s.samples.clone(); + let (hw, out) = std::thread::spawn(move || { + let out = decode_all(hw.as_mut(), &samples[..30]); + (hw, out) + }) + .join() + .unwrap(); + assert_same("first thread", &out, &reference); + let mut hw = hw; + hw.reset(); + let samples = s.samples.clone(); + let out = std::thread::spawn(move || decode_all(hw.as_mut(), &samples[..30])).join().unwrap(); + assert_same("second thread", &out, &reference); +} diff --git a/crates/platform/tests/setting.rs b/crates/platform/tests/setting.rs index 2636ef38..c177e55d 100644 --- a/crates/platform/tests/setting.rs +++ b/crates/platform/tests/setting.rs @@ -36,7 +36,10 @@ fn off_uses_the_software_decoder() { assert_eq!(d.name(), "FilmCraft HEVC"); return; } - assert_eq!(d.name(), "VideoToolbox HEVC"); + let (backend, hw_name) = if cfg!(target_os = "macos") { ("VideoToolbox", "VideoToolbox HEVC") } else { ("Media Foundation", "Media Foundation HEVC") }; + assert_eq!(d.name(), hw_name); + assert_eq!(filmcraft_codecs::hw::hw_backend(), Some(backend), "perf.stats reports the backend"); + assert!(filmcraft_platform::registered(), "register() put the factory in the registry"); let h1 = filmcraft_codecs::hw::hw_stats(); let src = filmcraft_codecs::open_bytes("hevc_main.mp4", bytes).unwrap(); for i in 0..30 { diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index ae2472c9..a5afd761 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -36,6 +36,9 @@ image = { version = "0.25", default-features = false, features = ["png"] } # Thread/process CPU clocks for frame-job profiling (safe wrappers; see frames.rs). [target.'cfg(unix)'.dependencies] rustix = { version = "1", default-features = false, features = ["std", "time"] } +# the same clocks on Windows (GetThreadTimes / GetProcessTimes behind a safe API; MIT OR Apache-2.0) +[target.'cfg(windows)'.dependencies] +cpu-time = "1" [dev-dependencies] pollster = "0.4" diff --git a/crates/ui-egui/examples/bench/fixtures.rs b/crates/ui-egui/examples/bench/fixtures.rs index c661f4a7..32b5b826 100644 --- a/crates/ui-egui/examples/bench/fixtures.rs +++ b/crates/ui-egui/examples/bench/fixtures.rs @@ -16,7 +16,9 @@ pub fn ffmpeg() -> Option { if let Some(p) = std::env::var_os("FILMCRAFT_FFMPEG") { return Some(PathBuf::from(p)); } - ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].iter().map(PathBuf::from).find(|p| p.exists()) + let fixed = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].iter().map(PathBuf::from).find(|p| p.exists()); + // otherwise whatever `ffmpeg` / `ffmpeg.exe` is on PATH (Windows has no fixed location) + fixed.or_else(|| std::env::split_paths(&std::env::var_os("PATH")?).map(|d| d.join(format!("ffmpeg{}", std::env::consts::EXE_SUFFIX))).find(|p| p.is_file())) } /// A fixture: file name, lavfi video source, seconds, with audio, video encoder arguments. @@ -32,6 +34,22 @@ const X264: &[&str] = &["-c:v", "libx264", "-preset", "fast", "-crf", "20", "-pi const X265: &[&str] = &["-c:v", "libx265", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p", "-tag:v", "hvc1", "-x265-params", "log-level=error"]; const VP9: &[&str] = &["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8", "-b:v", "0", "-crf", "32", "-row-mt", "1", "-pix_fmt", "yuv420p"]; const AV1: &[&str] = &["-c:v", "libsvtav1", "-preset", "10", "-crf", "35", "-pix_fmt", "yuv420p", "-svtav1-params", "keyint=48"]; +const X265_MAIN10: &[&str] = &[ + "-c:v", + "libx265", + "-preset", + "fast", + "-crf", + "22", + "-profile:v", + "main10", + "-pix_fmt", + "yuv420p10le", + "-tag:v", + "hvc1", + "-x265-params", + "log-level=error", +]; const PRORES: &[&str] = &["-c:v", "prores_ks", "-profile:v", "3", "-pix_fmt", "yuv422p10le", "-vendor", "apl0"]; const SPECS: &[Spec] = &[ @@ -49,6 +67,7 @@ const SPECS: &[Spec] = &[ Spec { name: "dec_h264_2160.mp4", src: "testsrc2=s=3840x2160:r=24000/1001:d=3,noise=alls=6:allf=t", secs: 3, audio: false, codec: X264 }, Spec { name: "dec_hevc_1080.mp4", src: "testsrc2=s=1920x1080:r=24000/1001:d=5,noise=alls=6:allf=t", secs: 5, audio: false, codec: X265 }, Spec { name: "dec_hevc_2160.mp4", src: "testsrc2=s=3840x2160:r=24000/1001:d=3,noise=alls=6:allf=t", secs: 3, audio: false, codec: X265 }, + Spec { name: "dec_hevc10_2160.mp4", src: "testsrc2=s=3840x2160:r=24000/1001:d=3,noise=alls=6:allf=t", secs: 3, audio: false, codec: X265_MAIN10 }, Spec { name: "dec_vp9_1080.webm", src: "testsrc2=s=1920x1080:r=24000/1001:d=5,noise=alls=6:allf=t", secs: 5, audio: false, codec: VP9 }, Spec { name: "dec_vp9_2160.webm", src: "testsrc2=s=3840x2160:r=24000/1001:d=3,noise=alls=6:allf=t", secs: 3, audio: false, codec: VP9 }, Spec { name: "dec_av1_1080.mp4", src: "testsrc2=s=1920x1080:r=24000/1001:d=5,noise=alls=6:allf=t", secs: 5, audio: false, codec: AV1 }, diff --git a/crates/ui-egui/examples/bench/main.rs b/crates/ui-egui/examples/bench/main.rs index e06af5bf..ffae87de 100644 --- a/crates/ui-egui/examples/bench/main.rs +++ b/crates/ui-egui/examples/bench/main.rs @@ -193,6 +193,7 @@ fn machine_info() -> Value { .ok() .and_then(|c| c.lines().find(|l| l.starts_with("model name")).and_then(|l| l.split(':').nth(1)).map(|s| s.trim().to_string())) }) + .or_else(|| sh("powershell", &["-NoProfile", "-Command", "(Get-CimInstance Win32_Processor | Select-Object -First 1).Name"])) .unwrap_or_else(|| "unknown CPU".into()); let mem_gb = sh("sysctl", &["-n", "hw.memsize"]).and_then(|m| m.parse::().ok()).map(|b| b as f64 / (1u64 << 30) as f64); let os = sh("uname", &["-sr"]).unwrap_or_default(); @@ -249,6 +250,9 @@ pub fn section_markdown(v: &Value) -> String { ("cpu_ms_per_frame", "CPU ms/frame"), ("first_frame_ms", "first frame ms"), ("realtime", "× real time"), + ("hw_frames", "hw frames"), + ("hw_sessions", "hw sessions"), + ("hw_fallbacks", "hw fallbacks"), ("load", "load"), ], 1, @@ -267,6 +271,9 @@ pub fn section_markdown(v: &Value) -> String { ("seeks", "seeks"), ("skipped", "skipped"), ("draft_frames", "draft"), + ("hw_frames", "hw frames"), + ("hw_sessions", "hw sessions"), + ("hw_fallbacks", "hw fallbacks"), ("load", "load"), ], 1, diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index bce9bd12..97ea0521 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -29,6 +29,7 @@ const DECODE: &[(&str, &str, &str)] = &[ ("dec_h264_2160.mp4", "H.264", "2160p"), ("dec_hevc_1080.mp4", "HEVC", "1080p"), ("dec_hevc_2160.mp4", "HEVC", "2160p"), + ("dec_hevc10_2160.mp4", "HEVC Main 10", "2160p"), ("dec_vp9_1080.webm", "VP9", "1080p"), ("dec_vp9_2160.webm", "VP9", "2160p"), ("dec_av1_1080.mp4", "AV1", "1080p"), @@ -52,6 +53,7 @@ pub fn decode(o: &Opts) -> Vec { let mut first = Vec::new(); let mut frames = 0; let mut rate = FrameRate::FPS_23_976; + let hw0 = filmcraft_codecs::hw::hw_stats(); for _ in 0..o.repeat { let src = match filmcraft_codecs::open_bytes(name, bytes.clone()) { Ok(s) => s, @@ -82,12 +84,16 @@ pub fn decode(o: &Opts) -> Vec { continue; } let best = fps.iter().cloned().fold(0.0, f64::max); + let hw = filmcraft_codecs::hw::hw_stats(); let row = json!({ "fixture": name, "codec": codec, "size": size, "frames": frames, "fps": best, "fps_min": fps.iter().cloned().fold(f64::MAX, f64::min), "fps_runs": fps, "cpu_ms_per_frame": median(&cpu), "first_frame_ms": median(&first), "realtime": best / rate.as_f64(), "decode_threads": rayon::current_num_threads(), "load": load_avg(), - "hw_frames": filmcraft_codecs::hw::hw_stats().frames, + // OS hardware decoder activity during this row: pictures, decoders created, mid-stream + // fallbacks, streams handed to software (all zero with --hw off) + "hw_frames": hw.frames - hw0.frames, "hw_sessions": hw.sessions - hw0.sessions, + "hw_fallbacks": hw.fallbacks - hw0.fallbacks, "hw_declined": hw.declined - hw0.declined, }); eprintln!("decode {name}: {best:.1} fps"); rows.push(row); @@ -191,6 +197,7 @@ pub fn playback(o: &Opts) -> Vec { bench.draft = draft; let mut display = Display::new(o.gpu); let path = if display.gpu_available() { "gpu" } else { "cpu" }; + let hw0 = filmcraft_codecs::hw::hw_stats(); let r = bench.play(&s, seq, &mut display, scale, Tick::ZERO, &format!("{scenario} {res} #{}", rep + 1)); let mut v = playback::to_json(&r, scenario, res, path); let frames = (r.shown + r.dropped).max(1) as f64; @@ -200,6 +207,11 @@ pub fn playback(o: &Opts) -> Vec { v["seeks"] = json!(r.gop.seeks); v["skipped"] = json!(r.gop.skipped); v["draft_frames"] = json!(r.gop.draft); + // OS hardware decoder activity during the run (zero with --hw off) + let hw = filmcraft_codecs::hw::hw_stats(); + v["hw_frames"] = json!(hw.frames - hw0.frames); + v["hw_sessions"] = json!(hw.sessions - hw0.sessions); + v["hw_fallbacks"] = json!(hw.fallbacks - hw0.fallbacks); v["load"] = json!(r.load); v["cores_needed"] = json!(r.process_cpu / frames * fps / 1000.0); eprintln!("playback {scenario} {res}: {}/{} shown/dropped (load {})", r.shown, r.dropped, r.load); diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index e58fec90..6e0ef1ef 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -174,7 +174,11 @@ pub fn thread_cpu_time() -> Option { let t = rustix::time::clock_gettime(rustix::time::ClockId::ThreadCPUTime); Some(Duration::new(t.tv_sec as u64, t.tv_nsec as u32)) } - #[cfg(not(all(unix, not(target_arch = "wasm32"))))] + #[cfg(windows)] + { + cpu_time::ThreadTime::try_now().ok().map(|t| t.as_duration()) + } + #[cfg(not(any(windows, all(unix, not(target_arch = "wasm32")))))] { None } @@ -187,7 +191,11 @@ pub fn process_cpu_time() -> Option { let t = rustix::time::clock_gettime(rustix::time::ClockId::ProcessCPUTime); Some(Duration::new(t.tv_sec as u64, t.tv_nsec as u32)) } - #[cfg(not(all(unix, not(target_arch = "wasm32"))))] + #[cfg(windows)] + { + cpu_time::ProcessTime::try_now().ok().map(|t| t.as_duration()) + } + #[cfg(not(any(windows, all(unix, not(target_arch = "wasm32")))))] { None } From 932cdb77f5495f630bebcdf82e7ad8f90ca2820d Mon Sep 17 00:00:00 2001 From: Davi Andrei <67552001+SaymonMaster14@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:59:57 -0300 Subject: [PATCH 2/2] Docs: Windows hardware decoding (platform README, ADR 0001, architecture, performance, roadmap) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MMorBHwct72udS2LiC3ZHp --- AGENTS.md | 2 +- README.md | 2 +- ROADMAP.md | 6 ++--- docs/adr/0001-platform-ffi.md | 19 ++++++++++---- docs/architecture.md | 20 ++++++++++++--- docs/performance.md | 47 +++++++++++++++++++++++++++++++++++ docs/testing.md | 4 ++- 7 files changed, 85 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5379e83f..8ab29b1f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ panic path, and fix a crash before building on top of it. The cross-app standard code alone (a literal that always parses): a single-item `#[allow(clippy::expect_used)]` with the reason. 3. **No `unsafe`** (`unsafe_code = "forbid"` for the workspace). The single exception is `crates/platform`, which needs `unsafe` to call the operating system's media APIs (hardware - video decoding through VideoToolbox; later encoding and other OSes) and does nothing else. It + video decoding through VideoToolbox and Media Foundation / Direct3D 11; later encoding and Linux) and does nothing else. It uses `unsafe_code = "deny"` with `#[allow(unsafe_code)]` only on its FFI modules, a `// SAFETY:` comment on every `unsafe` block, a safe `Result`-returning public API, and the pure-Rust decoder as the tested fallback ([ADR 0001](docs/adr/0001-platform-ffi.md)). diff --git a/README.md b/README.md index cba7e686..e0751d46 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ We track two numbers ([ROADMAP.md](ROADMAP.md#honest-assessment-2026-10-05)): The biggest gaps today: -- **Speed on big footage.** Hardware decoding works on macOS; Windows and Linux have no hardware path yet. Blend modes and the most common effects run on the GPU, but Lumetri, keys, export and encoding still run on the CPU ([#30](https://github.com/storytold/filmcraft/issues/30)). +- **Speed on big footage.** Hardware decoding works on macOS and Windows; Linux has no hardware path yet. Blend modes and the most common effects run on the GPU, but Lumetri, keys, export and encoding still run on the CPU ([#30](https://github.com/storytold/filmcraft/issues/30)). - **No plugins.** No VST3 / Audio Units or OpenFX hosting. - **Delivery codecs.** H.264 is our only delivery-codec export; no HEVC or AV1 export yet. - **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS. diff --git a/ROADMAP.md b/ROADMAP.md index 553a6fcf..8a393420 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ before choosing work. | **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. | | **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. | | **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. | -| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS only (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects, export and encoding are CPU. Linux and Windows have no hardware path. 8K and AV1 aren't real-time. The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | +| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264 and HEVC Main / Main 10, #30). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects, export and encoding are CPU. Linux has no hardware path. 8K and AV1 aren't real-time. The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | | **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. | | **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. | | **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. | @@ -48,7 +48,7 @@ before choosing work. In priority order. Agents should prefer this work over adding more checklist items. 1. **Hardware acceleration** (#30), the most visible gap to users: - - hardware decode on Linux (VA-API) and Windows (Media Foundation); + - hardware decode on Linux (VA-API); VP9 / AV1 on Windows; - zero-copy decoded frames into wgpu; - the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done); - GPU export, then hardware encode (H.264 / HEVC). @@ -154,7 +154,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Running now -- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Next: VA-API (Linux) and Media Foundation (Windows) decode, zero-copy upload, Lumetri on the GPU. +- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. - **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land. ## Log diff --git a/docs/adr/0001-platform-ffi.md b/docs/adr/0001-platform-ffi.md index 7720cc5c..f76beb9b 100644 --- a/docs/adr/0001-platform-ffi.md +++ b/docs/adr/0001-platform-ffi.md @@ -25,15 +25,21 @@ Containment rules: 1. The crate does not use `lints.workspace = true`. Its own `[lints]` table copies the workspace lints except `unsafe_code = "deny"` (not `forbid`), and adds - `clippy::undocumented_unsafe_blocks = "deny"`. Only the FFI modules (`videotoolbox`) carry - `#[allow(unsafe_code)]`; the rest of the crate (the fallback logic in `hybrid`) has no `unsafe`. + `clippy::undocumented_unsafe_blocks = "deny"`. Only the FFI modules (`videotoolbox`, and + `media_foundation::gpu` / `media_foundation::mft` on Windows) carry `#[allow(unsafe_code)]`; the + rest of the crate (the fallback logic in `hybrid`, the decoder logic in `media_foundation`, + `annexb`, `biplanar`) has no `unsafe`. 2. Every `unsafe` block has a `// SAFETY:` comment saying why it is sound. 3. The public API is safe: no `pub unsafe fn`, no raw pointers or FFI types in public signatures; failures are `Result`s. The crate keeps the never-crash `deny(clippy::unwrap_used, …)` attribute of every clean crate. 4. No panic crosses the FFI boundary: the VideoToolbox output callback runs under `catch_unwind` - and reports a panic as a failed frame. -5. The crate compiles on every target. OS bindings are target-specific dependencies; elsewhere + and reports a panic as a failed frame. The Windows backend has no callbacks into Rust (it drives + a synchronous decoder MFT with `ProcessInput` / `ProcessOutput`), so nothing can unwind into + COM. +5. The crate compiles on every target. OS bindings are target-specific dependencies (the + `objc2-*` crates on macOS, the official `windows` crate on Windows: MIT OR Apache-2.0, already + in the lockfile through wgpu / winit, with only the Win32 features the backend uses); elsewhere `register()` reports `Unavailable` and does nothing. It is not part of the wasm check (L5). ## The fallback guarantee @@ -44,7 +50,10 @@ Registering a hardware decoder never makes a file undecodable, and never changes decoding is Off, for formats the hardware path does not take (field-coded H.264, bit depths other than 8 / 10, 4:4:4, mismatched luma / chroma depth), and when the OS cannot create a hardware session for the stream (hardware decoding is *required*, so the OS's own software - decoder is never used instead of ours). + decoder is never used instead of ours). On Windows that means the decoder MFT must be + Direct3D-aware, the GPU's DXVA decoder must list the stream, and every picture must come back as + a Direct3D 11 texture: Microsoft's decoder MFTs quietly decode in software when DXVA is not + available, and a system-memory picture is treated as a failure. - A hardware decoder that fails mid-stream (decode error, session invalidated by a GPU change or sleep, in-band parameter sets that differ from the sample entry) becomes the software decoder for that stream without the caller noticing: `HybridDecoder` replays the samples since the last diff --git a/docs/architecture.md b/docs/architecture.md index 619e1c7d..98786203 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,7 +72,7 @@ and `filmcraft-cli`. | `engine` | L4 | `Session`, command registry, undo history, media pool, jobs, interchange glue | | `ui-egui` | L5 | the egui frontend: docking, panels, timeline, monitors, playback, control-channel handlers | | `automation` | L5 | MCP server (`rmcp`, stdio), headless or bridged to the running app | -| `platform` | L5 | OS media FFI only: hardware video decoding (VideoToolbox H.264 / HEVC on macOS; a no-op elsewhere) behind `codecs::VideoDecoder`, with transparent fallback to our decoders. The one crate allowed `unsafe` ([ADR 0001](adr/0001-platform-ffi.md), [README](../crates/platform/README.md)) | +| `platform` | L5 | OS media FFI only: hardware video decoding (VideoToolbox H.264 / HEVC on macOS; Media Foundation / Direct3D 11 H.264 / HEVC on Windows; a no-op elsewhere) behind `codecs::VideoDecoder`, with transparent fallback to our decoders. The one crate allowed `unsafe` ([ADR 0001](adr/0001-platform-ffi.md), [README](../crates/platform/README.md)) | | `filmcraft` | L6 | desktop binary: eframe/wgpu window, cpal audio output, file dialogs, native macOS menu, TCP control server | | `filmcraft-cli` | L6 | headless CLI: `exec`, `run`, `inspect`, `describe`, `commands`, `import`, `export`, `render`, `probe`, `mcp`; `--bridge` targets the running app | | `filmcraft-web` | L6 | the browser app (wasm32): eframe web runner on WebGPU/WebGL2, Blob-backed services, OPFS recovery, WebAudio, WebCodecs, `window.filmcraft` API ([web.md](web.md)) | @@ -200,7 +200,7 @@ pub struct CommandSpec { ```text file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware seek │ decoder registry: h264, hevc, vp9, av1, prores, dnx, mjpeg (+ any registered first: - │ platform's VideoToolbox H.264 / HEVC on macOS, falling back to ours) + │ platform's VideoToolbox (macOS) or Media Foundation (Windows) H.264 / HEVC, falling back to ours) ▼ media::MediaSource ──► frame cache (byte-budgeted LRU, shared) ▼ @@ -231,7 +231,8 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see hardware decoder can take precedence. The GOP cache never holds its lock while decoding. - **Hardware decoding.** The apps (desktop, CLI / headless MCP, bench) call `filmcraft_platform::register()` at startup, which on macOS registers a VideoToolbox factory for - `avcC` / `hvcC` streams (8 / 10-bit, 4:2:0 and 4:2:2; elsewhere it does nothing). The factory + `avcC` / `hvcC` streams (8 / 10-bit, 4:2:0 and 4:2:2; elsewhere it does nothing), and on Windows a + Media Foundation one (below). The factory declines (our decoder is used) when Settings ▸ Playback ▸ Hardware decoding is Off (`codecs::hw::set_hardware_decoding`, applied by the engine whenever preferences change), for formats it does not take, and when the OS cannot create a *hardware* session for the stream. @@ -246,7 +247,18 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see `codecs::software_video_decoder` (built-in factories only), replays the samples since the last restart point and continues in software for that instance; it is logged and counted. A source already open keeps its decoder when the setting changes, until it is reopened. `perf.stats` - `decode.hardware` reports hardware vs software frames, sessions, declines and fallbacks. + `decode.hardware` reports hardware vs software frames, sessions, declines and fallbacks, and the + registered `backend`. + On Windows the decoder (`platform::media_foundation`) is a Direct3D-aware decoder MFT driven at + the level of single access units (no Source Reader: the demuxer above already delivers the + samples): the `avcC` / `hvcC` sample becomes Annex B, the MFT is given the process's Direct3D 11 + video device through an `IMFDXGIDeviceManager` so it decodes with DXVA, and its NV12 (8-bit) / + P010 (10-bit) texture is read back through a staging texture (the one GPU to CPU copy) into + planar `Yuv8` / `Yuv16`. H.264 Baseline / Main / High and HEVC Main / Main 10, 4:2:0, progressive; + everything else, and any stream the GPU's DXVA decoder does not list, is declined. A decoder that + would hand back system-memory pictures (Microsoft's decoder MFTs do that when DXVA is not + available) fails the stream, so the hybrid continues with our decoder and Windows' software + decoding is never used in its place. Zero-copy (the texture straight to wgpu) is not done yet. Decoders run slices on rayon, so an export worker waiting inside a decode can pick up another frame of the same source. A request that finds the shared decoder busy decodes with a private decoder. diff --git a/docs/performance.md b/docs/performance.md index f284313b..4082248f 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -125,6 +125,53 @@ the draft plan's box decimation of the planes, cheap next to software decoding, CPU cost. Zero-copy upload of the decoded `CVPixelBuffer` into wgpu (no copy-out at all) is the next step (issue #30). +## Results (HW2: Windows Media Foundation / Direct3D 11 hardware decoding, Off → Auto) + +Same binary (release build of the HW2 commit), Settings ▸ Playback ▸ Hardware decoding switched +with `--hw off` / `--hw auto`, 2026-10-07: Intel Xeon E5-2680 v4 (14 cores / 28 threads, 2.4 GHz), +NVIDIA GeForce RTX 5060 8 GB (driver 617.14), Windows 11 Pro 26200, idle machine. Two alternating +rounds; the table gives the range of the two (decode: best of 3 within each). Reproduce with +`cargo xtask bench --sections decode --only dec_h264 --repeat 3 --hw off|auto` (likewise `dec_hevc`) +and `--sections playback --only "h264-2160 full" --hw off|auto`; set `FILMCRAFT_FFMPEG` if ffmpeg is not +on `PATH` for the fixture generator. Pictures are identical either way (bit-exact parity tests, +`crates/platform/tests/media_foundation.rs`). The decoder MFTs were Microsoft's H.264 decoder +(`Microsoft H264 Video Decoder MFT`) and the `HEVCVideoExtension` decoder, both decoding with DXVA +on the RTX 5060's NVDEC: `nvidia-smi dmon -s u` showed the decoder engine at 18–27 % during an +Auto run and 0 % otherwise, and every picture came back as a Direct3D 11 texture. + +### Decode (every frame through the media stack) + +| codec | size | CPU ms/frame Off → **Auto** | fps Off → **Auto** | +|---|---|---|---| +| H.264 | 1080p | 77–80 → **3.3** | 162–166 → **347–353** | +| H.264 | 2160p | 310–315 → **11.9** | 39–43 → **93** | +| HEVC Main | 1080p | 69–73 → **3.5** | 89–93 → **307–308** | +| HEVC Main | 2160p | 315–348 → **12–13** | 26–28 → **87–90** | +| HEVC Main 10 | 2160p | 314–336 → **17.6** | 32 → **57** | + +Counters: every Auto row had `hw frames` = frames × 3 repeats (360 at 1080p, 216 at 2160p), 3 +sessions, 0 fallbacks, 0 declined; every Off row had 0 hardware frames and 0 sessions. + +What is left on the CPU is the readback: `examples/mfprobe.rs --time` splits a 2160p H.264 picture +into 1.1 ms DXVA decode (feeding the MFT), 2.8 ms GPU to CPU copy (staging texture + map) and 7 ms +CPU conversion (Annex B, biplanar to planar Y'CbCr); Main 10 is 1.4 / 4.5 / 10 ms (twice the bytes). +A zero-copy path removes the last two. + +### Program-monitor playback (8 s, GPU path) + +| case | shown/dropped Off → **Auto** | CPU ms/frame Off → **Auto** | +|---|---|---| +| H.264 1080p Full | 192/0 → **192/0** | 86–89 → **8.5–8.8** | +| H.264 2160p Full | 192/0 → **189–190/2–3** | 338–353 → **17–20** | +| H.264 2160p 1/2 | 192/0 → **190/2** | 355–361 → **22** | +| HEVC 2160p Full | 112–152/40–80 → **191–192/0–1** | 355–364 → **18** | + +This machine has 28 software-decoding threads, so Off keeps up with 4K H.264 (at 340 ms of CPU per +frame, about 8 cores busy); Auto uses a twentieth of that CPU but drops 2–3 of 192 frames +(readback latency at the start of the run). 4K HEVC is where Off drops frames and Auto does not. +Draft playback (1/2 draft: 410–416 → 93–105 CPU ms/frame) is dominated by the draft plan's decimation, +as on macOS, because the hardware ignores draft mode. + ## Results (GPU1: blend modes on the GPU compositor, #30, before → after) Before = this change with the old whole-frame CPU fallback for non-Normal blend modes put back diff --git a/docs/testing.md b/docs/testing.md index 83dd97fe..56126ace 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -66,7 +66,9 @@ are never linked or shipped ([AGENTS.md](../AGENTS.md) §2). the matrix. - Fixture sources are synthetic: `testsrc2`, `mandelbrot`, SMPTE bars, noise and fades, sine tones. H.264 and HEVC fixtures need ffmpeg built with libx264 and libx265. VideoToolbox fixtures are - generated only on macOS. + generated only on macOS. The Media Foundation parity tests (`crates/platform/tests/media_foundation.rs`, + Windows only) make their 640x360, 1080p and 2160p H.264 / HEVC / Main 10 fixtures with ffmpeg and skip + without a Direct3D 11 video device or the HEVC Video Extensions. ### Pass criteria per codec