diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index cf533031..7b5964e4 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -53,6 +53,11 @@ fn settings() -> RenderSettings { } fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { + run_frames(p, cid, footage, om, path, FRAMES) +} + +/// [`run`] for a comp of `frames` frames. +fn run_frames(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path, frames: u64) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); let job = Job { @@ -77,10 +82,10 @@ fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, true }) .expect("export"); - assert_eq!(last.done, FRAMES); - assert_eq!(last.total, FRAMES); + assert_eq!(last.done, frames); + assert_eq!(last.total, frames); assert!(calls >= 2); - assert_eq!(r.frames, FRAMES); + assert_eq!(r.frames, frames); r } @@ -400,6 +405,58 @@ fn movies_carry_audio() { } } +/// `src`'s audio decoded by ffmpeg as interleaved stereo `f32` at 48 kHz (external oracle). +fn ffmpeg_audio(src: &Path) -> Vec { + let out = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(src) + .args(["-f", "f32le", "-ac", "2", "-ar", "48000", "-"]) + .output() + .expect("ffmpeg"); + assert!(out.status.success(), "ffmpeg failed to decode {}", src.display()); + out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +/// Movies of a 29.97 fps comp whose footage has AAC audio at 44.1 kHz sound like the footage +/// (both decoded by ffmpeg): reading footage audio at another rate ramped from a wrong position +/// at the start of every batch of frames, so exports stuttered (#274). +#[test] +fn movies_carry_resampled_footage_audio() { + let d = out_dir("audio-44k"); + let clip = d.join("clip.mp4"); + let made = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x48:rate=30000/1001:duration=3", "-f", "lavfi", "-i"]) + .args(["aevalsrc=0.5*sin(2*PI*440*t)|0.5*sin(2*PI*(200+400*t/3)*t):s=44100:d=3", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest"]) + .arg(&clip) + .status() + .is_ok_and(|s| s.success()); + if !made { + eprintln!("ffmpeg not found: skipping"); + return; + } + let fps = FrameRate::new(30000, 1001); + let mut p = Project::default(); + let mut comp = Comp::new(W, H, fps, fps.tick_of(60)); + let fid = p.add_item("clip.mp4", Label::SeaFoam, None, ItemKind::Footage(effectcraft_media::probe(&clip).expect("probe clip"))); + let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (64, 48), None); + comp.layers.push(l); + let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let want = ffmpeg_audio(&clip); + let pool = MediaPool::new(); + // AAC is lossy; PCM differs from ffmpeg's resampler by little more than 16-bit rounding. + for (fmt, name, tol) in [(OutputFormat::H264, "a.mp4", 0.06), (OutputFormat::ProRes, "a.mov", 0.01)] { + let path = d.join(name); + assert!(run_frames(&p, cid, &pool, &OutputModule::for_format(fmt), &path, 60).audio); + let got = ffmpeg_audio(&path); + // 60 frames at 29.97 fps: 96 096 samples at 48 kHz. + assert!((got.len() as i64 / 2 - 96_096).abs() < 1024, "{name}: {} samples", got.len() / 2); + // (Up to the last AAC frames, where the cut to silence smears back into the sound.) + let worst = got.iter().zip(&want).take(94_000 * 2).map(|(g, w)| (g - w).abs()).fold(0.0f32, f32::max); + eprintln!("{name}: max error {worst:.4}"); + assert!(worst < tol, "{name}: max error {worst} against the footage"); + } +} + #[test] fn webm_vp9_opus_roundtrip() { let d = out_dir("webm"); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 46f379ea..e9cf4bc0 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -378,31 +378,35 @@ impl MediaPool { self.decode_audio(footage, start, frames, rate) } - /// [`MediaPool::audio_samples`] straight from the decoder. + /// [`MediaPool::audio_samples`] straight from the decoder. The source is read at its own + /// rate and resampled here (linear, by absolute sample position, so a range comes out the + /// same however it is split into reads). FilmCraft's MP4 reader, asked for another rate, + /// applied the edit list's priming offset twice and ramped the start of every read: AAC + /// footage not at the mix rate stuttered in previews and exports (#274). fn decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec { - let mut out = vec![0.0; frames * 2]; let path: Arc = footage.path.as_str().into(); - let Some(src) = self.inner.source(&path) else { return out }; + let Some(src) = self.inner.source(&path) else { return vec![0.0; frames * 2] }; let s0 = start.to_units_floor(rate as i64); - let skip = (-s0).max(0) as usize; - if skip >= frames { - return out; + let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate); + if native == 0 || native == rate { + return read_stereo(&src, &path, s0, frames, rate); } - let buf = match src.audio(s0.max(0), frames - skip, rate) { - Ok(b) => b, - Err(e) => { - log::warn!("media: audio of {path}: {e}"); - return out; + // Output sample `n` sits at source position n × native / rate. + let (native_i, rate_i) = (i128::from(native), i128::from(rate)); + let first = (i128::from(s0) * native_i).div_euclid(rate_i); + let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; + let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return vec![0.0; frames * 2] }; + let buf = read_stereo(&src, &path, first64, count, native); + let mut out = Vec::with_capacity(frames * 2); + for k in 0..frames as i128 { + let pos = (i128::from(s0) + k) * native_i; + let i = usize::try_from(pos.div_euclid(rate_i) - first).unwrap_or(0) * 2; + let f = (pos.rem_euclid(rate_i) as f64 / rate as f64) as f32; + for c in 0..2 { + let a = buf.get(i + c).copied().unwrap_or(0.0); + let b = buf.get(i + 2 + c).copied().unwrap_or(a); + out.push(a + (b - a) * f); } - }; - let chans = buf.channels.len(); - if chans == 0 { - return out; - } - let (l, r) = (&buf.channels[0], &buf.channels[if chans > 1 { 1 } else { 0 }]); - for i in 0..(frames - skip).min(l.len()).min(r.len()) { - out[(skip + i) * 2] = l[i]; - out[(skip + i) * 2 + 1] = r[i]; } out } @@ -663,6 +667,30 @@ impl FootageSource for MediaPool { } } +/// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which +/// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both +/// channels; silence before the start and where decoding fails. +fn read_stereo(src: &SharedSource, path: &str, s0: i64, frames: usize, rate: u32) -> Vec { + let mut out = vec![0.0; frames * 2]; + let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); + if skip >= frames { + return out; + } + let buf = match src.audio(s0.max(0), frames - skip, rate) { + Ok(b) => b, + Err(e) => { + log::warn!("media: audio of {path}: {e}"); + return out; + } + }; + let (Some(l), Some(r)) = (buf.channels.first(), buf.channels.get(1).or(buf.channels.first())) else { return out }; + for (o, (a, b)) in out[skip * 2..].chunks_exact_mut(2).zip(l.iter().zip(r)) { + o[0] = *a; + o[1] = *b; + } + out +} + fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(|e| e.into_inner()) } diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index 38fa7cc7..e7819610 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -251,6 +251,68 @@ fn h264_audio_samples() { assert!(pre.iter().all(|v| *v == 0.0)); } +/// H.264 + AAC at 44.1 kHz in MP4 (ffmpeg writes an edit list that skips the AAC priming): +/// 440 Hz left, a 200 → 1000 Hz sweep right, 29.97 fps. +fn aac_44k_mp4() -> Option { + ffmpeg_fixture( + "aac_44k.mp4", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=size=64x48:rate=30000/1001:duration=3", + "-f", + "lavfi", + "-i", + "aevalsrc=0.5*sin(2*PI*440*t)|0.5*sin(2*PI*(200+400*t/3)*t):s=44100:d=3", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + ], + ) +} + +/// `src`'s audio decoded by ffmpeg as interleaved stereo `f32` at `rate` Hz (external oracle). +fn ffmpeg_audio(src: &Path, rate: u32) -> Vec { + let out = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(src) + .args(["-f", "f32le", "-ac", "2", "-ar", &rate.to_string(), "-"]) + .output() + .expect("ffmpeg"); + assert!(out.status.success(), "ffmpeg failed to decode {}", src.display()); + out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +/// Footage audio at a rate other than the file's, read in one-frame pieces as export and the +/// preview mix do, matches ffmpeg's decode: a resampled MP4 read ramped from a wrong position +/// at the start of every piece (peaks near 100 against a 0.5 tone), so AAC footage stuttered in +/// previews and exports (#274). +#[test] +fn resampled_audio_read_in_frame_pieces_matches_ffmpeg() { + let (Some(a), Some(b)) = (aac_44k_mp4(), h264_mp4()) else { return }; + // 44.1 kHz read at 48 kHz, 48 kHz read at 44.1 kHz; NTSC frames are fractional in samples. + for (p, rate) in [(a, 48_000u32), (b, 44_100)] { + let f = probe(&p).expect("probe"); + let want = ffmpeg_audio(&p, rate); + let fps = FrameRate::new(30000, 1001); + let pool = MediaPool::new(); + let (mut got, mut cursor) = (Vec::new(), 0i64); + for k in 1..=55 { + let end = fps.tick_of(k).to_units_floor(rate as i64); + got.extend(pool.audio_samples(&f, Tick::from_units(cursor, rate as i64), (end - cursor) as usize, rate)); + cursor = end; + } + let worst = got.iter().zip(&want).map(|(g, w)| (g - w).abs()).fold(0.0f32, f32::max); + eprintln!("{} at {rate} Hz: max error {worst:.4}", p.display()); + assert!(want.len() >= got.len() && worst < 0.01, "{} at {rate} Hz: max error {worst}", p.display()); + } +} + #[test] fn prores_4444_with_alpha() { let Some(p) = prores_mov() else { return };