Media: footage audio at another sample rate no longer stutters (#274)

AAC audio in an MP4/MOV whose sample rate differs from the mix rate
(a 44.1 kHz clip in the 48 kHz preview or export, or the reverse)
stuttered and sounded low-pitched in previews and in exported movies.

Cause: FilmCraft's MP4 reader, asked for audio at a rate other than
the file's, applied the edit list's priming offset (1024 samples for
AAC) twice when it resampled: each read started about 1024 source
samples early and extrapolated a ramp over its first ~940 samples
(peaks near 100 against a 0.5 tone). Export and the preview mix ask
for audio a frame or a device buffer at a time, so the ramp repeated
dozens of times a second. Same-rate reads take FilmCraft's copy path
and were exact. FilmCraft fixed this upstream in 5bf2203, 87 commits
past the pinned revision.

Fix: MediaPool reads footage audio at the source's own rate and
resamples it itself (linear, by absolute sample position, so a range
comes out the same however it is split into reads), for every
container.

Test: resampled_audio_read_in_frame_pieces_matches_ffmpeg (44.1 kHz
AAC read at 48 kHz and 48 kHz at 44.1 kHz in 29.97 fps frame pieces,
against ffmpeg's decode: max error 44 before, 0.002 after) and
movies_carry_resampled_footage_audio (a 29.97 fps comp with 44.1 kHz
AAC footage exported to MP4/AAC and MOV/PCM, decoded by ffmpeg: max
error 8.7 before, 0.03 / 0.002 after).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/export/tests/roundtrip.rs

@@ -53,6 +53,11 @@ fn settings() -> RenderSettings {
 }
 
 fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report {
+    run_frames(p, cid, footage, om, path, FRAMES)
+}
+
+/// [`run`] for a comp of `frames` frames.
+fn run_frames(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path, frames: u64) -> effectcraft_export::Report {
     let s = settings();
     let path = path.to_string_lossy().to_string();
     let job = Job {
@@ -77,10 +82,10 @@ fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule,
         true
     })
     .expect("export");
-    assert_eq!(last.done, FRAMES);
-    assert_eq!(last.total, FRAMES);
+    assert_eq!(last.done, frames);
+    assert_eq!(last.total, frames);
     assert!(calls >= 2);
-    assert_eq!(r.frames, FRAMES);
+    assert_eq!(r.frames, frames);
     r
 }
 
@@ -400,6 +405,58 @@ fn movies_carry_audio() {
     }
 }
 
+/// `src`'s audio decoded by ffmpeg as interleaved stereo `f32` at 48 kHz (external oracle).
+fn ffmpeg_audio(src: &Path) -> Vec<f32> {
+    let out = Command::new("ffmpeg")
+        .args(["-hide_banner", "-loglevel", "error", "-i"])
+        .arg(src)
+        .args(["-f", "f32le", "-ac", "2", "-ar", "48000", "-"])
+        .output()
+        .expect("ffmpeg");
+    assert!(out.status.success(), "ffmpeg failed to decode {}", src.display());
+    out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
+}
+
+/// Movies of a 29.97 fps comp whose footage has AAC audio at 44.1 kHz sound like the footage
+/// (both decoded by ffmpeg): reading footage audio at another rate ramped from a wrong position
+/// at the start of every batch of frames, so exports stuttered (#274).
+#[test]
+fn movies_carry_resampled_footage_audio() {
+    let d = out_dir("audio-44k");
+    let clip = d.join("clip.mp4");
+    let made = Command::new("ffmpeg")
+        .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x48:rate=30000/1001:duration=3", "-f", "lavfi", "-i"])
+        .args(["aevalsrc=0.5*sin(2*PI*440*t)|0.5*sin(2*PI*(200+400*t/3)*t):s=44100:d=3", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest"])
+        .arg(&clip)
+        .status()
+        .is_ok_and(|s| s.success());
+    if !made {
+        eprintln!("ffmpeg not found: skipping");
+        return;
+    }
+    let fps = FrameRate::new(30000, 1001);
+    let mut p = Project::default();
+    let mut comp = Comp::new(W, H, fps, fps.tick_of(60));
+    let fid = p.add_item("clip.mp4", Label::SeaFoam, None, ItemKind::Footage(effectcraft_media::probe(&clip).expect("probe clip")));
+    let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (64, 48), None);
+    comp.layers.push(l);
+    let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into()));
+    let want = ffmpeg_audio(&clip);
+    let pool = MediaPool::new();
+    // AAC is lossy; PCM differs from ffmpeg's resampler by little more than 16-bit rounding.
+    for (fmt, name, tol) in [(OutputFormat::H264, "a.mp4", 0.06), (OutputFormat::ProRes, "a.mov", 0.01)] {
+        let path = d.join(name);
+        assert!(run_frames(&p, cid, &pool, &OutputModule::for_format(fmt), &path, 60).audio);
+        let got = ffmpeg_audio(&path);
+        // 60 frames at 29.97 fps: 96 096 samples at 48 kHz.
+        assert!((got.len() as i64 / 2 - 96_096).abs() < 1024, "{name}: {} samples", got.len() / 2);
+        // (Up to the last AAC frames, where the cut to silence smears back into the sound.)
+        let worst = got.iter().zip(&want).take(94_000 * 2).map(|(g, w)| (g - w).abs()).fold(0.0f32, f32::max);
+        eprintln!("{name}: max error {worst:.4}");
+        assert!(worst < tol, "{name}: max error {worst} against the footage");
+    }
+}
+
 #[test]
 fn webm_vp9_opus_roundtrip() {
     let d = out_dir("webm");

crates/media/src/pool.rs

@@ -378,31 +378,35 @@ impl MediaPool {
         self.decode_audio(footage, start, frames, rate)
     }
 
-    /// [`MediaPool::audio_samples`] straight from the decoder.
+    /// [`MediaPool::audio_samples`] straight from the decoder. The source is read at its own
+    /// rate and resampled here (linear, by absolute sample position, so a range comes out the
+    /// same however it is split into reads). FilmCraft's MP4 reader, asked for another rate,
+    /// applied the edit list's priming offset twice and ramped the start of every read: AAC
+    /// footage not at the mix rate stuttered in previews and exports (#274).
     fn decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec<f32> {
-        let mut out = vec![0.0; frames * 2];
         let path: Arc<str> = footage.path.as_str().into();
-        let Some(src) = self.inner.source(&path) else { return out };
+        let Some(src) = self.inner.source(&path) else { return vec![0.0; frames * 2] };
         let s0 = start.to_units_floor(rate as i64);
-        let skip = (-s0).max(0) as usize;
-        if skip >= frames {
-            return out;
+        let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate);
+        if native == 0 || native == rate {
+            return read_stereo(&src, &path, s0, frames, rate);
         }
-        let buf = match src.audio(s0.max(0), frames - skip, rate) {
-            Ok(b) => b,
-            Err(e) => {
-                log::warn!("media: audio of {path}: {e}");
-                return out;
+        // Output sample `n` sits at source position n × native / rate.
+        let (native_i, rate_i) = (i128::from(native), i128::from(rate));
+        let first = (i128::from(s0) * native_i).div_euclid(rate_i);
+        let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1;
+        let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return vec![0.0; frames * 2] };
+        let buf = read_stereo(&src, &path, first64, count, native);
+        let mut out = Vec::with_capacity(frames * 2);
+        for k in 0..frames as i128 {
+            let pos = (i128::from(s0) + k) * native_i;
+            let i = usize::try_from(pos.div_euclid(rate_i) - first).unwrap_or(0) * 2;
+            let f = (pos.rem_euclid(rate_i) as f64 / rate as f64) as f32;
+            for c in 0..2 {
+                let a = buf.get(i + c).copied().unwrap_or(0.0);
+                let b = buf.get(i + 2 + c).copied().unwrap_or(a);
+                out.push(a + (b - a) * f);
             }
-        };
-        let chans = buf.channels.len();
-        if chans == 0 {
-            return out;
-        }
-        let (l, r) = (&buf.channels[0], &buf.channels[if chans > 1 { 1 } else { 0 }]);
-        for i in 0..(frames - skip).min(l.len()).min(r.len()) {
-            out[(skip + i) * 2] = l[i];
-            out[(skip + i) * 2 + 1] = r[i];
         }
         out
     }
@@ -663,6 +667,30 @@ impl FootageSource for MediaPool {
     }
 }
 
+/// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which
+/// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both
+/// channels; silence before the start and where decoding fails.
+fn read_stereo(src: &SharedSource, path: &str, s0: i64, frames: usize, rate: u32) -> Vec<f32> {
+    let mut out = vec![0.0; frames * 2];
+    let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0);
+    if skip >= frames {
+        return out;
+    }
+    let buf = match src.audio(s0.max(0), frames - skip, rate) {
+        Ok(b) => b,
+        Err(e) => {
+            log::warn!("media: audio of {path}: {e}");
+            return out;
+        }
+    };
+    let (Some(l), Some(r)) = (buf.channels.first(), buf.channels.get(1).or(buf.channels.first())) else { return out };
+    for (o, (a, b)) in out[skip * 2..].chunks_exact_mut(2).zip(l.iter().zip(r)) {
+        o[0] = *a;
+        o[1] = *b;
+    }
+    out
+}
+
 fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
     m.lock().unwrap_or_else(|e| e.into_inner())
 }

crates/media/tests/footage.rs

@@ -251,6 +251,68 @@ fn h264_audio_samples() {
     assert!(pre.iter().all(|v| *v == 0.0));
 }
 
+/// H.264 + AAC at 44.1 kHz in MP4 (ffmpeg writes an edit list that skips the AAC priming):
+/// 440 Hz left, a 200 → 1000 Hz sweep right, 29.97 fps.
+fn aac_44k_mp4() -> Option<PathBuf> {
+    ffmpeg_fixture(
+        "aac_44k.mp4",
+        &[
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=64x48:rate=30000/1001:duration=3",
+            "-f",
+            "lavfi",
+            "-i",
+            "aevalsrc=0.5*sin(2*PI*440*t)|0.5*sin(2*PI*(200+400*t/3)*t):s=44100:d=3",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-shortest",
+        ],
+    )
+}
+
+/// `src`'s audio decoded by ffmpeg as interleaved stereo `f32` at `rate` Hz (external oracle).
+fn ffmpeg_audio(src: &Path, rate: u32) -> Vec<f32> {
+    let out = Command::new("ffmpeg")
+        .args(["-hide_banner", "-loglevel", "error", "-i"])
+        .arg(src)
+        .args(["-f", "f32le", "-ac", "2", "-ar", &rate.to_string(), "-"])
+        .output()
+        .expect("ffmpeg");
+    assert!(out.status.success(), "ffmpeg failed to decode {}", src.display());
+    out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
+}
+
+/// Footage audio at a rate other than the file's, read in one-frame pieces as export and the
+/// preview mix do, matches ffmpeg's decode: a resampled MP4 read ramped from a wrong position
+/// at the start of every piece (peaks near 100 against a 0.5 tone), so AAC footage stuttered in
+/// previews and exports (#274).
+#[test]
+fn resampled_audio_read_in_frame_pieces_matches_ffmpeg() {
+    let (Some(a), Some(b)) = (aac_44k_mp4(), h264_mp4()) else { return };
+    // 44.1 kHz read at 48 kHz, 48 kHz read at 44.1 kHz; NTSC frames are fractional in samples.
+    for (p, rate) in [(a, 48_000u32), (b, 44_100)] {
+        let f = probe(&p).expect("probe");
+        let want = ffmpeg_audio(&p, rate);
+        let fps = FrameRate::new(30000, 1001);
+        let pool = MediaPool::new();
+        let (mut got, mut cursor) = (Vec::new(), 0i64);
+        for k in 1..=55 {
+            let end = fps.tick_of(k).to_units_floor(rate as i64);
+            got.extend(pool.audio_samples(&f, Tick::from_units(cursor, rate as i64), (end - cursor) as usize, rate));
+            cursor = end;
+        }
+        let worst = got.iter().zip(&want).map(|(g, w)| (g - w).abs()).fold(0.0f32, f32::max);
+        eprintln!("{} at {rate} Hz: max error {worst:.4}", p.display());
+        assert!(want.len() >= got.len() && worst < 0.01, "{} at {rate} Hz: max error {worst}", p.display());
+    }
+}
+
 #[test]
 fn prores_4444_with_alpha() {
     let Some(p) = prores_mov() else { return };