Merge pull request #1146 from haranishi/fix/959-shuttle-maintain-pitch

fix(audio): シャトル再生で音程維持設定を反映 (#959)

crates/ui-egui/src/lib.rs

@@ -51,6 +51,7 @@ pub mod perf;
 pub mod play_ahead;
 mod rtl;
 pub mod scrub;
+mod shuttle;
 pub mod source_playback;
 pub mod state;
 pub mod theme;
@@ -697,21 +698,24 @@ impl FilmcraftApp {
         }
         self.frames.set_cache_budget(p.memory.frame_cache_mb as usize * (1 << 20));
         self.ui.play_after_render = p.timeline.play_after_rendering;
-        if prev.as_ref().is_none_or(|q| q.audio_hardware != p.audio_hardware) {
+        let hardware_changed = prev.as_ref().is_none_or(|q| q.audio_hardware != p.audio_hardware);
+        let pitch_changed = prev.as_ref().is_some_and(|q| q.audio.maintain_pitch_shuttling != p.audio.maintain_pitch_shuttling);
+        if hardware_changed {
             if let Some(input) = self.session.voiceover.input.as_mut() {
                 input.configure_host(&p.audio_hardware.device_class);
             }
             let rate = self.session.active_sequence().map(|q| q.settings.sample_rate);
-            let playing = self.playback.playing && self.playback.preroll.is_none();
             if let Some(a) = self.audio.as_mut() {
                 a.stop();
                 a.configure(&p.audio_hardware, rate);
             }
-            if playing {
-                self.playback.anchor_tick = self.session.playhead();
-                self.playback.anchor_time = ctx.input(|i| i.time);
-                self.start_audio();
-            }
+        }
+        if (hardware_changed || pitch_changed) && self.playback.playing && self.playback.preroll.is_none() {
+            self.playback.anchor_tick = self.session.playhead();
+            let now = ctx.input(|i| i.time);
+            self.playback.anchor_time = now;
+            self.playback.audio_seen = (0, now);
+            self.start_audio();
         }
         if let Some(note) = self.audio.as_ref().and_then(|a| a.note()) {
             self.ui.status = note;
@@ -995,15 +999,13 @@ impl FilmcraftApp {
         let start_tick = self.session.playhead();
         let Some(a) = self.audio.as_mut() else { return };
         let sr = a.sample_rate();
-        // Mix at a lower effective rate for forward shuttle playback, then hand the
-        // samples to the device at its real rate. This keeps sound on the same
-        // timeline as the accelerated picture (with the expected pitch shift).
-        let Some(mix_rate) = shuttle_mix_rate(sr, self.playback.speed) else {
+        // Output frames still follow the accelerated timeline. The pitch preference selects
+        // either the original rate conversion or pitch-corrected program audio.
+        let Some((mix_rate, mix)) = shuttle::mix(&self.session, seq_id, sr, self.playback.speed) else {
             a.stop();
             return;
         };
         let cursor = start_tick.to_units_floor(i64::from(mix_rate));
-        let mix = playback_mix(&self.session, seq_id, mix_rate);
         // the old stream stops before the new mixer resets the underrun counters
         a.stop();
         // Desktop: mix ahead on a thread so the device callback never waits on decoding.
@@ -2248,6 +2250,115 @@ mod audio_recovery_tests {
         app.stop();
     }
 
+    /// Capture the block primed by the real shuttle playback path, without relying on a
+    /// physical device or consuming the producer faster than a device would.
+    #[test]
+    fn maintain_pitch_shuttling_controls_the_samples_sent_to_the_device() {
+        use std::sync::Mutex;
+        struct Capture(Arc<Mutex<Vec<f32>>>);
+        impl AudioOut for Capture {
+            fn start(&mut self, mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
+                let mut buf = vec![0.0; 2048];
+                fill(&mut buf, 2);
+                *self.0.lock().unwrap() = buf;
+                Ok(48_000)
+            }
+            fn stop(&mut self) {}
+            fn sample_rate(&self) -> u32 {
+                48_000
+            }
+            fn played_frames(&self) -> Option<u64> {
+                Some(4_800)
+            }
+        }
+        let amplitude = |samples: &[f32], ch: usize, hz: f64| {
+            let (mut re, mut im) = (0.0, 0.0);
+            for (i, frame) in samples.as_chunks::<2>().0.iter().enumerate() {
+                let phase = std::f64::consts::TAU * hz * i as f64 / 48_000.0;
+                re += f64::from(frame[ch]) * phase.cos();
+                im += f64::from(frame[ch]) * phase.sin();
+            }
+            2.0 * re.hypot(im) / (samples.len() / 2) as f64
+        };
+        let mut session = Session::default();
+        session.execute("file.newSequence", json!({"width":16,"height":16})).unwrap();
+        let inter: Vec<f32> =
+            (0..48_000 * 8).flat_map(|i| [440.0, 660.0].map(|hz| (0.2 * (std::f64::consts::TAU * hz * i as f64 / 48_000.0).sin()) as f32)).collect();
+        let bytes: Arc<[u8]> = filmcraft_engine::previews::write_wav_f32(&inter, 48_000).into();
+        let item = filmcraft_engine::commands::import_bytes(&mut session, "/shuttle-tone.wav", bytes, None).unwrap();
+        session.execute("timeline.place", json!({"item":item.0,"audioTrack":"A1","seconds":0.0})).unwrap();
+        let mut app = FilmcraftApp::new(session);
+        let samples = Arc::new(Mutex::new(Vec::new()));
+        app.audio = Some(Box::new(Capture(samples.clone())));
+        let mut normal = None;
+        for maintain in [true, false] {
+            app.session.prefs.audio.maintain_pitch_shuttling = maintain;
+            for speed in [0.5, 1.0, 1.5, 2.0, 4.0] {
+                app.session.set_playhead(Tick::from_seconds_f64(1.0));
+                let expected = app.session.sequence_rate().snap(app.session.playhead() + Tick::from_seconds_f64(0.1 * speed));
+                app.play(speed);
+                app.end_preroll(0.0);
+                assert!(app.playback.audio_clock);
+                let buf = samples.lock().unwrap();
+                for (ch, tone) in [440.0, 660.0].into_iter().enumerate() {
+                    let hz = if maintain { tone } else { tone * speed };
+                    let level = amplitude(&buf, ch, hz);
+                    assert!(level > 0.08, "maintain={maintain}, speed={speed}, channel={ch}: {hz} Hz level={level}");
+                    if speed != 1.0 {
+                        let wrong = if maintain { tone * speed } else { tone };
+                        assert!(amplitude(&buf, ch, wrong) < level * 0.3, "wrong pitch {wrong} Hz at {speed}x, maintain={maintain}");
+                    }
+                }
+                if speed == 1.0 {
+                    if let Some(previous) = &normal {
+                        assert_eq!(&*buf, previous, "1x must bypass pitch processing");
+                    } else {
+                        normal = Some(buf.clone());
+                    }
+                }
+                drop(buf);
+                app.advance_playback(&egui::Context::default());
+                assert_eq!(app.session.playhead(), expected, "the device clock must advance the timeline at {speed}x, snapped to a video frame");
+                app.stop();
+            }
+        }
+    }
+
+    #[test]
+    fn changing_pitch_preference_restarts_audio_at_the_current_playhead() {
+        let mut session = Session::default();
+        session.execute("file.newSequence", json!({"width":16,"height":16})).unwrap();
+        let ctx = egui::Context::default();
+        let mut app = FilmcraftApp::new(session);
+        let starts = Arc::new(AtomicUsize::new(0));
+        app.audio = Some(Box::new(Device { ready: true, starts: starts.clone(), reading: Some(0) }));
+        app.session.prefs.audio_hardware.default_output = "available".into();
+        app.apply_prefs(&ctx);
+        app.play(2.0);
+        app.end_preroll(0.0);
+        assert_eq!(starts.load(Ordering::Relaxed), 1);
+        app.session.set_playhead(Tick::from_seconds_f64(2.0));
+        let position = app.session.playhead();
+        for (maintain, expected_starts) in [(false, 2), (true, 3)] {
+            app.session.execute("prefs.set", json!({"key":"audio.maintainPitchShuttling","value":maintain})).unwrap();
+            app.apply_prefs(&ctx);
+            assert_eq!(starts.load(Ordering::Relaxed), expected_starts);
+            assert!(app.playback.audio_clock);
+            assert_eq!(app.playback.speed, 2.0);
+            assert_eq!(app.playback.anchor_tick, app.session.playhead());
+            assert_eq!(app.session.playhead(), position);
+            app.apply_prefs(&ctx);
+            assert_eq!(starts.load(Ordering::Relaxed), expected_starts, "unchanged settings do not restart");
+        }
+        for speed in [-1.0, 8.0] {
+            app.play(speed);
+            app.end_preroll(0.0);
+            assert!(!app.playback.audio_clock, "unsupported speed {speed} remains silent");
+        }
+        assert_eq!(starts.load(Ordering::Relaxed), 3);
+        app.stop();
+    }
+
     #[test]
     fn changing_hardware_recovers_wall_clock_playback_without_rewinding() {
         let mut session = Session::default();

crates/ui-egui/src/shuttle.rs

@@ -0,0 +1,257 @@
+//! Pitch-preserving shuttle playback. Pitch is corrected before changing the sample rate,
+//! so faster playback does not first discard the upper part of the audible spectrum.
+
+use filmcraft_audio_dsp::{AudioEffect, effects::PitchShifter, resample::StreamResampler};
+use filmcraft_engine::Session;
+use filmcraft_project::ItemId;
+
+type Mixer = Box<dyn FnMut(i64, &mut [f32], usize) + Send>;
+const BLOCK: usize = 1024;
+const MAX_CHANNELS: usize = 64;
+// Two shifters have at most 16384 frames of latency; leave room for priming and interpolation.
+const FRAME_MARGIN: i128 = 65_536;
+
+pub(crate) fn mix(session: &Session, seq_id: ItemId, device_rate: u32, speed: f64) -> Option<(u32, Mixer)> {
+    let rate = super::shuttle_mix_rate(device_rate, speed)?;
+    if !session.prefs.audio.maintain_pitch_shuttling || rate == device_rate {
+        return Some((rate, Box::new(super::playback_mix(session, seq_id, rate))));
+    }
+    // Slow playback needs headroom for the upward pitch correction before rate conversion.
+    let internal_rate = if rate > device_rate { device_rate.checked_mul(2)? } else { device_rate };
+    let semitones = (12.0 * (f64::from(rate) / f64::from(device_rate)).log2()) as f32;
+    Some((rate, Box::new(corrected(super::playback_mix(session, seq_id, internal_rate), internal_rate, rate, semitones))))
+}
+
+fn corrected(mix: impl FnMut(i64, &mut [f32], usize) + Send + 'static, from: u32, to: u32, semitones: f32) -> impl FnMut(i64, &mut [f32], usize) + Send {
+    let mut source = PitchedStream { mix: Box::new(mix), rate: from, semitones, channels: 0, stages: Vec::new(), latency: 0, next: None };
+    let mut resampler = StreamResampler::new(from, to);
+    move |cursor, buf: &mut [f32], ch| {
+        if ch == 0 || ch > MAX_CHANNELS || !buf.len().is_multiple_of(ch) {
+            buf.fill(0.0);
+            return;
+        }
+        let n = buf.len() / ch;
+        if n == 0 {
+            return;
+        }
+        // StreamResampler uses i64 source coordinates. Reject extremes before its interpolation
+        // and before the pitch stream's pre-roll/lookahead arithmetic can overflow.
+        let first = (i128::from(cursor) * i128::from(from)).div_euclid(i128::from(to.max(1)));
+        let last = ((i128::from(cursor) + n as i128) * i128::from(from)).div_euclid(i128::from(to.max(1)));
+        if first < i128::from(i64::MIN) + FRAME_MARGIN || last > i128::from(i64::MAX) - FRAME_MARGIN || i128::from(cursor) + n as i128 > i128::from(i64::MAX) {
+            buf.fill(0.0);
+            source.next = None;
+            resampler = StreamResampler::new(from, to);
+            return;
+        }
+        if source.channels != ch {
+            source.configure(ch);
+            resampler = StreamResampler::new(from, to);
+        }
+        for (b, chunk) in buf.chunks_mut(BLOCK * ch).enumerate() {
+            let start = cursor + (b * BLOCK) as i64;
+            let planar = resampler.process(start, chunk.len() / ch, |s, frames| source.pull(s, frames));
+            for (i, frame) in chunk.chunks_exact_mut(ch).enumerate() {
+                for (sample, plane) in frame.iter_mut().zip(&planar) {
+                    *sample = plane.get(i).copied().filter(|x| x.is_finite()).unwrap_or(0.0);
+                }
+            }
+        }
+    }
+}
+
+struct PitchedStream {
+    mix: Mixer,
+    rate: u32,
+    semitones: f32,
+    channels: usize,
+    stages: Vec<PitchShifter>,
+    latency: usize,
+    // Expected next source output frame; mixer input is latency frames ahead of it.
+    next: Option<i64>,
+}
+
+impl PitchedStream {
+    fn configure(&mut self, channels: usize) {
+        self.channels = channels;
+        self.stages.clear();
+        let mut remaining = self.semitones;
+        // The existing effect supports ±12 semitones. Two stages cover 4x (-24 semitones).
+        for _ in 0..2 {
+            let shift = remaining.clamp(-12.0, 12.0);
+            let mut stage = PitchShifter::new(self.rate as f32, channels);
+            stage.set_param("semitones", shift);
+            stage.reset();
+            self.stages.push(stage);
+            remaining -= shift;
+            if remaining.abs() < 1e-4 {
+                break;
+            }
+        }
+        self.latency = self.stages.iter().map(AudioEffect::latency).sum();
+        self.next = None;
+    }
+
+    fn process(&mut self, start: i64, n: usize) -> Vec<Vec<f32>> {
+        let mut interleaved = vec![0.0; n * self.channels];
+        (self.mix)(start, &mut interleaved, self.channels);
+        let mut planar = vec![vec![0.0; n]; self.channels];
+        for (i, frame) in interleaved.chunks_exact(self.channels).enumerate() {
+            for (plane, sample) in planar.iter_mut().zip(frame) {
+                plane[i] = if sample.is_finite() { *sample } else { 0.0 };
+            }
+        }
+        let mut lanes: Vec<_> = planar.iter_mut().map(Vec::as_mut_slice).collect();
+        for stage in &mut self.stages {
+            stage.process(&mut lanes);
+        }
+        planar
+    }
+
+    fn pull(&mut self, start: i64, n: usize) -> Vec<Vec<f32>> {
+        if self.next != Some(start) {
+            for stage in &mut self.stages {
+                stage.reset();
+            }
+            // Prime from earlier timeline audio and discard the delay. First returned audio is
+            // aligned with `start`, rather than shifting sound behind the accelerated picture.
+            let pre = self.latency * 2;
+            let Some(mut input) = start.checked_sub(pre as i64) else { return vec![vec![0.0; n]; self.channels] };
+            let mut left = pre + self.latency;
+            while left > 0 {
+                let count = left.min(BLOCK);
+                self.process(input, count);
+                input += count as i64;
+                left -= count;
+            }
+        }
+        let mut result = vec![Vec::with_capacity(n); self.channels];
+        let mut done = 0;
+        while done < n {
+            let count = (n - done).min(BLOCK);
+            let block = self.process(start + self.latency as i64 + done as i64, count);
+            for (out, plane) in result.iter_mut().zip(block) {
+                out.extend(plane);
+            }
+            done += count;
+        }
+        self.next = Some(start + n as i64);
+        result
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tone_mix(rate: u32, tones: Vec<f64>, burst: bool) -> impl FnMut(i64, &mut [f32], usize) + Send {
+        move |start, buf: &mut [f32], ch| {
+            for (i, frame) in buf.chunks_exact_mut(ch).enumerate() {
+                let time = (start + i as i64) as f64 / f64::from(rate);
+                for (sample, hz) in frame.iter_mut().zip(&tones) {
+                    let on = !burst || (0.5..0.6).contains(&time);
+                    *sample = if on { (0.3 * (std::f64::consts::TAU * hz * time).sin()) as f32 } else { 0.0 };
+                }
+            }
+        }
+    }
+
+    fn amplitude(samples: &[f32], hz: f64, sr: u32) -> f64 {
+        let (mut re, mut im) = (0.0, 0.0);
+        for (i, x) in samples.iter().enumerate() {
+            let phase = std::f64::consts::TAU * hz * i as f64 / f64::from(sr);
+            re += f64::from(*x) * phase.cos();
+            im += f64::from(*x) * phase.sin();
+        }
+        2.0 * re.hypot(im) / samples.len() as f64
+    }
+
+    fn rates(sr: u32, speed: f64) -> (u32, u32, f32) {
+        let to = super::super::shuttle_mix_rate(sr, speed).unwrap();
+        let from = if speed < 1.0 { sr * 2 } else { sr };
+        (from, to, (12.0 * (f64::from(to) / f64::from(sr)).log2()) as f32)
+    }
+
+    #[test]
+    fn original_tones_survive_shuttling_including_high_frequencies() {
+        for sr in [44_100, 48_000, 96_000] {
+            for speed in [0.5, 0.75, 1.5, 2.0, 4.0] {
+                let (from, to, semitones) = rates(sr, speed);
+                let tones = vec![440.0, 660.0, 6300.0, 9000.0, 18_000.0, 1000.0];
+                let mut mix = corrected(tone_mix(from, tones.clone(), false), from, to, semitones);
+                let mut buf = vec![0.0; 16_000 * tones.len()];
+                mix(i64::from(to), &mut buf, tones.len());
+                for (ch, hz) in tones.iter().enumerate() {
+                    let samples: Vec<f32> = buf.chunks_exact(tones.len()).map(|f| f[ch]).collect();
+                    let level = amplitude(&samples, *hz, sr);
+                    assert!(level > 0.1, "{sr} Hz, {speed}x, channel {ch}: {hz} Hz level={level}");
+                    if hz * speed < f64::from(sr) / 2.0 {
+                        assert!(amplitude(&samples, hz * speed, sr) < level * 0.15, "uncorrected pitch at {speed}x: {hz} Hz");
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn block_boundaries_and_seeks_do_not_change_the_output() {
+        let (sr, speed) = (48_000, 4.0);
+        let (from, to, semitones) = rates(sr, speed);
+        let make = || corrected(tone_mix(from, vec![440.0, 660.0], false), from, to, semitones);
+        let mut whole_mix = make();
+        let mut whole = vec![0.0; 24_000];
+        whole_mix(i64::from(to), &mut whole, 2);
+        let mut cut_mix = make();
+        let mut cut = Vec::new();
+        let mut n = 0;
+        for size in [1, 7, 333, 1024].into_iter().cycle() {
+            let frames = size.min(12_000 - n);
+            let mut block = vec![0.0; frames * 2];
+            cut_mix(i64::from(to) + n as i64, &mut block, 2);
+            cut.extend(block);
+            n += frames;
+            if n == 12_000 {
+                break;
+            }
+        }
+        assert_eq!(cut, whole);
+        let mut sought = vec![0.0; 4096];
+        cut_mix(0, &mut sought, 2);
+        let mut fresh = vec![0.0; 4096];
+        make()(0, &mut fresh, 2);
+        assert_eq!(sought, fresh, "a backward seek must reset the pitch history");
+    }
+
+    #[test]
+    fn pitch_latency_is_compensated_against_the_timeline() {
+        for speed in [0.5, 2.0, 4.0] {
+            let sr = 48_000;
+            let (from, to, semitones) = rates(sr, speed);
+            let mut mix = corrected(tone_mix(from, vec![440.0], true), from, to, semitones);
+            let mut buf = vec![0.0; (f64::from(sr) * 1.4) as usize];
+            mix(0, &mut buf, 1);
+            let energy: f64 = buf.iter().map(|x| f64::from(*x).powi(2)).sum();
+            let center: f64 = buf.iter().enumerate().map(|(i, x)| i as f64 / f64::from(sr) * f64::from(*x).powi(2)).sum::<f64>() / energy;
+            let expected = 0.55 / speed;
+            assert!(energy > 1.0);
+            assert!((center - expected).abs() < 0.025 / speed, "{speed}x: burst at {center}s, expected {expected}s");
+            assert!(buf.iter().take((0.4 / speed * f64::from(sr)) as usize).all(|x| x.abs() < 1e-6), "audio must not arrive early");
+        }
+    }
+
+    #[test]
+    fn hostile_coordinates_and_channel_counts_produce_silence() {
+        let (from, to, semitones) = rates(48_000, 4.0);
+        let mut mix = corrected(tone_mix(from, vec![440.0, 660.0], false), from, to, semitones);
+        for (cursor, ch, len) in [(i64::MIN, 2, 2048), (i64::MAX, 2, 2048), (0, 0, 2048), (0, 65, 65), (0, 2, 3)] {
+            let mut buf = vec![1.0; len];
+            mix(cursor, &mut buf, ch);
+            assert!(buf.iter().all(|x| *x == 0.0));
+        }
+        mix(0, &mut [], 2);
+        let mut buf = vec![0.0; 2048];
+        mix(i64::from(to), &mut buf, 2);
+        assert!(buf.iter().all(|x| x.is_finite()));
+        assert!(buf.iter().any(|x| x.abs() > 0.1));
+    }
+}