h264enc: never crash — no unwrap/expect/unreachable, encode returns Result

- `Encoder::encode` returns `Result` (it was `try_encode(..).expect(..)`, panicking on short
  planes); tests and the example unwrap explicitly.
- P/B macroblock analysis without a reference codes intra instead of `expect`/`unwrap`;
  spatial direct without a list-1 picture predicts zero motion.
- Rate control prior, DPB sliding window, DTS queue, reconstruction buffers, CAVLC writer,
  SAD rows and forward 8x8 transforms without `unwrap`/`expect`; skip-macroblock arms in
  `write_mb` debug-assert instead of `unreachable!`.
- `#![deny(clippy::unwrap_used, …)]` outside tests.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/h264enc/examples/h264enc_synth.rs

@@ -64,7 +64,7 @@ fn main() {
         enc.set_recon_capture(false);
     }
     for (i, f) in frames.iter().enumerate() {
-        pk.extend(enc.encode(&f.frame(), i as i64));
+        pk.extend(enc.encode(&f.frame(), i as i64).unwrap());
     }
     pk.extend(enc.flush());
     let el = t0.elapsed().as_secs_f64();

crates/h264enc/src/cabac_mb.rs

@@ -296,7 +296,8 @@ pub fn write_mb<S: BinSink>(s: &mut S, nb: &NbCtx, c: &MbCode) {
             s.decision(if bins[1] != 1 { 16 } else { 17 }, bins[2]);
         }
         MbKind::BDirect | MbKind::BInter => write_b_bins(s, nb, B_BINS[c.b_type_number() as usize]),
-        _ => unreachable!("skip/none macroblocks are not written with write_mb"),
+        // Skip / none macroblocks are not written with write_mb.
+        _ => debug_assert!(false, "write_mb on a skipped macroblock"),
     }
     let is_inxn = c.kind.is_inxn();
     if c.kind == MbKind::PInter && c.part == Part::P8x8 {

crates/h264enc/src/cavlc.rs

@@ -151,7 +151,8 @@ pub fn write_mb(w: &mut BitWriter, nb: &NbCtx, c: &MbCode) {
             Part::P8x8 => 3,
         }),
         MbKind::BDirect | MbKind::BInter => w.write_ue(c.b_type_number()),
-        _ => unreachable!(),
+        // Skip / none macroblocks are not written with write_mb.
+        _ => debug_assert!(false, "write_mb on a skipped macroblock"),
     }
     if c.kind == MbKind::PInter && c.part == Part::P8x8 {
         for _ in 0..4 {

crates/h264enc/src/dsp.rs

@@ -16,8 +16,9 @@ pub fn sad(a: &[u8], sa: usize, b: &[u8], sb: usize, w: usize, h: usize) -> u32
 pub fn sad16(a: &[u8], sa: usize, b: &[u8], sb: usize, h: usize) -> u32 {
     let mut s = 0u32;
     for r in 0..h {
-        let ra: &[u8; 16] = a[r * sa..r * sa + 16].try_into().unwrap();
-        let rb: &[u8; 16] = b[r * sb..r * sb + 16].try_into().unwrap();
+        let (Some(ra), Some(rb)) = (a.get(r * sa..).and_then(|x| x.first_chunk::<16>()), b.get(r * sb..).and_then(|x| x.first_chunk::<16>())) else {
+            break;
+        };
         let mut acc = 0u16;
         for i in 0..16 {
             acc += ra[i].abs_diff(rb[i]) as u16;

crates/h264enc/src/lib.rs

@@ -13,10 +13,12 @@
 //! let mut enc = Encoder::new(cfg).unwrap();
 //! let (y, u, v) = (vec![128u8; 1280 * 720], vec![128u8; 640 * 360], vec![128u8; 640 * 360]);
 //! let frame = YuvFrame { y: &y, u: &u, v: &v, y_stride: 1280, uv_stride: 640 };
-//! let mut packets = enc.encode(&frame, 0);
+//! let mut packets = enc.encode(&frame, 0).unwrap();
 //! packets.extend(enc.flush());
 //! ```
 
+#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
+
 mod cabac;
 mod cabac_mb;
 mod cavlc;
@@ -504,12 +506,11 @@ impl Encoder {
         Some(PassStats { frames: self.rc.pass1.clone(), fps: self.cfg.fps_num as f64 / self.cfg.fps_den as f64 })
     }
 
-    /// Encode one picture. Returns zero or more packets in decoding order.
-    ///
-    /// # Panics
-    /// If the planes are smaller than the configured size (see [`Encoder::try_encode`]).
-    pub fn encode(&mut self, frame: &YuvFrame, pts: i64) -> Vec<Packet> {
-        self.try_encode(frame, pts).expect("invalid input frame")
+    /// Encode one picture. Returns zero or more packets in decoding order, or
+    /// [`Error::BadFrame`] if the planes are smaller than the configured size (same as
+    /// [`Encoder::try_encode`]).
+    pub fn encode(&mut self, frame: &YuvFrame, pts: i64) -> Result<Vec<Packet>, Error> {
+        self.try_encode(frame, pts)
     }
 
     pub fn try_encode(&mut self, frame: &YuvFrame, pts: i64) -> Result<Vec<Packet>, Error> {
@@ -722,7 +723,8 @@ impl Encoder {
         let bounds: Vec<(usize, usize)> = (0..n).map(|s| (s * mbh / n, (s + 1) * mbh / n)).filter(|(a, b)| b > a).collect();
         let capture = self.recon.is_some();
         let mut slice_results: Vec<Vec<SliceOut>> = (0..jobs.len()).map(|_| Vec::new()).collect();
-        let mut bufs: Vec<(Frame, Vec<MbInfo>)> = jobs.iter_mut().map(|j| (j.rec.take().unwrap(), std::mem::take(&mut j.mbs))).collect();
+        // Every job carries its reconstruction buffer (`rec: Some(..)` above).
+        let mut bufs: Vec<(Frame, Vec<MbInfo>)> = jobs.iter_mut().filter_map(|j| Some((j.rec.take()?, std::mem::take(&mut j.mbs)))).collect();
         {
             let this = &*self;
             let fencs: Vec<FrameEnc> = jobs
@@ -810,7 +812,7 @@ impl Encoder {
         // ---- finalise each picture in coding order
         for job in jobs {
             let Job { input, st, idr, qpf, qp, cplx, hdr: _, nal_ref_idc: _, l0: _, l1: _, poc, rec, mbs, results, aq: _ } = job;
-            let mut rec = rec.unwrap();
+            let Some(mut rec) = rec else { continue };
             let is_ref = st != SliceType::B;
             let annexb = self.cfg.format == PacketFormat::AnnexB;
             let mut nals: Vec<Vec<u8>> = Vec::new();
@@ -871,9 +873,10 @@ impl Encoder {
                 let rp = RefPic { frame: rec, hpel, mbs, poc, id: self.ref_uid };
                 if self.dpb.len() >= self.max_refs {
                     // sliding window: drop the oldest
-                    let oldest = self.dpb.iter().enumerate().min_by_key(|(_, r)| r.id).map(|(i, _)| i).unwrap();
-                    let r = self.dpb.remove(oldest);
-                    self.recycle_ref(r);
+                    if let Some(oldest) = self.dpb.iter().enumerate().min_by_key(|(_, r)| r.id).map(|(i, _)| i) {
+                        let r = self.dpb.remove(oldest);
+                        self.recycle_ref(r);
+                    }
                 }
                 self.dpb.push(rp);
                 self.next_frame_num = (self.next_frame_num + 1) % (1 << LOG2_MAX_FRAME_NUM);
@@ -884,8 +887,9 @@ impl Encoder {
                 self.idr_count += 1;
             }
             // decoding timestamp base: the k-th smallest presentation time (offset applied in `release`)
-            let pos = self.display_pts.iter().enumerate().min_by_key(|(_, p)| **p).map(|(i, _)| i).unwrap();
-            let dts_base = self.display_pts.remove(pos).unwrap();
+            // One display time is queued per input picture, so the queue is never empty here.
+            let pos = self.display_pts.iter().enumerate().min_by_key(|(_, p)| **p).map(|(i, _)| i);
+            let dts_base = pos.and_then(|p| self.display_pts.remove(p)).unwrap_or(input.pts);
             self.coded_count += 1;
             let frame_type = if idr {
                 FrameType::Idr

crates/h264enc/src/ratecontrol.rs

@@ -175,12 +175,12 @@ impl RateControl {
                 let s_here = self.coef[ti] * cplx.powf(QCOMP) / type_factor(t);
                 // Virtual history of one second of P frames (inter cost ~ 40% of the first frame's) so the first
                 // I frame gets a realistic share of the budget instead of a single frame's worth.
-                if self.prior.is_none() {
-                    let n = self.fps.max(1.0);
+                let (fps, coef1) = (self.fps, self.coef[1]);
+                let (pn0, ps0) = *self.prior.get_or_insert_with(|| {
+                    let n = fps.max(1.0);
                     let c0 = if t == SliceType::I { 0.4 * cplx } else { cplx };
-                    self.prior = Some((n, n * self.coef[1] * c0.powf(QCOMP)));
-                }
-                let (pn0, ps0) = self.prior.unwrap();
+                    (n, n * coef1 * c0.powf(QCOMP))
+                });
                 let fade = (1.0 - self.frames as f64 / pn0).max(0.0);
                 let (pn, ps) = (pn0 * fade, ps0 * fade);
                 let s = self.sum_s + s_here + ps;

crates/h264enc/src/slice.rs

@@ -236,7 +236,7 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
         let rbsp = if let Some(c) = self.cabac.take() {
             c.finish()
         } else {
-            let mut w = self.cavlc.take().unwrap();
+            let mut w = self.cavlc.take().unwrap_or_default();
             if self.skip_run > 0 {
                 w.write_ue(self.skip_run);
             }
@@ -1489,7 +1489,8 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
 
     fn analyse_p(&self, mx: usize, my: usize, qp: u8, av: Avail, e: &MbEdges, src: &Src) -> Box<Cand> {
         let f = self.f;
-        let r = f.l0.expect("P slice without reference");
+        // P slices always have a reference; without one the macroblock is coded intra.
+        let Some(r) = f.l0 else { return self.analyse_intra(mx, my, qp, av, e, src, u64::MAX) };
         let lam = f.lam.sad[qp as usize];
         let (px, py) = (mx * 16, my * 16);
         let skip_mv = self.pskip_mv(mx, my);
@@ -1640,7 +1641,8 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
                 }
             }
         }
-        let col = &self.f.l1.unwrap().mbs[my * self.f.p.mbw + mx];
+        let Some(l1) = self.f.l1 else { return ([[0; 4]; 2], [[Mv::ZERO; 4]; 2]) };
+        let col = &l1.mbs[my * self.f.p.mbw + mx];
         let mut out_ref = [[-1i8; 4]; 2];
         let mut out_mv = [[Mv::ZERO; 4]; 2];
         for q in 0..4 {
@@ -1751,7 +1753,8 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
 
     fn analyse_b(&self, mx: usize, my: usize, qp: u8, av: Avail, e: &MbEdges, src: &Src) -> Box<Cand> {
         let f = self.f;
-        let (r0, r1) = (f.l0.unwrap(), f.l1.unwrap());
+        // B slices always have both references; without them the macroblock is coded intra.
+        let (Some(r0), Some(r1)) = (f.l0, f.l1) else { return self.analyse_intra(mx, my, qp, av, e, src, u64::MAX) };
         let lam = f.lam.sad[qp as usize];
         let (px, py) = (mx * 16, my * 16);
         // --- direct
@@ -1837,7 +1840,7 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
             res[1].2 as u64 + ((lam * 3) >> 8) as u64,
             satd(&src.y, 16, &bi.0, 16, 16, 16) as u64 + ((lam * (bits(0) + bits(1) + 5)) >> 8) as u64,
         ];
-        let dir = (0..3).min_by_key(|&d| costs[d]).unwrap();
+        let dir = (0..3).min_by_key(|&d| costs[d]).unwrap_or(0);
         let mut ci = Cand::new();
         ci.code.kind = MbKind::BInter;
         ci.code.part = Part::P16x16;
@@ -1870,7 +1873,7 @@ impl<'a, 'b> SliceEnc<'a, 'b> {
                 }
             });
         }
-        let mut best = best.unwrap();
+        let mut best = best.unwrap_or_else(|| self.analyse_intra(mx, my, qp, av, e, src, u64::MAX));
         // --- 16x8 / 8x16 partitions with a per-partition prediction direction
         if f.p.partitions && best.cost > 256 * 4 {
             for part in [Part::P16x8, Part::P8x16] {

crates/h264enc/src/syntax.rs

@@ -61,7 +61,7 @@ impl MbCode {
                 Part::P16x16 => 1 + self.bdir[0] as u32,
                 Part::P16x8 | Part::P8x16 => {
                     const PAIRS: [(u8, u8); 9] = [(0, 0), (1, 1), (0, 1), (1, 0), (0, 2), (1, 2), (2, 0), (2, 1), (2, 2)];
-                    let k = PAIRS.iter().position(|&p| p == (self.bdir[0], self.bdir[1])).unwrap() as u32;
+                    let k = PAIRS.iter().position(|&p| p == (self.bdir[0], self.bdir[1])).unwrap_or(0) as u32;
                     4 + 2 * k + (self.part == Part::P8x16) as u32
                 }
                 Part::P8x8 => 22,

crates/h264enc/src/transform.rs

@@ -91,8 +91,8 @@ fn fdct8_1d(s: [i32; 8]) -> [i32; 8] {
 /// Forward 8x8 transform X = A x A^T (unnormalised; rows of A are the H.264 8x8 basis scaled by 8).
 pub fn fdct8(d: &[i32; 64]) -> [i32; 64] {
     let mut t = [0i32; 64];
-    for y in 0..8 {
-        let o = fdct8_1d(d[y * 8..y * 8 + 8].try_into().unwrap());
+    for (y, row) in d.as_chunks::<8>().0.iter().enumerate() {
+        let o = fdct8_1d(*row);
         t[y * 8..y * 8 + 8].copy_from_slice(&o);
     }
     let mut out = [0i32; 64];
@@ -140,9 +140,8 @@ pub fn idct8(d: &[i32; 64]) -> [i32; 64] {
         [b0 + b7, b2 + b5, b4 + b3, b6 + b1, b6 - b1, b4 - b3, b2 - b5, b0 - b7]
     }
     let mut g = [0i32; 64];
-    for i in 0..8 {
-        let row: [i32; 8] = d[i * 8..i * 8 + 8].try_into().unwrap();
-        let o = one(row);
+    for (i, row) in d.as_chunks::<8>().0.iter().enumerate() {
+        let o = one(*row);
         g[i * 8..i * 8 + 8].copy_from_slice(&o);
     }
     let mut r = [0i32; 64];

crates/h264enc/tests/ffmpeg_oracle.rs

@@ -31,7 +31,7 @@ fn encode(c: EncoderConfig, frames: &[Yuv]) -> Run {
     enc.set_recon_capture(true);
     let mut packets = Vec::new();
     for (i, f) in frames.iter().enumerate() {
-        packets.extend(enc.encode(&f.frame(), i as i64));
+        packets.extend(enc.encode(&f.frame(), i as i64).unwrap());
     }
     packets.extend(enc.flush());
     let mut recon = enc.take_recon();
@@ -169,7 +169,7 @@ fn two_pass_vbr_hits_target() {
     c1.pass = Pass::First;
     let mut enc = Encoder::new(c1.clone()).unwrap();
     for (i, f) in frames.iter().enumerate() {
-        enc.encode(&f.frame(), i as i64);
+        enc.encode(&f.frame(), i as i64).unwrap();
     }
     enc.flush();
     let stats = enc.pass_stats().unwrap();
@@ -243,7 +243,7 @@ fn length_prefixed_with_avcc() {
     let pps = &avcc[11 + sps_len..11 + sps_len + pps_len];
     let mut packets = Vec::new();
     for (i, f) in frames.iter().enumerate() {
-        packets.extend(enc.encode(&f.frame(), i as i64));
+        packets.extend(enc.encode(&f.frame(), i as i64).unwrap());
     }
     packets.extend(enc.flush());
     // convert to Annex-B using the avcC parameter sets