Merge pull request #22 from storytold/pr/never-crash-av1enc

Never crash: AV1 encoder without expect()/unreachable!()

crates/av1enc/src/decide.rs

@@ -378,7 +378,13 @@ impl<'a> Decider<'a> {
                 }
             }
         }
-        let (_, y_mode, tx_type_y, ty, recon) = best.expect("an intra mode");
+        // Every ranked mode tries DCT_DCT, so `best` is always set; without a candidate the block
+        // can't be coded intra (an infinite cost keeps it from being chosen).
+        let Some((_, y_mode, tx_type_y, ty, recon)) = best else {
+            let leaf =
+                Leaf { bsize, is_inter: false, y_mode: DC_PRED, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv: [0, 0], skip: true, coefs: Default::default() };
+            return (leaf, f64::INFINITY);
+        };
         self.put(0, c * 4, r * 4, n, recon.as_deref().unwrap_or(&best_pred[..n * n]));
         let tx_type_y = if ty.nonzero { tx_type_y } else { DCT_DCT };
         self.mark_decoded(0, r, c, n4);
@@ -414,16 +420,17 @@ impl<'a> Decider<'a> {
         let uv_mode = best_uv.1;
         let uv_tx = tx_size_for(clog2);
         let uv_type = intra_uv_tx_type(uv_mode, uv_tx);
-        let mut tuv = Vec::with_capacity(2);
-        for i in 0..2 {
+        let mut code_uv = |i: usize| {
             predict_intra(&ced[i], &ce[i], uv_mode, &mut pred);
-            tuv.push(self.code_tx(1 + i, c * 2, r * 2, clog2, uv_type, &pred[..cn * cn], true));
+            let t = self.code_tx(1 + i, c * 2, r * 2, clog2, uv_type, &pred[..cn * cn], true);
             self.mark_decoded(1 + i, r, c, n4);
-        }
-        let skip = !ty.nonzero && !tuv[0].nonzero && !tuv[1].nonzero;
-        let dist = ty.dist + tuv[0].dist + tuv[1].dist;
-        let bits = if skip { 1.0 } else { ty.bits + tuv[0].bits + tuv[1].bits + 1.0 } + MODE_BITS_INTRA + 2.0;
-        let [u, v]: [TxResult; 2] = tuv.try_into().ok().expect("two chroma planes");
+            t
+        };
+        let u = code_uv(0);
+        let v = code_uv(1);
+        let skip = !ty.nonzero && !u.nonzero && !v.nonzero;
+        let dist = ty.dist + u.dist + v.dist;
+        let bits = if skip { 1.0 } else { ty.bits + u.bits + v.bits + 1.0 } + MODE_BITS_INTRA + 2.0;
         let leaf = Leaf { bsize, is_inter: false, y_mode, uv_mode, tx_type_y, mv: [0, 0], skip, coefs: [ty.levels, u.levels, v.levels] };
         (leaf, dist + self.lambda * bits)
     }
@@ -432,8 +439,9 @@ impl<'a> Decider<'a> {
         let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize;
         let n = n4 * 4;
         let log2 = n.trailing_zeros();
+        // Inter blocks are only decided with a reference frame; without one, code intra.
+        let Some(refr) = self.refr else { return self.intra_leaf(r, c, bsize) };
         let (mv, mv_bits) = self.motion_search(r, c, n, parent_mv);
-        let refr = self.refr.expect("reference frame");
         let mut pred = vec![0u16; n * n];
         let mut tx = Vec::with_capacity(3);
         let mut dist = 0.0;
@@ -461,7 +469,16 @@ impl<'a> Decider<'a> {
         }
         let skip = tx.iter().all(|t| !t.nonzero);
         let bits = mode_bits + if skip { 1.0 } else { 1.0 + tx.iter().map(|t| t.bits).sum::<f64>() };
-        let [y, u, v]: [TxResult; 3] = tx.try_into().ok().expect("three planes");
+        let [y, u, v]: [TxResult; 3] = match tx.try_into() {
+            Ok(planes) => planes,
+            // One transform block per plane was coded above.
+            Err(_) => {
+                return (
+                    Leaf { bsize, is_inter: true, y_mode: NEWMV, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv, skip: true, coefs: Default::default() },
+                    f64::INFINITY,
+                );
+            }
+        };
         let leaf = Leaf { bsize, is_inter: true, y_mode: NEWMV, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv, skip, coefs: [y.levels, u.levels, v.levels] };
         (leaf, dist + self.lambda * bits)
     }
@@ -470,7 +487,7 @@ impl<'a> Decider<'a> {
     /// Returns the vector and an estimate of its coding cost in bits.
     fn motion_search(&self, r: usize, c: usize, n: usize, parent_mv: Option<[i32; 2]>) -> ([i32; 2], f64) {
         let g = self.g;
-        let refr = self.refr.expect("reference frame");
+        let Some(refr) = self.refr else { return ([0, 0], 0.0) };
         let mi = |rr: usize, cc: usize| self.mvs[rr * g.mi_cols + cc];
         let mut pred_mv = [0, 0];
         let mut cands: Vec<[i32; 2]> = vec![[0, 0]];
@@ -579,7 +596,7 @@ impl<'a> Decider<'a> {
     /// Approximate luma prediction for motion search: exact at whole and half-sample
     /// positions (precomputed planes), averaged neighbours at quarter-sample positions.
     fn search_block(&self, mv: [i32; 2], x: usize, y: usize, n: usize, out: &mut [u16]) {
-        let refr = self.refr.expect("reference frame");
+        let Some(refr) = self.refr else { return };
         let (w, h) = (self.g.width as i32, self.g.height as i32);
         // per axis: up to two (integer offset, half) taps
         let axis = |v: i32| -> ([(i32, bool); 2], usize) {

crates/av1enc/src/lib.rs

@@ -8,6 +8,8 @@
 //!
 //! The public API below is the contract `crates/export` codes against; keep it stable.
 
+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
+
 mod bitw;
 mod cdf;
 mod decide;
@@ -444,7 +446,16 @@ impl Encoder {
         let src = self.load_source(frame);
         let (frame_obu, rf) = match self.cfg.rate {
             RateControl::ConstantQ(q) => self.encode_frame(&src, key, q.max(1)),
-            RateControl::Bitrate { .. } => self.encode_rate_controlled(&src, key),
+            RateControl::Bitrate { .. } => match self.rc.take() {
+                Some(mut rc) => {
+                    let out = self.encode_rate_controlled(&src, key, &mut rc);
+                    self.rc = Some(rc);
+                    out
+                }
+                // The state is created with the encoder for bitrate mode; without it, encode at a
+                // middle quantizer rather than fail.
+                None => self.encode_frame(&src, key, 128),
+            },
         };
         self.refr = Some(rf);
         self.frame_num += 1;
@@ -456,9 +467,8 @@ impl Encoder {
         Packet { data, keyframe: key }
     }
 
-    fn encode_rate_controlled(&mut self, src: &[Plane; 3], key: bool) -> (Vec<u8>, RefFrame) {
+    fn encode_rate_controlled(&mut self, src: &[Plane; 3], key: bool, rc: &mut RateState) -> (Vec<u8>, RefFrame) {
         let first = self.frame_num == 0;
-        let rc = self.rc.as_ref().expect("rate control state");
         let (frame_bits, debt, q_inter) = (rc.frame_bits, rc.debt, rc.q);
         // key frames get a lower quantizer and a larger share of the budget
         let key_boost = 1.0 + (self.cfg.keyint.clamp(1, 60) as f64 - 1.0).sqrt() * 0.75;
@@ -481,8 +491,8 @@ impl Encoder {
                 q = (q + 30.0 * err).clamp(1.0, 255.0);
             }
         }
-        let (obu, rf, bits) = best.expect("encoded");
-        let rc = self.rc.as_mut().expect("rate control state");
+        // At least one pass ran, so `best` is set.
+        let Some((obu, rf, bits)) = best else { return self.encode_frame(src, key, q.round().clamp(1.0, 255.0) as u8) };
         rc.debt += bits - budget;
         // adapt the inter quantizer: about 30 index steps per doubling of the rate
         let err = (bits / target).log2().clamp(-1.0, 1.0);

crates/av1enc/src/transform.rs

@@ -290,7 +290,11 @@ pub(crate) fn kinds(tx_type: usize) -> (Kind, Kind) {
         ADST_DCT => (Kind::Adst, Kind::Dct),
         DCT_ADST => (Kind::Dct, Kind::Adst),
         ADST_ADST => (Kind::Adst, Kind::Adst),
-        _ => unreachable!("unsupported transform type {tx_type}"),
+        // The encoder only chooses the four types above.
+        _ => {
+            debug_assert!(false, "unsupported transform type {tx_type}");
+            (Kind::Dct, Kind::Dct)
+        }
     }
 }
 

crates/av1enc/src/writer.rs

@@ -211,7 +211,8 @@ impl<'a> TileWriter<'a> {
                 debug_assert_eq!(leaf.bsize, bsize);
                 self.block(r, c, leaf);
             }
-            Node::Out => unreachable!("decided block inside the frame"),
+            // Blocks inside the frame are always decided; nothing is coded outside it.
+            Node::Out => debug_assert!(false, "undecided block inside the frame"),
         }
     }
 
@@ -460,10 +461,11 @@ impl<'a> TileWriter<'a> {
                 if set > 0 && self.qidx > 0 {
                     let sqr = TX_SIZE_SQR[tx_sz] as usize;
                     if b.is_inter {
-                        let s = TX_TYPE_INTER_INV_SET3.iter().position(|&v| v as usize == t).expect("inter tx type in set");
+                        // The decision only picks types of this set (DCT_DCT otherwise).
+                        let s = TX_TYPE_INTER_INV_SET3.iter().position(|&v| v as usize == t).unwrap_or_default();
                         self.sw.symbol(s, &mut self.cdf.inter_tx_type_set3[sqr]);
                     } else {
-                        let s = TX_TYPE_INTRA_INV_SET2.iter().position(|&v| v as usize == t).expect("intra tx type in set");
+                        let s = TX_TYPE_INTRA_INV_SET2.iter().position(|&v| v as usize == t).unwrap_or_default();
                         self.sw.symbol(s, &mut self.cdf.intra_tx_type_set2[sqr][b.y_mode]);
                     }
                 } else {