GPU errors and device loss: keep running, autosave at once, tell the user (#652)

Squash of PR #652 by @aldozar.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/engine/src/autosave.rs

@@ -162,6 +162,12 @@ impl Autosaver {
     /// `dir`: write drawings changed `minutes` ago or earlier and not autosaved since, delete the
     /// files of drawings that were saved (no unsaved changes left) or closed.
     pub fn plan(&mut self, s: &Session, now: f64, minutes: u32, dir: &str) -> Vec<Job> {
+        self.plan_with(s, now, minutes, dir, false)
+    }
+
+    /// [`Autosaver::plan`], with `urgent` writing every drawing with changes not yet autosaved
+    /// now instead of `minutes` after the change (autosave still off when `minutes` is 0).
+    fn plan_with(&mut self, s: &Session, now: f64, minutes: u32, dir: &str, urgent: bool) -> Vec<Job> {
         let mut jobs = Vec::new();
         let live: HashSet<u64> = s.docs.iter().map(|d| d.uid).collect();
         let busy = &self.busy;
@@ -193,7 +199,7 @@ impl Autosaver {
                 continue;
             }
             let since = *t.since.get_or_insert(now);
-            if minutes == 0 || dir.trim().is_empty() || now - since < f64::from(minutes) * 60.0 {
+            if minutes == 0 || dir.trim().is_empty() || (!urgent && now - since < f64::from(minutes) * 60.0) {
                 continue;
             }
             let path = t
@@ -351,6 +357,20 @@ mod native {
             self.next_due(now, minutes).map(Duration::from_secs_f64)
         }
 
+        /// Autosave every drawing with changes not yet autosaved right away, without waiting for
+        /// SAVETIME (the app may not survive what just happened, e.g. a GPU error). Returns how
+        /// many drawings are being written.
+        pub fn save_now(&mut self, s: &Session) -> usize {
+            let now = self.start.elapsed().as_secs_f64();
+            let jobs = self.plan_with(s, now, savetime(), &savefilepath(), true);
+            let writes = jobs.iter().filter(|j| matches!(j, Job::Write { .. })).count();
+            self.perform(jobs);
+            // The lock file marks the files as this running app's (Drawing Recovery skips them).
+            self.last_beat = None;
+            self.heartbeat(&savefilepath(), now);
+            writes
+        }
+
         /// Carry out jobs from [`Autosaver::plan`]: writes on a background thread, deletes now.
         pub fn perform(&mut self, jobs: Vec<Job>) {
             for job in jobs {

crates/engine/tests/autosave.rs

@@ -198,6 +198,13 @@ fn autosave_files_are_written_in_the_background_and_removed_on_exit() {
     assert!(dir.join(format!("{}.lock", a.instance)).exists());
     assert!(autosave::recoverable(&dir, "someone-else").is_empty());
 
+    // After a GPU error a change made a moment ago is autosaved right away, not SAVETIME later.
+    assert!(a.tick(&s).is_none(), "no change waiting");
+    draw(&mut s);
+    assert!(a.tick(&s).is_some(), "SAVETIME (3 minutes) is not up yet");
+    assert_eq!(a.save_now(&s), 1);
+    assert_eq!(a.save_now(&s), 0, "being written or written: nothing more to save");
+
     a.shutdown();
     assert!(!std::path::Path::new(&path).exists());
     assert!(!dir.join(format!("{}.lock", a.instance)).exists());

crates/ui-egui/src/autosave.rs

@@ -29,6 +29,14 @@ pub fn load_prefs(json: &str) {
     }
 }
 
+/// What the GPU error notice says (see [`gpu_notice`]).
+#[derive(Clone, Debug, PartialEq)]
+pub struct GpuNotice {
+    pub fault: crate::gpu::GpuFault,
+    /// Drawings autosaved because of it (0: nothing unsaved, or autosave is off).
+    pub autosaved: usize,
+}
+
 /// Drawing Recovery: autosave files left behind by an app that stopped, listed once at start.
 #[derive(Clone, Debug, Default)]
 pub struct Recovery {
@@ -81,6 +89,19 @@ impl CadApp {
         }
     }
 
+    /// Every frame: after a GPU error or device loss, autosave unsaved drawings at once (the
+    /// session may not survive it) and show [`gpu_notice`].
+    pub fn gpu_fault_tick(&mut self, ctx: &egui::Context) {
+        let Some(fault) = self.gpu_faults.take() else { return };
+        #[cfg(not(target_arch = "wasm32"))]
+        let saved = self.autosave.as_mut().map_or(0, |a| a.save_now(&self.session));
+        #[cfg(target_arch = "wasm32")]
+        let saved = 0;
+        let autosaved = self.gpu_notice.as_ref().map_or(0, |n| n.autosaved) + saved;
+        self.gpu_notice = Some(GpuNotice { fault, autosaved });
+        ctx.request_repaint();
+    }
+
     /// The app closes normally: this app's autosave files are not needed any more.
     pub fn shutdown_autosave(&mut self) {
         #[cfg(not(target_arch = "wasm32"))]
@@ -163,6 +184,48 @@ pub fn show(app: &mut CadApp, ctx: &egui::Context) {
     }
 }
 
+/// The notice after a GPU error or device loss: what happened, what was autosaved, and what to
+/// do. Not modal, so drawings can still be saved while the device works.
+pub fn gpu_notice(app: &mut CadApp, ctx: &egui::Context) {
+    let Some(n) = &app.gpu_notice else { return };
+    let t = Tokens::get();
+    let mut dismiss = false;
+    egui::Window::new("Graphics error")
+        .id(egui::Id::new("cc_gpu_notice"))
+        .collapsible(false)
+        .resizable(false)
+        .anchor(egui::Align2::CENTER_TOP, vec2(0.0, 48.0))
+        .show(ctx, |ui| {
+            ui.set_width(440.0);
+            if n.fault.lost {
+                ui.label("The graphics device stopped working (a driver reset or a GPU that stopped responding). CADCraft can't draw again until it restarts.");
+            } else {
+                let times = if n.fault.errors == 1 { String::new() } else { format!(" ({} times)", n.fault.errors) };
+                ui.label(format!("The graphics driver reported an error{times}. CADCraft kept running, but the drawing may not display correctly."));
+            }
+            ui.add_space(4.0);
+            match n.autosaved {
+                0 => ui.label("Nothing needed autosaving."),
+                1 => ui.label("A recovery copy of the open drawing with unsaved changes was autosaved."),
+                k => ui.label(format!("Recovery copies of {k} drawings with unsaved changes were autosaved.")),
+            };
+            ui.label(RichText::new("Save your work and restart CADCraft. Recovery copies are offered at the next start if needed.").strong());
+            ui.add_space(4.0);
+            ui.collapsing("Details", |ui| {
+                ui.label(RichText::new(&n.fault.message).color(t.text_dim).monospace());
+            });
+            // A row just as tall as the button (a plain right-to-left layout takes the rest).
+            ui.allocate_ui_with_layout(vec2(ui.available_width(), 0.0), egui::Layout::right_to_left(egui::Align::Center), |ui| {
+                if ui.button("Dismiss").clicked() {
+                    dismiss = true;
+                }
+            });
+        });
+    if dismiss {
+        app.gpu_notice = None;
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #[test]

crates/ui-egui/src/gpu.rs

@@ -494,9 +494,79 @@ fn upload(device: &wgpu::Device, label: &str, bytes: &[u8], stride: usize, usage
         .collect()
 }
 
-/// Create the canvas pipelines in the renderer's callback resources. Call once with the app's
-/// render state (eframe `CreationContext::wgpu_render_state`).
-pub fn install(rs: &egui_wgpu::RenderState) -> GpuTarget {
+/// GPU trouble wgpu reported: uncaptured errors (validation, out of memory) and the device being
+/// lost (a driver reset or a hung GPU). wgpu's default handler panics on the first uncaptured
+/// error, closing the app with any unsaved work (#602); the handlers [`install`] registers record
+/// them here instead, and the app autosaves and tells the user (see `CadApp::gpu_fault_tick`).
+#[derive(Debug, Default)]
+pub struct GpuFaults {
+    state: Mutex<FaultState>,
+}
+
+#[derive(Debug, Default)]
+struct FaultState {
+    errors: u64,
+    errors_seen: u64,
+    lost: bool,
+    lost_seen: bool,
+    last: String,
+}
+
+/// What [`GpuFaults::take`] reports.
+#[derive(Clone, Debug, PartialEq)]
+pub struct GpuFault {
+    /// Uncaptured errors so far (an invalid resource can fail again every frame).
+    pub errors: u64,
+    /// The device was lost: nothing more can be drawn until the app restarts.
+    pub lost: bool,
+    /// The latest error or device-lost message.
+    pub message: String,
+}
+
+impl GpuFaults {
+    /// Errors logged in full before only every 1000th is (a repeating error would flood the log).
+    const LOGGED: u64 = 10;
+
+    pub fn error(&self, message: String) {
+        let Ok(mut s) = self.state.lock() else { return };
+        s.errors += 1;
+        if s.errors <= Self::LOGGED || s.errors % 1000 == 0 {
+            log::error!("wgpu error #{}: {message}", s.errors);
+        }
+        s.last = message;
+    }
+
+    pub fn device_lost(&self, reason: wgpu::DeviceLostReason, message: String) {
+        // `Destroyed` is the device being dropped on purpose (the app closing), not a failure.
+        if reason == wgpu::DeviceLostReason::Destroyed {
+            return;
+        }
+        log::error!("wgpu device lost ({reason:?}): {message}");
+        let Ok(mut s) = self.state.lock() else { return };
+        s.lost = true;
+        s.last = message;
+    }
+
+    /// Faults reported since the last call, if any.
+    pub fn take(&self) -> Option<GpuFault> {
+        let Ok(mut s) = self.state.lock() else { return None };
+        if s.errors == s.errors_seen && s.lost == s.lost_seen {
+            return None;
+        }
+        s.errors_seen = s.errors;
+        s.lost_seen = s.lost;
+        Some(GpuFault { errors: s.errors, lost: s.lost, message: s.last.clone() })
+    }
+}
+
+/// Create the canvas pipelines in the renderer's callback resources, and report wgpu errors and
+/// device loss to `faults` instead of panicking. Call once with the app's render state (eframe
+/// `CreationContext::wgpu_render_state`).
+pub fn install(rs: &egui_wgpu::RenderState, faults: &Arc<GpuFaults>) -> GpuTarget {
+    let f = Arc::clone(faults);
+    rs.device.on_uncaptured_error(Arc::new(move |e: wgpu::Error| f.error(e.to_string())));
+    let f = Arc::clone(faults);
+    rs.device.set_device_lost_callback(move |reason, message| f.device_lost(reason, message));
     let target = GpuTarget { format: rs.target_format };
     let res = Resources::new(&rs.device, target);
     rs.renderer.write().callback_resources.insert(res);

crates/ui-egui/src/lib.rs

@@ -211,6 +211,10 @@ pub struct CadApp {
     pub autosave: Option<cadcraft_engine::autosave::Autosaver>,
     /// Drawing Recovery: autosave files a crashed app left behind.
     pub recovery: autosave::Recovery,
+    /// GPU errors and device loss reported by wgpu ([`gpu::GpuFaults`]).
+    pub gpu_faults: std::sync::Arc<gpu::GpuFaults>,
+    /// The notice shown after one (see [`autosave::gpu_notice`]).
+    pub gpu_notice: Option<autosave::GpuNotice>,
 }
 
 impl CadApp {
@@ -242,6 +246,8 @@ impl CadApp {
             closing: None,
             autosave: None,
             recovery: autosave::Recovery::default(),
+            gpu_faults: std::sync::Arc::default(),
+            gpu_notice: None,
         }
     }
 
@@ -252,7 +258,7 @@ impl CadApp {
     /// Draw the canvas on the GPU with the app's wgpu render state (eframe's
     /// `CreationContext::wgpu_render_state`). Without it the canvas draws on the CPU.
     pub fn set_wgpu(&mut self, rs: &egui_wgpu::RenderState) {
-        self.canvas.gpu = Some(gpu::install(rs));
+        self.canvas.gpu = Some(gpu::install(rs, &self.gpu_faults));
     }
 
     pub fn with_control(mut self, rx: Receiver<ControlRequest>) -> Self {
@@ -425,6 +431,7 @@ impl CadApp {
         }
         self.collect_screenshots(ctx);
         self.issue_screenshots(ctx);
+        self.gpu_fault_tick(ctx);
         self.autosave_tick(ctx);
         menus::shortcuts(self, ctx);
         #[cfg(not(target_arch = "wasm32"))]
@@ -492,6 +499,7 @@ impl CadApp {
         dialogs::show(self, ui.ctx());
         closing::prompt(self, ui.ctx());
         autosave::show(self, ui.ctx());
+        autosave::gpu_notice(self, ui.ctx());
         self.frame_ms = now_ms() - t0;
     }
 

crates/ui-egui/tests/gpu_faults.rs

@@ -0,0 +1,90 @@
+//! A wgpu error or device loss no longer closes the app (#602): it is recorded, unsaved drawings
+//! are autosaved at once instead of SAVETIME minutes later, and a notice says what happened.
+//! SAVEFILEPATH is process-wide, so everything runs in one test.
+
+use cadcraft_engine::Session;
+use cadcraft_engine::cmd::file::{IoHooks, set_io};
+use cadcraft_engine::doc::Drawing;
+use cadcraft_ui_egui::gpu::GpuFaults;
+use cadcraft_ui_egui::{CadApp, Services};
+use egui_wgpu::wgpu::DeviceLostReason;
+
+/// The text the notice paints (the second frame: a new window sizes itself in the first).
+fn painted_text(app: &mut CadApp, ctx: &egui::Context) -> String {
+    fn collect(shape: &egui::Shape, text: &mut String) {
+        match shape {
+            egui::Shape::Text(t) => {
+                text.push_str(t.galley.text());
+                text.push('\n');
+            }
+            egui::Shape::Vec(v) => v.iter().for_each(|s| collect(s, text)),
+            _ => {}
+        }
+    }
+    let mut text = String::new();
+    for _ in 0..2 {
+        let mut out = ctx.run_ui(egui::RawInput::default(), |ui| cadcraft_ui_egui::autosave::gpu_notice(app, ui.ctx()));
+        out.textures_delta.clear(); // no renderer here
+        text.clear();
+        out.shapes.iter().for_each(|s| collect(&s.shape, &mut text));
+    }
+    text
+}
+
+#[test]
+fn faults_are_recorded_autosaved_and_shown() {
+    // The registry reports each new batch once; dropping the device on purpose is not a fault.
+    let f = GpuFaults::default();
+    assert_eq!(f.take(), None);
+    f.error("first".into());
+    f.error("second".into());
+    let got = f.take().unwrap();
+    assert_eq!((got.errors, got.lost, got.message.as_str()), (2, false, "second"));
+    assert_eq!(f.take(), None);
+    f.device_lost(DeviceLostReason::Destroyed, "dropped".into());
+    assert_eq!(f.take(), None);
+    f.device_lost(DeviceLostReason::Unknown, "driver reset".into());
+    assert!(f.take().is_some_and(|g| g.lost && g.message == "driver reset"));
+
+    set_io(IoHooks { read: |_, _| Ok(Drawing::new_metric()), write: |_, _| Ok(b"0\nEOF\n".to_vec()), plot: None });
+    let dir = std::env::temp_dir().join(format!("cadcraft-gpu-faults-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    let mut app = CadApp::new(Session::new(), Services::default());
+    app.enable_autosave(&dir);
+    app.run("circle", serde_json::json!({"center": [0, 0], "radius": 1})).unwrap();
+    let ctx = egui::Context::default();
+
+    app.gpu_fault_tick(&ctx);
+    assert!(app.gpu_notice.is_none(), "no fault, no notice");
+
+    app.gpu_faults.error("Buffer size 302884104 is greater than the maximum buffer size".into());
+    app.gpu_fault_tick(&ctx);
+    let n = app.gpu_notice.clone().unwrap();
+    assert_eq!((n.fault.errors, n.fault.lost, n.autosaved), (1, false, 1));
+    // The change was autosaved now, not SAVETIME minutes from now.
+    let written = || std::fs::read_dir(&dir).map(|d| d.flatten().filter(|e| e.path().extension().is_some_and(|x| x == "sv$")).count()).unwrap_or(0);
+    for _ in 0..500 {
+        if written() == 1 {
+            break;
+        }
+        std::thread::sleep(std::time::Duration::from_millis(10));
+    }
+    assert_eq!(written(), 1);
+    let text = painted_text(&mut app, &ctx);
+    assert!(text.contains("graphics driver reported an error") && text.contains("recovery copy"), "{text}");
+
+    // The same error again (an invalid buffer used every frame): counted, nothing new to save.
+    app.gpu_faults.error("again".into());
+    app.gpu_fault_tick(&ctx);
+    let n = app.gpu_notice.clone().unwrap();
+    assert_eq!((n.fault.errors, n.autosaved), (2, 1));
+    assert!(painted_text(&mut app, &ctx).contains("(2 times)"));
+
+    app.gpu_faults.device_lost(DeviceLostReason::Unknown, "lost".into());
+    app.gpu_fault_tick(&ctx);
+    assert!(app.gpu_notice.as_ref().is_some_and(|n| n.fault.lost));
+    assert!(painted_text(&mut app, &ctx).contains("can't draw again until it restarts"));
+
+    app.shutdown_autosave();
+    let _ = std::fs::remove_dir_all(&dir);
+}