Tests: one GPU harness at a time in every GPU test binary (WARP crash) (#3212)

On Windows, GPU UI test binaries crashed with STATUS_ACCESS_VIOLATION
whenever two of their tests ran at once: arbitrary_rotation_preview 3
runs in 4 on main, and color_managed_canvas, crop_default_mode,
drag_preview_canvas and full_res_preview at random. egui_kittest
prefers a CPU adapter, which on Windows is WARP over DX12, and a dump
puts the fault inside d3d10warp.dll with two WARP devices rendering on
parallel test threads. Bare egui harnesses with no PhotoCraft code
crash the same way, and Vulkan or GL never does. CI's Windows runners
have no GPU, so they run on WARP too.

Three binaries already held a per-binary lock for the same class of
driver bug (Mesa llvmpipe and RADV, #194; libvulkan with NVIDIA and
lavapipe). That lock now lives once in photocraft-testkit
(`gpu_lock`, with the reasons) and every test in the twelve GPU test
binaries without it takes it on its first line. Under nextest each
test is its own process and the lock is free.

Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/testkit/src/gpu.rs

@@ -0,0 +1,64 @@
+//! One GPU harness at a time per test process.
+//!
+//! `cargo test` runs a binary's tests on parallel threads, and several graphics drivers crash
+//! when one process drives wgpu devices from more than one thread at once:
+//!
+//! - **WARP** (Windows): egui_kittest prefers a CPU adapter, which on Windows is WARP over DX12,
+//!   so every kittest GPU test there runs on it. Two or more WARP devices rendering in parallel
+//!   fault inside `d3d10warp.dll` (`STATUS_ACCESS_VIOLATION`); the GitHub Windows runners have no
+//!   GPU, so CI runs on WARP too.
+//! - **Mesa** llvmpipe over GL and RADV (#194).
+//! - **libvulkan** with the NVIDIA and lavapipe drivers both installed: `wgpu::Instance::new` on
+//!   several threads segfaults.
+//!
+//! So every test that builds a GPU harness (or its own wgpu device) holds [`gpu_lock`] for its
+//! whole run, harness included. Under nextest each test is its own process and the lock costs
+//! nothing; `.config/nextest.toml` caps how many of those run at once instead.
+
+use std::sync::{Mutex, MutexGuard, PoisonError};
+
+static GPU_LOCK: Mutex<()> = Mutex::new(());
+
+/// Hold this for as long as the test uses a GPU harness or device: take it on the test's first
+/// line (`let _gpu = gpu_lock();`) so it is released last. A test that failed while holding it
+/// doesn't block the rest.
+pub fn gpu_lock() -> MutexGuard<'static, ()> {
+    GPU_LOCK.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_panicked_holder_does_not_block_the_next_test() {
+        let _ = std::thread::spawn(|| {
+            let _gpu = gpu_lock();
+            panic!("a GPU test failed");
+        })
+        .join();
+        drop(gpu_lock());
+    }
+
+    #[test]
+    fn holders_take_turns() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        static INSIDE: AtomicUsize = AtomicUsize::new(0);
+        static MAX: AtomicUsize = AtomicUsize::new(0);
+        let threads: Vec<_> = (0..8)
+            .map(|_| {
+                std::thread::spawn(|| {
+                    let _gpu = gpu_lock();
+                    let now = INSIDE.fetch_add(1, Ordering::SeqCst) + 1;
+                    MAX.fetch_max(now, Ordering::SeqCst);
+                    std::thread::sleep(std::time::Duration::from_millis(2));
+                    INSIDE.fetch_sub(1, Ordering::SeqCst);
+                })
+            })
+            .collect();
+        for t in threads {
+            assert!(t.join().is_ok());
+        }
+        assert_eq!(MAX.load(Ordering::SeqCst), 1);
+    }
+}

crates/testkit/src/lib.rs

@@ -1,5 +1,8 @@
 //! photocraft-testkit
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 
+pub mod gpu;
 #[cfg(not(target_arch = "wasm32"))]
 pub mod perf;
+
+pub use gpu::gpu_lock;

crates/ui-egui/tests/arbitrary_rotation_preview.rs

@@ -87,6 +87,7 @@ fn blue(h: &mut Harness, name: &str) -> (usize, u32, u32) {
 
 #[test]
 fn size_changing_rotation_preview_is_visible_and_preserves_the_panned_center() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     for (width, height, zoom, center, degrees, factor) in [
@@ -121,6 +122,7 @@ fn size_changing_rotation_preview_is_visible_and_preserves_the_panned_center() {
 
 #[test]
 fn preview_toggle_and_cancel_keep_the_document_while_ok_applies_rotation() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     setup(&mut h, 400, 240, 1.0, [200.0, 120.0]);

crates/ui-egui/tests/camera_raw_localization.rs

@@ -6,6 +6,7 @@ use serde_json::json;
 
 #[test]
 fn all_seven_languages_render_camera_raw_and_switch_without_resetting_view_or_filter() {
+    let _gpu = photocraft_testkit::gpu_lock();
     for lang in i18n::Lang::all() {
         let gpu = std::env::var_os("PHOTOCRAFT_LOCALE_SNAPSHOTS").is_some();
         let builder = Harness::builder().with_step_dt(1.0 / 60.0).with_size(vec2(1440.0, 1000.0));

crates/ui-egui/tests/camera_raw_scope.rs

@@ -4,15 +4,7 @@ use egui_kittest::{Harness, kittest::Queryable};
 use photocraft_ui_egui::{PhotocraftApp, camera_raw_ui, control, theme::ThemeKind};
 use serde_json::{Value, json};
 
-/// Concurrent wgpu instances in one process crash on some drivers (see `canvas_16f.rs`). With the
-/// NVIDIA and lavapipe Vulkan drivers both installed, `wgpu::Instance::new` on several test
-/// threads segfaults inside libvulkan, so every test that builds a GPU harness holds this lock for
-/// its whole run, harness included.
-static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
-
-fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
-    GPU_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
-}
+use photocraft_testkit::gpu_lock;
 
 fn menu(app: &mut PhotocraftApp, ctx: &egui::Context, p: Value) -> Result<Value, String> {
     camera_raw_ui::menu(app, ctx, "filter.cameraRaw", &p).ok_or("missing handler")?

crates/ui-egui/tests/canvas_16f.rs

@@ -17,13 +17,7 @@ use photocraft_ui_egui::PhotocraftApp;
 use photocraft_ui_egui::gpu_canvas::{self, GpuCanvas};
 use serde_json::json;
 
-/// Concurrent wgpu devices in one process crash on some drivers (Mesa llvmpipe over GL, RADV;
-/// see #194), so every test here holds this lock for its whole run, devices included.
-static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
-
-fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
-    GPU_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
-}
+use photocraft_testkit::gpu_lock;
 
 /// A headless GPU canvas on a device created like the app's, if the adapter has a float canvas.
 fn canvas() -> Option<(GpuCanvas, RenderState)> {

crates/ui-egui/tests/checker_canvas.rs

@@ -125,6 +125,7 @@ fn render_on(h: &mut Harness, gpu: bool, zoom: f32, doc: u32, black: bool, grid:
 
 #[test]
 fn checker_cells_are_8_screen_pixels_at_every_zoom() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     // The 8 px cell never changes with the zoom.
@@ -154,6 +155,7 @@ fn checker_cells_are_8_screen_pixels_at_every_zoom() {
 
 #[test]
 fn checker_phase_moves_with_the_image() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     // The document origin moves four screen pixels between these two views, and the checker
@@ -174,6 +176,7 @@ fn lit_columns(img: &Shot, x0: u32, y: u32, len: u32) -> Vec<u32> {
 
 #[test]
 fn pixel_grid_shows_above_500_percent_and_lightens_a_dark_pixel_by_about_a_quarter() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     let (flat, x0, y0, _) = render(&mut h, 5.0, 48, true, true);
@@ -190,6 +193,7 @@ fn pixel_grid_shows_above_500_percent_and_lightens_a_dark_pixel_by_about_a_quart
 
 #[test]
 fn pixel_grid_is_not_drawn_over_empty_checker() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     for zoom in [8.0f32, 16.0] {
@@ -205,6 +209,7 @@ fn pixel_grid_is_not_drawn_over_empty_checker() {
 
 #[test]
 fn cpu_pixel_grid_shows_above_500_percent_over_content() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness_with(false) else { return };
     h.run_steps(4);
     let (flat, x0, y0, _) = render_on(&mut h, false, 5.0, 48, true, true);
@@ -222,6 +227,7 @@ fn cpu_pixel_grid_shows_above_500_percent_over_content() {
 
 #[test]
 fn cpu_pixel_grid_is_not_drawn_over_empty_checker() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness_with(false) else { return };
     h.run_steps(4);
     // The CPU checker has soft cell edges of its own, so compare with and without the grid.

crates/ui-egui/tests/color_managed_canvas.rs

@@ -110,6 +110,7 @@ fn close(a: [u8; 3], b: [f32; 3], tol: f32) -> bool {
 
 #[test]
 fn display_p3_is_converted_on_gpu_and_cpu_canvases() {
+    let _gpu = photocraft_testkit::gpu_lock();
     // P3 (0.8, 0.5, 0.3) → sRGB (216.7, 122.9, 64.3) (see engine display_color_tests); raw would be (204, 128, 77).
     let Some((gpu, cpu)) = render("display-p3", [0.8, 0.5, 0.3], true) else { return };
     assert!(close(cpu, [216.7, 122.9, 64.3], 2.0), "CPU canvas {cpu:?}");
@@ -118,6 +119,7 @@ fn display_p3_is_converted_on_gpu_and_cpu_canvases() {
 
 #[test]
 fn srgb_documents_are_unchanged() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some((gpu, cpu)) = render("srgb", [0.8, 0.5, 0.3], false) else { return };
     assert!(close(cpu, [204.0, 128.0, 77.0], 1.0), "CPU canvas {cpu:?}");
     assert!(close(gpu, [204.0, 128.0, 77.0], 1.0), "GPU canvas {gpu:?}");

crates/ui-egui/tests/crop_beyond_canvas.rs

@@ -103,6 +103,7 @@ fn tinted(p: [u8; 3], c: usize) -> bool {
 
 #[test]
 fn editing_a_crop_frame_shows_the_pixels_past_the_canvas() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     {

crates/ui-egui/tests/crop_default_mode.rs

@@ -169,6 +169,7 @@ fn a_resize_renders_the_frame_and_image_with_the_same_camera() {
 
 #[test]
 fn dragging_inside_moves_the_image_and_the_box_stays_centred() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = setup() else { return };
     assert!(!photocraft_ui_egui::crop_ui::pending(h.state()), "the untouched frame");
     let c = h.state().last_canvas_rect.center();
@@ -199,6 +200,7 @@ fn dragging_inside_moves_the_image_and_the_box_stays_centred() {
 
 #[test]
 fn a_resize_keeps_the_box_centre_after_a_space_pan() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = setup() else { return };
     // Space held: the temporary Hand pans the view 80, 40.
     let c = h.state().last_canvas_rect.center();

crates/ui-egui/tests/drag_preview_canvas.rs

@@ -185,6 +185,7 @@ fn edges(h: &Harness, a: [f32; 2], b: [f32; 2]) -> [(Pos2, Pos2); 4] {
 
 #[test]
 fn marquee_previews_show_while_dragging_on_white_and_black_at_1x_and_2x() {
+    let _gpu = photocraft_testkit::gpu_lock();
     for ppp in [1.0, 2.0] {
         let Some(mut h) = harness(ppp) else { return };
         h.run_steps(4);
@@ -240,6 +241,7 @@ fn marquee_previews_show_while_dragging_on_white_and_black_at_1x_and_2x() {
 
 #[test]
 fn lasso_and_crop_previews_show_while_dragging() {
+    let _gpu = photocraft_testkit::gpu_lock();
     for ppp in [1.0, 2.0] {
         let Some(mut h) = harness(ppp) else { return };
         h.run_steps(4);
@@ -272,6 +274,7 @@ fn lasso_and_crop_previews_show_while_dragging() {
 
 #[test]
 fn a_held_marquee_drag_keeps_requesting_frames() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness(1.0) else { return };
     h.run_steps(4);
     setup(&mut h, "white", "rectMarquee");

crates/ui-egui/tests/export_as_scaling.rs

@@ -68,6 +68,7 @@ fn evidence(h: &mut Harness, depth: u8, stage: &str) {
 
 #[test]
 fn export_as_scales_a_narrow_image_without_editing_the_source() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some((mut h, writes)) = harness() else { return };
     h.run_steps(4);
     for depth in [8, 16, 32] {

crates/ui-egui/tests/full_res_preview.rs

@@ -111,6 +111,7 @@ fn mean(r: &[i32]) -> f32 {
 
 #[test]
 fn adjustment_layer_drag_previews_at_full_resolution_when_low_res_is_off() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     setup(&mut h, 1.0);
     if !on_gpu(&mut h) {
@@ -135,6 +136,7 @@ fn adjustment_layer_drag_previews_at_full_resolution_when_low_res_is_off() {
 
 #[test]
 fn filter_dialog_previews_at_full_resolution_when_low_res_is_off() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     setup(&mut h, 1.0);
     if !on_gpu(&mut h) {
@@ -151,6 +153,7 @@ fn filter_dialog_previews_at_full_resolution_when_low_res_is_off() {
 
 #[test]
 fn adjustment_dialog_zoomed_out_skips_the_reduced_copy_when_low_res_is_off() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     setup(&mut h, 0.5);
     if !on_gpu(&mut h) {

crates/ui-egui/tests/gpu_device_loss.rs

@@ -11,12 +11,7 @@ use photocraft_ui_egui::gpu_canvas::{self, GpuCanvas};
 use photocraft_ui_egui::{PhotocraftApp, gpu_status};
 use serde_json::json;
 
-/// Concurrent wgpu devices in one process crash on some drivers (see `canvas_16f.rs`).
-static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
-
-fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
-    GPU_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
-}
+use photocraft_testkit::gpu_lock;
 
 fn render_state() -> Option<RenderState> {
     let rs =

crates/ui-egui/tests/live_stroke_canvas.rs

@@ -120,6 +120,7 @@ fn max_diff(a: &[[u8; 4]], b: &[[u8; 4]]) -> u8 {
 
 #[test]
 fn the_canvas_shows_the_committed_stroke_while_it_is_drawn() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     let mut cases = Vec::new();

crates/ui-egui/tests/paint_bucket_all_layers.rs

@@ -88,6 +88,7 @@ fn pixels(h: &Harness, layer: LayerId) -> Vec<[f32; 4]> {
 
 #[test]
 fn all_layers_checkbox_limits_bucket_fill_to_the_visible_color() {
+    let _gpu = photocraft_testkit::gpu_lock();
     let Some(mut h) = harness() else { return };
     h.run_steps(4);
     for all_layers in [false, true] {

crates/ui-egui/tests/ui_scale_canvas.rs

@@ -115,10 +115,12 @@ fn assert_document_size_is_physical(gpu: bool) {
 
 #[test]
 fn ui_scale_keeps_the_document_pixel_size_on_the_gpu_canvas() {
+    let _gpu = photocraft_testkit::gpu_lock();
     assert_document_size_is_physical(true);
 }
 
 #[test]
 fn ui_scale_keeps_the_document_pixel_size_on_the_cpu_canvas() {
+    let _gpu = photocraft_testkit::gpu_lock();
     assert_document_size_is_physical(false);
 }