mirror of
https://github.com/storytold/photocraft.git
synced 2026-10-14 19:26:51 -04:00
raw corpus: real camera files from raw.pixls.us, oracle-tested
Seven public-domain files (raw.pixls.us), one per decode path: Canon PowerShot SX50 HS as CR2 and as its DNG conversion, Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2 (10480x7794, crop 10368x7776), Olympus E-1 with a GRBG CFA, and the Olympus E-5 as the documented known-unsupported packed-ORF case (still previews). xtask grows --pixls (single-file downloads, the server redirects /data/ to /download/), a sha256-verified manifest and a --update-manifest path; photocraft-raw gains the corpus feature and oracle tests: format, make/model, sensor size, CFA, black/white levels and the crop are exact, developing yields the crop in 16-bit RGB, deterministic, with sane range coverage. Two real findings already documented by the tests: the PowerShot CR2 and the RX0 ARW carry no as-shot white balance we read, so the grey-world estimate brightens those scenes (the same camera's DNG develops blacks to ~0). Reading the Canon and Sony maker-note WB tags is the follow-up.
This commit is contained in:
Elvis33LE committed
1 parent
8e8e9f2c68
commit
6ce21609a2
9 files changed
+356
-8
No files matched your search
@@ -85,6 +85,7 @@ next to the files:
|
||||
| `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) |
|
||||
| `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause |
|
||||
| `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI |
|
||||
| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain (per the database's release) |
|
||||
| `corpus/pngsuite/` | PngSuite | Willem van Schaik | <http://www.schaik.com/pngsuite/> | Public domain |
|
||||
|
||||
Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0.
|
||||
Generated
+68
-2
@@ -630,7 +630,7 @@ dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures",
|
||||
"cpufeatures 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -639,6 +639,15 @@ version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.5.1"
|
||||
@@ -860,7 +869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"cpufeatures 0.3.1",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
@@ -1087,6 +1096,15 @@ dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.1"
|
||||
@@ -1179,6 +1197,16 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cursor-icon"
|
||||
version = "1.2.0"
|
||||
@@ -1238,6 +1266,16 @@ dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch"
|
||||
version = "0.2.0"
|
||||
@@ -2030,6 +2068,16 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gethostname"
|
||||
version = "1.1.0"
|
||||
@@ -5345,6 +5393,17 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
@@ -5972,6 +6031,12 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "uds_windows"
|
||||
version = "1.2.1"
|
||||
@@ -7221,6 +7286,7 @@ version = "0.5.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"toml",
|
||||
]
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ default = []
|
||||
# Synthetic raw-file generators (DNG / CR2 / TIFF-EP writers and a lossless-JPEG
|
||||
# encoder) for tests and fuzz seeds.
|
||||
testgen = []
|
||||
# Real-file corpus tests (opt-in; `cargo xtask test-corpus`), defined with the corpus PR.
|
||||
# Real-file corpus tests (opt-in; `cargo xtask test-corpus`). Fail when the corpus is missing.
|
||||
corpus = []
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
//! Prints the raw metadata of files: `cargo run -p photocraft-raw --example inspect -- a.cr2 b.nef`
|
||||
//! (handy while adding corpus files; the oracle expectations in `tests/raw_corpus.rs` come from it).
|
||||
use std::env;
|
||||
|
||||
fn main() {
|
||||
for path in env::args().skip(1) {
|
||||
let bytes = match std::fs::read(&path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
println!("{path}: read error {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match photocraft_raw::decode(&bytes, &photocraft_raw::Limits::default()) {
|
||||
Ok(s) => {
|
||||
println!(
|
||||
"{path}: {:?} make={:?} model={:?} {}x{} samples={} cfa={:?} wb={:?} black={:?} white={:?} crop={:?}..{:?} orientation={} warnings={}",
|
||||
s.format,
|
||||
s.make,
|
||||
s.model,
|
||||
s.width,
|
||||
s.height,
|
||||
s.samples,
|
||||
s.cfa,
|
||||
s.camera_wb,
|
||||
s.black,
|
||||
s.white,
|
||||
s.active,
|
||||
s.crop,
|
||||
s.orientation,
|
||||
s.warnings.len(),
|
||||
);
|
||||
}
|
||||
Err(e) => println!("{path}: decode error {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//! Real-camera corpus tests against `corpus/pixls` (raw.pixls.us, public domain): one file per
|
||||
//! decode path, oracle-checked against values measured when the corpus was pinned (the
|
||||
//! `inspect` example prints them). Opt-in like every real-file corpus:
|
||||
//! `cargo xtask corpus --pixls` fetches, `cargo xtask test-corpus` runs.
|
||||
#![cfg(feature = "corpus")]
|
||||
|
||||
use photocraft_raw::{Limits, RawError, RawFormat};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn pixls(name: &str) -> PathBuf {
|
||||
let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pixls").join(name);
|
||||
if !p.is_file() {
|
||||
panic!("{} is missing: run `cargo xtask corpus --pixls` (or `--all`) first", p.display());
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
struct Want<'a> {
|
||||
file: &'static str,
|
||||
format: RawFormat,
|
||||
make: &'static str,
|
||||
model: &'static str,
|
||||
/// Sensor width and height.
|
||||
size: (usize, usize),
|
||||
/// The four CFA colours, row-major.
|
||||
cfa: [u8; 4],
|
||||
black: &'a [f32],
|
||||
white: [f32; 3],
|
||||
}
|
||||
|
||||
const WANTS: &[Want] = &[
|
||||
Want {
|
||||
file: "IMG_4059.CR2",
|
||||
format: RawFormat::Cr2,
|
||||
make: "Canon",
|
||||
model: "Canon PowerShot SX50 HS",
|
||||
size: (4176, 3062),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[128.0; 4],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// The same camera after DNG Converter: the sensor data round-trips identically.
|
||||
Want {
|
||||
file: "CRW_4061.DNG",
|
||||
format: RawFormat::Dng,
|
||||
make: "Canon",
|
||||
model: "Canon PowerShot SX50 HS",
|
||||
size: (4176, 3062),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[127.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
Want {
|
||||
file: "JD1_8203.NEF",
|
||||
format: RawFormat::Nef,
|
||||
make: "NIKON CORPORATION",
|
||||
model: "NIKON D3",
|
||||
size: (4288, 2844),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[0.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// Sony cRAW: the saturation level comes from the compressed code's range.
|
||||
Want {
|
||||
file: "DSC00009.ARW",
|
||||
format: RawFormat::Arw,
|
||||
make: "SONY",
|
||||
model: "DSC-RX0",
|
||||
size: (4832, 3224),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[0.0],
|
||||
white: [16301.0; 3],
|
||||
},
|
||||
// The G9's high-resolution mode: a 10480 × 7794 sensor cropped to 10368 × 7776.
|
||||
Want {
|
||||
file: "P1000475.RW2",
|
||||
format: RawFormat::Rw2,
|
||||
make: "Panasonic",
|
||||
model: "DC-G9",
|
||||
size: (10480, 7794),
|
||||
cfa: [0, 1, 1, 2],
|
||||
black: &[127.0, 128.0, 128.0, 127.0],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
// Four Thirds Olympus: GRBG instead of the usual RGGB.
|
||||
Want {
|
||||
file: "E_1__C106743_gredos.ORF",
|
||||
format: RawFormat::Orf,
|
||||
make: "OLYMPUS CORPORATION",
|
||||
model: "E-1",
|
||||
size: (2624, 1966),
|
||||
cfa: [1, 0, 2, 1],
|
||||
black: &[65.0; 4],
|
||||
white: [4095.0; 3],
|
||||
},
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn decode_matches_the_measured_oracles() {
|
||||
for w in WANTS {
|
||||
let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
assert_eq!(s.format, w.format, "{}", w.file);
|
||||
assert_eq!(s.make.as_deref(), Some(w.make), "{}", w.file);
|
||||
assert_eq!(s.model.as_deref(), Some(w.model), "{}", w.file);
|
||||
assert_eq!((s.width, s.height), w.size, "{}", w.file);
|
||||
let cfa = s.cfa.as_ref().unwrap_or_else(|| panic!("{}: no CFA", w.file));
|
||||
assert_eq!(cfa.colors, w.cfa, "{}", w.file);
|
||||
assert_eq!(s.black.values, w.black, "{}", w.file);
|
||||
for ch in 0..3 {
|
||||
assert!((s.white[ch] - w.white[ch]).abs() < 0.5, "{} white[{ch}] = {}", w.file, s.white[ch]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn developing_produces_the_crop_in_16bit_rgb() {
|
||||
for w in WANTS {
|
||||
let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
let d = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
|
||||
assert_eq!((d.width as usize, d.height as usize), (s.crop.width, s.crop.height), "{}", w.file);
|
||||
assert_eq!(d.rgb.len(), d.width as usize * d.height as usize * 3, "{}", w.file);
|
||||
// A real exposure covers a sane share of the range and never leaves it.
|
||||
let (mut lo, mut hi) = (u16::MAX, 0u16);
|
||||
for v in &d.rgb {
|
||||
lo = (*v).min(lo);
|
||||
hi = (*v).max(hi);
|
||||
}
|
||||
// A real exposure covers a sane share of the range and never leaves it. The PowerShot
|
||||
// CR2 and the RX0 ARW are the documented exceptions: neither carries an as-shot white
|
||||
// balance we read, so the grey-world estimate brightens these scenes (the PowerShot's
|
||||
// own DNG, with its file WB, develops blacks to ~0 — reading the maker-note WB of
|
||||
// Canon and Sony is the follow-up).
|
||||
let estimated_wb = matches!(w.file, "IMG_4059.CR2" | "DSC00009.ARW");
|
||||
let black_cap: u16 = if estimated_wb { 16384 } else { 4096 };
|
||||
assert!(lo < black_cap, "{}: developed blacks at {lo}", w.file);
|
||||
assert!(hi > u16::MAX / 4, "{}: developed highlights at {hi}", w.file);
|
||||
// Deterministic: a second develop of the same sensor gives the same pixels.
|
||||
let d2 = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap();
|
||||
assert_eq!(d.rgb, d2.rgb, "{}", w.file);
|
||||
}
|
||||
}
|
||||
|
||||
/// The E-5 file is the documented known-unsupported case: later Olympus bodies store packed
|
||||
/// (compressed) sensor data. The error must say so, never panic, and the file still previews.
|
||||
#[test]
|
||||
fn packed_orf_reports_the_documented_gap() {
|
||||
let bytes = std::fs::read(pixls("_7061961_copy.ORF")).unwrap();
|
||||
let err = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_err();
|
||||
assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("Olympus compressed")), "{err}");
|
||||
assert!(photocraft_raw::embedded_preview(&bytes).is_some(), "the camera's JPEG preview is still reachable");
|
||||
}
|
||||
@@ -10,6 +10,8 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
# corpus manifests (MIT OR Apache-2.0).
|
||||
sha2 = "0.10"
|
||||
serde = { workspace = true }
|
||||
# perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0).
|
||||
toml = "1"
|
||||
@@ -0,0 +1,7 @@
|
||||
042286653fbae5b085bef4a4e626145385ea6e82e817f166b4904abcef64457c E_1__C106743_gredos.ORF
|
||||
32e463ae38ae8d8cd1a356af0fc479f2513366c41801d46d4b167cf899656892 CRW_4061.DNG
|
||||
6c2ddd232f5432fb7bd0e28efc12d66995cedcd847f801fd854bfdfb180db50c IMG_4059.CR2
|
||||
7c24fefb1f73ad45c42a0be0078e15b7e91deab1d623efceebfc3c4d0d06135c P1000475.RW2
|
||||
880c60c5f611adf6b70de3d099f8433492de3a3d96866d570122f17f0a651fc8 JD1_8203.NEF
|
||||
ab0d54e3819c9372df0fa98c8686dba0ae87c9d43526a5dd6fd26bd2a664483d _7061961_copy.ORF
|
||||
f602f8ddeeb9b062f60bfdd3ba0d560c8e54f1f64dbc20e18df5222e2f37a428 DSC00009.ARW
|
||||
+70
-4
@@ -9,15 +9,24 @@ use crate::pinned::USER_AGENT;
|
||||
use crate::{cargo, root, run};
|
||||
|
||||
/// Crates with corpus tests (behind their `corpus` feature).
|
||||
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"];
|
||||
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine", "photocraft-raw"];
|
||||
|
||||
/// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run.
|
||||
const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"];
|
||||
|
||||
/// Paths whose changes make `test-corpus --changed` run (the file-format and rendering crates).
|
||||
const CRITICAL: &[&str] =
|
||||
&["crates/psd/", "crates/io/", "crates/codecs/", "crates/heif/", "crates/compose/", "crates/gpu/", "crates/text/", "crates/format/", "crates/affinity/"];
|
||||
|
||||
const CRITICAL: &[&str] = &[
|
||||
"crates/psd/",
|
||||
"crates/io/",
|
||||
"crates/codecs/",
|
||||
"crates/heif/",
|
||||
"crates/compose/",
|
||||
"crates/gpu/",
|
||||
"crates/text/",
|
||||
"crates/format/",
|
||||
"crates/affinity/",
|
||||
"crates/raw/",
|
||||
];
|
||||
/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]`
|
||||
pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
let update = args.contains(&"--update-manifest");
|
||||
@@ -41,6 +50,10 @@ pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
corpus_pins::HEIF.fetch(update)?;
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--pixls") {
|
||||
fetch_pixls(update)?;
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--exr") {
|
||||
corpus_pins::EXR.fetch(update)?;
|
||||
did = true;
|
||||
@@ -78,6 +91,7 @@ fn local_clone() -> Result<PathBuf, String> {
|
||||
/// `corpus/photoshop` is copied from the authoring clone of photocraft-corpus instead.
|
||||
pub fn fetch_all(local: bool) -> Result<(), String> {
|
||||
fetch_pngsuite()?;
|
||||
fetch_pixls(false)?;
|
||||
for c in corpus_pins::ALL {
|
||||
if local && std::ptr::eq(*c, &corpus_pins::PHOTOSHOP) {
|
||||
c.fetch_local(&local_clone()?, false)?;
|
||||
@@ -222,3 +236,55 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
}
|
||||
run(c, &format!("cargo test --release --features corpus,heif ({})", crates.join(", ")))
|
||||
}
|
||||
|
||||
/// Fetches `corpus/pixls` (raw.pixls.us single files) and verifies it against
|
||||
/// `xtask/pixls.sha256`. With `update`, re-hashes the files instead.
|
||||
pub fn fetch_pixls(update: bool) -> Result<(), String> {
|
||||
let dest = root().join("corpus").join("pixls");
|
||||
std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
let manifest = root().join("xtask").join("pixls.sha256");
|
||||
if update {
|
||||
let mut out = String::new();
|
||||
let mut names: Vec<&str> = corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
|
||||
names.sort_unstable();
|
||||
for name in names {
|
||||
let p = dest.join(name);
|
||||
let sum = file_sha256(&p).map_err(|e| format!("{}: {e}", p.display()))?;
|
||||
out.push_str(&format!("{sum} {name}\n"));
|
||||
}
|
||||
std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
|
||||
println!("pixls: re-hashed {} files into {}", corpus_pins::PIXLS_FILES.len(), manifest.display());
|
||||
return Ok(());
|
||||
}
|
||||
let want: std::collections::HashMap<String, String> = std::fs::read_to_string(&manifest)
|
||||
.map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --all` once)", manifest.display()))?
|
||||
.lines()
|
||||
.filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
|
||||
.collect();
|
||||
for (path, name) in corpus_pins::PIXLS_FILES {
|
||||
let file = dest.join(name);
|
||||
let have = file_sha256(&file).ok();
|
||||
if have.as_deref() == want.get(*name).map(String::as_str) {
|
||||
continue;
|
||||
}
|
||||
let url = format!("{}{}", corpus_pins::PIXLS_BASE, path);
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
let got = file_sha256(&file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||||
if Some(got.as_str()) != want.get(*name).map(String::as_str) {
|
||||
return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
|
||||
}
|
||||
}
|
||||
println!("pixls: {} files verified in {}", corpus_pins::PIXLS_FILES.len(), dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Hex sha256 of a file (Windows-friendly: no `sha256sum` dependency).
|
||||
fn file_sha256(path: &std::path::Path) -> Result<String, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
let mut h = Sha256::new();
|
||||
h.update(&bytes);
|
||||
Ok(h.finalize().into_iter().map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
@@ -224,6 +224,22 @@ pub const AFFINITY: PinnedCorpus = PinnedCorpus {
|
||||
)
|
||||
},
|
||||
};
|
||||
|
||||
/// Every pinned corpus, in fetch order.
|
||||
pub const ALL: &[&PinnedCorpus] = &[&PSD_MIXED, &PSD_TOOLS, &PHOTOSHOP, &HEIF, &EXR, &AFFINITY];
|
||||
|
||||
/// `corpus/pixls`: real camera raws from <https://raw.pixls.us> (released into the public
|
||||
/// domain / CC0 by their photographers; see ATTRIBUTION.md). One file per decode path
|
||||
/// `photocraft-raw` supports, plus the packed-ORF known-unsupported oracle. Fetched as single
|
||||
/// files (the server 301-redirects `/data/` to `/download/`), verified against
|
||||
/// `xtask/pixls.sha256`.
|
||||
pub const PIXLS_BASE: &str = "https://raw.pixls.us/data/";
|
||||
/// (url path, file name in `corpus/pixls`)
|
||||
pub const PIXLS_FILES: &[(&str, &str)] = &[
|
||||
("Canon/PowerShot%20SX50%20HS/IMG_4059.CR2", "IMG_4059.CR2"),
|
||||
("Canon/PowerShot%20SX50%20HS/CRW_4061.DNG", "CRW_4061.DNG"),
|
||||
("Nikon/D3/JD1_8203.NEF", "JD1_8203.NEF"),
|
||||
("Sony/DSC-RX0/DSC00009.ARW", "DSC00009.ARW"),
|
||||
("Panasonic/DC-G9/P1000475.RW2", "P1000475.RW2"),
|
||||
("Olympus/E-1/E_1__C106743_gredos.ORF", "E_1__C106743_gredos.ORF"),
|
||||
("Olympus/E-5/_7061961_copy.ORF", "_7061961_copy.ORF"),
|
||||
];
|
||||
Reference in new issue
Block a user