mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-20 06:04:11 -04:00
Compare commits
123
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d999e382ba | ||
|
|
fb7ac08b9a | ||
|
|
31a677fe3d | ||
|
|
a945533f63 | ||
|
|
d43dd1fc8f | ||
|
|
6f5d662df3 | ||
|
|
70265ed17a | ||
|
|
19008bdee9 | ||
|
|
56c9b49634 | ||
|
|
2c60d2ab27 | ||
|
|
4b6db59ade | ||
|
|
cbdbd50f84 | ||
|
|
fe7edd7b26 | ||
|
|
1e4aa93d58 | ||
|
|
7614fb14f1 | ||
|
|
2983a40385 | ||
|
|
583e3ceefc | ||
|
|
5b66a4825e | ||
|
|
dee8358c38 | ||
|
|
8d41bf69f7 | ||
|
|
f4e7d07e9b | ||
|
|
4e77935e6e | ||
|
|
7134099101 | ||
|
|
ad22d059db | ||
|
|
29384a0404 | ||
|
|
70100a34c5 | ||
|
|
363dbbdd9f | ||
|
|
80e2d0e64a | ||
|
|
d3b6266d07 | ||
|
|
c9f383b368 | ||
|
|
389be0f6a9 | ||
|
|
f819c8439b | ||
|
|
e6a7501adc | ||
|
|
0a0cb19011 | ||
|
|
7b65a6a59f | ||
|
|
d43889145b | ||
|
|
6985cfa54c | ||
|
|
592cb335cb | ||
|
|
d6a9a2e8f9 | ||
|
|
11308a5c8a | ||
|
|
9bf08f6ce6 | ||
|
|
5c85bc3ee4 | ||
|
|
0c3c500ea4 | ||
|
|
601ddce6ef | ||
|
|
b9159b0fce | ||
|
|
863adcac6f | ||
|
|
4f0b110bcb | ||
|
|
20ccedf56e | ||
|
|
43a47e5a3b | ||
|
|
cebb2be4d4 | ||
|
|
f48b2ad895 | ||
|
|
0d2c77c258 | ||
|
|
5589827178 | ||
|
|
2218d97e89 | ||
|
|
f9b0c8c8a1 | ||
|
|
2b173f0bee | ||
|
|
e6587c3f82 | ||
|
|
d54673bb95 | ||
|
|
a8bc882092 | ||
|
|
9c4fce176f | ||
|
|
16beb4db40 | ||
|
|
f6d465c72c | ||
|
|
fafa70724b | ||
|
|
035d572ac8 | ||
|
|
e0c02d5e89 | ||
|
|
8e268431e0 | ||
|
|
62eb6627a6 | ||
|
|
edc369565e | ||
|
|
80e439f6dc | ||
|
|
5be931e48a | ||
|
|
7b934ca2b2 | ||
|
|
346681561a | ||
|
|
3accb56196 | ||
|
|
0a9675c6d9 | ||
|
|
312dd04830 | ||
|
|
b67bb01952 | ||
|
|
f34c7a8a89 | ||
|
|
3885b61e6b | ||
|
|
0802892132 | ||
|
|
c8f741fd0b | ||
|
|
c5deea05e4 | ||
|
|
9bec9db541 | ||
|
|
fa715c8c94 | ||
|
|
6938be39e4 | ||
|
|
dde403946b | ||
|
|
b5dc500c54 | ||
|
|
0f67225e83 | ||
|
|
d020c55062 | ||
|
|
25d466fd69 | ||
|
|
7ac6dd81fe | ||
|
|
79d726f856 | ||
|
|
a43cb68593 | ||
|
|
8a4d6f2ebf | ||
|
|
112e202fed | ||
|
|
de038ca3d4 | ||
|
|
0da998f24d | ||
|
|
e8856c828c | ||
|
|
b17483a3ce | ||
|
|
e6dceb9f89 | ||
|
|
5e9f7f1a65 | ||
|
|
9ddb8d1824 | ||
|
|
8291f08ab7 | ||
|
|
1d7699056e | ||
|
|
2179473536 | ||
|
|
132b0e3ac8 | ||
|
|
4212907656 | ||
|
|
888a54f52a | ||
|
|
aed214da7e | ||
|
|
31f4ad43fe | ||
|
|
ef216d2110 | ||
|
|
f906998174 | ||
|
|
1130bda456 | ||
|
|
48d69bfae5 | ||
|
|
e65b97e239 | ||
|
|
540a281ccd | ||
|
|
61ec5c596c | ||
|
|
f625c74c7a | ||
|
|
2b86ca0537 | ||
|
|
d598c304aa | ||
|
|
0242a75b5a | ||
|
|
80a8acaa30 | ||
|
|
c78b457c0d | ||
|
|
76f12dcabd |
No files matched your search
@@ -0,0 +1,30 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Maintain Rust / Cargo dependencies
|
||||
- package-ecosystem: "cargo"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "06:00"
|
||||
timezone: "America/New_York"
|
||||
# Consolidate updates into a single PR rather than one PR per crate
|
||||
groups:
|
||||
cargo-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
commit-message:
|
||||
prefix: "chore(deps)"
|
||||
|
||||
# Maintain GitHub Actions workflows
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "06:00"
|
||||
timezone: "America/New_York"
|
||||
groups:
|
||||
actions:
|
||||
patterns:
|
||||
- "*"
|
||||
commit-message:
|
||||
prefix: "ci(deps)"
|
||||
@@ -1,16 +1,23 @@
|
||||
# Builds and tests the workspace on FreeBSD, inside a FreeBSD VM on a Linux runner
|
||||
# (GitHub has no FreeBSD runners). GPU tests skip themselves when the VM has no adapter.
|
||||
# FreeBSD build and package check in a QEMU VM.
|
||||
name: FreeBSD
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
paths:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
- "**/Cargo.toml"
|
||||
- .github/workflows/freebsd.yml
|
||||
- ".github/workflows/freebsd.yml"
|
||||
- "packaging/freebsd/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- ".github/workflows/freebsd.yml"
|
||||
- "packaging/freebsd/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
@@ -22,34 +29,8 @@ permissions:
|
||||
|
||||
jobs:
|
||||
freebsd:
|
||||
name: FreeBSD 14 (build + test)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 300
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Optional font build input, so the Japanese font tests run (they skip without it).
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
|
||||
path: craft-fonts
|
||||
- name: Build and test in a FreeBSD VM
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
|
||||
with:
|
||||
release: "14.3"
|
||||
usesh: true
|
||||
cpu: 4
|
||||
mem: 12288
|
||||
# Nothing to bring back; copying the target dir out of the VM fails on large builds.
|
||||
copyback: false
|
||||
prepare: |
|
||||
pkg install -y git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
run: |
|
||||
. "$HOME/.cargo/env"
|
||||
# The checkout is synced in under another owner; let tests that call git read it.
|
||||
git config --global --add safe.directory "*"
|
||||
rustc --version
|
||||
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked
|
||||
CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked
|
||||
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
|
||||
uses: storytold/craft-common-actions/.github/workflows/freebsd.yml@main
|
||||
with:
|
||||
require-craft-fonts: true
|
||||
craft-fonts-ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
|
||||
@@ -1,7 +1,5 @@
|
||||
# Quick checks for the release pipeline (no Rust build): workflow syntax, packaging
|
||||
# shell/PowerShell scripts, and the Linux desktop/AppStream/Flatpak metadata (seconds), plus a
|
||||
# Windows job that builds and installs throwaway per-user MSI probes for the install-folder
|
||||
# searches (a few minutes).
|
||||
# Quick static checks for the release pipeline (seconds, no Rust build): workflow syntax,
|
||||
# packaging shell/PowerShell scripts, and the Linux desktop/AppStream/Flatpak metadata.
|
||||
name: Packaging lint
|
||||
|
||||
on:
|
||||
@@ -21,64 +19,10 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
ACTIONLINT_VERSION: 1.7.12
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends shellcheck desktop-file-utils appstream libxml2-utils python3-yaml
|
||||
curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" \
|
||||
| tar -xz -C "$RUNNER_TEMP" actionlint
|
||||
- name: actionlint
|
||||
run: |
|
||||
"$RUNNER_TEMP/actionlint" -color
|
||||
- name: shellcheck
|
||||
run: shellcheck packaging/*.sh packaging/*/*.sh
|
||||
- name: PowerShell
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in Get-ChildItem packaging/windows/*.ps1) {
|
||||
$errors = $null
|
||||
[System.Management.Automation.Language.Parser]::ParseFile($f.FullName, [ref]$null, [ref]$errors) | Out-Null
|
||||
if ($errors) { $errors | ForEach-Object { Write-Output "::error file=$($f.Name)::$_" }; exit 1 }
|
||||
Write-Output "ok $($f.Name)"
|
||||
}
|
||||
- name: XML, desktop entry, AppStream, Flatpak manifest
|
||||
run: |
|
||||
xmllint --noout packaging/windows/lightcraft.wxs packaging/macos/*.plist* packaging/linux/*.xml*
|
||||
# Windows Installer needs Icon ids ending in .ico or .exe, or shortcuts show a blank icon.
|
||||
if grep -oE '<Icon Id="[^"]*"' packaging/windows/lightcraft.wxs | grep -vE '\.(ico|exe)"$'; then
|
||||
echo "::error::Icon Id must end in .ico or .exe"; exit 1
|
||||
fi
|
||||
# The interactive installer asks where to install and ends on a completion page (issues #18, #399).
|
||||
for want in '<UIRef Id="LightcraftUI" />' 'Value="InstallDirDlg"' '<Property Id="WIXUI_INSTALLDIR" Value="INSTALLFOLDER" />' \
|
||||
'Key="Software\LightCraft" Name="InstallFolder"' 'Value="LaunchLightcraft"'; do
|
||||
grep -qF "$want" packaging/windows/lightcraft.wxs || { echo "::error::lightcraft.wxs lost: $want"; exit 1; }
|
||||
done
|
||||
# The completion page the Publish events point at must actually exist: a Publish alone
|
||||
# does not pull the dialog fragment into the MSI, and without the DialogRef the
|
||||
# installer finishes silently (issues #790, #892).
|
||||
grep -qF '<DialogRef Id="ExitDialog" />' packaging/windows/lightcraft.wxs \
|
||||
|| { echo "::error::ExitDialog is not referenced; installs and removes end without a completion dialog"; exit 1; }
|
||||
desktop-file-validate packaging/linux/ai.storyteller.lightcraft.desktop
|
||||
sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.lightcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
|
||||
appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.lightcraft.metainfo.xml"
|
||||
python3 - <<'EOF'
|
||||
import yaml
|
||||
src = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.yml"))
|
||||
bundle = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.lightcraft.bundle.yml"))
|
||||
for m in (src, bundle):
|
||||
assert m["id"] == "ai.storyteller.lightcraft" and m["command"] == "lightcraft", m["id"]
|
||||
for key in ("runtime", "runtime-version", "sdk", "finish-args"):
|
||||
assert src[key] == bundle[key], f"Flatpak manifests disagree on {key}: {src[key]} vs {bundle[key]}"
|
||||
print("Flatpak manifests ok")
|
||||
EOF
|
||||
- name: FreeBSD tarball layout (stub binaries)
|
||||
run: bash packaging/freebsd/package.sh --dry-run
|
||||
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
|
||||
uses: storytold/craft-common-actions/.github/workflows/packaging-lint.yml@main
|
||||
with:
|
||||
extra-checks-script: packaging/check-app-specific.sh
|
||||
|
||||
windows-install-folder:
|
||||
runs-on: windows-latest
|
||||
|
||||
+25
-479
@@ -1,13 +1,5 @@
|
||||
# Release pipeline: every push to `release` builds platform artifacts (signed where credentials
|
||||
# are configured) and creates or updates the draft GitHub Release "LightCraft v<version>".
|
||||
# See docs/releasing.md.
|
||||
#
|
||||
# The signing jobs (macOS, Windows) and the draft-release job run in the `release` environment
|
||||
# (restricted to the `release` branch), which holds the signing secrets. Every secret is optional:
|
||||
# missing ones produce unsigned artifacts and a warning, never a failed build. The jobs that sign
|
||||
# nothing (version, Linux, Linux riscv64, Flatpak, FreeBSD, web) run without it, so a dispatch on
|
||||
# any branch is a dry run of them; the release job needs the signed jobs, so a branch run never
|
||||
# drafts a release.
|
||||
# Release pipeline: every push to `release` builds signed installers for every platform and
|
||||
# creates or updates the draft GitHub Release "LightCraft v<version>". See docs/releasing.md.
|
||||
name: Release
|
||||
|
||||
on:
|
||||
@@ -21,479 +13,33 @@ on:
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
# Never cancel a run mid-notarization or mid-upload; a newer push waits its turn.
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUST_BACKTRACE: 1
|
||||
LIGHTCRAFT_BUILD_SHA: ${{ github.sha }}
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
|
||||
jobs:
|
||||
version:
|
||||
timeout-minutes: 10
|
||||
name: version
|
||||
runs-on: ubuntu-latest
|
||||
# No `environment: release` here or on the other jobs that sign nothing: no release secrets.
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
date: ${{ steps.v.outputs.date }}
|
||||
prerelease: ${{ steps.v.outputs.prerelease }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- id: v
|
||||
name: Resolve version
|
||||
run: |
|
||||
. packaging/env.sh
|
||||
v="${INPUT_VERSION#v}"
|
||||
v="${v:-$VERSION}"
|
||||
if ! printf '%s' "$v" | grep -Eq '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'; then
|
||||
echo "::error::'$v' is not a version like 1.2.3 or 1.2.3-rc.1"
|
||||
exit 1
|
||||
fi
|
||||
case "$v" in *-*) pre=true ;; *) pre=false ;; esac
|
||||
{
|
||||
echo "version=$v"
|
||||
echo "date=$(date -u +%Y-%m-%d)"
|
||||
echo "prerelease=$pre"
|
||||
} >>"$GITHUB_OUTPUT"
|
||||
echo "### LightCraft v$v (\`${GITHUB_SHA::9}\`)" >>"$GITHUB_STEP_SUMMARY"
|
||||
|
||||
macos:
|
||||
timeout-minutes: 90
|
||||
name: macOS universal
|
||||
needs: version
|
||||
runs-on: macos-15-xlarge # M2 Pro larger runner; use only in release.yml (actions_macos_xl budget caps spend)
|
||||
environment: release
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: aarch64-apple-darwin, x86_64-apple-darwin
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-macos-universal
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Import signing certificate
|
||||
env:
|
||||
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
|
||||
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
|
||||
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
|
||||
run: packaging/macos/import-cert.sh
|
||||
- name: Build, sign, notarize
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
APPLE_ID: ${{ secrets.APPLE_ID }}
|
||||
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
|
||||
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
|
||||
run: packaging/macos/package.sh --arch universal
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-macos
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
- name: Remove temporary keychain
|
||||
if: always()
|
||||
run: |
|
||||
if [ -n "${MACOS_KEYCHAIN:-}" ]; then security delete-keychain "$MACOS_KEYCHAIN" || true; fi
|
||||
|
||||
windows:
|
||||
timeout-minutes: 90
|
||||
name: Windows ${{ matrix.arch }}
|
||||
needs: version
|
||||
runs-on: windows-latest
|
||||
environment: release
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { arch: x64, target: x86_64-pc-windows-msvc }
|
||||
- { arch: x86, target: i686-pc-windows-msvc }
|
||||
# Windows on ARM: cross-compiled on the x64 runner, so signing and WiX run as above.
|
||||
# .github/workflows/windows-arm64.yml installs and runs it on ARM64 hardware.
|
||||
- { arch: arm64, target: aarch64-pc-windows-msvc }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-windows-${{ matrix.arch }}
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
shell: bash
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
# package.ps1 adds the matching WixToolset.UI/Util.wixext (5.0.2) to the global extension cache.
|
||||
- name: Install WiX
|
||||
shell: pwsh
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Build, sign, package
|
||||
shell: pwsh
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
|
||||
AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }}
|
||||
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
|
||||
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
|
||||
run: ./packaging/windows/package.ps1 -Arch ${{ matrix.arch }}
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-windows-${{ matrix.arch }}
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
linux:
|
||||
timeout-minutes: 90
|
||||
name: Linux ${{ matrix.arch }}
|
||||
needs: version
|
||||
# The oldest GitHub-hosted image, so the binaries only need glibc >= 2.35 (Ubuntu 22.04+,
|
||||
# Debian 12+, Fedora 36+, RHEL 10). See docs/releasing.md.
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { arch: x86_64, runner: ubuntu-22.04, deb: amd64 }
|
||||
- { arch: aarch64, runner: ubuntu-22.04-arm, deb: arm64 }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
NFPM_VERSION: 2.47.0
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- name: Install build and packaging tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev \
|
||||
desktop-file-utils appstream python3-yaml zsync
|
||||
curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" \
|
||||
"https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_${{ matrix.deb }}.deb"
|
||||
sudo dpkg -i "$RUNNER_TEMP/nfpm.deb"
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-linux-${{ matrix.arch }}
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and package (AppImage, deb, rpm, tar.gz)
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/linux/package.sh
|
||||
- name: Check the packages
|
||||
run: |
|
||||
dpkg-deb --info dist/release/*.deb
|
||||
dpkg-deb --contents dist/release/*.deb | grep -E 'usr/bin/lightcraft$|metainfo|applications'
|
||||
ldd target/release/lightcraft
|
||||
python3 -c "import yaml; m = yaml.safe_load(open('packaging/linux/flatpak/ai.storyteller.lightcraft.yml')); assert m['id'] == 'ai.storyteller.lightcraft'"
|
||||
for appimage in dist/release/*.AppImage; do
|
||||
chmod +x "$appimage"
|
||||
APPIMAGE_EXTRACT_AND_RUN=1 "$appimage" --version
|
||||
# Update information is embedded, and the .zsync for delta updates sits beside it.
|
||||
"$appimage" --appimage-updateinformation | grep -F 'gh-releases-zsync|'
|
||||
test -s "$appimage.zsync"
|
||||
done
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-linux-${{ matrix.arch }}
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
flatpak:
|
||||
timeout-minutes: 90
|
||||
name: Flatpak ${{ matrix.arch }}
|
||||
needs: [version, linux]
|
||||
# Repackages the Linux job's tarball (the same binaries as the AppImage/deb/rpm); no Rust
|
||||
# build here. 24.04 for a newer flatpak-builder (1.4) than 22.04 ships.
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { arch: x86_64, runner: ubuntu-24.04 }
|
||||
- { arch: aarch64, runner: ubuntu-24.04-arm }
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: release-linux-${{ matrix.arch }}
|
||||
path: dist/linux
|
||||
- name: Install flatpak-builder
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# librsvg2-common: the SVG pixbuf loader. flatpak-builder runs the host's
|
||||
# `appstreamcli compose`, which renders the scalable icon and fails without it.
|
||||
sudo apt-get install -y --no-install-recommends flatpak flatpak-builder librsvg2-common
|
||||
# Ubuntu 24.04's AppArmor blocks the unprivileged user namespaces bubblewrap needs.
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
- name: Build the Flatpak bundle
|
||||
run: packaging/linux/flatpak-bundle.sh "dist/linux/lightcraft-$LIGHTCRAFT_VERSION-linux-${{ matrix.arch }}.tar.gz"
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-flatpak-${{ matrix.arch }}
|
||||
path: dist/release/*.flatpak
|
||||
if-no-files-found: error
|
||||
|
||||
linux-riscv64:
|
||||
timeout-minutes: 90
|
||||
name: Linux riscv64
|
||||
needs: version
|
||||
# Cross-compiled on x86_64 (no hosted RISC-V runners) against the runner's own Ubuntu release
|
||||
# from ports.ubuntu.com, so the binaries need that release's glibc (24.04: 2.39). 22.04's
|
||||
# riscv64 ports lag its amd64 updates, which breaks the multiarch install. tar.gz only; the
|
||||
# CLI is smoke-tested under QEMU.
|
||||
runs-on: ubuntu-24.04
|
||||
# No `environment: release`: this job signs nothing, so it gets no release secrets.
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: riscv64gc-unknown-linux-gnu
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-linux-riscv64
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Install cross-compilation dependencies
|
||||
run: |
|
||||
. /etc/os-release
|
||||
sudo dpkg --add-architecture riscv64
|
||||
# Pin the existing (archive.ubuntu.com) sources to amd64; riscv64 comes from ports.
|
||||
if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
|
||||
sudo sed -i '/^Types: deb/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
fi
|
||||
if [ -f /etc/apt/sources.list ]; then
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /g' /etc/apt/sources.list
|
||||
sudo sed -i 's/^deb-src /deb-src [arch=amd64] /g' /etc/apt/sources.list
|
||||
fi
|
||||
for suite in "$VERSION_CODENAME" "$VERSION_CODENAME-updates" "$VERSION_CODENAME-security"; do
|
||||
echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports $suite main universe restricted multiverse"
|
||||
done | sudo tee /etc/apt/sources.list.d/riscv64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
gcc-riscv64-linux-gnu binutils-riscv64-linux-gnu libc6-dev-riscv64-cross libc6-dev:riscv64 qemu-user \
|
||||
libxkbcommon-dev:riscv64 libwayland-dev:riscv64 libx11-dev:riscv64 libxrandr-dev:riscv64 \
|
||||
libxi-dev:riscv64 libgl1-mesa-dev:riscv64
|
||||
- name: Build and package (tar.gz)
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER: riscv64-linux-gnu-gcc
|
||||
PKG_CONFIG_SYSROOT_DIR: /
|
||||
PKG_CONFIG_PATH: /usr/lib/riscv64-linux-gnu/pkgconfig:/usr/share/pkgconfig
|
||||
PKG_CONFIG_ALLOW_CROSS: "1"
|
||||
CROSS_ARCH: riscv64
|
||||
CROSS_TARGET: riscv64gc-unknown-linux-gnu
|
||||
CROSS_COMPILE: riscv64-linux-gnu-
|
||||
EMULATOR: qemu-riscv64
|
||||
QEMU_LD_PREFIX: /usr/riscv64-linux-gnu
|
||||
run: packaging/linux/package.sh --formats tar
|
||||
- name: Check the tarball
|
||||
run: |
|
||||
tar -tzf dist/release/lightcraft-*-linux-riscv64.tar.gz \
|
||||
| grep -E '/bin/lightcraft$|/bin/lightcraft-cli$|/share/applications/ai\.storyteller\.lightcraft\.desktop$'
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-linux-riscv64
|
||||
path: dist/release/*.tar.gz
|
||||
if-no-files-found: error
|
||||
|
||||
freebsd:
|
||||
name: FreeBSD x86_64
|
||||
needs: version
|
||||
# GitHub has no FreeBSD runners: build in a FreeBSD 14.3 VM, the same image and packages as
|
||||
# .github/workflows/freebsd.yml.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- name: Build and package in a FreeBSD VM (tar.gz)
|
||||
# vmactions/freebsd-vm v1.5.9, pinned by commit (it runs our build).
|
||||
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
|
||||
with:
|
||||
release: "14.3"
|
||||
usesh: true
|
||||
cpu: 4
|
||||
mem: 12288
|
||||
# Passed into the VM by name; never templated into the script below.
|
||||
envs: "LIGHTCRAFT_VERSION LIGHTCRAFT_BUILD_DATE LIGHTCRAFT_BUILD_SHA INPUT_VERSION"
|
||||
# The FreeBSD CI packages, plus bash for the packaging script.
|
||||
prepare: |
|
||||
pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
|
||||
run: |
|
||||
set -e
|
||||
. "$HOME/.cargo/env"
|
||||
git config --global --add safe.directory "$PWD"
|
||||
# Build outside the checkout so only dist/ is copied back out of the VM (copying the
|
||||
# target dir back fails on large builds). 4 jobs: more runs the VM out of memory.
|
||||
export CARGO_TARGET_DIR=/var/tmp/lightcraft-target CARGO_BUILD_JOBS=4
|
||||
if [ -n "$INPUT_VERSION" ]; then cargo xtask version set "$LIGHTCRAFT_VERSION"; fi
|
||||
# craft-fonts was checked out next to the sources and synced into the VM with them.
|
||||
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
|
||||
bash packaging/freebsd/package.sh
|
||||
tar -tzf dist/release/lightcraft-*-freebsd-x86_64.tar.gz \
|
||||
| grep -E '/bin/lightcraft$|/bin/lightcraft-cli$|/share/applications/ai\.storyteller\.lightcraft\.desktop$|/share/doc/lightcraft/LICENSE-MIT$'
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-freebsd-x86_64
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
web:
|
||||
timeout-minutes: 90
|
||||
name: Web (wasm)
|
||||
needs: version
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
LIGHTCRAFT_VERSION: ${{ needs.version.outputs.version }}
|
||||
LIGHTCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
|
||||
WASM_BINDGEN_VERSION: 0.2.129
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
# Fonts (optional build input; required for releases): see craftrules standards/fonts.md.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: storytold/craft-fonts
|
||||
ref: 3d9ab3a132ed0eff457e424226a435a901da30d3 # bump deliberately
|
||||
path: craft-fonts
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: wasm32-unknown-unknown
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: release-web
|
||||
- name: Install wasm-bindgen CLI
|
||||
run: |
|
||||
# Must match the wasm-bindgen version in Cargo.lock exactly (cargo xtask web checks it).
|
||||
if ! { command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION"; }; then
|
||||
cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
|
||||
fi
|
||||
wasm-bindgen --version
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
run: cargo xtask version set "$LIGHTCRAFT_VERSION"
|
||||
- name: Build and zip
|
||||
env:
|
||||
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
|
||||
CRAFT_FONTS_REQUIRED: "1"
|
||||
run: packaging/web/package.sh
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-web
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
release:
|
||||
timeout-minutes: 10
|
||||
name: Draft GitHub Release
|
||||
needs: [version, macos, windows, linux, linux-riscv64, flatpak, freebsd, web]
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
VERSION: ${{ needs.version.outputs.version }}
|
||||
PRERELEASE: ${{ needs.version.outputs.prerelease }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: release-*
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
- name: Checksums
|
||||
working-directory: artifacts
|
||||
run: |
|
||||
sha256sum -- * | tee SHA256SUMS.txt
|
||||
- name: Create or update the draft release
|
||||
working-directory: artifacts
|
||||
run: |
|
||||
tag="v$VERSION"
|
||||
title="LightCraft v$VERSION"
|
||||
pre=()
|
||||
if [ "$PRERELEASE" = true ]; then pre=(--prerelease); fi
|
||||
if draft="$(gh release view "$tag" --json isDraft --jq .isDraft 2>/dev/null)"; then
|
||||
if [ "$draft" != "true" ]; then
|
||||
echo "::error::Release $tag is already published. Bump the version (cargo xtask version set) before pushing to release."
|
||||
exit 1
|
||||
fi
|
||||
echo "Updating draft $tag: replacing its assets"
|
||||
gh release view "$tag" --json assets --jq '.assets[].name' | while read -r asset; do
|
||||
gh release delete-asset "$tag" "$asset" --yes
|
||||
done
|
||||
gh release edit "$tag" --draft --title "$title" --target "$GITHUB_SHA" --prerelease="$PRERELEASE"
|
||||
gh release upload "$tag" --clobber -- *
|
||||
else
|
||||
echo "Creating draft $tag"
|
||||
gh release create "$tag" --draft --title "$title" --target "$GITHUB_SHA" --generate-notes "${pre[@]}" -- *
|
||||
fi
|
||||
{
|
||||
echo "Draft release **$title** at \`${GITHUB_SHA::9}\`:"
|
||||
echo
|
||||
sed 's/^/ /' SHA256SUMS.txt
|
||||
} >>"$GITHUB_STEP_SUMMARY"
|
||||
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
|
||||
uses: storytold/craft-common-actions/.github/workflows/release.yml@main
|
||||
with:
|
||||
app-name: lightcraft
|
||||
app-title: LightCraft
|
||||
craft-fonts-ref: 3d9ab3a132ed0eff457e424226a435a901da30d3
|
||||
version: ${{ inputs.version }}
|
||||
secrets:
|
||||
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
|
||||
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
|
||||
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
|
||||
APPLE_ID: ${{ secrets.APPLE_ID }}
|
||||
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
|
||||
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
|
||||
AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }}
|
||||
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
|
||||
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
|
||||
@@ -1,22 +1,27 @@
|
||||
# Windows on ARM: package the ARM64 release the way release.yml does (cross-compiled on an x64
|
||||
# runner, unsigned here), then install that MSI on a Windows 11 ARM64 runner, check the installed
|
||||
# binaries are ARM64, run the CLI natively, and uninstall. (A native ARM64 `cargo test` job is left
|
||||
# out for now: PdfCraft saw an intermittent access violation in egui UI tests there.)
|
||||
# Windows ARM64 build and smoke test.
|
||||
#
|
||||
# Release packages on arm64-windows are produced on x64 runners via cross-compilation
|
||||
# (target: aarch64-pc-windows-msvc) using MSVC's ARM64 cross tools, then installed and run
|
||||
# on GitHub Actions' native windows-11-arm runner.
|
||||
name: Windows ARM64
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
paths:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
- "**/Cargo.toml"
|
||||
- "**/build.rs"
|
||||
- .github/workflows/windows-arm64.yml
|
||||
- packaging/windows/**
|
||||
schedule:
|
||||
- cron: "41 4 * * 1" # weekly, so drift in the runner images shows up
|
||||
- ".github/workflows/windows-arm64.yml"
|
||||
- "packaging/windows/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- ".github/workflows/windows-arm64.yml"
|
||||
- "packaging/windows/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
@@ -26,70 +31,10 @@ concurrency:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
package:
|
||||
name: package (cross-compiled on x64)
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
targets: aarch64-pc-windows-msvc
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: windows-arm64-package
|
||||
- name: Install WiX
|
||||
shell: pwsh
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Build and package (unsigned)
|
||||
shell: pwsh
|
||||
env:
|
||||
DIST: ${{ github.workspace }}\dist\arm64
|
||||
run: ./packaging/windows/package.ps1 -Arch arm64
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist/arm64/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install:
|
||||
name: install and run on ARM64
|
||||
needs: package
|
||||
runs-on: windows-11-arm
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: lightcraft-windows-arm64
|
||||
path: dist
|
||||
- name: Install the MSI, check it, run it, uninstall it
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$msi = (Get-ChildItem dist/*.msi | Select-Object -First 1).FullName
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/i', "`"$msi`"", '/qn', '/l*v', 'install.log' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { Get-Content install.log -Tail 60; throw "msiexec /i exited $($p.ExitCode)" }
|
||||
$dir = Join-Path $env:ProgramFiles 'LightCraft'
|
||||
foreach ($exe in 'lightcraft.exe', 'lightcraft-cli.exe') {
|
||||
$path = Join-Path $dir $exe
|
||||
if (-not (Test-Path $path)) { throw "$exe not installed in $dir" }
|
||||
$bytes = [System.IO.File]::ReadAllBytes($path)
|
||||
$machine = [BitConverter]::ToUInt16($bytes, [BitConverter]::ToInt32($bytes, 0x3C) + 4)
|
||||
if ($machine -ne 0xAA64) { throw "$exe is for machine 0x$('{0:X}' -f $machine), not ARM64" }
|
||||
Write-Output "ok $path (ARM64)"
|
||||
}
|
||||
# The installed CLI runs natively on ARM64.
|
||||
$cli = Join-Path $dir 'lightcraft-cli.exe'
|
||||
$out = & $cli --version 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -ne 0) { Write-Output $out; throw "lightcraft-cli --version exited $LASTEXITCODE" }
|
||||
Write-Output "ok lightcraft-cli --version: $($out.Trim())"
|
||||
$p = Start-Process msiexec.exe -ArgumentList '/x', "`"$msi`"", '/qn' -Wait -PassThru
|
||||
if ($p.ExitCode -ne 0) { throw "msiexec /x exited $($p.ExitCode)" }
|
||||
if (Test-Path (Join-Path $dir 'lightcraft.exe')) { throw 'uninstall left lightcraft.exe behind' }
|
||||
Write-Output 'ok uninstalled'
|
||||
windows-arm64:
|
||||
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
|
||||
uses: storytold/craft-common-actions/.github/workflows/windows-arm64.yml@main
|
||||
with:
|
||||
app-name: lightcraft
|
||||
app-dir: LightCraft
|
||||
Generated
+41
-33
@@ -1072,6 +1072,11 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-docpath"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=82a24b006eefa036b43c09a67fb9a3fbd17ffe63#82a24b006eefa036b43c09a67fb9a3fbd17ffe63"
|
||||
|
||||
[[package]]
|
||||
name = "craft-ui"
|
||||
version = "0.1.0"
|
||||
@@ -3073,7 +3078,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"eframe",
|
||||
"egui",
|
||||
@@ -3098,7 +3103,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-catalog"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"lightcraft-develop",
|
||||
@@ -3115,7 +3120,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-cli"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"dhat",
|
||||
"lightcraft-codecs",
|
||||
@@ -3131,10 +3136,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-codecs"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"avif-serialize",
|
||||
"bytemuck",
|
||||
"dhat",
|
||||
"image",
|
||||
"image-webp",
|
||||
"jixel",
|
||||
@@ -3161,7 +3167,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-color"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-geom",
|
||||
"proptest",
|
||||
@@ -3170,7 +3176,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-denoise"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"faer-core",
|
||||
"lightcraft-denoise-core",
|
||||
@@ -3187,7 +3193,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-denoise-core"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-raster",
|
||||
"miniz_oxide 0.8.9",
|
||||
@@ -3202,7 +3208,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-develop"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-color",
|
||||
"lightcraft-geom",
|
||||
@@ -3214,9 +3220,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-engine"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"craft-docpath",
|
||||
"harfrust",
|
||||
"jxl-oxide",
|
||||
"libc",
|
||||
@@ -3256,7 +3263,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-faces"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-testkit",
|
||||
"matrixmultiply",
|
||||
@@ -3271,7 +3278,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-fetch"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"log",
|
||||
"rustls",
|
||||
@@ -3282,7 +3289,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-geocode"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"miniz_oxide 0.8.9",
|
||||
"serde",
|
||||
@@ -3290,7 +3297,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-geom"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
@@ -3298,12 +3305,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-gpu"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"lightcraft-color",
|
||||
"lightcraft-denoise",
|
||||
"lightcraft-develop",
|
||||
"lightcraft-faces",
|
||||
"lightcraft-geom",
|
||||
"lightcraft-pipeline",
|
||||
"lightcraft-raster",
|
||||
@@ -3317,7 +3325,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-heif"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"heic-rs",
|
||||
"rayon",
|
||||
@@ -3325,14 +3333,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-inpaint"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-lensdb"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-develop",
|
||||
"lightcraft-geom",
|
||||
@@ -3341,7 +3349,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-mcp"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-codecs",
|
||||
"lightcraft-engine",
|
||||
@@ -3352,7 +3360,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-merge"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-codecs",
|
||||
"lightcraft-color",
|
||||
@@ -3368,7 +3376,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-meta"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-geom",
|
||||
"lightcraft-tiff",
|
||||
@@ -3380,7 +3388,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-pipeline"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"image",
|
||||
"lightcraft-color",
|
||||
@@ -3398,7 +3406,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-preview"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"jpeg-encoder",
|
||||
"lightcraft-raster",
|
||||
@@ -3410,7 +3418,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-raster"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"lightcraft-color",
|
||||
@@ -3422,7 +3430,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-raw"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"dhat",
|
||||
"jpeg-encoder",
|
||||
@@ -3446,7 +3454,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-scenes"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"image",
|
||||
"lightcraft-color",
|
||||
@@ -3457,7 +3465,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-segment"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"candle-core",
|
||||
"candle-nn",
|
||||
@@ -3471,7 +3479,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-slideshow"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"lightcraft-raster",
|
||||
"serde",
|
||||
@@ -3480,15 +3488,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-sysmem"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-testkit"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-tiff"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
@@ -3497,7 +3505,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-ui-egui"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"craft-ui",
|
||||
"egui",
|
||||
@@ -3526,7 +3534,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-web"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"eframe",
|
||||
@@ -6996,7 +7004,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
|
||||
|
||||
[[package]]
|
||||
name = "xtask"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
|
||||
+2
-1
@@ -3,7 +3,7 @@ resolver = "3"
|
||||
members = ["crates/*", "apps/*", "xtask"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
edition = "2024"
|
||||
license = "MIT OR Apache-2.0"
|
||||
rust-version = "1.90"
|
||||
@@ -11,6 +11,7 @@ repository = "https://github.com/storytold/lightcraft"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-ui = { git = "https://github.com/storytold/craft-libs", rev = "5c874d51eeb3de70e735c8ce87a59a7dedd47675", features = ["serde"] }
|
||||
craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "82a24b006eefa036b43c09a67fb9a3fbd17ffe63" }
|
||||
# internal
|
||||
lightcraft-denoise-core = { path = "crates/denoise-core" }
|
||||
lightcraft-denoise = { path = "crates/denoise", default-features = false }
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<b>Photo library and raw development; an open-source, clean-room reimplementation of Adobe Lightroom, rebuilt in pure Rust.</b><br>
|
||||
<b>Photo library and raw development; an open-source, non-destructive photo library and raw developer, built in pure Rust.</b><br>
|
||||
Native on macOS, Windows and Linux. In the browser via WebAssembly. Drivable end to end by AI agents over MCP.
|
||||
</p>
|
||||
|
||||
@@ -129,7 +129,7 @@ Split-tone shadows, midtones and highlights independently with drag-anywhere col
|
||||
|
||||
## Before & after
|
||||
|
||||
Hold <kbd>\\</kbd> to peek at the original, press <kbd>Y</kbd> for side by side, or <kbd>Shift</kbd>+<kbd>Y</kbd> for a
|
||||
Press <kbd>\\</kbd> to toggle the Before view, <kbd>Y</kbd> for side by side, or <kbd>Shift</kbd>+<kbd>Y</kbd> for a
|
||||
split view. Every image below is a real screenshot of LightCraft, captured automatically by an agent through the
|
||||
[control channel](#built-for-agents).
|
||||
|
||||
@@ -150,9 +150,13 @@ split view. Every image below is a real screenshot of LightCraft, captured autom
|
||||
|
||||
Paint with a **Brush** (size, feather, flow, density, erase), drop **Linear** and **Radial Gradients** with draggable
|
||||
pins, or select by **Luminance Range**, **Color Range**, **Sky**, **Subject** and **Background**. Combine components
|
||||
with **Add / Subtract / Intersect**, invert any of them, and dial in 15 local adjustments per mask (Temp, Tint,
|
||||
Exposure, Contrast, Highlights, Shadows, Whites, Blacks, Texture, Clarity, Dehaze, Hue, Saturation, Sharpness, Noise)
|
||||
plus an overall Amount.
|
||||
with **Add / Subtract / Intersect**, invert any of them, and dial in 17 local adjustments per mask (Temp, Tint,
|
||||
Exposure, Contrast, Highlights, Shadows, Whites, Blacks, Texture, Clarity, Dehaze, Hue, Saturation, Sharpness, Noise,
|
||||
Moiré, Defringe), plus an overall Amount.
|
||||
|
||||
Desktop **Object** and **Describe** masks use SAM 3 for click-based and text-prompt selection, with a separately
|
||||
installed model; see [model setup and terms](docs/ai-masks.md). Subject, Sky and Background selections still use
|
||||
classical heuristics. Existing model-generated masks render and export without the model installed.
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/images/masking.jpg" alt="Masking panel with a linear Sky mask and a radial Sun glow mask; the radial gradient is drawn as a red overlay around the sun on a lake scene" width="100%">
|
||||
@@ -164,7 +168,7 @@ plus an overall Amount.
|
||||
|
||||
## Presets, profiles & the Color Mixer
|
||||
|
||||
Eighteen hand-built presets ship in the box (*Golden Hour, Teal & Orange, Faded Matte, Selenium Tone, Crisp
|
||||
Forty-one hand-built presets ship in the box (*Golden Hour, Teal & Orange, Faded Matte, Selenium Tone, Crisp
|
||||
Landscape* and more), each with an **Amount** slider from 0 to 200 %. Save your own from any group of settings, mark
|
||||
favourites, and copy, paste or sync edits across a whole selection with exactly the groups you choose.
|
||||
|
||||
@@ -209,6 +213,10 @@ A library that stays out of your way: **All Photos**, **Recently Added**, **Pick
|
||||
import date, edit date, name, rating, size or at random (a stable shuffle; View → Sort → Reshuffle for a new one). The justified **Photo Grid** and **Square Grid** views are virtualized,
|
||||
so they stay smooth whether you have forty photos or forty thousand.
|
||||
|
||||
The **People** view groups named faces, including names read from XMP; face boxes can be edited in the loupe.
|
||||
On desktop, automatic detection and recognition use optional models installed through **Settings → Faces**; no weights are
|
||||
bundled. See [face models and setup](docs/faces.md).
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/grid-demo.jpg" alt="Justified Photo Grid of 24 demo photos with ratings and flags, and a sidebar of nested albums under a Travel 2026 folder"><br><sub>Photo Grid with albums nested in a folder, ratings and flags. <i>Demo library.</i></sub></td>
|
||||
@@ -281,7 +289,13 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
print.cell.add width=5 height=7 print.cell.add width=2.5 height=3.5 print.layout.autoLayout print.exportPdf path=Package.pdf
|
||||
```
|
||||
|
||||
Printer colour management and page overlays (identity plate, watermark, page numbers) are not there yet.
|
||||
The Print Job panel supports printer/paper ICC profiles, rendering intent, black point compensation,
|
||||
print adjustment, sharpening and 8/16-bit PDF output. Direct printing through an OS print dialog and page
|
||||
overlays (identity plate, watermark, page numbers) are not there yet.
|
||||
- **Slideshows:** **View → Slideshow** plays the photos in view full screen, with fades, repeat and random order.
|
||||
**Settings → Slideshow** configures layouts, templates, backgrounds, overlays and title screens. Native exports
|
||||
support PDF, JPEG sequences and AV1 video in MP4 or WebM; music and a dedicated Slideshow module editor remain
|
||||
missing. See [docs/slideshow.md](docs/slideshow.md).
|
||||
- **Every widget is addressable** (`ui.widgets`) and clickable by name, so agents operate the real UI, not a
|
||||
side door.
|
||||
- The screenshots in this README were produced end to end by the [`docs/showcase/`](docs/showcase/) scripts.
|
||||
@@ -293,7 +307,7 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
|
||||
- **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2/CR3, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans,
|
||||
Panasonic RW2 / Leica RWL, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP,
|
||||
PSD composites and JPEG XL open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
|
||||
PSD composites, JPEG XL, GIF (first frame) and BMP open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice,
|
||||
as in PhotoCraft).
|
||||
- **Scene-referred & wide-gamut.** Linear Rec.2020 float internally, Bradford-adapted white balance, gamut mapping
|
||||
instead of clipping, a filmic shoulder for raw and pixel-exact pass-through for JPEGs you haven't touched.
|
||||
@@ -319,17 +333,21 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (details in the [roadmap](ROADMAP.md) and
|
||||
[docs/target-app-parity.md](docs/target-app-parity.md)):
|
||||
|
||||
- **By feature count we're at ~81%** of Lightroom (core features 98%), tracked row by row in
|
||||
- **By feature count we're at ~82%** (core features 98%), tracked row by row in
|
||||
[docs/parity-checklist.md](docs/parity-checklist.md).
|
||||
- **As a day-to-day Lightroom replacement we're nearer 61% for the whole product, ~51% for a typical working photographer and ~65% for
|
||||
- **As a day-to-day tool we're nearer 61% for the whole product, ~51% for a typical working photographer and ~65% for
|
||||
casual use.** It's great for JPEG/DNG and most Sony / Panasonic /
|
||||
Fujifilm / Nikon / older-Canon raws on one machine; see [docs/raw-parity.md](docs/raw-parity.md) for your camera.
|
||||
- **The biggest gaps:**
|
||||
- **camera colour calibration:** Sony, Nikon, Panasonic, Fujifilm and Canon CR3 raws have guarded estimates from their camera JPEGs, with built-in ILCA-99M2, ILCE-7CR, ILCE-7M4, X-H2S, X-T4, Nikon Z 6II, Z 8 and D5300 profiles; measured calibration is missing, and other raws or rejected fits retain a neutral matrix;
|
||||
- **compressed Olympus raws and unsupported CR3 variants:** these use embedded JPEG previews when present. Fujifilm lossless/lossy compressed RAF now decodes sensor data; [verification and existing-library reload instructions](docs/raf-compression.md);
|
||||
- **AI masks and denoise:** subject and sky selection are classical heuristics;
|
||||
- **AI quality and model setup:** Subject and Sky selection are classical heuristics; Object/Describe use an
|
||||
optional SAM 3 model. AI Denoise supports Bayer RAW with separately installed weights, while X-Trans and
|
||||
linear-RGB denoise and Super Resolution remain missing; see [AI masks](docs/ai-masks.md) and [AI Denoise](docs/denoise.md);
|
||||
- **network libraries:** a preview reads the raw's header — or, for a CR3 or a TIFF raw whose preview tag sits past it, exactly the ranges the sparse planner names — and its embedded JPEG only (56.5 MB → 5.9 MB, 10 s → 1 s on the test share), but the *developed* picture still needs the sensor data over the wire; see [docs/raw-preview-reads.md](docs/raw-preview-reads.md);
|
||||
- **on-screen HDR display, video and the Classic Print / Book / Map modules.**
|
||||
- **on-screen HDR display, video import/editing, and the Book / Map module views.** Print layouts and
|
||||
colour-managed PDF output work; direct printer output and page overlays remain missing. Slideshow playback
|
||||
and export work, with a dedicated module editor and music still missing.
|
||||
- **What's next:** see [the ranked gaps](docs/gaps.md) and [the roadmap](docs/roadmap.md).
|
||||
|
||||
| Area | Status |
|
||||
@@ -337,16 +355,16 @@ LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (det
|
||||
| Library: albums, folders, smart albums, stacks (incl. auto-stack), virtual copies, ratings, flags, labels, filter bar, search, sort, grids, filmstrip | ✅ |
|
||||
| Culling: Compare (synced zoom) and Survey views, auto-advance, instant previews | ✅ |
|
||||
| Light, Color, Effects (vignette styles), Tone Curve (+ refine saturation, targeted adjustment), Color Mixer (+ targeted), Point Color, Color Grading, Calibration, B&W | ✅ |
|
||||
| Masking: brush, linear, radial, luminance/colour range, add/subtract/intersect | ✅ (AI subject/sky use classical heuristics for now) |
|
||||
| Masking: brush, linear, radial, luminance/colour range, add/subtract/intersect; Object and Describe selections | ✅ · Object/Describe require optional SAM 3; Subject/Sky remain heuristic ([details](docs/ai-masks.md)) |
|
||||
| Crop, straighten tool + auto straighten, flip, rotate, aspect ratios, overlays | ✅ |
|
||||
| Profiles (Color, Neutral, Vivid, Landscape, Portrait, Monochrome: our own looks), presets, versions, history, copy/paste/sync settings | ✅ |
|
||||
| Camera colour: DNG files use their own matrices | ✅ DNG · 🚧 own Sony/Fujifilm profiles; measured calibration database missing |
|
||||
| Camera colour: DNG matrices, guarded camera-JPEG look fits and built-in profiles for selected cameras | 🚧 measured calibration and per-model coverage remain incomplete ([raw parity](docs/raw-parity.md)) |
|
||||
| Native macOS menu bar (generated from the command registry), control channel + every widget addressable, headless UI snapshots | ✅ |
|
||||
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed, incl. lossy after split), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed, 12.8 bpp), Samsung SRW (uncompressed); embedded previews for every format incl. CR3 | 🚧 · other CR3 variants, compressed ORF / SRW and Nikon HE NEF decode ⬜ ([raw parity](docs/raw-parity.md)) |
|
||||
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise 🚧 (Bayer, user-installed weights), Super Resolution ⬜ |
|
||||
| Remove / Heal / Clone spots (auto source), Visualize Spots, Red Eye and Pet Eye (auto pupil detection, catchlight) | ✅ · content-aware fill, spot pin editing 🚧 |
|
||||
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ |
|
||||
| HDR: HDR editing with a headroom limit, SDR rendition, Visualize HDR; export as ISO 21496-1 gain map JPEG, PQ AVIF or 32-bit float TIFF | ✅ · HDR display ⬜ |
|
||||
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise 🚧 (Bayer, user-installed weights; [setup](docs/denoise.md)), Super Resolution ⬜ |
|
||||
| Remove / Heal / Clone spots (auto source), non-generative Fill, editable spot pins, Visualize Spots, Red Eye and Pet Eye | ✅ · Fill uses PatchMatch; no generative fill |
|
||||
| Export: JPEG / PNG / TIFF / flat PSD / WebP / AVIF / JPEG XL / raw or rendered DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ |
|
||||
| HDR: HDR editing with a headroom limit, SDR rendition, Visualize HDR; gain map JPEG, PQ/HLG AVIF, PQ PNG/JPEG XL and float TIFF export | ✅ · HDR display ⬜ |
|
||||
| Library persistence (crash-safe op log + snapshots, background compaction, failed saves reported), disk thumbnail cache | ✅ |
|
||||
| Import: Add in place / Copy / Move, rename and folder templates, devices, duplicate detection, watched folders; Local folder browsing | ✅ |
|
||||
| MCP server (headless or live app, persistent libraries), CLI, control channel | ✅ |
|
||||
@@ -354,7 +372,11 @@ LightCraft is young and moving fast: **alpha**. **Where we honestly stand** (det
|
||||
| Optics (distortion, vignetting, auto + manual CA, defringe, lens corrections embedded in DNG files, Panasonic / Leica RW2 / RWL distortion data and Nikon Z 6II / Z 8 / Z f distortion data), Geometry (transforms, Constrain Crop), Upright (Auto/Level/Vertical/Full/Guided) | ✅ · camera lens profiles (our own) ⬜ |
|
||||
| Photo Merge: HDR (auto-align, deghost), Panorama (spherical/cylindrical/perspective, boundary warp, auto crop), HDR Panorama → DNG | ✅ |
|
||||
| GPU pipeline (wgpu compute, CPU-exact within 1/255), CPU fallback on device limits / errors | ✅ · WebGPU in the browser 🚧 |
|
||||
| AI: segmentation masks, AI denoise, super resolution, faces; video | 🚧 (see [gaps](docs/gaps.md)) |
|
||||
| People view, XMP face names and editable face boxes; automatic face detection and recognition | ✅ · detection/recognition require optional models ([setup](docs/faces.md)) |
|
||||
| GPX tracklog auto-tagging and offline reverse geocoding | ✅ · place names require the GeoNames dataset; Map view ⬜ |
|
||||
| Print layouts, picture packages, templates and colour-managed PDF export | ✅ layouts/PDF · 🚧 direct printer output and page overlays |
|
||||
| Slideshow playback, layouts, overlays, templates and PDF/JPEG/AV1 export | ✅ playback · 🚧 dedicated module editor and music ([details](docs/slideshow.md)) |
|
||||
| Video import/editing and Book module | ⬜ |
|
||||
| Web build (same UI in the browser via WASM): persistent library in OPFS/IndexedDB, Web Worker rendering, export downloads | ✅ · WebGPU, Safari/Firefox testing 🚧 |
|
||||
|
||||
<sub>✅ works today · 🚧 in progress · ⬜ not started</sub>
|
||||
@@ -373,7 +395,7 @@ cargo run --release -p lightcraft -- --memory # a throwaway in-memory
|
||||
cargo run --release -p lightcraft -- --control 7980 # with the automation channel
|
||||
cargo xtask web --serve # the same app in the browser: http://127.0.0.1:8080/
|
||||
cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light.exposure=0.5
|
||||
cargo xtask ci # fmt, clippy, tests, layering, wasm checks
|
||||
cargo xtask ci # fmt, clippy, HEIF, tests, parity, layers, assets, WASM
|
||||
```
|
||||
|
||||
CI defaults to line-table debug information and one build job per 1.5 GB of RAM
|
||||
@@ -426,7 +448,7 @@ CRAFT_FONTS_DIR=../craft-fonts cargo run --release -p lightcraft
|
||||
Without it LightCraft builds and runs the same, but Chinese and Japanese text have no glyphs. Fonts are never committed to this
|
||||
repo — the standard for what goes there lives with it: see the [craft-fonts README](https://github.com/storytold/craft-fonts#readme).
|
||||
|
||||
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語, Português (Brasil), Español, Deutsch, Русский, Français, Українська, Čeština, ქართული) or **Settings → General →
|
||||
**Interface language:** **Edit → Language** (English, 简体中文, 繁體中文(台灣), 日本語, Português (Brasil), Español, Deutsch, Русский, Français, Українська, Polski, Čeština, Italiano, ქართული) or **Settings → General →
|
||||
Language**; the choice applies immediately and persists. See [docs/localization.md](docs/localization.md).
|
||||
|
||||
**Logs:** the desktop app writes its log to standard error and to `logs/lightcraft.log` in its settings folder
|
||||
@@ -491,9 +513,9 @@ egui frontend is one swappable crate; nothing below it knows a UI exists. `mcp`
|
||||
|
||||
Humans and agents follow the same rules, so read [AGENTS.md](AGENTS.md) first. The short version:
|
||||
|
||||
- **Clean-room.** Never read Adobe binaries or GPL raw/photo code (darktable, RawTherapee, LibRaw, rawspeed, dcraw…);
|
||||
- **Clean-room.** Never read proprietary binaries or GPL raw/photo code (darktable, RawTherapee, LibRaw, rawspeed, dcraw…);
|
||||
work from public specs and black-box observation.
|
||||
- **No Adobe assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from Adobe products. Every
|
||||
- **No proprietary assets, ever:** no icons, screenshots, presets, profiles, LUTs or fonts from commercial products. Every
|
||||
image, icon and font in the repo is original, public domain, Creative Commons, OFL or permissively licensed, and has
|
||||
an entry in [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) added in the same commit. New fonts go to
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts), not here.
|
||||
@@ -536,7 +558,7 @@ option to start it. For unattended installs: `msiexec /i lightcraft-<ver>-window
|
||||
|
||||
| Format | x86_64 | aarch64 (ARM64) | Notes |
|
||||
|---|---|---|---|
|
||||
| AppImage | `lightcraft-<ver>-linux-x86_64.AppImage` | `lightcraft-<ver>-linux-aarch64.AppImage` | Runs anywhere; updates itself with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) |
|
||||
| AppImage | `lightcraft-<ver>-linux-x86_64.AppImage` | `lightcraft-<ver>-linux-aarch64.AppImage` | Portable; updates with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) |
|
||||
| Flatpak | `lightcraft-<ver>-linux-x86_64.flatpak` | `lightcraft-<ver>-linux-aarch64.flatpak` | Sandboxed; `flatpak install --user <file>` |
|
||||
| Debian/Ubuntu | `lightcraft-<ver>-linux-x86_64.deb` | `lightcraft-<ver>-linux-aarch64.deb` | |
|
||||
| Fedora/RHEL/openSUSE | `lightcraft-<ver>-linux-x86_64.rpm` | `lightcraft-<ver>-linux-aarch64.rpm` | |
|
||||
@@ -623,8 +645,6 @@ ArtCraft Team and are not covered by this license. They may be used only unmodif
|
||||
part of this repository and LightCraft, under [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt).
|
||||
Forks and modified versions must remove them.
|
||||
|
||||
<sub>Adobe, Photoshop, Illustrator, Premiere Pro, Lightroom, Acrobat, After Effects and InDesign are trademarks or registered trademarks of Adobe Inc. in the United States and/or other countries. LightCraft is an independent, open-source project and is not affiliated with, sponsored by or endorsed by Adobe Inc.; these names are used only to describe the workflows it is compatible with.</sub>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
|
||||
<sub>Made by the <a href="https://getartcraft.com/">ArtCraft</a> team and community.</sub>
|
||||
|
||||
+5
-7
@@ -18,7 +18,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<b>照片图库与 raw 显影;Adobe Lightroom 的开源净室实现,用纯 Rust 重建。</b><br>
|
||||
<b>照片图库与 raw 显影;开源、非破坏性的照片图库与 raw 显影工具,用纯 Rust 构建。</b><br>
|
||||
在 macOS、Windows 和 Linux 上原生运行,也可通过 WebAssembly 在浏览器中使用,并能由 AI 智能体经 MCP 端到端驱动。
|
||||
</p>
|
||||
|
||||
@@ -299,9 +299,9 @@ lightcraft --control 7980 ~/Pictures/trip
|
||||
|
||||
LightCraft 很年轻,而且进展很快。**我们如实所处的位置**(细节见[路线图](ROADMAP.md#where-we-stand)):
|
||||
|
||||
- **按功能数量算,我们达到了 Lightroom 的约 79%**(核心功能 98%),逐行记录在
|
||||
- **按功能数量算,我们达到了约 79%**(核心功能 98%),逐行记录在
|
||||
[docs/parity.md](docs/parity.md)。
|
||||
- **作为日常替代 Lightroom 的工具,我们更接近 60–70%。** 在单台机器上处理 JPEG/DNG 以及大多数尼康/索尼/
|
||||
- **作为日常使用的工具,我们更接近 60–70%。** 在单台机器上处理 JPEG/DNG 以及大多数尼康/索尼/
|
||||
较老佳能的 raw 都很出色。
|
||||
- **最大的缺口:**
|
||||
- **相机色彩校准:** 索尼、尼康、松下、富士和佳能 CR3 的 raw 会从各自的相机 JPEG 得到受保护的估计值,内置 ILCA-99M2、ILCE-7CR、ILCE-7M4、X-H2S、X-T4、尼康 Z 6II、Z 8 和 D5300 配置文件;实测校准仍然缺失,其他 raw 或被拒绝的拟合会保留中性矩阵;
|
||||
@@ -453,9 +453,9 @@ egui 前端是一个可替换的 crate;它下面没有任何东西知道 UI
|
||||
|
||||
人和智能体遵循同样的规则,所以请先读 [AGENTS.md](AGENTS.md)。简短版本:
|
||||
|
||||
- **净室。** 绝不阅读 Adobe 的二进制文件或 GPL 的 raw/照片代码(darktable、RawTherapee、LibRaw、rawspeed、dcraw……);
|
||||
- **净室。** 绝不阅读专有软件的二进制文件或 GPL 的 raw/照片代码(darktable、RawTherapee、LibRaw、rawspeed、dcraw……);
|
||||
只依据公开规范和黑盒观察来工作。
|
||||
- **绝不使用 Adobe 资源:** 不使用 Adobe 产品的图标、截图、预设、配置文件、LUT 或字体。仓库中的每一个
|
||||
- **绝不使用专有资源:** 不使用商业产品的图标、截图、预设、配置文件、LUT 或字体。仓库中的每一个
|
||||
图像、图标和字体都是原创、公有领域、知识共享、OFL 或宽松许可的,并在同一提交中
|
||||
于 [assets/ATTRIBUTION.md](assets/ATTRIBUTION.md) 添加条目。新字体提交到
|
||||
[storytold/craft-fonts](https://github.com/storytold/craft-fonts),而不是这里。
|
||||
@@ -579,8 +579,6 @@ ArtCraft 团队的商标,不在本许可范围内。它们只能在未修改
|
||||
本仓库和 LightCraft 的一部分,按 [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt) 使用。
|
||||
分支和修改版本必须移除它们。
|
||||
|
||||
<sub>Adobe、Photoshop、Illustrator、Premiere Pro、Lightroom、Acrobat、After Effects 和 InDesign 是 Adobe Inc. 在美国和/或其他国家的商标或注册商标。LightCraft 是一个独立的开源项目,与 Adobe Inc. 无隶属关系,也未获其赞助或认可;这些名称仅用于说明它所兼容的工作流。</sub>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://getartcraft.com/"><img alt="ArtCraft" src="docs/brand/artcraft-mark.svg" width="28"></a><br>
|
||||
<sub>由 <a href="https://getartcraft.com/">ArtCraft</a> 团队和社区制作。</sub>
|
||||
|
||||
+3
-2
@@ -46,7 +46,7 @@ Hours are calibrated on this repo's PR history; see
|
||||
|---|---:|---:|---:|---|
|
||||
| Features | ~61% | ~77% (checklist 81.1%) | 600–1,100 h | [target-app-parity](docs/target-app-parity.md#features) |
|
||||
| ↳ RAW cameras, colour, lens profiles | ~60% | 83% of 1,446 Adobe models decode, 38% verified | 255–480 h | [raw-parity](docs/raw-parity.md) |
|
||||
| UI/UX fidelity | ~70% | ~85% | 34–58 h | [ui-parity](docs/ui-parity.md) |
|
||||
| UI/UX fidelity | ~70% | ~85% | 32–54 h | [ui-parity](docs/ui-parity.md) |
|
||||
| File formats (non-raw) | ~70% | ~85% | 30–55 h | [file-format-parity](docs/file-format-parity.md) |
|
||||
| Hardware | ~40% | ~50% | 40–80 h | [hardware-parity](docs/hardware-parity.md) |
|
||||
| Localization | ~55% of the 12 key languages | 7 of 12 present | 130–210 h (+145–255 h new scripts) | [localization-parity](docs/localization-parity.md) |
|
||||
@@ -105,7 +105,7 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
|
||||
1. Camera colour of our own: measured calibration for the popular models (50–90 h)
|
||||
2. CR3 for every Canon body (25–45 h)
|
||||
3. Render-fidelity suite against Lightroom, then tuning (30–60 h)
|
||||
4. Full-resolution zoom (8–15 h)
|
||||
4. ~~Full-resolution zoom~~ (done 2026-10-12)
|
||||
5. AI masks: Subject / Sky / People (40–70 h + model decision)
|
||||
6. Compressed ORF and HE NEF (28–55 h)
|
||||
7. Lens profile database of our own (40–80 h)
|
||||
@@ -192,6 +192,7 @@ Ranked toward beta; detail in [docs/roadmap.md](docs/roadmap.md), gaps in [docs/
|
||||
| 2026-10-12 | minor | Mamiya MEF decoded and imported: ZD verified (241); headline numbers unchanged |
|
||||
| 2026-10-12 | minor | Epson ERF decoded: 3 more verified models (240); headline numbers unchanged |
|
||||
| 2026-10-12 | minor | Georgian (`ka`) interface language added (every catalog message) |
|
||||
| 2026-10-14 | minor | Full-resolution zoom (gap 4) closed: soft proofing gets zoom windows; headline numbers unchanged |
|
||||
| 2026-10-12 | minor | UI/UX remaining 40–70 h → 34–58 h (accessibility audit landed); headline numbers unchanged |
|
||||
| 2026-10-14 | minor | Camera-coverage sweep (#1327): 30 models verified on 40 CC0 files (coverage.py 515 → 545 verified); sensor verification does not establish colour parity (#1277); headline numbers unchanged |
|
||||
| 2026-10-12 | minor | 22 more CR3 bodies verified, EOS R6 Mark III white balance: RAW decoding 77% → 83%, RAW ~56% → ~60%; mainstream 49% → 51%, essentials 64% → 65%; full unchanged at ~61% |
|
||||
|
||||
@@ -45,3 +45,15 @@ sync_github_name = true
|
||||
[people.Silvertree2010]
|
||||
display_name = "Netter"
|
||||
sync_github_name = false
|
||||
|
||||
[people.nledford]
|
||||
real_name = "Nathaniel Ledford"
|
||||
|
||||
[people.marcoarnulfo]
|
||||
real_name = "Marco Arnulfo"
|
||||
display_name = "Marco Arnulfo"
|
||||
sync_github_name = false
|
||||
|
||||
[people.egegl]
|
||||
real_name = "Ege Gursel"
|
||||
sync_github_name = true
|
||||
@@ -49,7 +49,9 @@ pub fn shift_iso(iso: &str, secs: i64) -> Option<String> {
|
||||
if !(first..=last).contains(&t) {
|
||||
return None;
|
||||
}
|
||||
let tail = iso.trim().get(19..).unwrap_or("");
|
||||
let iso = iso.trim();
|
||||
// the seconds are optional (`10:00+02:00`): without them the tail starts right after the minutes
|
||||
let tail = iso.get(if iso.get(16..17) == Some(":") { 19 } else { 16 }..).unwrap_or("");
|
||||
Some(format!("{}{tail}", civil(t)))
|
||||
}
|
||||
|
||||
@@ -222,6 +224,15 @@ mod tests {
|
||||
assert_eq!(normalize_iso("2026-02-30"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shift_keeps_the_zone_of_a_time_without_seconds() {
|
||||
assert_eq!(shift_iso("2026-01-01T10:00+02:00", 3600).as_deref(), Some("2026-01-01T11:00:00+02:00"));
|
||||
assert_eq!(shift_iso("2026-01-01T10:00Z", 60).as_deref(), Some("2026-01-01T10:01:00Z"));
|
||||
assert_eq!(shift_iso("2026-01-01T10:00", 3600).as_deref(), Some("2026-01-01T11:00:00"));
|
||||
assert_eq!(shift_iso("2026-01-01T10:00:00.5-05:00", 3600).as_deref(), Some("2026-01-01T11:00:00.5-05:00"));
|
||||
assert_eq!(iso_seconds("2026-01-01T10:00+02:00"), iso_seconds("2026-01-01T10:00"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runs_by_day_month_year_keep_stacks_together() {
|
||||
let mut c = Catalog::new();
|
||||
|
||||
@@ -271,7 +271,7 @@ fn resync(line: &[u8], last: u64) -> Option<(usize, (u64, Op))> {
|
||||
const START: &[u8] = b"{\"seq\":";
|
||||
(1..line.len().saturating_sub(START.len() - 1)).filter(|&i| line[i..].starts_with(START)).find_map(|i| {
|
||||
let rec = std::str::from_utf8(&line[i..]).ok().map(|l| l.trim_end_matches('\r')).and_then(decode_record)?;
|
||||
(rec.0 <= last + 1).then_some((i, rec))
|
||||
(rec.0 <= last.saturating_add(1)).then_some((i, rec))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -298,6 +298,9 @@ fn write_snapshot(store: &mut dyn Store, seq: u64, catalog: &Catalog) -> std::io
|
||||
Ok(SnapshotTiming { serialize_ms, write_sync_ms: (ms_since(t0) - serialize_ms).max(0.0), bytes, ..Default::default() })
|
||||
}
|
||||
|
||||
/// The highest `seq` a snapshot may hold; beyond it the counter is treated as corrupt, far from overflow.
|
||||
const MAX_SEQ: u64 = u64::MAX / 2;
|
||||
|
||||
/// Just the identification of `catalog.snap` (read before the catalog itself).
|
||||
#[derive(serde::Deserialize)]
|
||||
struct SnapHeader {
|
||||
@@ -331,6 +334,10 @@ impl Journal {
|
||||
return Err(newer(format!("{SNAPSHOT} is catalog format v{}", h.version)));
|
||||
}
|
||||
let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
|
||||
// a seq this high can't come from real use; the next record's seq would wrap
|
||||
if s.seq > MAX_SEQ {
|
||||
return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: seq {} is out of range", s.seq)));
|
||||
}
|
||||
let mut catalog = s.catalog;
|
||||
report.repaired = catalog.repair();
|
||||
for r in &report.repaired {
|
||||
@@ -392,7 +399,7 @@ impl Journal {
|
||||
let _ = op;
|
||||
report.stale += 1;
|
||||
}
|
||||
Some((seq, op)) if seq == j.seq + 1 => {
|
||||
Some((seq, op)) if j.seq.checked_add(1) == Some(seq) => {
|
||||
if catalog.apply(op).is_err() {
|
||||
report.failed += 1;
|
||||
} else {
|
||||
@@ -476,7 +483,7 @@ impl Journal {
|
||||
let mut buf = String::new();
|
||||
let mut seq = self.seq;
|
||||
for op in ops {
|
||||
seq += 1;
|
||||
seq = seq.checked_add(1).ok_or_else(|| CatalogError::Corrupt(format!("{LOG}: op counter overflow")))?;
|
||||
buf.push_str(&encode_record(seq, op));
|
||||
buf.push('\n');
|
||||
}
|
||||
@@ -527,7 +534,7 @@ impl Journal {
|
||||
log::warn!("catalog: background snapshot failed ({e}); writing one now");
|
||||
}
|
||||
let t0 = web_time::Instant::now();
|
||||
let seq = self.seq + unlogged;
|
||||
let seq = self.seq.checked_add(unlogged).ok_or_else(|| CatalogError::Corrupt(format!("{SNAPSHOT}: op counter overflow")))?;
|
||||
let mut timing = match write_snapshot(self.store.as_mut(), seq, catalog) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
|
||||
@@ -643,6 +643,7 @@ mod edited_tests {
|
||||
("NIKON CORPORATION NIKON D850", "NEF", false),
|
||||
("Canon Canon EOS 5D Mark III", "CR2", false),
|
||||
("Canon Canon EOS R6m2", "CR3", false),
|
||||
("Canon Canon EOS R6 Mark II", "CR3", false),
|
||||
("Canon Canon EOS R7", "CR3", true),
|
||||
("FUJIFILM X-T4", "RAF", false),
|
||||
("ILCE-7M3", "ARW", false),
|
||||
|
||||
@@ -157,13 +157,20 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
Some((_, v)) => (x - v).abs() < 1e-9,
|
||||
None => false,
|
||||
};
|
||||
// Fields stored as `f32` compare at f32 precision: 2.8f32 widened to f64 is not 2.8.
|
||||
let cmp32 = |x: f32, s: &str| match num(s) {
|
||||
Some(('>', v)) => x > v as f32,
|
||||
Some(('<', v)) => x < v as f32,
|
||||
Some((_, v)) => x == v as f32,
|
||||
None => false,
|
||||
};
|
||||
return match field {
|
||||
"rating" | "stars" => cmp(p.rating as f64, val),
|
||||
"flag" => Flag::parse(val) == Some(p.flag),
|
||||
"label" | "color" => p.label.is_some_and(|l| format!("{l:?}").eq_ignore_ascii_case(val)),
|
||||
"iso" => p.meta.iso.is_some_and(|i| cmp(i as f64, val)),
|
||||
"f" | "aperture" => p.meta.aperture.is_some_and(|a| cmp(a as f64, val)),
|
||||
"focal" => p.meta.focal_mm.is_some_and(|a| cmp(a as f64, val)),
|
||||
"f" | "aperture" => p.meta.aperture.is_some_and(|a| cmp32(a, val)),
|
||||
"focal" => p.meta.focal_mm.is_some_and(|a| cmp32(a, val)),
|
||||
"camera" => p.meta.camera.to_lowercase().contains(val),
|
||||
"lens" => p.meta.lens.to_lowercase().contains(val),
|
||||
"keyword" | "kw" => p.meta.keywords.iter().any(|k| crate::keywords::is_under(k, val)),
|
||||
@@ -173,11 +180,18 @@ fn token_matches(p: &Photo, tok: &str) -> bool {
|
||||
"date" => p.captured.as_deref().is_some_and(|c| c.starts_with(val)),
|
||||
"copy" | "virtual" => (val == "true" || val == "yes") == p.copy_of.is_some(),
|
||||
"name" | "file" => p.file_name.to_lowercase().contains(val),
|
||||
_ => false,
|
||||
// Not a known field: the colon is part of a plain word ("10:30", "Re:").
|
||||
_ => text_matches(p, &t),
|
||||
};
|
||||
}
|
||||
text_matches(p, &t)
|
||||
}
|
||||
|
||||
/// Whether the lower-cased word `t` is in the file name, title, caption, camera, lens, location,
|
||||
/// format or a keyword of `p`.
|
||||
fn text_matches(p: &Photo, t: &str) -> bool {
|
||||
let hay = [&p.file_name, &p.meta.title, &p.meta.caption, &p.meta.camera, &p.meta.lens, &p.meta.location, &p.format];
|
||||
hay.iter().any(|h| h.to_lowercase().contains(&t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(&t))
|
||||
hay.iter().any(|h| h.to_lowercase().contains(t)) || p.meta.keywords.iter().any(|k| k.to_lowercase().contains(t))
|
||||
}
|
||||
|
||||
/// Whether `p` has a named face region called `name` (case-insensitive).
|
||||
|
||||
@@ -294,7 +294,8 @@ pub fn iso_seconds(s: &str) -> Option<i64> {
|
||||
if !(1..=12).contains(&m) || !(1..=31).contains(&d) {
|
||||
return None;
|
||||
}
|
||||
let (hh, mm, ss) = if s.len() >= 16 { (num(11, 13)?, num(14, 16)?, num(17, 19).unwrap_or(0)) } else { (0, 0, 0) };
|
||||
let (hh, mm, ss) =
|
||||
if s.len() >= 16 { (num(11, 13)?, num(14, 16)?, if s.get(16..17) == Some(":") { num(17, 19).unwrap_or(0) } else { 0 }) } else { (0, 0, 0) };
|
||||
// days from civil (proleptic Gregorian)
|
||||
let y2 = if m <= 2 { y - 1 } else { y };
|
||||
let era = y2.div_euclid(400);
|
||||
|
||||
@@ -123,6 +123,48 @@ fn filter_search_sort() {
|
||||
assert_eq!(c.keywords(), vec![("ocean".to_string(), 1), ("summer".to_string(), 1)]);
|
||||
}
|
||||
|
||||
/// A search word with a colon whose left part is not a known field ("10:30", "Re:") is plain
|
||||
/// text, not a fielded token that matches nothing.
|
||||
#[test]
|
||||
fn free_text_with_a_colon_searches_the_text_fields() {
|
||||
let mut c = Catalog::new();
|
||||
let a = photo(&mut c, "meeting.jpg", "2026-05-01T10:00:00");
|
||||
let _b = photo(&mut c, "other.jpg", "2026-04-01T10:00:00");
|
||||
let mut m = c.photo(a).unwrap().meta.clone();
|
||||
m.caption = "Meeting at 10:30 with Re: notes".into();
|
||||
m.iso = Some(1600);
|
||||
c.apply(Op::SetMeta { id: a, meta: Box::new(m) }).unwrap();
|
||||
let q = |text: &str| c.query(&Filter { text: text.into(), ..Default::default() }, &Sort::default());
|
||||
assert_eq!(q("meeting"), vec![a]);
|
||||
assert_eq!(q("10:30"), vec![a], "a free-text token with a colon");
|
||||
assert_eq!(q("re:"), vec![a]);
|
||||
assert_eq!(q("RE: notes"), vec![a], "case-insensitive, one token per word");
|
||||
assert_eq!(q("iso:>800"), vec![a], "known fields stay fielded");
|
||||
assert_eq!(q("iso:>2000"), Vec::<PhotoId>::new());
|
||||
assert_eq!(q("11:45"), Vec::<PhotoId>::new(), "an unknown field matching no text finds nothing");
|
||||
}
|
||||
|
||||
/// `f:`/`aperture:` and `focal:` are stored as `f32`; a typed value that isn't exact in binary
|
||||
/// (2.8, 5.6) must still match, and `<`/`>` must not count the photo's own value.
|
||||
#[test]
|
||||
fn search_tokens_match_f32_aperture_and_focal() {
|
||||
let mut c = Catalog::new();
|
||||
let a = photo(&mut c, "a.jpg", "2026-05-01T10:00:00");
|
||||
let mut m = c.photo(a).unwrap().meta.clone();
|
||||
m.aperture = Some(2.8);
|
||||
m.focal_mm = Some(5.6);
|
||||
c.apply(Op::SetMeta { id: a, meta: Box::new(m) }).unwrap();
|
||||
let q = |t: &str| c.query(&Filter { text: t.into(), ..Default::default() }, &Sort::default());
|
||||
assert_eq!(q("f:2.8"), vec![a], "f:2.8");
|
||||
assert_eq!(q("aperture:=2.8"), vec![a], "aperture:=2.8");
|
||||
assert_eq!(q("focal:5.6"), vec![a], "focal:5.6");
|
||||
assert!(q("f:2.9").is_empty(), "f:2.9");
|
||||
assert!(q("f:<2.8").is_empty(), "f:<2.8");
|
||||
assert!(q("f:>2.8").is_empty(), "f:>2.8");
|
||||
assert!(q("focal:<5.6").is_empty(), "focal:<5.6");
|
||||
assert_eq!(q("f:>2.7"), vec![a], "f:>2.7");
|
||||
}
|
||||
|
||||
/// A `captured` string that is not ASCII (the Lightroom importer copies `captureTime` into an
|
||||
/// `AddPhoto` op verbatim, and a JSON snapshot is valid UTF-8 whatever it holds) must not be
|
||||
/// sliced on a byte index in `date_groups`: grouping counts the well-formed dates and ignores
|
||||
|
||||
@@ -376,3 +376,18 @@ fn a_damaged_process_value_loads_as_v1_with_the_edits() {
|
||||
let d = &loaded.photo(id).unwrap().develop;
|
||||
assert_eq!((d.process, d.light.exposure), (ProcessVersion::V1, 0.8));
|
||||
}
|
||||
|
||||
/// A snapshot whose `seq` is near `u64::MAX` is refused as corrupt: the next record's seq would
|
||||
/// wrap (and every later edit would be skipped as stale on reopen) or overflow-panic in debug.
|
||||
#[test]
|
||||
fn snapshot_with_seq_near_u64_max_is_corrupt() {
|
||||
for seq in [u64::MAX, u64::MAX - 1, u64::MAX / 2 + 1] {
|
||||
let m = MemStore::new();
|
||||
m.set(
|
||||
SNAPSHOT,
|
||||
format!("{{\"format\":\"lightcraft-catalog\",\"version\":1,\"seq\":{seq},\"catalog\":{}}}\n", Catalog::new().to_snapshot()).into_bytes(),
|
||||
);
|
||||
let e = Journal::open(Box::new(m)).err();
|
||||
assert!(matches!(e, Some(CatalogError::Corrupt(_))), "seq {seq}: {e:?}");
|
||||
}
|
||||
}
|
||||
@@ -55,3 +55,6 @@ jxl-oxide = { workspace = true }
|
||||
proptest = { workspace = true }
|
||||
# Lossless synthetic HEIC files for the `heif` feature's tests.
|
||||
lightcraft-heif = { workspace = true, features = ["testdata"] }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
|
||||
dhat = { workspace = true }
|
||||
@@ -73,6 +73,10 @@ ICC profiles are parsed with `moxcms` (v2/v4, `curv`/`para` TRCs, `chad`); sRGB,
|
||||
Adobe RGB (1998), ProPhoto (ROMM) and Rec.2020 are recognised by colorants. `icc::write_matrix_trc`
|
||||
emits v4 display profiles (D50 PCS, Bradford `chad`) for export embedding.
|
||||
|
||||
Before colour conversion, positive-infinity float samples become 1.0; NaN and negative infinity
|
||||
become 0.0. This also applies to alpha. Finite samples keep their existing range: matrix/TRC paths
|
||||
preserve values above 1.0 and below 0.0, while CMS inputs and alpha remain bounded to 0..1.
|
||||
|
||||
## Dependencies and licences
|
||||
|
||||
All pure Rust, verified with `cargo info` / `cargo tree`:
|
||||
|
||||
@@ -44,11 +44,23 @@ impl Buf {
|
||||
match self {
|
||||
Buf::U8(v) => v[i] as f32 / 255.0,
|
||||
Buf::U16(v) => v[i] as f32 / 65535.0,
|
||||
Buf::F32(v) => v[i],
|
||||
Buf::F32(v) => normalize_float(v[i]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Clip non-finite device samples before colour conversion, preserving finite extended range.
|
||||
#[inline]
|
||||
fn normalize_float(value: f32) -> f32 {
|
||||
if value == f32::INFINITY {
|
||||
1.0
|
||||
} else if value.is_finite() {
|
||||
value
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Interleaved device samples as produced by a format decoder.
|
||||
pub(crate) struct Raw {
|
||||
pub width: usize,
|
||||
@@ -218,10 +230,7 @@ fn linearize(raw: &Raw, ch: usize, trc: &[Trc; 3]) -> Rgb32f {
|
||||
par_rows(&mut img.data, w, |y, row| {
|
||||
for (x, o) in row.iter_mut().enumerate() {
|
||||
let i = (y * w + x) * ch;
|
||||
let s = |c: usize| {
|
||||
let val = v[i + c];
|
||||
if val.is_finite() { val } else { 0.0 }
|
||||
};
|
||||
let s = |c: usize| normalize_float(v[i + c]);
|
||||
*o = if gray {
|
||||
let l = if lin { s(0) } else { trc[1].to_linear(s(0)) };
|
||||
[l, l, l]
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::convert::{Buf, Meta, Model, Raw, check_size, finish};
|
||||
use crate::space::{NamedSpace, SourceSpace, SpaceOrigin, Trc};
|
||||
use crate::{DecodeOptions, Decoded, Error, Format, Result};
|
||||
use jxl_oxide::color::{ColourSpace, EnumColourEncoding, Primaries, RenderingIntent, TransferFunction, WhitePoint};
|
||||
use jxl_oxide::{AuxBoxData, JxlImage};
|
||||
use jxl_oxide::{AllocTracker, AuxBoxData, InitializeResult, JxlImage};
|
||||
|
||||
const F: Format = Format::Jxl;
|
||||
|
||||
@@ -16,10 +16,60 @@ fn err(e: impl std::fmt::Display) -> Error {
|
||||
Error::Malformed(F, e.to_string())
|
||||
}
|
||||
|
||||
/// Tracked allocation budget per allowed pixel for frames, extra channels and modular trees.
|
||||
/// This bounds allocations covered by `AllocTracker`, not every allocation made by the decoder (#1517).
|
||||
const TRACKED_BYTES_PER_PIXEL: u64 = 4 * 4 * 8;
|
||||
|
||||
/// The allocation budget never drops below this, so small images keep the decoder's fixed
|
||||
/// overhead (group buffers, entropy tables) whatever `max_pixels` is.
|
||||
const MIN_TRACKED_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// Bytes handed to the decoder at a time (the size `JxlImageBuilder::read` reads in).
|
||||
const INPUT_CHUNK: usize = 4096;
|
||||
|
||||
/// Initializes from bounded input, checks dimensions before rendering, then loads remaining frames
|
||||
/// within a tracked allocation budget.
|
||||
fn open(bytes: &[u8], opts: &DecodeOptions) -> Result<JxlImage> {
|
||||
let budget = opts.max_pixels.saturating_mul(TRACKED_BYTES_PER_PIXEL).max(MIN_TRACKED_BYTES);
|
||||
let tracker = AllocTracker::with_limit(usize::try_from(budget).unwrap_or(usize::MAX));
|
||||
let mut uninit = JxlImage::builder().alloc_tracker(tracker).build_uninit();
|
||||
// Initialization can load already-buffered frame data, so small feeds limit work before
|
||||
// dimension checking.
|
||||
let mut rest = bytes;
|
||||
let mut window = INPUT_CHUNK;
|
||||
let mut img = loop {
|
||||
let consumed = uninit.feed_bytes(rest.get(..window).unwrap_or(rest)).map_err(err)?;
|
||||
rest = rest.get(consumed..).unwrap_or_default();
|
||||
match uninit.try_init().map_err(err)? {
|
||||
InitializeResult::Initialized(img) => break img,
|
||||
InitializeResult::NeedMoreData(more) => uninit = more,
|
||||
}
|
||||
if consumed == 0 {
|
||||
// the parser wants more than the window holds at once
|
||||
if window >= rest.len() {
|
||||
return Err(err("truncated JPEG XL header"));
|
||||
}
|
||||
window = window.saturating_mul(2);
|
||||
}
|
||||
};
|
||||
check_size(F, img.width() as u64, img.height() as u64, opts)?;
|
||||
// Match `JxlImageBuilder::read`: bounded feeds until the final frame, not merely the first
|
||||
// keyframe. `feed_bytes` copies codestream data into an untracked Vec, including any tail
|
||||
// after the last frame; feeding the remaining file at once bypasses the allocation budget.
|
||||
while !img.is_loading_done() && !rest.is_empty() {
|
||||
let consumed = img.feed_bytes(rest.get(..INPUT_CHUNK).unwrap_or(rest)).map_err(err)?;
|
||||
if consumed == 0 {
|
||||
break;
|
||||
}
|
||||
rest = rest.get(consumed..).unwrap_or_default();
|
||||
}
|
||||
img.finalize().map_err(err)?;
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
let mut img = JxlImage::builder().read(std::io::Cursor::new(bytes)).map_err(err)?;
|
||||
let mut img = open(bytes, opts)?;
|
||||
let (w, h) = (img.width(), img.height());
|
||||
check_size(F, w as u64, h as u64, opts)?;
|
||||
let header = img.image_header();
|
||||
let bits = header.metadata.bit_depth.bits_per_sample() as u8;
|
||||
let float = matches!(header.metadata.bit_depth, jxl_oxide::image::BitDepth::FloatSample { .. });
|
||||
|
||||
@@ -95,6 +95,11 @@ pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result<Decoded> {
|
||||
fn limits(opts: &DecodeOptions) -> Limits {
|
||||
let mut limits = Limits::default();
|
||||
limits.decoding_buffer_size = opts.max_pixels.saturating_mul(4 * 8).min(usize::MAX as u64) as usize;
|
||||
// A strip or tile whose declared byte count exceeds this is refused: the default 128 MiB refuses a
|
||||
// single-strip uncompressed 16-bit export of ~22 MP (#1983). Raised to the decoded image's cap
|
||||
// (32 bytes per allowed pixel), never lowered: a strip or tile can outgrow the pixels it shows
|
||||
// (JPEG headers, padding to the tile size).
|
||||
limits.intermediate_buffer_size = limits.intermediate_buffer_size.max(limits.decoding_buffer_size);
|
||||
limits.ifd_value_size = 32 << 20;
|
||||
limits
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Non-finite samples in procedural 32-bit float TIFFs (issue #1952, item 6).
|
||||
|
||||
use lightcraft_codecs::{icc, *};
|
||||
|
||||
fn tiff(samples: &[f32], channels: u8, profile: Option<&[u8]>) -> Vec<u8> {
|
||||
encode_tiff(
|
||||
&EncodeImage::new((samples.len() / channels as usize) as u32, 1, channels, Samples::F32(samples)),
|
||||
TiffCompression::None,
|
||||
&EncodeMeta { icc: profile, ..Default::default() },
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_tiff_nonfinite_rgb_clips_without_losing_finite_hdr() {
|
||||
let samples =
|
||||
[[f32::INFINITY; 3], [f32::NAN; 3], [f32::NEG_INFINITY; 3], [-0.25; 3], [0.25, 0.5, 0.75], [2.0, 4.0, 8.0], [f32::INFINITY, 0.5, f32::NAN]];
|
||||
let bytes = tiff(samples.as_flattened(), 3, None);
|
||||
let decoded = decode(&bytes, DecodeOptions::default()).unwrap();
|
||||
assert!(decoded.float);
|
||||
assert_eq!(decoded.bit_depth, 32);
|
||||
assert_eq!(decoded.image.data, [[1.0; 3], [0.0; 3], [0.0; 3], [-0.25; 3], [0.25, 0.5, 0.75], [2.0, 4.0, 8.0], [1.0, 0.5, 0.0]]);
|
||||
let display = decoded.to_srgb8();
|
||||
assert_eq!(display.data[0], [255; 4]);
|
||||
for pixel in &display.data[1..4] {
|
||||
assert_eq!(*pixel, [0, 0, 0, 255]);
|
||||
}
|
||||
assert_eq!(display.data[6], [255, 188, 0, 255]);
|
||||
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
|
||||
|
||||
// A box-filtered pair must average clipped white with its ordinary neighbor.
|
||||
let pair = tiff(&[f32::INFINITY, f32::INFINITY, f32::INFINITY, 0.25, 0.25, 0.25], 3, None);
|
||||
let small = decode(&pair, DecodeOptions::fit(1, 1)).unwrap();
|
||||
assert_eq!(small.image.data, [[0.625; 3]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_tiff_nonfinite_gray_and_icc_curves_match_clipped_samples() {
|
||||
let samples = [f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.25, 1.0, 2.0, -0.25];
|
||||
let clipped = [1.0, 0.0, 0.0, 0.25, 1.0, 2.0, -0.25];
|
||||
let profiles = [
|
||||
None,
|
||||
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Linear)),
|
||||
Some(icc::write_named(NamedSpace::Srgb)),
|
||||
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Gamma(2.0))),
|
||||
Some(icc::write_matrix_trc(&lightcraft_color::SRGB, &Trc::Table(vec![0, 16384, 65535]))),
|
||||
Some(moxcms::ColorProfile::new_gray_with_gamma(2.0).encode().unwrap()),
|
||||
Some(b"invalid ICC profile".to_vec()),
|
||||
];
|
||||
for channels in [1, 3] {
|
||||
let samples: Vec<_> = samples.iter().flat_map(|v| std::iter::repeat_n(*v, channels as usize)).collect();
|
||||
let clipped: Vec<_> = clipped.iter().flat_map(|v| std::iter::repeat_n(*v, channels as usize)).collect();
|
||||
for profile in &profiles {
|
||||
let profile = profile.as_deref();
|
||||
let decoded = decode(&tiff(&samples, channels, profile), DecodeOptions::default()).unwrap();
|
||||
let reference = decode(&tiff(&clipped, channels, profile), DecodeOptions::default()).unwrap();
|
||||
assert_eq!(decoded.grayscale, channels == 1);
|
||||
assert_eq!(decoded.image, reference.image, "channels={channels}, space={:?}", decoded.space);
|
||||
assert_eq!(decoded.to_srgb8(), reference.to_srgb8());
|
||||
assert!(decoded.image.data[5].iter().all(|v| *v > 1.0), "finite HDR survives the curve");
|
||||
assert!(decoded.image.data[6].iter().all(|v| *v < 0.0), "finite negative samples survive the curve");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_tiff_nonfinite_alpha_is_finite_and_bounded() {
|
||||
let samples = [
|
||||
[f32::INFINITY, f32::INFINITY, f32::INFINITY, f32::INFINITY],
|
||||
[f32::NAN, f32::NEG_INFINITY, 0.5, f32::NAN],
|
||||
[0.25, 0.5, 2.0, f32::NEG_INFINITY],
|
||||
[2.0, 3.0, 4.0, 0.5],
|
||||
];
|
||||
let decoded = decode(&tiff(samples.as_flattened(), 4, None), DecodeOptions::default()).unwrap();
|
||||
assert_eq!(decoded.alpha.as_ref().unwrap().data, [1.0, 0.0, 0.0, 0.5]);
|
||||
assert_eq!(decoded.image.data, [[1.0; 3], [0.0, 0.0, 0.5], [0.25, 0.5, 2.0], [2.0, 3.0, 4.0]]);
|
||||
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
|
||||
|
||||
// Associated alpha takes the same path before unpremultiplication.
|
||||
let mut cursor = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
use tiff::encoder::{TiffEncoder, colortype};
|
||||
let mut encoder = TiffEncoder::new(&mut cursor).unwrap();
|
||||
let mut image = encoder.new_image::<colortype::RGBA32Float>(2, 1).unwrap();
|
||||
image.encoder().write_tag(tiff::tags::Tag::ExtraSamples, &[1u16][..]).unwrap();
|
||||
image.write_data(&[f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.5, 1.0, 2.0, 3.0, 0.5]).unwrap();
|
||||
}
|
||||
let decoded = decode(&cursor.into_inner(), DecodeOptions::default()).unwrap();
|
||||
assert_eq!(decoded.alpha.as_ref().unwrap().data, [0.5, 0.5]);
|
||||
assert_eq!(decoded.image.data, [[2.0, 0.0, 0.0], [2.0, 4.0, 6.0]]);
|
||||
}
|
||||
|
||||
fn lut_profile() -> Vec<u8> {
|
||||
use moxcms::{ColorProfile, LutMultidimensionalType, LutWarehouse, Matrix3d, ToneReprCurve, Vector3d, Xyzd};
|
||||
let mut profile = ColorProfile::new_srgb();
|
||||
profile.cicp = None;
|
||||
profile.red_colorant = Xyzd::default();
|
||||
profile.green_colorant = Xyzd::default();
|
||||
profile.blue_colorant = Xyzd::default();
|
||||
profile.red_trc = None;
|
||||
profile.green_trc = None;
|
||||
profile.blue_trc = None;
|
||||
// A procedural mAB profile without a CLUT: linear RGB to D50 XYZ, then identity curves.
|
||||
profile.lut_a_to_b_colorimetric = Some(LutWarehouse::Multidimensional(LutMultidimensionalType {
|
||||
num_input_channels: 3,
|
||||
num_output_channels: 3,
|
||||
grid_points: [0; 16],
|
||||
clut: None,
|
||||
a_curves: vec![],
|
||||
m_curves: vec![ToneReprCurve::Parametric(vec![1.0]); 3],
|
||||
matrix: Matrix3d { v: NamedSpace::Srgb.to_xyz_d50().0.map(|r| r.map(|v| v * 32768.0 / 65535.0)) },
|
||||
b_curves: vec![ToneReprCurve::Parametric(vec![1.0]); 3],
|
||||
bias: Vector3d::default(),
|
||||
}));
|
||||
profile.encode().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_tiff_nonfinite_cms_samples_match_clipped_samples() {
|
||||
let profile = lut_profile();
|
||||
let samples = [f32::INFINITY, f32::NAN, f32::NEG_INFINITY, 0.25, 0.5, 0.75];
|
||||
let clipped = [1.0, 0.0, 0.0, 0.25, 0.5, 0.75];
|
||||
let decoded = decode(&tiff(&samples, 3, Some(&profile)), DecodeOptions::default()).unwrap();
|
||||
let reference = decode(&tiff(&clipped, 3, Some(&profile)), DecodeOptions::default()).unwrap();
|
||||
assert_eq!(decoded.space.origin, SpaceOrigin::IccCms);
|
||||
assert_eq!(decoded.image, reference.image);
|
||||
assert!(decoded.to_working().data.iter().flatten().all(|v| v.is_finite()));
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
#![cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
|
||||
|
||||
use lightcraft_codecs::exif::minimal_exif;
|
||||
use lightcraft_codecs::*;
|
||||
use lightcraft_raster::Rgba8;
|
||||
|
||||
fn codestream(bytes: &[u8]) -> Vec<u8> {
|
||||
if bytes.starts_with(b"\xff\x0a") {
|
||||
return bytes.to_vec();
|
||||
}
|
||||
let mut raw = Vec::new();
|
||||
let mut at = 12;
|
||||
while at + 8 <= bytes.len() {
|
||||
let len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
|
||||
let end = if len == 0 { bytes.len() } else { at + len };
|
||||
match &bytes[at + 4..at + 8] {
|
||||
b"jxlc" => raw.extend_from_slice(&bytes[at + 8..end]),
|
||||
b"jxlp" => raw.extend_from_slice(&bytes[at + 12..end]),
|
||||
_ => {}
|
||||
}
|
||||
at = end;
|
||||
}
|
||||
assert!(raw.starts_with(b"\xff\x0a"));
|
||||
raw
|
||||
}
|
||||
|
||||
fn box_bytes(out: &mut Vec<u8>, kind: &[u8; 4], data: &[u8]) {
|
||||
out.extend_from_slice(&u32::try_from(data.len() + 8).unwrap().to_be_bytes());
|
||||
out.extend_from_slice(kind);
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_in_final_feed_matches_builder() {
|
||||
let pixels = Rgba8::from_fn(8, 8, |x, y| [(x * 20) as u8, (y * 20) as u8, 80, 255]);
|
||||
let encoded = encode_jxl(&EncodeImage::rgba8(&pixels), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
|
||||
let raw = codestream(&encoded);
|
||||
let exif = minimal_exif(1);
|
||||
let mut exif_box = vec![0; 4]; // TIFF header offset, relative to the end of this field
|
||||
exif_box.extend_from_slice(&exif);
|
||||
// Split the codestream across two feeds, with Exif and XMP following its last frame.
|
||||
let mut bytes = b"\0\0\0\x0cJXL \x0d\x0a\x87\x0a".to_vec();
|
||||
box_bytes(&mut bytes, b"ftyp", b"jxl \0\0\0\0jxl ");
|
||||
let split = raw.len() / 2;
|
||||
let mut part = 0u32.to_be_bytes().to_vec();
|
||||
part.extend_from_slice(&raw[..split]);
|
||||
box_bytes(&mut bytes, b"jxlp", &part);
|
||||
let padding = vec![0; 4096 - bytes.len() - 8];
|
||||
box_bytes(&mut bytes, b"free", &padding);
|
||||
let mut part = 0x8000_0001u32.to_be_bytes().to_vec();
|
||||
part.extend_from_slice(&raw[split..]);
|
||||
box_bytes(&mut bytes, b"jxlp", &part);
|
||||
box_bytes(&mut bytes, b"Exif", &exif_box);
|
||||
box_bytes(&mut bytes, b"xml ", b"<xmp/>");
|
||||
assert!(bytes.len() < 8192, "metadata must fit in the completing input chunk");
|
||||
let old = jxl_oxide::JxlImage::builder().read(bytes.as_slice()).unwrap();
|
||||
let old_exif = match old.aux_boxes().first_exif().unwrap() {
|
||||
jxl_oxide::AuxBoxData::Data(ex) => ex.payload().to_vec(),
|
||||
_ => panic!("builder must preserve Exif in its final feed"),
|
||||
};
|
||||
let old_xml = match old.aux_boxes().first_xml() {
|
||||
jxl_oxide::AuxBoxData::Data(xml) => xml.to_vec(),
|
||||
_ => panic!("builder must preserve XMP in its final feed"),
|
||||
};
|
||||
let decoded = decode(&bytes, DecodeOptions::default()).unwrap();
|
||||
assert_eq!(decoded.exif.as_deref(), Some(old_exif.as_slice()));
|
||||
assert_eq!(decoded.xmp.as_deref(), Some(std::str::from_utf8(&old_xml).unwrap()));
|
||||
assert_eq!(decoded.to_srgb8(), pixels);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progressive_truncation_matches_builder() {
|
||||
let pixels = Rgba8::from_fn(96, 64, |x, y| {
|
||||
let mut seed = (y * 96 + x + 1) as u32;
|
||||
let mut rgb = [0; 4];
|
||||
for channel in &mut rgb[..3] {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
*channel = seed as u8;
|
||||
}
|
||||
rgb[3] = 255;
|
||||
rgb
|
||||
});
|
||||
for lossless in [false, true] {
|
||||
let encoded =
|
||||
encode_jxl(&EncodeImage::rgba8(&pixels), &JxlOptions { lossless, progressive: true, ..Default::default() }, &EncodeMeta::default())
|
||||
.unwrap();
|
||||
let raw = codestream(&encoded);
|
||||
assert!(raw.len() > 4096, "fixture must exercise post-header feeds");
|
||||
for cut in [0, 1, 2, 12, 4095, 4096, 4097, raw.len() / 2, raw.len() - 1, raw.len()] {
|
||||
let bytes = &raw[..cut];
|
||||
// `finalize` permits partial images: rendering determines whether frame 0 is usable.
|
||||
let old_ok = jxl_oxide::JxlImage::builder().read(bytes).is_ok_and(|image| image.render_frame(0).is_ok());
|
||||
assert_eq!(decode(bytes, DecodeOptions::default()).is_ok(), old_ok, "lossless={lossless}, cut={cut}/{}", raw.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
#![cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
|
||||
|
||||
use lightcraft_codecs::*;
|
||||
use lightcraft_raster::Rgba8;
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOC: dhat::Alloc = dhat::Alloc;
|
||||
|
||||
fn codestream(bytes: &[u8]) -> Vec<u8> {
|
||||
if bytes.starts_with(b"\xff\x0a") {
|
||||
return bytes.to_vec();
|
||||
}
|
||||
let mut raw = Vec::new();
|
||||
let mut at = 12;
|
||||
while at + 8 <= bytes.len() {
|
||||
let len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
|
||||
let end = if len == 0 { bytes.len() } else { at + len };
|
||||
match &bytes[at + 4..at + 8] {
|
||||
b"jxlc" => raw.extend_from_slice(&bytes[at + 8..end]),
|
||||
b"jxlp" => raw.extend_from_slice(&bytes[at + 12..end]),
|
||||
_ => {}
|
||||
}
|
||||
at = end;
|
||||
}
|
||||
assert!(raw.starts_with(b"\xff\x0a"));
|
||||
raw
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_jxl_tail_does_not_allocate_a_tail_sized_buffer() {
|
||||
let image = Rgba8::from_fn(8, 8, |x, y| [(x * 20) as u8, (y * 20) as u8, 80, 255]);
|
||||
let encoded = encode_jxl(&EncodeImage::rgba8(&image), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
|
||||
let mut raw = codestream(&encoded);
|
||||
let opts = DecodeOptions { max_pixels: 64, ..Default::default() };
|
||||
decode(&raw, opts).unwrap();
|
||||
raw.resize(raw.len() + (1 << 20), 0);
|
||||
let _profiler = dhat::Profiler::builder().testing().build();
|
||||
let decoded = decode(&raw, opts).unwrap();
|
||||
assert_eq!((decoded.width, decoded.height), (8, 8));
|
||||
let peak = dhat::HeapStats::get().max_bytes;
|
||||
eprintln!("peak heap bytes: {peak}");
|
||||
assert!(peak < (128 << 10), "tiny image used {peak} heap bytes");
|
||||
}
|
||||
@@ -263,6 +263,90 @@ fn tiff_cmyk_and_orientation() {
|
||||
assert!(red[0] > 0.99 && red[1] < 0.01 && red[2] < 0.01, "{red:?}");
|
||||
}
|
||||
|
||||
/// An uncompressed 16-bit RGB export in a single strip, the shape of the file in #1983 (3923 × 5877,
|
||||
/// 138 MB). Its one strip is larger than the `tiff` crate's default 128 MiB intermediate buffer, so
|
||||
/// the thumbnail (and the full decode) failed with "decoder limits exceeded".
|
||||
#[test]
|
||||
fn tiff_single_strip_larger_than_128_mib() {
|
||||
use tiff::encoder::{TiffEncoder, colortype};
|
||||
let (w, h) = (3923u32, 5877u32);
|
||||
let mut cur = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
let mut enc = TiffEncoder::new(&mut cur).unwrap();
|
||||
let mut im = enc.new_image::<colortype::RGB16>(w, h).unwrap();
|
||||
im.rows_per_strip(h).unwrap();
|
||||
im.write_data(&vec![0x8000u16; (w * h * 3) as usize]).unwrap();
|
||||
}
|
||||
let bytes = cur.into_inner();
|
||||
assert!(bytes.len() > 128 << 20);
|
||||
let t = decode_thumbnail(&bytes, 256).unwrap();
|
||||
assert_eq!((t.source_width, t.source_height), (w, h));
|
||||
let px = t.image.get(10, 10);
|
||||
assert!(px[0].abs_diff(128) <= 2 && px[0] == px[1] && px[1] == px[2], "{px:?}");
|
||||
}
|
||||
|
||||
/// A little-endian TIFF with one IFD of single-value `(tag, type, value)` entries, followed by `data`.
|
||||
fn ifd_tiff(entries: &[(u16, u16, u32)], data: &[u8]) -> Vec<u8> {
|
||||
let mut b = b"II*\0".to_vec();
|
||||
b.extend(8u32.to_le_bytes());
|
||||
b.extend((entries.len() as u16).to_le_bytes());
|
||||
for &(tag, ty, value) in entries {
|
||||
b.extend(tag.to_le_bytes());
|
||||
b.extend(ty.to_le_bytes());
|
||||
b.extend(1u32.to_le_bytes());
|
||||
b.extend(value.to_le_bytes());
|
||||
}
|
||||
b.extend(0u32.to_le_bytes());
|
||||
b.extend_from_slice(data);
|
||||
b
|
||||
}
|
||||
|
||||
/// A strip or tile can be larger than the pixels it shows (a JPEG header, padding to the tile size), so
|
||||
/// a small `max_pixels` must not shrink the intermediate buffer below the `tiff` crate's 128 MiB
|
||||
/// default. Both files below were refused with "decoder limits exceeded" when the buffer was capped at
|
||||
/// 32 bytes per allowed pixel. Fixtures from AlexChambers' review of #1989.
|
||||
#[test]
|
||||
fn tiff_small_pixel_budget_reads_headers_and_padded_tiles() {
|
||||
let opts = |max_pixels| DecodeOptions { max_size: None, max_pixels };
|
||||
|
||||
// 1 × 1 gray, one JPEG-compressed strip (~400 bytes for one pixel), with a one-pixel budget.
|
||||
let mut jpeg = Vec::new();
|
||||
jpeg_encoder::Encoder::new(&mut jpeg, 90).encode(&[128], 1, 1, jpeg_encoder::ColorType::Luma).unwrap();
|
||||
let at = 8 + 2 + 10 * 12 + 4;
|
||||
let entries = [
|
||||
(256, 4, 1),
|
||||
(257, 4, 1),
|
||||
(258, 3, 8),
|
||||
(259, 3, 7),
|
||||
(262, 3, 1),
|
||||
(273, 4, at),
|
||||
(277, 3, 1),
|
||||
(278, 4, 1),
|
||||
(279, 4, jpeg.len() as u32),
|
||||
(284, 3, 1),
|
||||
];
|
||||
let d = decode(&ifd_tiff(&entries, &jpeg), opts(1)).unwrap();
|
||||
assert_eq!((d.image.width, d.image.height), (1, 1));
|
||||
|
||||
// 16 × 16 gray stored in one 256 × 256 tile (64 KiB), with a budget of exactly 256 pixels.
|
||||
let at = 8 + 2 + 11 * 12 + 4;
|
||||
let entries = [
|
||||
(256, 4, 16),
|
||||
(257, 4, 16),
|
||||
(258, 3, 8),
|
||||
(259, 3, 1),
|
||||
(262, 3, 1),
|
||||
(277, 3, 1),
|
||||
(284, 3, 1),
|
||||
(322, 4, 256),
|
||||
(323, 4, 256),
|
||||
(324, 4, at),
|
||||
(325, 4, 65536),
|
||||
];
|
||||
let d = decode(&ifd_tiff(&entries, &[128; 65536]), opts(256)).unwrap();
|
||||
assert_eq!((d.image.width, d.image.height), (16, 16));
|
||||
}
|
||||
|
||||
/// A one-strip, uncompressed, single-sample TIFF (little-endian) whose samples are `bits` wide and
|
||||
/// already packed into `strip`, one padded row after another.
|
||||
fn packed_tiff(width: u16, height: u16, bits: u16, photometric: u16, strip: &[u8]) -> Vec<u8> {
|
||||
@@ -734,6 +818,21 @@ fn jxl_lossless_exact_with_metadata() {
|
||||
assert!(max_diff(&d.image, &want.image) < 1e-5, "16-bit lossless differs by {}", max_diff(&d.image, &want.image));
|
||||
}
|
||||
|
||||
/// #1517: the pixel limit rejects an oversized image; selected truncations never panic.
|
||||
#[test]
|
||||
#[cfg(all(feature = "jxl", not(target_arch = "wasm32")))]
|
||||
fn jxl_size_limit_and_truncation() {
|
||||
let b = encode_jxl(&EncodeImage::rgba8(&gradient(97, 61)), &JxlOptions { lossless: true, ..Default::default() }, &EncodeMeta::default()).unwrap();
|
||||
let r = decode(&b, DecodeOptions { max_pixels: 97 * 61 - 1, ..Default::default() });
|
||||
assert!(matches!(r, Err(Error::TooLarge(97, 61))), "{r:?}");
|
||||
let d = decode(&b, DecodeOptions { max_pixels: 97 * 61, ..Default::default() }).unwrap();
|
||||
assert_eq!((d.width, d.height), (97, 61));
|
||||
for cut in [0, 1, 2, 5, 12, 40, b.len() / 2, b.len() - 1] {
|
||||
let _ = decode(&b[..cut], DecodeOptions::default());
|
||||
}
|
||||
assert!(decode(&b[..12.min(b.len())], DecodeOptions::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "jxl")]
|
||||
fn jxl_lossy_quality_and_effort() {
|
||||
|
||||
+11
-2
@@ -180,8 +180,8 @@ pub struct Rgb8(pub u8, pub u8, pub u8);
|
||||
|
||||
impl Rgb8 {
|
||||
pub fn from_hex(s: &str) -> Option<Rgb8> {
|
||||
let s = s.trim_start_matches('#');
|
||||
if s.len() != 6 {
|
||||
let s = s.strip_prefix('#').unwrap_or(s);
|
||||
if s.len() != 6 || !s.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return None;
|
||||
}
|
||||
let v = u32::from_str_radix(s, 16).ok()?;
|
||||
@@ -196,6 +196,15 @@ impl Rgb8 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rgb8_from_hex_rejects_signs_and_extra_hashes() {
|
||||
assert_eq!(Rgb8::from_hex("#0f0f0f"), Some(Rgb8(15, 15, 15)));
|
||||
assert_eq!(Rgb8::from_hex("ff8000"), Some(Rgb8(255, 128, 0)));
|
||||
for bad in ["+12345", "#+12345", "#+f+f+f", "-12345", "##123456", "#12345", ""] {
|
||||
assert_eq!(Rgb8::from_hex(bad), None, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srgb_matrix_matches_standard() {
|
||||
let m = SRGB.to_xyz();
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::manifest::Gain;
|
||||
use crate::tiles::{starts, weight};
|
||||
|
||||
/// Most pixels one picture may have (guards the allocations that follow from its size).
|
||||
pub const MAX_PIXELS: usize = 100_000_000;
|
||||
pub const MAX_PIXELS: usize = 110_000_000;
|
||||
/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
|
||||
const MIN_SIGNAL: f32 = 0.003;
|
||||
|
||||
@@ -642,4 +642,19 @@ mod tests {
|
||||
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
|
||||
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_102_mp_inputs_and_rejects_inputs_above_the_limit() {
|
||||
let layout = Layout { dx: 0, dy: 0 };
|
||||
let model = Mock { tile: 32, scale: 1.0 };
|
||||
let p = params(32, 8, Gain::None);
|
||||
assert!(matches!(
|
||||
denoise_bayer(&[], 11_648, 8_736, layout, &model, &p, &Control::default()),
|
||||
Err(Error::Input(message)) if message == "the mosaic does not have width × height samples"
|
||||
));
|
||||
assert!(matches!(
|
||||
denoise_bayer(&[], MAX_PIXELS + 1, 1, layout, &model, &p, &Control::default()),
|
||||
Err(Error::Input(message)) if message == "the picture has no size, or is too large"
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ impl DevelopSettings {
|
||||
/// Merge a partial JSON object into these settings (fields not mentioned are kept). A white balance that
|
||||
/// names no [`WbScale`] keeps the meaning it always had (`Legacy`), whatever scale these settings were on: a
|
||||
/// partial from before the scale existed, or one that never said, must not become Kelvin by landing on
|
||||
/// Kelvin settings.
|
||||
/// Kelvin settings. Out-of-range values are brought back into range ([`DevelopSettings::keep_in_range`]).
|
||||
pub fn merged(&self, partial: &Value) -> Result<DevelopSettings, serde_json::Error> {
|
||||
let mut v = self.to_json();
|
||||
presets::deep_merge(&mut v, partial);
|
||||
@@ -52,9 +52,34 @@ impl DevelopSettings {
|
||||
{
|
||||
out.wb.scale = WbScale::Legacy;
|
||||
}
|
||||
out.keep_in_range(self);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// What crs XMP import does to the values it reads, for settings that arrive as JSON (presets, `develop.merge`,
|
||||
/// `render --settings`; issue #1946): every slider clamped through its control spec, point curves in 0..1, and a
|
||||
/// crop with no area left as it was in `before`. Unrotated, the crop is clipped to the photo first; a turned crop
|
||||
/// may reach past 0..1 and is left to the crop tools, which know the photo's size.
|
||||
fn keep_in_range(&mut self, before: &DevelopSettings) {
|
||||
for c in controls::CONTROLS {
|
||||
if let Some(v) = controls::get(self, c.id) {
|
||||
controls::set(self, c.id, v);
|
||||
}
|
||||
}
|
||||
let c = &mut self.curve;
|
||||
for p in [&mut c.master, &mut c.red, &mut c.green, &mut c.blue].into_iter().flatten() {
|
||||
*p = lightcraft_geom::Point::new(p.x.clamp(0.0, 1.0), p.y.clamp(0.0, 1.0));
|
||||
}
|
||||
let g = &mut self.crop.geometry;
|
||||
if g.angle == 0.0 {
|
||||
g.rect = g.rect.intersect(&lightcraft_geom::Rect::UNIT);
|
||||
}
|
||||
if g.rect.is_empty() {
|
||||
let old = before.crop.geometry.rect;
|
||||
g.rect = if old.is_empty() { lightcraft_geom::Rect::UNIT } else { old };
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable 64-bit hash of the settings (FNV-1a over canonical JSON), for preview cache keys.
|
||||
pub fn hash64(&self) -> u64 {
|
||||
let s = serde_json::to_string(self).unwrap_or_default();
|
||||
@@ -255,6 +280,30 @@ mod tests {
|
||||
assert_eq!(m.light.exposure, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_settings_are_kept_in_range() {
|
||||
// issue #1946: `develop.merge`, `render --settings` and presets stored whatever the JSON said
|
||||
let mut base = DevelopSettings::default();
|
||||
base.crop.geometry.rect = lightcraft_geom::Rect::new(0.1, 0.2, 0.8, 0.9);
|
||||
let m = base
|
||||
.merged(&json!({"light": {"exposure": 1e308}, "vignette": {"amount": -1e308}, "geometry": {"scale": 0}, "optics": {"distortion": 1e308}}))
|
||||
.unwrap();
|
||||
assert_eq!((m.light.exposure, m.vignette.amount, m.geometry.scale, m.optics.distortion), (5.0, -100.0, 50.0, 100.0));
|
||||
// point curves stay in 0..1, as an imported XMP curve does
|
||||
let m = base.merged(&json!({"curve": {"master": [{"x": -3, "y": 0}, {"x": 1, "y": 1e308}]}})).unwrap();
|
||||
assert_eq!(m.curve.master, vec![lightcraft_geom::Point::new(0.0, 0.0), lightcraft_geom::Point::new(1.0, 1.0)]);
|
||||
// a crop with no area is left out, as XMP import leaves out one whose edges are inverted
|
||||
for rect in
|
||||
[json!({"x0": 0.9, "x1": 0.1, "y0": 0, "y1": 1}), json!({"x0": 1e308, "x1": -1e308, "y0": 0, "y1": 1}), json!({"x0": 0.5, "x1": 0.5})]
|
||||
{
|
||||
let m = base.merged(&json!({"crop": {"geometry": {"rect": rect}}})).unwrap();
|
||||
assert_eq!(m.crop.geometry.rect, base.crop.geometry.rect, "{rect}");
|
||||
}
|
||||
// unrotated, the crop stays inside the photo
|
||||
let m = base.merged(&json!({"crop": {"geometry": {"rect": {"x0": -1e308, "y0": 0.25, "x1": 1e308, "y1": 0.75}}}})).unwrap();
|
||||
assert_eq!(m.crop.geometry.rect, lightcraft_geom::Rect::new(0.0, 0.25, 1.0, 0.75));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unedited_and_reset() {
|
||||
let mut s = DevelopSettings::default();
|
||||
|
||||
@@ -184,17 +184,11 @@ fn scale_patch(base: &Value, patch: &Value, t: f64) -> Value {
|
||||
}
|
||||
|
||||
/// Apply a partial settings object with an amount (1.0 = 100 %). Out-of-range values are clamped
|
||||
/// through the control specs.
|
||||
/// through the control specs, as every merge does ([`DevelopSettings::merged`]).
|
||||
pub fn apply_partial(s: &DevelopSettings, partial: &Value, amount: f64) -> DevelopSettings {
|
||||
let patch = if (amount - 1.0).abs() < 1e-9 { Cow::Borrowed(partial) } else { Cow::Owned(scale_patch(&s.to_json(), partial, amount)) };
|
||||
// (a white balance without a scale keeps its old meaning: `DevelopSettings::merged`)
|
||||
let mut out = s.merged(&patch).unwrap_or_else(|_| s.clone());
|
||||
for c in crate::controls::CONTROLS {
|
||||
if let Some(v) = crate::controls::get(&out, c.id) {
|
||||
crate::controls::set(&mut out, c.id, v);
|
||||
}
|
||||
}
|
||||
out
|
||||
s.merged(&patch).unwrap_or_else(|_| s.clone())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
|
||||
@@ -9,6 +9,8 @@ description = "LightCraft engine: session, command registry, history, render job
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Paths stored in a Lightroom catalog are followed only to local files or the catalog's share.
|
||||
craft-docpath = { workspace = true }
|
||||
lightcraft-geom = { workspace = true }
|
||||
lightcraft-color = { workspace = true }
|
||||
lightcraft-raster = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
//! What a Synchronize Folder scan costs on a folder whose photos are all in the library and
|
||||
//! unchanged: the walk, and the check of every photo and its sidecar. Reads the folder; writes
|
||||
//! nothing there, and probes no file (the library is made from the folder's own listing).
|
||||
//!
|
||||
//! `cargo run --release -p lightcraft-engine --example sync_scan_bench -- <folder>`
|
||||
use std::time::Instant;
|
||||
|
||||
use lightcraft_catalog::{Op, Photo, Source};
|
||||
use lightcraft_engine::Session;
|
||||
use lightcraft_engine::sync::{SyncInput, SyncProgress, scan_with};
|
||||
|
||||
fn main() {
|
||||
let Some(folder) = std::env::args().nth(1) else {
|
||||
eprintln!("usage: sync_scan_bench <folder>");
|
||||
return;
|
||||
};
|
||||
let mut s = Session::new();
|
||||
let t = Instant::now();
|
||||
let files = lightcraft_engine::import::expand(std::slice::from_ref(&folder), None);
|
||||
println!("{} files listed in {:.1} s (the library is made of them)", files.len(), t.elapsed().as_secs_f64());
|
||||
for f in &files {
|
||||
let id = s.catalog.alloc_photo_id();
|
||||
let name = std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let p = Photo::new(id, Source::File { path: f.clone() }, &name, "JPEG", 6, 4, "2026-01-01T00:00:00");
|
||||
if s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).is_err() {
|
||||
eprintln!("could not add {name}");
|
||||
}
|
||||
}
|
||||
for run in 0..2 {
|
||||
let input = match SyncInput::new(&mut s, &folder, true) {
|
||||
Ok(input) => input,
|
||||
Err(e) => {
|
||||
eprintln!("{e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let t = Instant::now();
|
||||
let changes = scan_with(input, &SyncProgress::default());
|
||||
println!(
|
||||
"run {run}: scan {:.1} s ({} new, {} missing, {} with sidecar news)",
|
||||
t.elapsed().as_secs_f64(),
|
||||
changes.new.len(),
|
||||
changes.missing.len(),
|
||||
changes.metadata.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ const SAME: f32 = 0.93;
|
||||
|
||||
fn secs(iso: &str) -> Option<i64> {
|
||||
let d = lightcraft_meta::DateTime::parse_iso(iso)?;
|
||||
Some(((d.year as i64 * 372 + d.month as i64 * 31 + d.day as i64) * 24 + d.hour as i64) * 3600 + d.minute as i64 * 60 + d.second as i64)
|
||||
Some(d.unix_seconds())
|
||||
}
|
||||
|
||||
fn analyze(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
|
||||
@@ -17,7 +17,7 @@ use lightcraft_faces::yunet::{Detector, Face, Options};
|
||||
use lightcraft_meta::{Region, RegionKind};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CommandSpec, bad, cmd, has_selection};
|
||||
use super::{CommandSpec, always, bad, cmd, has_selection};
|
||||
use crate::{AiModel, EngineError, Result, Session};
|
||||
|
||||
const C: &str = "faces.detect";
|
||||
@@ -54,10 +54,12 @@ impl NoDetector {
|
||||
}
|
||||
}
|
||||
|
||||
/// The YuNet loaded last, and from where.
|
||||
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
|
||||
|
||||
/// The installed YuNet, loaded once and kept (a new one is loaded when the model folder changes).
|
||||
/// (also used by the background scan, which aligns faces by the detector's landmarks and finds faces in photos that have none)
|
||||
pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, NoDetector> {
|
||||
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
|
||||
let home = s
|
||||
.face_models_dir
|
||||
.as_deref()
|
||||
@@ -82,10 +84,133 @@ pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, NoDete
|
||||
let d = Arc::new(
|
||||
Detector::new(&bytes).map_err(|e| NoDetector::Unusable(format!("{e}: remove the detector in Settings > Faces and download it again")))?,
|
||||
);
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let net = d.clone();
|
||||
offer_gpu(s, "detection", move |gpu| net.accelerate(gpu, true));
|
||||
}
|
||||
*loaded = Some((path, d.clone()));
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
/// Where a face model runs, for Settings ▸ Faces and `faces.gpu`.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct GpuState {
|
||||
/// The background check is still running.
|
||||
pub checking: bool,
|
||||
/// The GPU it runs on; `None` when it runs on the CPU.
|
||||
pub device: Option<String>,
|
||||
/// Why it runs on the CPU.
|
||||
pub why: Option<String>,
|
||||
}
|
||||
|
||||
/// Each model's [`GpuState`], by what it does ("detection", "recognition").
|
||||
static GPU_STATE: Mutex<Vec<(&'static str, GpuState)>> = Mutex::new(Vec::new());
|
||||
|
||||
pub(crate) fn gpu_state(what: &str) -> Option<GpuState> {
|
||||
GPU_STATE.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(w, _)| *w == what).map(|(_, st)| st.clone())
|
||||
}
|
||||
|
||||
fn set_gpu_state(what: &'static str, st: GpuState) {
|
||||
let mut all = GPU_STATE.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
match all.iter_mut().find(|(w, _)| *w == what) {
|
||||
Some((_, old)) => *old = st,
|
||||
None => all.push((what, st)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The user chose the CPU for face models (Settings ▸ Faces).
|
||||
pub(crate) fn cpu_only(s: &Session) -> bool {
|
||||
s.face_models_dir.as_deref().is_some_and(|d| super::face_models::read_settings(d).cpu_only)
|
||||
}
|
||||
|
||||
/// Check in the background whether the GPU runs a face model faster than the CPU and gives its results; the model
|
||||
/// runs on the CPU meanwhile, and stays there when the GPU is off, missing, wrong or slower, or the user chose the CPU.
|
||||
/// The check shows in the activity stack while it runs.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(crate) fn offer_gpu(
|
||||
s: &Session,
|
||||
what: &'static str,
|
||||
attach: impl FnOnce(Arc<dyn lightcraft_faces::net::Accelerator>) -> std::result::Result<(), String> + Send + 'static,
|
||||
) {
|
||||
if cpu_only(s) {
|
||||
set_gpu_state(what, GpuState { why: Some("the CPU is chosen in Settings".into()), ..GpuState::default() });
|
||||
return;
|
||||
}
|
||||
let task = s.activity.start("facesGpu", "Checking the GPU for faces", crate::activity::Cancel::No);
|
||||
task.detail(what);
|
||||
set_gpu_state(what, GpuState { checking: true, ..GpuState::default() });
|
||||
let spawned = std::thread::Builder::new().name(format!("faces-gpu-{what}")).spawn(move || {
|
||||
let _task = task;
|
||||
let gpu = match lightcraft_gpu::faces::accelerator() {
|
||||
Ok(gpu) => gpu,
|
||||
Err(why) => {
|
||||
log::info!("faces: {what} runs on the CPU ({why})");
|
||||
return set_gpu_state(what, GpuState { why: Some(why), ..GpuState::default() });
|
||||
}
|
||||
};
|
||||
let device = lightcraft_faces::net::Accelerator::device(&*gpu);
|
||||
match attach(gpu) {
|
||||
Ok(()) => {
|
||||
log::info!("faces: {what} runs on {device}");
|
||||
set_gpu_state(what, GpuState { device: Some(device), ..GpuState::default() });
|
||||
}
|
||||
Err(why) => {
|
||||
log::info!("faces: {what} stays on the CPU: {why}");
|
||||
set_gpu_state(what, GpuState { why: Some(why), ..GpuState::default() });
|
||||
}
|
||||
}
|
||||
});
|
||||
if let Err(e) = spawned {
|
||||
log::warn!("faces: could not check the GPU for {what}: {e}");
|
||||
set_gpu_state(what, GpuState { why: Some(format!("could not check the GPU: {e}")), ..GpuState::default() });
|
||||
}
|
||||
}
|
||||
|
||||
/// `faces.gpu`: where the face models run, and the switch that keeps them on the CPU.
|
||||
fn gpu(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
const C: &str = "faces.gpu";
|
||||
if let Some(cpu) = p.get("cpuOnly") {
|
||||
let cpu = cpu.as_bool().ok_or_else(|| bad(C, "`cpuOnly` is true or false"))?;
|
||||
let dir = s.face_models_dir.clone().ok_or_else(|| bad(C, "this build has nowhere to keep face models (the desktop app does)"))?;
|
||||
super::face_models::set_cpu_only(&dir, cpu)?;
|
||||
let detector = LOADED.lock().unwrap_or_else(PoisonError::into_inner).as_ref().map(|(_, d)| d.clone());
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let embedder = s.faces.embedder.as_ref().map(|(_, e)| e.clone());
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let embedder: Option<Arc<lightcraft_faces::runtime::Embedder>> = None;
|
||||
if cpu {
|
||||
for (what, net) in [("detection", detector.as_ref().map(|d| d.net())), ("recognition", embedder.as_ref().map(|e| e.net()))] {
|
||||
if let Some(net) = net {
|
||||
net.detach();
|
||||
}
|
||||
set_gpu_state(what, GpuState { why: Some("the CPU is chosen in Settings".into()), ..GpuState::default() });
|
||||
}
|
||||
} else {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
if let Some(d) = detector {
|
||||
offer_gpu(s, "detection", move |gpu| d.accelerate(gpu, true));
|
||||
}
|
||||
if let Some(e) = embedder {
|
||||
offer_gpu(s, "recognition", move |gpu| e.accelerate(gpu, true).map_err(|e| e.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(gpu_json(s))
|
||||
}
|
||||
|
||||
/// What `faces.gpu` and `faces.models.list` say about the GPU.
|
||||
pub(crate) fn gpu_json(s: &Session) -> Value {
|
||||
json!({
|
||||
"cpuOnly": cpu_only(s),
|
||||
"detection": gpu_state("detection"),
|
||||
"recognition": gpu_state("recognition"),
|
||||
})
|
||||
}
|
||||
|
||||
fn face_json(f: &Face) -> Value {
|
||||
json!({
|
||||
"rect": {"x0": f.x0, "y0": f.y0, "x1": f.x1, "y1": f.y1},
|
||||
@@ -173,15 +298,26 @@ fn detect(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
}
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![cmd!(
|
||||
"faces.detect",
|
||||
"Detect Faces",
|
||||
["Photo"],
|
||||
None,
|
||||
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
|
||||
has_selection,
|
||||
detect
|
||||
)]
|
||||
vec![
|
||||
cmd!(
|
||||
"faces.detect",
|
||||
"Detect Faces",
|
||||
["Photo"],
|
||||
None,
|
||||
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
|
||||
has_selection,
|
||||
detect
|
||||
),
|
||||
cmd!(
|
||||
query "faces.gpu",
|
||||
"Face Models on the GPU",
|
||||
[],
|
||||
None,
|
||||
"{cpuOnly?: bool} → {cpuOnly, detection, recognition}: where the face models run, each as {checking, device, why} (null until it is loaded). The GPU is used only after a background check shows it gives the CPU's results faster; `cpuOnly: true` keeps every face model on the CPU (Settings ▸ Faces), `false` checks the GPU again",
|
||||
always,
|
||||
gpu
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -27,6 +27,9 @@ pub(super) struct FaceSettings {
|
||||
pub(super) enabled: bool,
|
||||
/// The installed recogniser in use.
|
||||
pub(super) embedder: Option<String>,
|
||||
/// Keep every face model on the CPU, even when the GPU would be faster.
|
||||
#[serde(skip_serializing_if = "std::ops::Not::not")]
|
||||
pub(super) cpu_only: bool,
|
||||
}
|
||||
|
||||
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
|
||||
@@ -57,6 +60,13 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Keep the face models on the CPU (or let them use the GPU again).
|
||||
pub(super) fn set_cpu_only(dir: &Path, on: bool) -> Result<()> {
|
||||
let mut st = read_settings(dir);
|
||||
st.cpu_only = on;
|
||||
write_settings(dir, &st)
|
||||
}
|
||||
|
||||
fn write_settings(dir: &Path, st: &FaceSettings) -> Result<()> {
|
||||
std::fs::create_dir_all(dir).map_err(|e| fail("could not create the face models folder", e))?;
|
||||
write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| fail("settings", e))?)
|
||||
@@ -195,6 +205,8 @@ fn list(s: &mut Session, _: &Value) -> Result<Value> {
|
||||
"dir": dir.as_ref().map(|d| d.display().to_string()),
|
||||
"enabled": settings.enabled,
|
||||
"embedder": settings.embedder,
|
||||
// where the models run: {cpuOnly, detection, recognition} as `faces.gpu` gives it
|
||||
"gpu": super::face_detect::gpu_json(s),
|
||||
// Native builds always have the pure-Rust runner; model files/background scanning remain native-only.
|
||||
"runtime": cfg!(not(target_arch = "wasm32")),
|
||||
"models": models,
|
||||
|
||||
@@ -321,6 +321,8 @@ mod imp {
|
||||
return Ok(e.clone());
|
||||
}
|
||||
let embedder = Arc::new(Embedder::load(&dir.join(&id).join("model.onnx"), &installed.manifest).map_err(|e| bad(cmd, e.to_string()))?);
|
||||
let net = embedder.clone();
|
||||
super::super::face_detect::offer_gpu(s, "recognition", move |gpu| net.accelerate(gpu, true).map_err(|e| e.to_string()));
|
||||
let dim = match installed.manifest.output {
|
||||
lightcraft_faces::OutputSpec::Embedding { dim } => dim as usize,
|
||||
_ => return Err(bad(cmd, "the chosen model is not a recognition model")),
|
||||
@@ -672,6 +674,9 @@ mod imp {
|
||||
if let Some(w) = &s.faces.worker {
|
||||
w.allow(target, pace);
|
||||
}
|
||||
// the GPU is held back the same way: a light pace leaves it mostly free for the window's own renders
|
||||
let (runs, share) = crate::faces_worker::gpu_pace(pace);
|
||||
lightcraft_gpu::faces::set_pace(runs, share);
|
||||
while target > 0 && s.faces.in_flight.len() < target + PREFETCH {
|
||||
let Some(id) = s.faces.queue.pop() else { break };
|
||||
let Some(job) = prepare(s, id, true) else { continue };
|
||||
|
||||
@@ -1488,7 +1488,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
always, |_, p| {
|
||||
let path = str_param(p,"path").ok_or_else(|| bad("library.inspectLightroom","missing path"))?;
|
||||
let data = crate::lightroom_catalog::read(std::path::Path::new(path)).map_err(|e| bad("library.inspectLightroom",e))?;
|
||||
Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p|&p.path).collect::<Vec<_>>(),"warnings":data.warnings}))
|
||||
Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !p.original_is_file()).map(|p|&p.path).collect::<Vec<_>>(),"warnings":data.warnings}))
|
||||
}),
|
||||
cmd!(
|
||||
"library.importLightroom",
|
||||
|
||||
@@ -77,6 +77,7 @@ fn relative_ratings_and_history_use_auto_write() {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// Metadata-only fixtures: no import, image decoding or model/service dependency.
|
||||
for id in 1..=4 {
|
||||
std::fs::write(dir.join(format!("{id}.jpg")), b"original").unwrap();
|
||||
s.catalog
|
||||
.apply(Op::SetFile {
|
||||
id: PhotoId(id),
|
||||
|
||||
@@ -9,8 +9,13 @@ use crate::Result;
|
||||
|
||||
fn rename_args(s: &crate::Session, p: &Value, c: &str) -> Result<(Vec<lightcraft_catalog::PhotoId>, String, usize)> {
|
||||
let template = str_param(p, "template").ok_or_else(|| bad(c, "missing `template`"))?.to_string();
|
||||
let start = p.get("start").and_then(Value::as_u64).unwrap_or(1) as usize;
|
||||
Ok((s.targets(p)?, template, start))
|
||||
let start = p.get("start").and_then(Value::as_u64).unwrap_or(1);
|
||||
let ids = s.targets(p)?;
|
||||
// every number of the sequence `start..start + n` must fit
|
||||
let last = start.checked_add((ids.len() as u64).saturating_sub(1));
|
||||
let start = last.and_then(|l| usize::try_from(l).ok()).and_then(|_| usize::try_from(start).ok());
|
||||
let start = start.ok_or_else(|| bad(c, "start out of range"))?;
|
||||
Ok((ids, template, start))
|
||||
}
|
||||
|
||||
/// A named set of colour-label names (red, yellow, green, blue, purple; empty = the colour's name).
|
||||
|
||||
@@ -324,7 +324,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
"{ids?} → {settings: {layout, image, page, guides, job}, papers, pageWidth, pageHeight, printable, pageCount, cellsPerPage, gridCells?, overlaps}; lengths are inches",
|
||||
always, get),
|
||||
cmd!(query "print.set", "Set Print Settings", [], None,
|
||||
"{layout?: {style: singleImage|contactSheet|picturePackage|customPackage, page: {paper, customWidth, customHeight, orientation: portrait|landscape, margins: {top,right,bottom,left}}, grid: {columns, rows, spacingX, spacingY, keepSquare}, spacing}, image?: {zoomToFill, rotateToFit, borderEnabled, borderWidth, borderColor: [r,g,b], strokeEnabled, strokeWidthPt, strokeColor}, page?: {background: [r,g,b]}, guides?: {showRulers, units: in|cm|mm|pt, snap: off|cells|grid, gridSize, showMargins, showBleed, showPageGrid, showCells, showDimensions}, job?: {ppi: 72..720, color?: {profile: {kind: managed, space: srgb|adobeRgb} | {kind: icc, path}, intent: perceptual|relative, blackPointCompensation, brightness: -100..100, contrast: -100..100, sharpen: off|low|standard|high, media: matte|glossy, sixteenBit}}} — merges into the current print; cells and pages change with print.cell.* / print.layout.*; → print.get",
|
||||
"{layout?: {style: singleImage|contactSheet|picturePackage|customPackage, page: {paper, customWidth, customHeight, orientation: portrait|landscape, margins: {top,right,bottom,left}}, grid: {columns, rows, spacingX, spacingY, keepSquare}, spacing}, image?: {zoomToFill, rotateToFit, borderEnabled, borderWidth, borderColor: [r,g,b], strokeEnabled, strokeWidthPt, strokeColor}, page?: {background: [r,g,b], pageNumbers, pageInfo, cropMarks} (Page Options: single image and contact sheet only), guides?: {showRulers, units: in|cm|mm|pt, snap: off|cells|grid, gridSize, showMargins, showBleed, showPageGrid, showCells, showDimensions}, job?: {ppi: 72..720, color?: {profile: {kind: managed, space: srgb|adobeRgb} | {kind: icc, path}, intent: perceptual|relative, blackPointCompensation, brightness: -100..100, contrast: -100..100, sharpen: off|low|standard|high, media: matte|glossy, sixteenBit}}} — merges into the current print; cells and pages change with print.cell.* / print.layout.*; → print.get",
|
||||
always, set),
|
||||
cmd!(query "print.reset", "Reset Print Settings", [], None, "{} → print.get", always, reset),
|
||||
cmd!(query "print.cell.add", "Add Print Cell", [], None,
|
||||
|
||||
@@ -99,6 +99,21 @@ fn adjust(c: &XmpValue) -> Value {
|
||||
Value::Object(a)
|
||||
}
|
||||
|
||||
/// The correction's local sliders we have no counterpart for (e.g. `LocalGrain`), when set:
|
||||
/// reported, never dropped silently.
|
||||
fn unmapped_local(c: &XmpValue) -> Vec<String> {
|
||||
let XmpValue::Struct(m) = c else { return Vec::new() };
|
||||
m.keys()
|
||||
.filter_map(|k| {
|
||||
let field = k.strip_prefix("crs:").filter(|f| f.starts_with("Local"))?;
|
||||
// 100 = "curve saturation unchanged", written with every correction
|
||||
let neutral = if field == "LocalCurveRefineSaturation" { 100.0 } else { 0.0 };
|
||||
let set = num(c, k).is_some_and(|v| v != neutral);
|
||||
(set && !LOCAL.iter().any(|(crs, ..)| *crs == field)).then(|| field.to_string())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Perceptual lightness 0..1 (how range masks store luminance) → our range scale: log
|
||||
/// luminance on −8..+4 EV mapped to 0..1.
|
||||
fn lightness_to_range(l: f64) -> f64 {
|
||||
@@ -214,8 +229,12 @@ fn component(m: &XmpValue, container: &str, aspect: f64) -> Result<Value, String
|
||||
Some("2") => {
|
||||
let q: Vec<f64> = text(r, "crs:LumRange").unwrap_or("0 0 1 1").split_whitespace().filter_map(|v| v.parse().ok()).collect();
|
||||
let [a, b, c, d] = q[..] else { return Err("luminance range".into()) };
|
||||
// full strength at white (1, the top of their scale) takes in everything brighter: ours
|
||||
// goes on above scene white, so that end stays open instead of cutting off at 0 EV
|
||||
let open_top = c >= 1.0;
|
||||
let (a, b, c, d) = (lightness_to_range(a), lightness_to_range(b), lightness_to_range(c), lightness_to_range(d));
|
||||
json!({"kind": "luminanceRange", "lo": b, "hi": c, "lo_feather": (b - a).max(0.0), "hi_feather": (d - c).max(0.0)})
|
||||
let (hi, hi_feather) = if open_top { (1.0, 0.0) } else { (c, (d - c).max(0.0)) };
|
||||
json!({"kind": "luminanceRange", "lo": b, "hi": hi, "lo_feather": (b - a).max(0.0), "hi_feather": hi_feather})
|
||||
}
|
||||
Some("3") => {
|
||||
let q: Vec<f64> = text(r, "crs:DepthRange").unwrap_or("0 0 1 1").split_whitespace().filter_map(|v| v.parse().ok()).collect();
|
||||
@@ -252,6 +271,7 @@ pub fn masks(values: &Values, aspect: f64) -> (Vec<Value>, Vec<String>) {
|
||||
if parts.is_empty() {
|
||||
continue;
|
||||
}
|
||||
skipped.extend(unmapped_local(c));
|
||||
// brush dabs of one correction are one brush component
|
||||
let mut merged: Vec<Value> = Vec::new();
|
||||
for p in parts {
|
||||
@@ -506,6 +526,28 @@ mod tests {
|
||||
assert_ne!(render(&DevelopSettings { light: s.light, ..Default::default() }), render(&s), "the masks change the picture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn range_to_white_stays_open_and_unmapped_local_sliders_are_reported() {
|
||||
// written for this test: a highlights range (full from 0.85 up to white) carrying a local slider we lack
|
||||
let x = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about="" xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/">
|
||||
<crs:MaskGroupBasedCorrections><rdf:Seq>
|
||||
<rdf:li><rdf:Description crs:What="Correction" crs:LocalClarity2012="-0.3" crs:LocalGrain="0.4" crs:LocalCurveRefineSaturation="100">
|
||||
<crs:CorrectionMasks><rdf:Seq>
|
||||
<rdf:li crs:What="Mask/RangeMask"><crs:CorrectionRangeMask crs:Type="2" crs:LumRange="0.6 0.85 1 1"/></rdf:li>
|
||||
</rdf:Seq></crs:CorrectionMasks>
|
||||
</rdf:Description></rdf:li>
|
||||
</rdf:Seq></crs:MaskGroupBasedCorrections>
|
||||
</rdf:Description></rdf:RDF></x:xmpmeta>"#;
|
||||
let d = lightcraft_meta::parse_xmp(x).unwrap();
|
||||
let (partial, unmapped) = crate::crs::to_partial_report(&d.properties, Some(&d.values), crate::crs::Target::Any, 1.5);
|
||||
let s = DevelopSettings::default().merged(&partial).expect("valid settings");
|
||||
let MaskShape::LuminanceRange { hi, hi_feather, .. } = &s.masks[0].components[0].shape else { panic!() };
|
||||
assert_eq!((*hi, *hi_feather), (1.0, 0.0), "brighter than scene white is still in the range");
|
||||
assert_eq!(s.masks[0].adjust.clarity, -30.0);
|
||||
assert_eq!(unmapped, ["Mask: LocalGrain"], "a refine saturation of 100 is neutral, not reported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lrtemplate_corrections() {
|
||||
let t = r#"s = { title = "Grad", value = { settings = { Exposure2012 = 0.1, GradientBasedCorrections = {
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
//! Paths stored in a file LightCraft reads but the user did not choose path by path, such as the
|
||||
//! originals a Lightroom catalog lists.
|
||||
//!
|
||||
//! Such a path reaches the file system only through [`resolve`]: a relative path is taken from the
|
||||
//! document's folder, and only local files, or files on the network share the document itself is
|
||||
//! on, are looked at (see `craft_docpath`). A path that is not followed is handled like a missing
|
||||
//! file.
|
||||
|
||||
use std::fmt;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub use craft_docpath::Refused;
|
||||
|
||||
/// Why a stored path is not followed.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NotFollowed {
|
||||
/// The path is on another computer (other than the document's share), names a device, is a
|
||||
/// URL, or is not a usable path.
|
||||
Refused(Refused),
|
||||
/// The path is not an absolute path on this computer, such as a Windows drive path read on
|
||||
/// macOS or Linux.
|
||||
NotHere,
|
||||
}
|
||||
|
||||
impl fmt::Display for NotFollowed {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Refused(r) => r.fmt(f),
|
||||
Self::NotHere => f.write_str("the path is not a path on this computer"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for NotFollowed {}
|
||||
|
||||
/// The file to look at for `stored`, a path or `file:` URL stored in the document at `document`,
|
||||
/// or why it is not followed. A relative `document` is taken from the working directory, where it
|
||||
/// was opened from.
|
||||
pub fn resolve(stored: &str, document: Option<&Path>) -> Result<PathBuf, NotFollowed> {
|
||||
let document = document.map(|d| if d.is_relative() { std::path::absolute(d).unwrap_or_else(|_| d.to_path_buf()) } else { d.to_path_buf() });
|
||||
let path = craft_docpath::resolve(stored, document.as_deref()).map_err(NotFollowed::Refused)?;
|
||||
// A path that is still relative would be taken from the working directory, which the
|
||||
// document does not choose.
|
||||
if path.is_relative() {
|
||||
return Err(NotFollowed::NotHere);
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn relative_paths_are_taken_from_the_document_folder_and_other_computers_are_refused() {
|
||||
let doc = std::env::temp_dir().join("lrcat").join("catalog.lrcat");
|
||||
assert_eq!(resolve("Photos/a.jpg", Some(&doc)), Ok(std::env::temp_dir().join("lrcat").join("Photos").join("a.jpg")));
|
||||
assert_eq!(resolve("a.jpg", None), Err(NotFollowed::NotHere));
|
||||
assert_eq!(resolve(r"\\192.0.2.1\share\x.jpg", Some(&doc)), Err(NotFollowed::Refused(Refused::Network)));
|
||||
assert_eq!(resolve("file://192.0.2.1/s/x.jpg", Some(&doc)), Err(NotFollowed::Refused(Refused::Network)));
|
||||
assert_eq!(resolve(r"\\.\COM1", Some(&doc)), Err(NotFollowed::Refused(Refused::Device)));
|
||||
}
|
||||
}
|
||||
@@ -1310,7 +1310,15 @@ pub fn encode_with_metadata(img: &Rgba8, o: &ExportOptions, meta: Option<&Metada
|
||||
let exif = meta.map(lightcraft_meta::write_exif);
|
||||
let xmp = meta.map(|m| lightcraft_meta::write_xmp(m, None));
|
||||
let meta = EncodeMeta { icc: Some(&profile), exif: exif.as_deref(), xmp: xmp.as_deref(), ppi: Some(o.ppi) };
|
||||
let e = EncodeImage::rgba8(&img);
|
||||
// Opaque photo exports need no alpha channel (#1670). Keep real transparency for
|
||||
// callers of this encoder, and leave unrelated formats on their existing path.
|
||||
let rgb: Option<Vec<u8>> = (matches!(o.format, ExportFormat::Png | ExportFormat::Tiff | ExportFormat::Webp)
|
||||
&& img.data.iter().all(|p| p.last() == Some(&255)))
|
||||
.then(|| img.data.iter().flat_map(|p| p.iter().take(3).copied()).collect());
|
||||
let e = match &rgb {
|
||||
Some(data) => EncodeImage::new(img.width as u32, img.height as u32, 3, Samples::U8(data)),
|
||||
None => EncodeImage::rgba8(&img),
|
||||
};
|
||||
let r = match o.format {
|
||||
ExportFormat::Jpeg => {
|
||||
// 4:4:4 normally; 4:2:0 when targeting a file size (much smaller at equal visual quality).
|
||||
@@ -2281,6 +2289,43 @@ mod tests {
|
||||
img
|
||||
}
|
||||
|
||||
/// Issue #1670: removing a redundant alpha channel preserves samples and metadata.
|
||||
#[test]
|
||||
fn opaque_eight_bit_exports_are_rgb() {
|
||||
let img = test_image();
|
||||
let metadata = Metadata { copyright: Some("Synthetic export test".into()), ..Default::default() };
|
||||
for format in [ExportFormat::Png, ExportFormat::Tiff, ExportFormat::Webp] {
|
||||
for compression in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate, TiffCompression::PackBits] {
|
||||
let options = ExportOptions { format, tiff_compression: compression, ..Default::default() };
|
||||
let bytes = encode_with_metadata(&img, &options, Some(&metadata)).unwrap();
|
||||
let decoded = lightcraft_codecs::decode(&bytes, Default::default()).unwrap();
|
||||
assert!(!decoded.has_alpha, "{format:?}/{compression:?} must be RGB");
|
||||
assert_eq!((decoded.width, decoded.height, decoded.bit_depth), (96, 64, 8));
|
||||
assert_eq!(decoded.to_srgb8().data, img.data, "{format:?}/{compression:?} pixels");
|
||||
assert_eq!(decoded.icc.as_deref(), Some(icc::write_named(NamedSpace::Srgb).as_slice()));
|
||||
assert!(decoded.xmp.as_deref().unwrap_or_default().contains("Synthetic export test"));
|
||||
if format == ExportFormat::Png {
|
||||
assert_eq!(bytes[25], 2, "PNG IHDR declares RGB");
|
||||
}
|
||||
if format != ExportFormat::Tiff {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eight_bit_exports_preserve_real_alpha() {
|
||||
let mut img = test_image();
|
||||
img.data[0][3] = 128;
|
||||
for format in [ExportFormat::Png, ExportFormat::Tiff, ExportFormat::Webp] {
|
||||
let bytes = encode_image(&img, &ExportOptions { format, ..Default::default() }).unwrap();
|
||||
let decoded = lightcraft_codecs::decode(&bytes, Default::default()).unwrap();
|
||||
assert!(decoded.has_alpha, "{format:?} must retain real alpha");
|
||||
assert_eq!(decoded.to_srgb8().data, img.data, "{format:?} samples");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_have_magic() {
|
||||
let img = test_image();
|
||||
|
||||
@@ -167,6 +167,9 @@ impl Index {
|
||||
self.entries.insert((photo, key), (emb.into(), view));
|
||||
}
|
||||
}
|
||||
// a half-written last record (a crash while appending): written fresh at the next save, so later records are
|
||||
// not appended after it and shifted
|
||||
self.rewrite = !rest.is_empty();
|
||||
}
|
||||
|
||||
fn header(&self) -> Vec<u8> {
|
||||
@@ -201,12 +204,11 @@ impl Index {
|
||||
return Ok(());
|
||||
}
|
||||
if self.file_matches && !self.rewrite {
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
|
||||
let mut buf = Vec::new();
|
||||
for (k, e) in &self.unsaved {
|
||||
buf.extend(self.record(k, e));
|
||||
}
|
||||
f.write_all(&buf)?;
|
||||
append(&path, &buf)?;
|
||||
} else {
|
||||
let mut buf = self.header();
|
||||
for (k, e) in &self.entries {
|
||||
@@ -229,6 +231,22 @@ impl Drop for Index {
|
||||
}
|
||||
}
|
||||
|
||||
/// Append `buf` to the file at `path`. When the write fails (a full disk), the file is cut back to its old length so
|
||||
/// the part that got in does not shift what is appended later.
|
||||
fn append(path: &Path, buf: &[u8]) -> std::io::Result<()> {
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(path)?;
|
||||
let old_len = f.metadata()?.len();
|
||||
let written = f.write_all(buf);
|
||||
if written.is_err() {
|
||||
drop(f);
|
||||
// an append-only handle may not truncate (Windows): cut through a writable one
|
||||
if let Ok(w) = std::fs::OpenOptions::new().write(true).open(path) {
|
||||
let _ = w.set_len(old_len);
|
||||
}
|
||||
}
|
||||
written
|
||||
}
|
||||
|
||||
const SCAN_MAGIC: &[u8; 4] = b"LCFS";
|
||||
const SCAN_VERSION: u8 = 1;
|
||||
/// The detector a search was made with: a record made by another one is ignored (a better detector may find more).
|
||||
@@ -325,8 +343,11 @@ impl Scanned {
|
||||
let head = self.header();
|
||||
let Some(rest) = bytes.strip_prefix(head.as_slice()) else { return };
|
||||
self.file_matches = true;
|
||||
// a half-written last id (a crash while appending) is ignored
|
||||
self.set.extend(rest.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c)));
|
||||
// a half-written last id (a crash while appending) is ignored, and the record is written fresh at the next save
|
||||
// so later ids are not appended after it and shifted
|
||||
let (ids, torn) = rest.as_chunks::<8>();
|
||||
self.set.extend(ids.iter().map(|c| u64::from_le_bytes(*c)));
|
||||
self.rewrite = !torn.is_empty();
|
||||
}
|
||||
|
||||
/// Write what was added since the last save: appended when the file is this detector's, else the whole record
|
||||
@@ -341,9 +362,8 @@ impl Scanned {
|
||||
return Ok(());
|
||||
}
|
||||
if self.file_matches && !self.rewrite {
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
|
||||
let buf: Vec<u8> = self.unsaved.iter().flat_map(|id| id.to_le_bytes()).collect();
|
||||
f.write_all(&buf)?;
|
||||
append(&path, &buf)?;
|
||||
} else {
|
||||
let mut buf = self.header();
|
||||
buf.extend(self.set.iter().flat_map(|id| id.to_le_bytes()));
|
||||
@@ -487,6 +507,51 @@ mod tests {
|
||||
assert!(!t.contains(1) && t.contains(2) && t.contains(3));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A torn last record (a crash or a full disk while appending) does not shift the records appended after it
|
||||
/// (issue #2129).
|
||||
#[test]
|
||||
fn records_appended_after_a_torn_tail_stay_aligned() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-faces-torn-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let (emb, scan) = (dir.join("face-embeddings.bin"), dir.join("face-scanned.bin"));
|
||||
let junk = |p: &Path| {
|
||||
let mut f = std::fs::OpenOptions::new().append(true).open(p).unwrap();
|
||||
f.write_all(&[0x3f; 5]).unwrap();
|
||||
};
|
||||
let mut a = Index::default();
|
||||
a.reset("m@abc", 3, Some(emb.clone()));
|
||||
a.insert(7, &rect(0.1), vec![1.0, 0.0, 0.5]);
|
||||
a.save().unwrap();
|
||||
drop(a);
|
||||
junk(&emb);
|
||||
let mut b = Index::default();
|
||||
b.reset("m@abc", 3, Some(emb.clone()));
|
||||
b.insert(9, &rect(0.5), vec![0.5, 0.25, 0.125]);
|
||||
b.save().unwrap();
|
||||
drop(b);
|
||||
let mut c = Index::default();
|
||||
c.reset("m@abc", 3, Some(emb));
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(c.get(7, &rect(0.1)).map(|e| e.to_vec()), Some(vec![1.0, 0.0, 0.5]));
|
||||
assert_eq!(c.get(9, &rect(0.5)).map(|e| e.to_vec()), Some(vec![0.5, 0.25, 0.125]));
|
||||
let mut s = Scanned::default();
|
||||
s.reset(Some(scan.clone()));
|
||||
s.insert(42);
|
||||
s.save().unwrap();
|
||||
drop(s);
|
||||
junk(&scan);
|
||||
let mut t = Scanned::default();
|
||||
t.reset(Some(scan.clone()));
|
||||
t.insert(43);
|
||||
t.save().unwrap();
|
||||
drop(t);
|
||||
let mut u = Scanned::default();
|
||||
u.reset(Some(scan));
|
||||
assert!(u.contains(42) && u.contains(43) && u.len() == 2);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the recognition commands keep between calls: the loaded model and the embeddings made with it.
|
||||
|
||||
@@ -127,6 +127,17 @@ fn pool_threads_for(cores: usize, pace: Pace, cap: Option<usize>) -> usize {
|
||||
cap.map_or(n, |c| n.min(c.max(1)))
|
||||
}
|
||||
|
||||
/// How much of the GPU face models may take at `pace`: face model runs at once, and the share of the time it may be
|
||||
/// busy. Paused or light: one run at a time, busy a quarter of the time (the user is working, and the window renders
|
||||
/// on the same GPU); normal: two at once, busy most of the time; full: as fast as it goes.
|
||||
pub(crate) fn gpu_pace(pace: Pace) -> (usize, f32) {
|
||||
match pace {
|
||||
Pace::Pause | Pace::Light => (1, 0.25),
|
||||
Pace::Normal => (2, 0.75),
|
||||
Pace::Full => (4, 1.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// One photo's faces to embed (and, if `detect`, to find first).
|
||||
pub(crate) struct Prepared {
|
||||
/// `<model id>@<sha256>` the embeddings are for.
|
||||
@@ -201,7 +212,8 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
|
||||
let img = Rgb { data: &rgb, width: w, height: h };
|
||||
let (iw, ih) = embedder.input_size();
|
||||
let scaled = |f: &Face| f.landmarks.map(|(x, y)| (x * w as f32, y * h as f32));
|
||||
let mut results: Vec<(Rect, Option<Vec<f32>>, Rect)> = todo
|
||||
// every face's crop first, then all of them through the model at once (one GPU dispatch for a group photo)
|
||||
let mut faces: Vec<(Rect, Option<Vec<u8>>, Rect)> = todo
|
||||
.into_iter()
|
||||
.map(|rect| {
|
||||
let best = found.iter().map(|f| (overlap(&rect, f), f)).filter(|(o, _)| *o >= MATCH_IOU).max_by(|a, b| a.0.total_cmp(&b.0));
|
||||
@@ -218,20 +230,18 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
|
||||
ih,
|
||||
),
|
||||
};
|
||||
let v = crop.and_then(|c| embedder.embed(&c).ok());
|
||||
// shown by the detector's box where it found this face: every face then has the same tightness
|
||||
let view = best.and_then(|(_, f)| face_rect(f)).unwrap_or(rect);
|
||||
(rect, v, view)
|
||||
(rect, crop, view)
|
||||
})
|
||||
.collect();
|
||||
// a photo with no face regions: what the detector is sure of becomes its faces, embedded in the same pass
|
||||
let new_faces = if detect {
|
||||
detected.map(|faces| {
|
||||
detected.map(|found| {
|
||||
let mut rects = Vec::new();
|
||||
for f in faces.iter().filter(|f| f.score >= REGION_SCORE) {
|
||||
for f in found.iter().filter(|f| f.score >= REGION_SCORE) {
|
||||
let Some(rect) = face_rect(f) else { continue };
|
||||
let v = align_to_template(&img, &scaled(f), iw, ih).and_then(|c| embedder.embed(&c).ok());
|
||||
results.push((rect, v, rect));
|
||||
faces.push((rect, align_to_template(&img, &scaled(f), iw, ih), rect));
|
||||
rects.push(rect);
|
||||
}
|
||||
rects
|
||||
@@ -239,6 +249,10 @@ pub(crate) fn process(p: Prepared, embedder: &Embedder) -> Done {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let crops: Vec<&[u8]> = faces.iter().filter_map(|(_, c, _)| c.as_deref()).collect();
|
||||
let mut embedded = embedder.embed_batch(&crops).into_iter();
|
||||
let results: Vec<(Rect, Option<Vec<f32>>, Rect)> =
|
||||
faces.into_iter().map(|(rect, crop, view)| (rect, crop.and_then(|_| embedded.next().and_then(Result::ok)), view)).collect();
|
||||
(results, new_faces)
|
||||
}));
|
||||
let (results, found) = run.unwrap_or_else(|_| (fallback(rects), None));
|
||||
|
||||
@@ -707,8 +707,9 @@ pub(crate) fn decode_raw_preview(
|
||||
covering: Option<usize>,
|
||||
) -> Option<lightcraft_codecs::Decoded> {
|
||||
let jpeg = stand_in_image(bytes, covering)?;
|
||||
// the colour space is a property of the JPEG; a TIFF stand-in is read by the TIFF reader
|
||||
let is_jpeg = matches!(jpeg, std::borrow::Cow::Owned(_));
|
||||
// the colour space is a property of the JPEG; a TIFF stand-in (the file itself, or a Hasselblad RGB preview
|
||||
// wrapped as a TIFF) and a JPEG XL preview are read by their own decoders
|
||||
let is_jpeg = jpeg.starts_with(&[0xff, 0xd8]);
|
||||
match lightcraft_raw::embedded_preview_color_space(bytes).filter(|_| is_jpeg) {
|
||||
Some(space) => {
|
||||
let space = match space {
|
||||
|
||||
@@ -382,6 +382,17 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
|
||||
|
||||
/// [`expand`] with the walk bounded by `limits` → (files, whether a walk stopped at a bound).
|
||||
pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk::Limits) -> (Vec<String>, bool) {
|
||||
expand_noting(paths, skip, limits, None)
|
||||
}
|
||||
|
||||
/// [`expand_within`], handing the listing of every folder that is walked to `note`
|
||||
/// ([`crate::walk::files_in_noting`]).
|
||||
pub(crate) fn expand_noting(
|
||||
paths: &[String],
|
||||
skip: Option<&Path>,
|
||||
limits: crate::walk::Limits,
|
||||
mut note: Option<crate::walk::NoteFolder>,
|
||||
) -> (Vec<String>, bool) {
|
||||
let mut out = Vec::new();
|
||||
let mut truncated = false;
|
||||
for p in paths {
|
||||
@@ -390,7 +401,10 @@ pub fn expand_within(paths: &[String], skip: Option<&Path>, limits: crate::walk:
|
||||
continue;
|
||||
}
|
||||
if p.is_dir() {
|
||||
let w = crate::walk::files_in(p, skip, limits, is_supported);
|
||||
let w = match note.as_mut() {
|
||||
Some(note) => crate::walk::files_in_noting(p, skip, limits, is_supported, &mut **note),
|
||||
None => crate::walk::files_in(p, skip, limits, is_supported),
|
||||
};
|
||||
if w.truncated {
|
||||
log::warn!(
|
||||
"import: stopped looking in {} after {} entries (limits: {} entries, {} folder levels)",
|
||||
@@ -638,6 +652,11 @@ impl ScanInput {
|
||||
(input, paths.to_vec())
|
||||
}
|
||||
|
||||
/// The folder a scan never enters (the library's own).
|
||||
pub(crate) fn skip(&self) -> Option<&Path> {
|
||||
self.skip.as_deref()
|
||||
}
|
||||
|
||||
/// The library as the scan sees it (shared, for [`crate::sync`], which looks at the same one).
|
||||
pub(crate) fn library(&self) -> std::sync::Arc<lightcraft_catalog::Catalog> {
|
||||
std::sync::Arc::clone(&self.library)
|
||||
@@ -656,9 +675,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
|
||||
|
||||
/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it
|
||||
/// has) when `progress.cancel` is set.
|
||||
pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
pub fn scan_with(input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
|
||||
let files = expand(paths, input.skip.as_deref());
|
||||
scan_files(input, files, progress)
|
||||
}
|
||||
|
||||
/// [`scan_with`] for files that were found already (`paths` expanded, [`expand`]).
|
||||
pub(crate) fn scan_files(mut input: ScanInput, files: Vec<String>, progress: &ScanProgress) -> ScanOutput {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
let by_path = library_files(&input.library, &files);
|
||||
let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(Path::new(f))).cloned().collect();
|
||||
progress.total.store(todo.len(), Relaxed);
|
||||
|
||||
@@ -29,6 +29,7 @@ pub mod demo;
|
||||
pub mod denoise;
|
||||
pub mod devices;
|
||||
pub mod display;
|
||||
pub mod doc_path;
|
||||
mod errors;
|
||||
pub mod export;
|
||||
pub mod face_download;
|
||||
@@ -56,6 +57,7 @@ pub mod media;
|
||||
pub mod memory;
|
||||
pub mod merge;
|
||||
mod model_download;
|
||||
pub mod open_path;
|
||||
pub mod originals;
|
||||
mod pdf;
|
||||
pub mod preset_import;
|
||||
@@ -298,6 +300,10 @@ pub struct Session {
|
||||
pub xmp: sidecar::XmpPrefs,
|
||||
/// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json.
|
||||
pub last_export: Option<serde_json::Value>,
|
||||
/// The options of the last Photo Merge started, per command (`merge.hdr`, `merge.panorama`,
|
||||
/// `merge.hdrPanorama`): the merge dialog opens with them and "… with Last Settings" uses
|
||||
/// them. Persisted in prefs.json.
|
||||
pub last_merge: std::collections::BTreeMap<String, serde_json::Value>,
|
||||
/// The user's export presets (built-ins: [`export::builtin_presets`]), persisted in prefs.json.
|
||||
pub export_presets: Vec<export::ExportPreset>,
|
||||
/// The Print module's current layout, image settings, page, guides and job (`print.*`). Lives
|
||||
@@ -459,6 +465,7 @@ impl Session {
|
||||
library_identity: Arc::new(()),
|
||||
xmp: sidecar::XmpPrefs::default(),
|
||||
last_export: None,
|
||||
last_merge: Default::default(),
|
||||
export_presets: Vec::new(),
|
||||
print: Default::default(),
|
||||
print_templates: Vec::new(),
|
||||
|
||||
@@ -207,6 +207,8 @@ struct PrefsFile {
|
||||
/// Parameters of the last export (for Export with Previous).
|
||||
#[serde(alias = "last_export")]
|
||||
last_export: Option<serde_json::Value>,
|
||||
/// Options of the last Photo Merge, per command.
|
||||
last_merge: std::collections::BTreeMap<String, serde_json::Value>,
|
||||
/// The user's export presets.
|
||||
export_presets: Vec<crate::export::ExportPreset>,
|
||||
/// Print templates.
|
||||
@@ -492,6 +494,7 @@ impl Session {
|
||||
let prefs = settings.read::<PrefsFile>(files.as_mut(), "prefs.json").unwrap_or_default();
|
||||
self.xmp = prefs.xmp;
|
||||
self.last_export = prefs.last_export;
|
||||
self.last_merge = prefs.last_merge;
|
||||
self.export_presets = prefs.export_presets;
|
||||
self.print_templates = prefs.print_templates;
|
||||
self.metadata_presets = prefs.metadata_presets;
|
||||
@@ -746,6 +749,7 @@ impl Session {
|
||||
let v = serde_json::to_vec_pretty(&PrefsFile {
|
||||
xmp: self.xmp,
|
||||
last_export: self.last_export.clone(),
|
||||
last_merge: self.last_merge.clone(),
|
||||
export_presets: self.export_presets.clone(),
|
||||
print_templates: self.print_templates.clone(),
|
||||
metadata_presets: self.metadata_presets.clone(),
|
||||
|
||||
@@ -538,6 +538,7 @@ mod tests {
|
||||
keywords: vec!["Trips|Goa".into()],
|
||||
history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])],
|
||||
snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])],
|
||||
original: crate::lightroom_catalog::Original::Local,
|
||||
};
|
||||
CatalogImport {
|
||||
source: "source.lrcat".into(),
|
||||
@@ -621,6 +622,7 @@ mod tests {
|
||||
keywords: Vec::new(),
|
||||
history: vec![BTreeMap::from([("text".into(), serde_json::json!(format!("history-{index}-{}", "x".repeat(8 * 1024))))])],
|
||||
snapshots: Vec::new(),
|
||||
original: crate::lightroom_catalog::Original::Local,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(store_best_effort(&path, &input).unwrap(), None);
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//! Lightroom, Python and a C SQLite runtime are not needed. Original paths stay in place.
|
||||
//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
@@ -146,6 +146,31 @@ pub struct CatalogPhoto {
|
||||
pub keywords: Vec<String>,
|
||||
pub history: Vec<Row>,
|
||||
pub snapshots: Vec<Row>,
|
||||
/// Whether `path` may be looked at (derived when the catalog is read; not archived).
|
||||
#[serde(skip)]
|
||||
pub original: Original,
|
||||
}
|
||||
|
||||
/// Whether the original a catalog lists may be looked at. Its path is resolved with the catalog
|
||||
/// as the document ([`crate::doc_path::resolve`]).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Original {
|
||||
/// `path` is a local file path, or one on the network share the catalog is on.
|
||||
Local,
|
||||
/// `path` is not a path on this computer (a catalog made on another system): the photo is
|
||||
/// kept as an offline record for later relinking, and the path is not looked at.
|
||||
#[default]
|
||||
NotHere,
|
||||
/// `path` is refused (why): it is on another computer or a device, or its folder or file name
|
||||
/// is an absolute path. The photo is reported missing and not added to the library.
|
||||
Refused(String),
|
||||
}
|
||||
|
||||
impl CatalogPhoto {
|
||||
/// Whether the original is a file here; only a [`Original::Local`] path is looked at.
|
||||
pub fn original_is_file(&self) -> bool {
|
||||
self.original == Original::Local && Path::new(&self.path).is_file()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
@@ -343,25 +368,36 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {
|
||||
|
||||
/// Where a root folder is when its `absolutePath` isn't a folder here (the catalog came from
|
||||
/// another computer): `relativePathFromCatalog`, which Lightroom records for roots on the
|
||||
/// catalog's volume, resolved against the catalog's folder, if that is a folder.
|
||||
/// catalog's volume, resolved against the catalog's folder, if that is a folder. Both paths are
|
||||
/// looked at only when [`crate::doc_path::resolve`] follows them.
|
||||
fn relocated_root(absolute: &str, relative: &str, catalog: &Path) -> Option<PathBuf> {
|
||||
if relative.is_empty() || !Path::new(relative).is_relative() || Path::new(absolute).is_dir() {
|
||||
if relative.is_empty() || !Path::new(relative).is_relative() || crate::doc_path::resolve(absolute, Some(catalog)).is_ok_and(|p| p.is_dir()) {
|
||||
return None;
|
||||
}
|
||||
let dir = std::path::absolute(catalog).ok()?.parent()?.join(relative);
|
||||
let mut out = PathBuf::new();
|
||||
for part in dir.components() {
|
||||
match part {
|
||||
Component::ParentDir => {
|
||||
if !out.pop() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Component::CurDir => {}
|
||||
part => out.push(part),
|
||||
}
|
||||
crate::doc_path::resolve(relative, Some(catalog)).ok().filter(|dir| dir.is_dir())
|
||||
}
|
||||
|
||||
/// Whether a folder path or file name from the catalog is absolute: `Path::join` would put it in
|
||||
/// place of the root folder.
|
||||
fn is_absolute_part(part: &str) -> bool {
|
||||
let bytes = part.as_bytes();
|
||||
part.starts_with(['/', '\\'])
|
||||
|| (bytes.first().is_some_and(u8::is_ascii_alphabetic) && bytes.get(1) == Some(&b':'))
|
||||
|| Path::new(part).is_absolute()
|
||||
}
|
||||
|
||||
/// The path of an original under `root` and whether it may be looked at, resolved with the
|
||||
/// catalog at `catalog` as the document.
|
||||
fn original_path(root: &str, folder: &str, name: &str, catalog: &Path) -> (String, Original) {
|
||||
let stored = Path::new(root).join(folder).join(name).to_string_lossy().to_string();
|
||||
if is_absolute_part(folder) || is_absolute_part(name) {
|
||||
return (stored, Original::Refused("its folder or file name is an absolute path".into()));
|
||||
}
|
||||
match crate::doc_path::resolve(&stored, Some(catalog)) {
|
||||
Ok(found) => (found.to_string_lossy().to_string(), Original::Local),
|
||||
Err(crate::doc_path::NotFollowed::NotHere) => (stored, Original::NotHere),
|
||||
Err(crate::doc_path::NotFollowed::Refused(why)) => (stored, Original::Refused(why.to_string())),
|
||||
}
|
||||
out.is_dir().then_some(out)
|
||||
}
|
||||
|
||||
/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.
|
||||
@@ -457,6 +493,7 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
|
||||
out
|
||||
};
|
||||
let mut root_paths = HashMap::new();
|
||||
let mut refused_roots = HashSet::new();
|
||||
let mut root_ids: Vec<_> = roots.keys().copied().collect();
|
||||
root_ids.sort_unstable();
|
||||
for (id, root) in root_ids.iter().filter_map(|id| Some((*id, roots.get(id)?))) {
|
||||
@@ -469,6 +506,11 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
|
||||
}
|
||||
None => absolute.to_string(),
|
||||
};
|
||||
if let Err(crate::doc_path::NotFollowed::Refused(why)) = crate::doc_path::resolve(&resolved, Some(path)) {
|
||||
warnings
|
||||
.push(format!("root folder {}: {resolved} is not opened: {why}; its photos are listed as missing and not added", text(root, "name")));
|
||||
refused_roots.insert(id);
|
||||
}
|
||||
root_paths.insert(id, resolved);
|
||||
}
|
||||
let mut history = group(optional_table("Adobe_libraryImageDevelopHistoryStep", &mut warnings)?);
|
||||
@@ -487,13 +529,19 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
|
||||
let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?;
|
||||
let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?;
|
||||
let root = roots.get(&number(folder, "rootFolder")).ok_or_else(|| format!("image {id}: missing root record"))?;
|
||||
let root_path = root_paths.get(&number(root, "id_local")).map_or("", String::as_str);
|
||||
let root_id = number(root, "id_local");
|
||||
let root_path = root_paths.get(&root_id).map_or("", String::as_str);
|
||||
let folder_path = text(folder, "pathFromRoot");
|
||||
let name = text(file, "idx_filename");
|
||||
if root_path.is_empty() || name.is_empty() {
|
||||
return Err(format!("image {id}: incomplete original path"));
|
||||
}
|
||||
let path = Path::new(root_path).join(folder_path).join(name).to_string_lossy().to_string();
|
||||
let (photo_path, original) = original_path(root_path, folder_path, name, path);
|
||||
if let Original::Refused(why) = &original
|
||||
&& !refused_roots.contains(&root_id)
|
||||
{
|
||||
out.warnings.push(format!("image {id}: {photo_path} is not opened: {why}; listed as missing and not added"));
|
||||
}
|
||||
let packet = match xmp(metadata.get(&id).and_then(|r| r.get("xmp"))) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
@@ -504,19 +552,64 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
|
||||
out.photos.push(CatalogPhoto {
|
||||
source_id: id,
|
||||
uuid: text(&image, "id_global").into(),
|
||||
path,
|
||||
path: photo_path,
|
||||
image,
|
||||
settings: develops.get(&id).cloned().unwrap_or_default(),
|
||||
xmp: packet,
|
||||
keywords: keywords.remove(&id).unwrap_or_default(),
|
||||
history: history.remove(&id).unwrap_or_default(),
|
||||
snapshots: snapshots.remove(&id).unwrap_or_default(),
|
||||
original,
|
||||
});
|
||||
}
|
||||
refuse_non_media(&mut out);
|
||||
ensure_read_active(cancel)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Drop the catalog's images whose original is not a file type LightCraft reads, saying so in the
|
||||
/// import's warnings.
|
||||
///
|
||||
/// The catalog decides these paths, and a record it leaves behind is one click from the system
|
||||
/// opener (Photo ▸ Edit In ▸ Open Original in Other Editor). A record for `payload.exe`,
|
||||
/// `start.lnk` or `Evil.app` would make a `.lrcat` someone was sent a way to have a program run
|
||||
/// (issue #2135), so such an image never becomes a photo: not imported, not relinkable, not
|
||||
/// openable. A virtual copy of a refused original goes with it — it has no original of its own.
|
||||
///
|
||||
/// Called where the catalog is read and again where its photos become records, so neither route
|
||||
/// can keep one; the second pass finds nothing left to say.
|
||||
pub(crate) fn refuse_non_media(data: &mut CatalogImport) {
|
||||
// masters first: a virtual copy may come before its master in the table
|
||||
let refused: HashSet<i64> = data
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| number(&p.image, "masterImage") == 0 && !crate::open_path::is_media(Path::new(&p.path)))
|
||||
.map(|p| p.source_id)
|
||||
.collect();
|
||||
if refused.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut warnings = Vec::new();
|
||||
data.photos.retain(|p| {
|
||||
let master = number(&p.image, "masterImage");
|
||||
// `masterImage` is 0 for a master, so a refused id of 0 (a malformed catalog) must not
|
||||
// take every master of the catalog with it
|
||||
if master != 0 {
|
||||
if !refused.contains(&master) {
|
||||
return true;
|
||||
}
|
||||
warnings.push(format!("image {}: not imported; its original was not imported", p.source_id));
|
||||
return false;
|
||||
}
|
||||
if !refused.contains(&p.source_id) {
|
||||
return true;
|
||||
}
|
||||
warnings.push(format!("image {}: not imported; {} isn't one of the file types LightCraft reads", p.source_id, p.path));
|
||||
false
|
||||
});
|
||||
data.warnings.extend(warnings);
|
||||
}
|
||||
|
||||
fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
|
||||
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
|
||||
}
|
||||
@@ -665,11 +758,11 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
|
||||
let paths: Vec<_> = data
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file())
|
||||
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && p.original_is_file())
|
||||
.map(|p| p.path.clone())
|
||||
.collect();
|
||||
let missing_sources: HashSet<String> =
|
||||
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect();
|
||||
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !p.original_is_file()).map(|p| key(&p.path)).collect();
|
||||
let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;
|
||||
let applied =
|
||||
apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;
|
||||
@@ -692,8 +785,10 @@ pub(crate) struct ApplyPrepared<'a> {
|
||||
pub(crate) undo0: usize,
|
||||
}
|
||||
|
||||
pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
|
||||
pub(crate) fn apply_prepared(s: &mut crate::Session, mut data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
|
||||
let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;
|
||||
// nothing but a file type LightCraft reads becomes a record, whichever route got here (#2135)
|
||||
refuse_non_media(&mut data);
|
||||
validate(&data).map_err(error)?;
|
||||
let mut warnings = data.warnings.clone();
|
||||
let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect();
|
||||
@@ -713,12 +808,18 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
|
||||
let mut photos: Vec<_> = data.photos.iter().collect();
|
||||
photos.sort_by_key(|p| number(&p.image, "masterImage") != 0);
|
||||
let copy_masters: HashSet<_> = data.photos.iter().map(|p| number(&p.image, "masterImage")).filter(|id| *id != 0).collect();
|
||||
let refused: HashSet<_> = data.photos.iter().filter(|p| matches!(p.original, Original::Refused(_))).map(|p| p.source_id).collect();
|
||||
let mut planned: HashMap<PhotoId, Photo> = HashMap::new();
|
||||
for src in photos {
|
||||
let master = number(&src.image, "masterImage");
|
||||
if master == 0 && missing_sources.contains(&key(&src.path)) {
|
||||
missing.push(src.path.clone());
|
||||
}
|
||||
// A refused path must not become a library record the app would look at later; its
|
||||
// virtual copies go with it.
|
||||
if refused.contains(&src.source_id) || refused.contains(&master) {
|
||||
continue;
|
||||
}
|
||||
let indexed = index
|
||||
.photos
|
||||
.get(&identity(&data, src))
|
||||
@@ -940,6 +1041,7 @@ mod tests {
|
||||
keywords: vec!["Travel|Goa".into()],
|
||||
history: vec![row(json!({"text":"s = { Exposure2012 = 0.5 }"}))],
|
||||
snapshots: Vec::new(),
|
||||
original: Original::Local,
|
||||
};
|
||||
let mut copy = photo.clone();
|
||||
copy.source_id = 2;
|
||||
@@ -962,6 +1064,48 @@ mod tests {
|
||||
warnings: Vec::new(),
|
||||
}
|
||||
}
|
||||
/// A crafted `.lrcat` that names a program as a photo's original gets no record for it, so
|
||||
/// Photo ▸ Edit In ▸ Open Original has nothing to hand the system opener (#2135). Its virtual
|
||||
/// copies go with it; the catalog's real photos still import.
|
||||
#[test]
|
||||
fn a_catalog_original_that_is_not_a_photo_file_is_not_imported() {
|
||||
let mut data = sample();
|
||||
let mut exe = data.photos[1].clone();
|
||||
exe.source_id = 3;
|
||||
exe.uuid = "payload".into();
|
||||
exe.path = "/Users/sky/Downloads/payload.exe".into();
|
||||
exe.image = row(json!({"id_local":3,"fileFormat":"ARW","fileWidth":20,"fileHeight":10}));
|
||||
let mut exe_copy = exe.clone();
|
||||
exe_copy.source_id = 4;
|
||||
exe_copy.uuid = "payload-copy".into();
|
||||
exe_copy.path = "/Users/sky/Downloads/payload.exe".into();
|
||||
exe_copy.image = row(json!({"id_local":4,"masterImage":3,"copyName":"Mono"}));
|
||||
// the copy comes before its master, as a catalog's table order may have it
|
||||
data.photos.insert(0, exe_copy);
|
||||
data.photos.push(exe);
|
||||
for bad in [r"C:\Users\sky\start.lnk", "/Applications/Evil.app", "/Users/sky/go.command", "/Users/sky/run.bat"] {
|
||||
let mut one = data.clone();
|
||||
one.photos.retain(|p| p.source_id < 3);
|
||||
let mut planted = one.photos[1].clone();
|
||||
planted.source_id = 5;
|
||||
planted.uuid = "planted".into();
|
||||
planted.path = bad.into();
|
||||
planted.image = row(json!({"id_local":5,"fileFormat":"ARW"}));
|
||||
one.photos.push(planted);
|
||||
let mut s = crate::Session::new();
|
||||
let r = apply(&mut s, one, false).unwrap();
|
||||
assert_eq!(r["photos"], 2, "{bad} is not catalogued: {r}");
|
||||
assert!(s.catalog.photos().all(|p| !matches!(&p.source, Source::File { path } if path == bad)), "{bad} became a library record");
|
||||
}
|
||||
let mut s = crate::Session::new();
|
||||
let r = apply(&mut s, data, false).unwrap();
|
||||
assert_eq!(r["photos"], 2, "only the real master and its copy: {r}");
|
||||
assert!(s.catalog.photos().all(|p| !matches!(&p.source, Source::File { path } if path.ends_with(".exe"))), "the program was catalogued");
|
||||
let warnings = r["warnings"].to_string();
|
||||
assert!(warnings.contains("payload.exe") && warnings.contains("file types LightCraft reads"), "{warnings}");
|
||||
assert!(warnings.contains("image 4: not imported"), "the virtual copy of the refused original goes with it: {warnings}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offline_local_master_and_copy_join_library_with_one_undo_step() {
|
||||
let mut s = crate::Session::new();
|
||||
@@ -1508,11 +1652,57 @@ mod tests {
|
||||
// a relative path that leads nowhere keeps the recorded one
|
||||
std::fs::write(&path, native_catalog(&photo_tables(("C:/Users/me/Project/RAW/", "../Elsewhere/")))).unwrap();
|
||||
let catalog = read(&path).unwrap();
|
||||
assert!(catalog.photos[0].path.starts_with("C:/Users/me/Project/RAW/"), "{}", catalog.photos[0].path);
|
||||
assert!(Path::new(&catalog.photos[0].path).starts_with("C:/Users/me/Project/RAW/"), "{}", catalog.photos[0].path);
|
||||
assert!(catalog.warnings.is_empty(), "{:?}", catalog.warnings);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn originals_on_other_computers_are_never_looked_at_or_added() {
|
||||
let dir = std::env::temp_dir().join(format!("lightcraft-lrcat-remote-roots-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(dir.join("Photos")).unwrap();
|
||||
std::fs::write(dir.join("Photos/test.ARW"), b"original").unwrap();
|
||||
let source = dir.join("source.lrcat");
|
||||
let mut roots = vec![r"\\192.0.2.1\share\".to_string(), "file://192.0.2.1/s/".to_string()];
|
||||
if cfg!(unix) {
|
||||
// `//<local folder>/` is a local folder to Linux, but a network path to the resolver.
|
||||
roots.push(format!("/{}/", dir.display()));
|
||||
}
|
||||
for root in &roots {
|
||||
std::fs::write(&source, native_catalog(&photo_tables((root, "")))).unwrap();
|
||||
let catalog = read(&source).unwrap();
|
||||
assert!(matches!(catalog.photos[0].original, Original::Refused(_)), "{root}: {:?}", catalog.photos[0]);
|
||||
assert!(catalog.warnings.iter().any(|w| w.contains("not opened")), "{root}: {:?}", catalog.warnings);
|
||||
let mut s = crate::Session::new();
|
||||
let probes = Arc::new(AtomicUsize::new(0));
|
||||
let calls = probes.clone();
|
||||
s.media.file_probe = Some(Arc::new(move |_| {
|
||||
calls.fetch_add(1, Ordering::Relaxed);
|
||||
Err("a refused original must not be probed".into())
|
||||
}));
|
||||
let report = import(&mut s, &source, false).unwrap();
|
||||
assert_eq!(probes.load(Ordering::Relaxed), 0, "{root}");
|
||||
assert_eq!(report["missing"].as_array().map(Vec::len), Some(1), "{root}: {report}");
|
||||
assert!(s.catalog.is_empty(), "{root}");
|
||||
}
|
||||
// An absolute folder path would replace the root.
|
||||
let mut tables = photo_tables((dir.to_str().unwrap(), ""));
|
||||
tables[1].2[2] = SqlValue::Text(format!("{}/Photos/", dir.display()));
|
||||
std::fs::write(&source, native_catalog(&tables)).unwrap();
|
||||
assert!(matches!(read(&source).unwrap().photos[0].original, Original::Refused(_)));
|
||||
// A root beside the catalog is followed.
|
||||
std::fs::write(&source, native_catalog(&photo_tables((dir.to_str().unwrap(), "")))).unwrap();
|
||||
let catalog = read(&source).unwrap();
|
||||
assert_eq!(catalog.photos[0].original, Original::Local);
|
||||
assert!(catalog.photos[0].original_is_file());
|
||||
// A root that moved is found through its path relative to the catalog.
|
||||
std::fs::write(&source, native_catalog(&photo_tables((r"\\192.0.2.1\share\", "./")))).unwrap();
|
||||
let catalog = read(&source).unwrap();
|
||||
assert!(catalog.photos[0].original_is_file(), "{:?}", catalog.photos[0]);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_optional_table_warns_and_is_skipped() {
|
||||
let mut main = vec![0u8; 2 * 512];
|
||||
|
||||
@@ -329,7 +329,7 @@ fn original_paths(data: &CatalogImport, cancel: &AtomicBool) -> crate::Result<(V
|
||||
if !seen.insert(key.clone()) {
|
||||
continue;
|
||||
}
|
||||
if Path::new(&photo.path).is_file() {
|
||||
if photo.original_is_file() {
|
||||
paths.push(photo.path.clone());
|
||||
} else {
|
||||
missing.insert(key);
|
||||
@@ -393,6 +393,7 @@ mod tests {
|
||||
keywords: Vec::new(),
|
||||
history: Vec::new(),
|
||||
snapshots: Vec::new(),
|
||||
original: lightroom_catalog::Original::Local,
|
||||
}],
|
||||
collections: Vec::new(),
|
||||
members: Vec::new(),
|
||||
@@ -531,6 +532,7 @@ mod tests {
|
||||
serde_json::json!(format!("history-{source_id}-{}", "x".repeat(8 * 1024))),
|
||||
)])],
|
||||
snapshots: Vec::new(),
|
||||
original: lightroom_catalog::Original::Local,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -2189,6 +2189,40 @@ mod tests {
|
||||
assert!(mean(&a) > mean(&b) + 10.0, "two stops brighter after: {} vs {}", mean(&a), mean(&b));
|
||||
}
|
||||
|
||||
// LR-VIEW-ZOOM: what the loupe shows at 1:1 is what an export at full size writes. A window of
|
||||
// the zoomed frame (the photo's own pixels) is pixel-for-pixel the export, at the centre and
|
||||
// at a corner, with tone, colour and detail edits on a camera-sized photo
|
||||
#[test]
|
||||
fn one_to_one_window_pixels_are_export_pixels() {
|
||||
use lightcraft_pipeline::{OutputDepth, OutputSpace, PixelWindow};
|
||||
let mut s = crate::Session::with_demo();
|
||||
let id = s.active().unwrap();
|
||||
let p = s.catalog.photo(id).unwrap().clone();
|
||||
let (w, h) = (p.width as usize, p.height as usize);
|
||||
assert!(w.max(h) > 2560, "larger than the preview cap: {w}×{h}");
|
||||
let mut d = (*s.develop_of(id).unwrap()).clone();
|
||||
d.light.exposure = 0.7;
|
||||
d.light.contrast = 25.0;
|
||||
d.color.saturation = 20.0;
|
||||
s.set_develop(id, d, "Edits").unwrap();
|
||||
let export = s.render_export(id, w, h, OutputSpace::Srgb, OutputDepth::U8).unwrap().image;
|
||||
assert_eq!((export.width, export.height), (w, h), "a full-size export");
|
||||
for win in [PixelWindow { x: w / 2 - 256, y: h / 2 - 256, w: 512, h: 384 }, PixelWindow { x: 0, y: h - 300, w: 400, h: 300 }] {
|
||||
let tile = s.region_job(id, w, h, win, true).unwrap().run().rendered.unwrap().image;
|
||||
assert_eq!((tile.width, tile.height), (win.w, win.h));
|
||||
let mut worst = 0u8;
|
||||
for y in 0..win.h {
|
||||
for x in 0..win.w {
|
||||
let (a, b) = (tile.data[y * win.w + x], export.data[(win.y + y) * w + win.x + x]);
|
||||
for k in 0..3 {
|
||||
worst = worst.max(a[k].abs_diff(b[k]));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(worst <= 1, "window {win:?}: the 1:1 view differs from the export by up to {worst} levels");
|
||||
}
|
||||
}
|
||||
|
||||
/// The thumbnail of a photo whose preview-level source is decoded is cut from it: the file (here
|
||||
/// one nothing can decode) is not read again.
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
//! Whether a path the library holds may be handed to the operating system.
|
||||
//!
|
||||
//! A photo's `Source::File` path is not always something a user typed: it can come from an
|
||||
//! imported Lightroom catalog, an `.lrcat` someone was sent, a relink or a synchronised folder.
|
||||
//! Photo ▸ Edit In ▸ *Open Original in Other Editor* and the system default opener run whatever
|
||||
//! the path names, so a record pointing at `payload.exe`, `thing.lnk`, `run.bat`, `page.hta`
|
||||
//! (Windows) or `Evil.app`, `go.command` (macOS) would be executed on one click (issue #2135).
|
||||
//!
|
||||
//! [`openable`] is the one gate every such hand-over goes through: the path must be a regular
|
||||
//! file whose extension is one LightCraft itself reads ([`crate::import::EXTENSIONS`], which
|
||||
//! already covers the Edit In copy formats `tif`/`tiff`/`psd`), and what a link resolves to must
|
||||
//! be one too, so a `photo.jpg` pointing at `/bin/sh` is refused as well. What comes back is the
|
||||
//! absolute path to hand over; what comes back on refusal is the sentence to show the user.
|
||||
//!
|
||||
//! Nothing here opens, stats-for-content or reads the file: it only decides whether the path may
|
||||
//! leave LightCraft. Catalog import applies the same test ([`crate::lightroom_catalog`]), so a
|
||||
//! refused original never becomes a library record in the first place.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
/// The absolute path to hand to an opener or external editor, or the reason it is refused.
|
||||
///
|
||||
/// `label` names the thing in the error (a file name, a photo's name); an empty `label` uses
|
||||
/// the path's own file name.
|
||||
pub fn openable(path: &str, label: &str) -> Result<String, String> {
|
||||
let p = Path::new(path);
|
||||
let name = match label.trim() {
|
||||
"" => p.file_name().map_or_else(|| path.to_string(), |n| n.to_string_lossy().into_owned()),
|
||||
l => l.to_string(),
|
||||
};
|
||||
if !is_media(p) {
|
||||
return Err(format!("{name} isn't one of the file types LightCraft reads, so it isn't handed to another application"));
|
||||
}
|
||||
match std::fs::metadata(p) {
|
||||
Ok(m) if m.is_file() => {}
|
||||
Ok(_) => return Err(format!("{name} isn't a file, so it isn't handed to another application")),
|
||||
Err(e) => return Err(format!("the original is missing: {path} ({e})")),
|
||||
}
|
||||
// a link may name a photo and point at a program; what runs is what it points at
|
||||
if let Ok(target) = p.canonicalize()
|
||||
&& !is_media(&target)
|
||||
{
|
||||
return Err(format!("{name} points at {}, which isn't a file type LightCraft reads", target.display()));
|
||||
}
|
||||
let absolute = std::path::absolute(p).map_err(|e| format!("{name}: {e}"))?;
|
||||
Ok(absolute.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
/// Whether `path`'s extension is one LightCraft reads (the Edit In copy formats included).
|
||||
pub fn is_media(path: &Path) -> bool {
|
||||
crate::import::is_supported(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write(dir: &Path, name: &str) -> String {
|
||||
let p = dir.join(name);
|
||||
std::fs::write(&p, b"x").unwrap();
|
||||
p.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_file_types_lightcraft_reads_are_handed_over() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-openable-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
for good in ["DSC_1.NEF", "a.jpg", "copy-Edit.tif", "copy-Edit.psd"] {
|
||||
let path = write(&dir, good);
|
||||
assert_eq!(openable(&path, "").unwrap(), path, "{good} is a photo LightCraft reads");
|
||||
}
|
||||
for bad in ["payload.exe", "shortcut.lnk", "run.bat", "page.hta", "go.command", "no-extension"] {
|
||||
let path = write(&dir, bad);
|
||||
let e = openable(&path, "").unwrap_err();
|
||||
assert!(e.contains(bad) && e.contains("file types LightCraft reads"), "{bad}: {e}");
|
||||
}
|
||||
// a bundle (macOS `.app`) and any other folder are not files
|
||||
let bundle = dir.join("Evil.app");
|
||||
std::fs::create_dir_all(&bundle).unwrap();
|
||||
let e = openable(&bundle.to_string_lossy(), "").unwrap_err();
|
||||
assert!(e.contains("isn't one of the file types") || e.contains("isn't a file"), "{e}");
|
||||
let e = openable(&dir.join("gone.jpg").to_string_lossy(), "").unwrap_err();
|
||||
assert!(e.contains("missing"), "{e}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A relative path is made absolute: the opener runs with its own working directory.
|
||||
#[test]
|
||||
fn the_path_handed_over_is_absolute() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-openable-abs-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = write(&dir, "a.jpg");
|
||||
let dotted = Path::new(&path).parent().unwrap().join(".").join("a.jpg");
|
||||
let out = openable(&dotted.to_string_lossy(), "").unwrap();
|
||||
assert!(Path::new(&out).is_absolute(), "{out}");
|
||||
assert!(out.ends_with("a.jpg"), "{out}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A link named like a photo that points at a program is refused: what the opener runs is
|
||||
/// the target, not the name.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_link_to_a_program_is_refused() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-openable-link-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let target = write(&dir, "payload.command");
|
||||
let link = dir.join("holiday.jpg");
|
||||
std::os::unix::fs::symlink(&target, &link).unwrap();
|
||||
let e = openable(&link.to_string_lossy(), "").unwrap_err();
|
||||
assert!(e.contains("points at") && e.contains("payload.command"), "{e}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Composes print pages: places each cell's photo (fit or Zoom to Fill, optionally Rotate to Fit),
|
||||
//! draws Photo Border and Inner Stroke, fills the page background and returns [`PageRaster`]s.
|
||||
//! draws Photo Border and Inner Stroke, fills the page background, adds the Page Options (crop
|
||||
//! marks, page number, page info) and returns [`PageRaster`]s.
|
||||
//!
|
||||
//! Work is split like an export: [`prepare`] reads the session (which photos go in which cell, one
|
||||
//! render job per cell) and [`Prepared::run`] does the heavy part without it, so it can run on a
|
||||
@@ -28,6 +29,13 @@ pub const MAX_BORDER_IN: f64 = 2.0;
|
||||
pub const MAX_STROKE_PT: f64 = 36.0;
|
||||
/// Renders kept for cells that print the same photo at the same size.
|
||||
const RENDER_CACHE: usize = 4;
|
||||
/// A crop mark's length and its gap from the photo's corner, inches.
|
||||
pub const CROP_MARK_IN: f64 = 0.25;
|
||||
pub const CROP_MARK_GAP_IN: f64 = 0.0625;
|
||||
/// A crop mark's weight, points.
|
||||
pub const CROP_MARK_PT: f64 = 0.5;
|
||||
/// Size of the page number and page info text, points.
|
||||
pub const PAGE_TEXT_PT: f64 = 9.0;
|
||||
|
||||
/// Lightroom's Image Settings panel.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
@@ -85,11 +93,18 @@ impl ImageSettings {
|
||||
pub struct PageStyle {
|
||||
/// Page background colour, sRGB.
|
||||
pub background: [u8; 3],
|
||||
/// Page Options (Single Image and Contact Sheet, as in Lightroom; the packages ignore them):
|
||||
/// the page number at the bottom right of each page.
|
||||
pub page_numbers: bool,
|
||||
/// The print settings (sharpening, profile, resolution) at the bottom left of each page.
|
||||
pub page_info: bool,
|
||||
/// Marks at the corners of each photo to cut along.
|
||||
pub crop_marks: bool,
|
||||
}
|
||||
|
||||
impl Default for PageStyle {
|
||||
fn default() -> Self {
|
||||
Self { background: [255; 3] }
|
||||
Self { background: [255; 3], page_numbers: false, page_info: false, crop_marks: false }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,6 +115,41 @@ pub fn rec2020_from_srgb8(c: [u8; 3]) -> [u16; 3] {
|
||||
lin.map(|v| (trc.encode(v) * 65535.0 + 0.5) as u16)
|
||||
}
|
||||
|
||||
/// Whether a layout style prints the Page Options (page numbers, page info, crop marks):
|
||||
/// Lightroom offers them for Single Image and Contact Sheet; its packages have Cut Guides instead.
|
||||
pub fn has_page_options(style: Style) -> bool {
|
||||
!style.is_package()
|
||||
}
|
||||
|
||||
/// The colour of crop marks and page text on `background`: black, or white on a dark page.
|
||||
pub fn mark_ink(background: [u8; 3]) -> [u8; 3] {
|
||||
let [r, g, b] = background.map(f64::from);
|
||||
if 0.2126 * r + 0.7152 * g + 0.0722 * b < 128.0 { [255; 3] } else { [0; 3] }
|
||||
}
|
||||
|
||||
/// The Page Info line: print sharpening, profile and resolution (Lightroom names the printer
|
||||
/// where the resolution is; this print goes to a PDF).
|
||||
pub fn page_info_text(settings: &PrintSettings) -> String {
|
||||
use crate::print_color::{ManagedSpace, PrintMedia, PrintProfile, PrintSharpen};
|
||||
let c = &settings.job.color;
|
||||
let media = match c.media {
|
||||
PrintMedia::Matte => "Matte",
|
||||
PrintMedia::Glossy => "Glossy",
|
||||
};
|
||||
let sharpen = match c.sharpen {
|
||||
PrintSharpen::Off => "Off".to_string(),
|
||||
PrintSharpen::Low => format!("Low ({media})"),
|
||||
PrintSharpen::Standard => format!("Standard ({media})"),
|
||||
PrintSharpen::High => format!("High ({media})"),
|
||||
};
|
||||
let profile = match &c.profile {
|
||||
PrintProfile::Managed { space: ManagedSpace::Srgb } => "Managed by Printer (sRGB)".to_string(),
|
||||
PrintProfile::Managed { space: ManagedSpace::AdobeRgb } => "Managed by Printer (Adobe RGB)".to_string(),
|
||||
PrintProfile::Icc { path } => path.file_stem().map_or_else(|| path.display().to_string(), |n| n.to_string_lossy().into_owned()),
|
||||
};
|
||||
format!("Print Sharpening: {sharpen} · Profile: {profile} · {:.0} ppi", settings.job.effective_ppi())
|
||||
}
|
||||
|
||||
// ---- placement ------------------------------------------------------------------------------
|
||||
|
||||
/// How a photo of `iw` x `ih` goes into an `aw` x `ah` area.
|
||||
@@ -355,8 +405,8 @@ impl CellStyle {
|
||||
}
|
||||
|
||||
/// Draws `photo` into `cell` of `page`: border frame, the photo fitted or filled (and turned when
|
||||
/// asked), then the inner stroke on top of it.
|
||||
pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &CellStyle) -> Result<(), String> {
|
||||
/// asked), then the inner stroke on top of it. Returns where the photo landed.
|
||||
pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &CellStyle) -> Result<PxRect, String> {
|
||||
// The border may not eat the cell: at least one pixel of photo area stays.
|
||||
let border = style.border_px.min(cell.w.min(cell.h).saturating_sub(1) / 2);
|
||||
let (ax, ay) = (cell.x + border, cell.y + border);
|
||||
@@ -382,7 +432,105 @@ pub fn draw_cell(page: &mut Image16, photo: &Image16, cell: PxRect, style: &Cell
|
||||
page.fill_rect(dx, dy + s, s, h - 2 * s, style.stroke);
|
||||
page.fill_rect(dx + w - s, dy + s, s, h - 2 * s, style.stroke);
|
||||
}
|
||||
Ok(())
|
||||
Ok(PxRect { x: dx, y: dy, w, h })
|
||||
}
|
||||
|
||||
/// What the Page Options draw on every page, in page pixels.
|
||||
struct Marks {
|
||||
/// Crop mark length, gap from the corner and weight.
|
||||
crop: Option<(usize, usize, usize)>,
|
||||
numbers: bool,
|
||||
info: Option<String>,
|
||||
ink: [u16; 3],
|
||||
text_px: f32,
|
||||
/// The printable area: the text sits in the margin below it, aligned with its sides.
|
||||
area: Option<PxRect>,
|
||||
}
|
||||
|
||||
impl Marks {
|
||||
fn of(settings: &PrintSettings, ppi: f64, pw: usize, ph: usize) -> Self {
|
||||
let on = has_page_options(settings.layout.style);
|
||||
let page = &settings.page;
|
||||
let px = |inches: f64| (inches * ppi).round() as usize;
|
||||
Marks {
|
||||
crop: (on && page.crop_marks).then(|| (px(CROP_MARK_IN), px(CROP_MARK_GAP_IN), px(CROP_MARK_PT / 72.0).max(1))),
|
||||
numbers: on && page.page_numbers,
|
||||
info: (on && page.page_info).then(|| page_info_text(settings)),
|
||||
ink: rec2020_from_srgb8(mark_ink(page.background)),
|
||||
text_px: (PAGE_TEXT_PT / 72.0 * ppi) as f32,
|
||||
area: settings.layout.printable().ok().and_then(|a| px_rect(&a, ppi, pw, ph)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Draws the marks of page `number` (one-based) around `photos`, the photos on it.
|
||||
fn draw(&self, page: &mut Image16, photos: &[PxRect], number: usize) {
|
||||
if let Some((len, gap, weight)) = self.crop {
|
||||
for r in photos {
|
||||
crop_marks(page, r, len, gap, weight, photos, self.ink);
|
||||
}
|
||||
}
|
||||
let area = self.area.unwrap_or(PxRect { x: 0, y: 0, w: page.w, h: page.h });
|
||||
let (area_bottom, page_h) = ((area.y + area.h) as f32, page.h as f32);
|
||||
// centred in the bottom margin, and on the paper when the margin is thinner than the text
|
||||
let top = |h: f32| (area_bottom + (page_h - area_bottom - h) / 2.0).clamp(0.0, (page_h - h).max(0.0));
|
||||
if self.numbers
|
||||
&& let Some(t) = crate::text::TextLayout::new(&number.to_string(), self.text_px, false, crate::fonts::CRAFT_FONTS)
|
||||
{
|
||||
let x = ((area.x + area.w) as f32 - t.width).max(0.0);
|
||||
draw_text(page, &t, x, top(t.height), self.ink);
|
||||
}
|
||||
if let Some(info) = &self.info
|
||||
&& let Some(t) = crate::text::TextLayout::new(info, self.text_px, false, crate::fonts::CRAFT_FONTS)
|
||||
{
|
||||
draw_text(page, &t, area.x as f32, top(t.height), self.ink);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Crop marks for the photo at `r`: at each corner, a line along each of its edges that starts
|
||||
/// `gap` past the corner and runs `len` outward, `weight` thick. Marks never cover a photo.
|
||||
fn crop_marks(page: &mut Image16, r: &PxRect, len: usize, gap: usize, weight: usize, photos: &[PxRect], ink: [u16; 3]) {
|
||||
let (x0, y0, x1, y1) = (r.x as i64, r.y as i64, (r.x + r.w) as i64, (r.y + r.h) as i64);
|
||||
let (l, g, t) = (len as i64, gap as i64, weight.min(r.w).min(r.h) as i64);
|
||||
let segments = [
|
||||
(x0 - g - l, y0, l, t),
|
||||
(x1 + g, y0, l, t),
|
||||
(x0 - g - l, y1 - t, l, t),
|
||||
(x1 + g, y1 - t, l, t),
|
||||
(x0, y0 - g - l, t, l),
|
||||
(x0, y1 + g, t, l),
|
||||
(x1 - t, y0 - g - l, t, l),
|
||||
(x1 - t, y1 + g, t, l),
|
||||
];
|
||||
let on_photo = |x: usize, y: usize| photos.iter().any(|p| x >= p.x && x < p.x + p.w && y >= p.y && y < p.y + p.h);
|
||||
for (sx, sy, sw, sh) in segments {
|
||||
for y in sy.max(0)..(sy + sh).min(page.h as i64) {
|
||||
for x in sx.max(0)..(sx + sw).min(page.w as i64) {
|
||||
let (x, y) = (x as usize, y as usize);
|
||||
if !on_photo(x, y) {
|
||||
page.fill_rect(x, y, 1, 1, ink);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Blends text in `ink` into `page` with its top-left corner at (`x`, `y`).
|
||||
fn draw_text(page: &mut Image16, text: &crate::text::TextLayout, x: f32, y: f32, ink: [u16; 3]) {
|
||||
let (w, h) = (page.w, page.h);
|
||||
text.draw_in(x, y, w as f32, h as f32, |gx, gy, cov| {
|
||||
let (Ok(gx), Ok(gy)) = (usize::try_from(gx), usize::try_from(gy)) else { return };
|
||||
if gx >= w || gy >= h {
|
||||
return;
|
||||
}
|
||||
let i = (gy * w + gx) * 3;
|
||||
if let Some(px) = page.rgb.get_mut(i..i + 3) {
|
||||
let k = cov.clamp(0.0, 1.0);
|
||||
for (p, c) in px.iter_mut().zip(ink) {
|
||||
*p = (f32::from(*p) + (f32::from(c) - f32::from(*p)) * k).round().clamp(0.0, 65535.0) as u16;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---- the job ---------------------------------------------------------------------------------
|
||||
@@ -402,6 +550,7 @@ pub struct Prepared {
|
||||
ppi: f32,
|
||||
background: [u16; 3],
|
||||
style: CellStyle,
|
||||
marks: Marks,
|
||||
/// Print sharpening, applied to each photo once it's rendered at its print size.
|
||||
color: crate::print_color::PrintColorSettings,
|
||||
}
|
||||
@@ -430,6 +579,7 @@ impl Prepared {
|
||||
let mut out = Vec::with_capacity(self.pages.len());
|
||||
for (index, &(w, h)) in self.pages.iter().enumerate() {
|
||||
let mut page = Image16::filled(w, h, self.background)?;
|
||||
let mut placed = Vec::new();
|
||||
while let Some(item) = items.next_if(|i| i.page == index) {
|
||||
if !progress(done, &item.name) {
|
||||
return Err("Print cancelled".into());
|
||||
@@ -448,8 +598,9 @@ impl Prepared {
|
||||
}
|
||||
};
|
||||
let photo = &cache.get(at).ok_or("render cache lost an entry")?.1;
|
||||
draw_cell(&mut page, photo, item.cell, &self.style).map_err(|e| format!("{}: {e}", item.name))?;
|
||||
placed.push(draw_cell(&mut page, photo, item.cell, &self.style).map_err(|e| format!("{}: {e}", item.name))?);
|
||||
}
|
||||
self.marks.draw(&mut page, &placed, index + 1);
|
||||
out.push(PageRaster { width: w as u32, height: h as u32, ppi: self.ppi, rgb: page.rgb });
|
||||
}
|
||||
if !progress(done, "") {
|
||||
@@ -508,6 +659,7 @@ pub fn prepare(session: &mut Session, settings: &PrintSettings, photos: &[PhotoI
|
||||
ppi: ppi as f32,
|
||||
background: rec2020_from_srgb8(settings.page.background),
|
||||
style,
|
||||
marks: Marks::of(&settings, ppi, pw, ph),
|
||||
color: settings.job.color.clone(),
|
||||
})
|
||||
}
|
||||
@@ -825,6 +977,94 @@ mod tests {
|
||||
assert!(green_px > 1000, "the border frames the photo inside its cell: {green_px} border pixels");
|
||||
}
|
||||
|
||||
/// Pages for two photos, one per 4x6 page at 100 ppi, each filling its 0.5 in margins (so the
|
||||
/// margins are bare background unless something draws there).
|
||||
fn page_options_pages(s: &mut Session, style: Style, page: PageStyle) -> Vec<PageRaster> {
|
||||
let ids = first_ids(s, 2);
|
||||
let mut set = settings(style);
|
||||
if style.is_package() {
|
||||
set.layout.add_cell(3.0, 5.0).unwrap();
|
||||
set.layout.new_page().unwrap();
|
||||
let id = set.layout.add_cell(3.0, 5.0).unwrap();
|
||||
set.layout.set_cell_page(id, 1).unwrap();
|
||||
}
|
||||
set.image.zoom_to_fill = true;
|
||||
set.page = page;
|
||||
prepare(s, &set, &ids).unwrap().run(&mut |_, _| true).unwrap()
|
||||
}
|
||||
|
||||
fn rgb(p: &PageRaster, x: usize, y: usize) -> [u16; 3] {
|
||||
let i = (y * p.width as usize + x) * 3;
|
||||
[p.rgb[i], p.rgb[i + 1], p.rgb[i + 2]]
|
||||
}
|
||||
|
||||
/// The pixels of `x0..x1` x `y0..y1` that are not the white background.
|
||||
fn inked(p: &PageRaster, x0: usize, x1: usize, y0: usize, y1: usize) -> usize {
|
||||
(y0..y1).flat_map(|y| (x0..x1).map(move |x| (x, y))).filter(|&(x, y)| rgb(p, x, y) != WHITE).count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crop_marks_sit_outside_each_photo_corner() {
|
||||
let mut s = Session::with_demo();
|
||||
let plain = page_options_pages(&mut s, Style::SingleImage, PageStyle::default());
|
||||
assert_eq!(inked(&plain[0], 0, 400, 0, 50), 0, "no marks unless asked");
|
||||
let pages = page_options_pages(&mut s, Style::SingleImage, PageStyle { crop_marks: true, ..Default::default() });
|
||||
// The photo fills 50..350 x 50..550; a mark is a line 0.25 in long, 1/16 in from the corner,
|
||||
// along each edge of the photo.
|
||||
for p in &pages {
|
||||
for (x, y) in [(30, 50), (50, 30), (370, 50), (349, 30), (30, 549), (50, 570), (370, 549), (349, 570)] {
|
||||
assert_eq!(rgb(p, x, y), [0; 3], "a mark at ({x}, {y})");
|
||||
}
|
||||
for (x, y) in [(47, 50), (50, 47), (30, 60), (60, 30), (10, 50), (50, 10)] {
|
||||
assert_eq!(rgb(p, x, y), WHITE, "the gap, or past the mark, at ({x}, {y})");
|
||||
}
|
||||
}
|
||||
// a dark page gets white marks
|
||||
let dark = page_options_pages(&mut s, Style::SingleImage, PageStyle { background: [20; 3], crop_marks: true, ..Default::default() });
|
||||
assert_eq!(rgb(&dark[0], 30, 50), WHITE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_numbers_and_info_print_in_the_bottom_margin() {
|
||||
let mut s = Session::with_demo();
|
||||
let numbered = page_options_pages(&mut s, Style::SingleImage, PageStyle { page_numbers: true, ..Default::default() });
|
||||
// bottom right: the number, different on each page; nothing at the bottom left
|
||||
let (right, left) = ((200, 400), (0, 200));
|
||||
for p in &numbered {
|
||||
assert!(inked(p, right.0, right.1, 550, 600) > 10, "a page number");
|
||||
assert_eq!(inked(p, left.0, left.1, 550, 600), 0);
|
||||
assert_eq!(inked(p, 0, 400, 0, 50), 0, "nothing at the top");
|
||||
}
|
||||
let region = |p: &PageRaster| (550..600).flat_map(|y| (right.0..right.1).map(move |x| (x, y))).map(|(x, y)| rgb(p, x, y)).collect::<Vec<_>>();
|
||||
assert_ne!(region(&numbered[0]), region(&numbered[1]), "page 1 and page 2 are numbered differently");
|
||||
let info = page_options_pages(&mut s, Style::SingleImage, PageStyle { page_info: true, ..Default::default() });
|
||||
for p in &info {
|
||||
assert!(inked(p, left.0, left.1, 550, 600) > 10, "the print settings line");
|
||||
}
|
||||
// the packages have no Page Options in Lightroom: nothing is drawn
|
||||
let all = PageStyle { page_numbers: true, page_info: true, crop_marks: true, ..Default::default() };
|
||||
let package = page_options_pages(&mut s, Style::PicturePackage, all);
|
||||
assert_eq!(package.len(), 2);
|
||||
for p in &package {
|
||||
assert_eq!(inked(p, 0, 400, 550, 600) + inked(p, 0, 400, 0, 50), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_info_names_the_sharpening_profile_and_resolution() {
|
||||
use crate::print_color::{ManagedSpace, PrintMedia, PrintProfile, PrintSharpen};
|
||||
let mut set = PrintSettings::default();
|
||||
let line = page_info_text(&set);
|
||||
assert_eq!(line, "Print Sharpening: Standard (Glossy) · Profile: Managed by Printer (sRGB) · 240 ppi");
|
||||
set.job.color.sharpen = PrintSharpen::Off;
|
||||
set.job.color.media = PrintMedia::Matte;
|
||||
set.job.color.profile = PrintProfile::Icc { path: "/profiles/Lustre Paper.icc".into() };
|
||||
set.job.ppi = 300.0;
|
||||
assert_eq!(page_info_text(&set), "Print Sharpening: Off · Profile: Lustre Paper · 300 ppi");
|
||||
set.job.color.profile = PrintProfile::Managed { space: ManagedSpace::AdobeRgb };
|
||||
assert!(page_info_text(&set).contains("Managed by Printer (Adobe RGB)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_and_hostile_settings_are_refused() {
|
||||
let mut s = Session::with_demo();
|
||||
|
||||
@@ -916,7 +916,8 @@ fn plan_rename_core(
|
||||
) -> Vec<RenamePlan> {
|
||||
let mut taken: HashSet<String> = HashSet::new(); // lower-case target paths/names claimed in this batch
|
||||
let mut out = Vec::new();
|
||||
for (seq, p) in (start..).zip(photos) {
|
||||
for (i, p) in photos.iter().enumerate() {
|
||||
let seq = start.saturating_add(i);
|
||||
let want = expand(template, p, seq);
|
||||
let (stem, ext) = split_ext(&want);
|
||||
let (stem, ext) = (stem.to_string(), ext.to_string());
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
//! change when [`XmpPrefs::auto_write`] is on): rating, colour label, title, caption, copyright,
|
||||
//! creator, keywords, capture time and GPS in the standard namespaces, plus our complete develop
|
||||
//! settings (`lc:settings`, JSON), the pick/reject flag (`lc:flag`) and location (`lc:location`).
|
||||
//! A sidecar is written only beside an original that is an existing file at an absolute path;
|
||||
//! otherwise the save fails.
|
||||
//!
|
||||
//! An existing sidecar is never replaced wholesale (issue #92): LightCraft's properties
|
||||
//! ([`OWNED`], [`OWNED_IF_STATED`]) are merged into it ([`lightcraft_meta::merge_xmp`]) and
|
||||
@@ -66,6 +68,13 @@ pub fn sidecar_path(original: &str, naming: SidecarNaming) -> PathBuf {
|
||||
/// accepting `.XMP` and `.Xmp`, and a full name whose photo extension differs in case, such as
|
||||
/// `IMG_1.cr3.xmp` beside `IMG_1.CR3`, which a case-sensitive file system would otherwise miss).
|
||||
pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
|
||||
sidecar_candidates(original, naming).find(|p| p.is_file())
|
||||
}
|
||||
|
||||
/// The names `original`'s sidecar can have, in the order they are looked for: under `naming`,
|
||||
/// then under the other naming (the full name also with the photo extension in the other case),
|
||||
/// each as `.xmp`, `.XMP` and `.Xmp`.
|
||||
pub(crate) fn sidecar_candidates(original: &str, naming: SidecarNaming) -> impl Iterator<Item = PathBuf> {
|
||||
let other = if naming == SidecarNaming::Stem { SidecarNaming::Full } else { SidecarNaming::Stem };
|
||||
let ext = Path::new(original).extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let base = Path::new(original).with_extension("");
|
||||
@@ -73,7 +82,7 @@ pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
|
||||
let full_variants: Vec<String> = [recased(ext.to_lowercase()), recased(ext.to_uppercase())].into_iter().flatten().collect();
|
||||
[naming, other]
|
||||
.into_iter()
|
||||
.flat_map(|n| {
|
||||
.flat_map(move |n| {
|
||||
let mut v = vec![sidecar_path(original, n)];
|
||||
if n == SidecarNaming::Full {
|
||||
v.extend(full_variants.iter().map(|o| sidecar_path(o, n)));
|
||||
@@ -81,7 +90,6 @@ pub fn find_sidecar(original: &str, naming: SidecarNaming) -> Option<PathBuf> {
|
||||
v
|
||||
})
|
||||
.flat_map(|p| [p.clone(), p.with_extension("XMP"), p.with_extension("Xmp")])
|
||||
.find(|p| p.is_file())
|
||||
}
|
||||
|
||||
/// What a sidecar (or embedded XMP packet) says about a photo. `None` = not stated.
|
||||
@@ -626,6 +634,10 @@ impl Session {
|
||||
return Err(EngineError::Other(format!("{} is a virtual copy: its settings live only in the library", p.file_name)));
|
||||
}
|
||||
let orig = file_path(p).ok_or_else(|| EngineError::Other(format!("{} is not a file on disk", p.file_name)))?;
|
||||
// A relative path would be taken from the working directory.
|
||||
if Path::new(orig).is_relative() || !Path::new(orig).is_file() {
|
||||
return Err(EngineError::Other(format!("{}: the original {orig} is missing; no XMP sidecar is written", p.file_name)));
|
||||
}
|
||||
let naming = owners.naming(p, self.xmp.naming);
|
||||
let preferred = sidecar_path(orig, naming);
|
||||
let other = if naming == SidecarNaming::Stem { SidecarNaming::Full } else { SidecarNaming::Stem };
|
||||
@@ -780,6 +792,30 @@ impl Session {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn sidecars_are_written_only_beside_an_existing_original() {
|
||||
let dir = std::env::temp_dir().join(format!("lightcraft-sidecar-local-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("IMG_1.jpg"), b"original").unwrap();
|
||||
let originals = [
|
||||
(r"\\192.0.2.1\share\IMG_1.jpg".to_string(), false),
|
||||
(dir.join("IMG_2.jpg").to_string_lossy().to_string(), false),
|
||||
(dir.join("IMG_1.jpg").to_string_lossy().to_string(), true),
|
||||
];
|
||||
for (original, written) in originals {
|
||||
let _ = std::fs::remove_file(dir.join("IMG_1.xmp"));
|
||||
let mut s = Session::new();
|
||||
let p = Photo::new(PhotoId(1), Source::File { path: original.clone() }, "IMG_1.jpg", "JPEG", 4, 3, "2026-01-01T00:00:00");
|
||||
s.catalog.apply(Op::AddPhoto { photo: Box::new(p) }).unwrap();
|
||||
assert_eq!(s.save_sidecar(PhotoId(1)).is_ok(), written, "{original}");
|
||||
assert_eq!(dir.join("IMG_1.xmp").is_file(), written, "{original}");
|
||||
assert!(!dir.join("IMG_2.xmp").exists(), "{original}");
|
||||
assert!(!Path::new(r"\\192.0.2.1\share\IMG_1.xmp").exists(), "{original}");
|
||||
}
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
/// A sidecar written by Lightroom Classic (names flat, paths in `lr:hierarchicalSubject`)
|
||||
/// gives the photo its keywords as paths.
|
||||
#[test]
|
||||
|
||||
@@ -903,7 +903,7 @@ fn shadow(img: &mut Rgba8, tl: &crate::text::TextLayout, x: f64, y: f64, sigma:
|
||||
let my0 = (y - pad).floor().max(-pad);
|
||||
let mx1 = (x + tl.width as f64 + pad).ceil().min(iw + pad);
|
||||
let my1 = (y + tl.height as f64 + pad).ceil().min(ih + pad);
|
||||
if !(mx1 > mx0 && my1 > my0) || !mx0.is_finite() || !my0.is_finite() {
|
||||
if !(mx1 > mx0 && my1 > my0 && mx0.is_finite() && my0.is_finite()) {
|
||||
return;
|
||||
}
|
||||
let (w, h) = ((mx1 - mx0) as usize, (my1 - my0) as usize);
|
||||
|
||||
@@ -115,7 +115,7 @@ fn color(v: &Value) -> std::result::Result<Option<[u8; 3]>, String> {
|
||||
Value::String(s) if s.is_empty() || s.eq_ignore_ascii_case("none") => Ok(None),
|
||||
Value::String(s) => {
|
||||
let h = s.strip_prefix('#').unwrap_or(s);
|
||||
if h.len() != 6 || !h.is_ascii() {
|
||||
if h.len() != 6 || !h.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return Err(bad());
|
||||
}
|
||||
let byte = |i: usize| h.get(i..i + 2).and_then(|x| u8::from_str_radix(x, 16).ok()).ok_or_else(bad);
|
||||
@@ -747,6 +747,8 @@ mod tests {
|
||||
json!({"repeat": 1}),
|
||||
json!({"fadeColor": "#12345"}),
|
||||
json!({"fadeColor": "#ééé"}),
|
||||
json!({"fadeColor": "#+f+f+f"}),
|
||||
json!({"fadeColor": "-12345"}),
|
||||
json!({"fadeColor": [300, 0, 0]}),
|
||||
json!({"fadeColor": [1, 2]}),
|
||||
json!({"speed": 2}),
|
||||
|
||||
+167
-12
@@ -38,7 +38,7 @@
|
||||
//! not cancelled midway.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use lightcraft_catalog::{Catalog, Filter, Op, Photo, PhotoId, Sort, Source};
|
||||
@@ -203,18 +203,37 @@ pub fn scan_with(input: SyncInput, progress: &SyncProgress) -> FolderChanges {
|
||||
let namings = crate::sidecar::namings_in(&labels, naming, &ids);
|
||||
let photos: Vec<(Arc<Photo>, SidecarNaming)> =
|
||||
ids.iter().zip(namings).filter_map(|(id, naming)| labels.photo(*id).map(|p| (Arc::clone(p), naming))).collect();
|
||||
// The folder is walked once, and its listing answers for the library's photos in it: on a
|
||||
// network share that is one request for a folder's names where it was several for each
|
||||
// photo. Then both halves wait on the disk at once: the new files are probed (on their own
|
||||
// threads) beside the checks of the photos, which ask the disk only what the listing can't say.
|
||||
// (how many photos there are to check is known before the walk: the count shows while it runs)
|
||||
progress.to_check.store(photos.len(), std::sync::atomic::Ordering::Relaxed);
|
||||
let mut listing = Listing::default();
|
||||
let (files, _) = crate::import::expand_noting(
|
||||
std::slice::from_ref(&folder),
|
||||
scan.skip(),
|
||||
crate::walk::Limits::default(),
|
||||
Some(&mut |dir, names, complete| listing.note(dir, names, complete)),
|
||||
);
|
||||
let listing = &listing;
|
||||
// (no threads on the web: one after the other there)
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let (out, checks) = (crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files), check_photos(&photos, &labels, progress));
|
||||
let (out, checks) = (crate::import::scan_files(scan, files, &progress.files), check_photos(&photos, &labels, listing, progress));
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let (out, checks) = std::thread::scope(|sc| {
|
||||
let photos = &photos;
|
||||
let labels = &labels;
|
||||
let checks = sc.spawn(move || check_photos(photos, labels, progress));
|
||||
let out = crate::import::scan_with(scan, std::slice::from_ref(&folder), &progress.files);
|
||||
let checks = sc.spawn(move || check_photos(photos, labels, listing, progress));
|
||||
let out = crate::import::scan_files(scan, files, &progress.files);
|
||||
// (a check that panicked counts as nothing found; the scan still reports the rest)
|
||||
(out, checks.join().unwrap_or_default())
|
||||
});
|
||||
log::debug!(
|
||||
"synchronize: {} folders listed; the disk was asked {} times about the library's photos",
|
||||
listing.folders.len(),
|
||||
listing.asked.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let mut changes = FolderChanges { path: folder, seen, ..Default::default() };
|
||||
for ((p, _), check) in photos.iter().zip(checks) {
|
||||
let Some(file) = crate::cmd::missing::checked_path(p) else { continue };
|
||||
@@ -335,11 +354,136 @@ enum Check {
|
||||
News,
|
||||
}
|
||||
|
||||
fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
|
||||
/// The listings of the folders a scan walked: what is in each, by name. It answers "is this file
|
||||
/// there" for the library's photos and their sidecars without asking the disk, which on a
|
||||
/// network share is a round trip for every name (and a photo without a sidecar took five).
|
||||
///
|
||||
/// It answers only what holds on any file system. A name is there when the folder lists exactly
|
||||
/// that name ([`asks_the_disk`] aside). It is not there when the folder was listed to its end,
|
||||
/// every name in the folder is ASCII, the name asked about is plain ASCII ([`plain_name`]), and
|
||||
/// no name in the folder is that name in another case. Anything else is not answered, and the
|
||||
/// disk is asked as before: a name that differs in case only, a link, a folder that was not
|
||||
/// walked or not to its end, and every name that is not listed in a folder that holds a name
|
||||
/// outside ASCII. (File systems hold names outside ASCII equal in more ways than can be listed
|
||||
/// here: by case, where the capital sharp s is `SS`; by how accents are written; and some pass
|
||||
/// over marks altogether.)
|
||||
///
|
||||
/// Where the listing and a look-up would differ, the listing is the folder as the scan saw it:
|
||||
/// a file it lists is there though it could not be opened by its path (a folder that may be
|
||||
/// listed but not searched, a path longer than the system takes), and a share that drops after
|
||||
/// the walk does not turn every photo the listing answers for into a missing one.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Listing {
|
||||
folders: HashMap<PathBuf, Listed>,
|
||||
/// How often the disk was asked after all (for the tests, and the log).
|
||||
pub(crate) asked: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
/// One folder's listing.
|
||||
struct Listed {
|
||||
/// name → what it is.
|
||||
names: HashMap<String, crate::walk::EntryKind>,
|
||||
/// The names, lower-cased (only kept for a folder whose names are all ASCII): what a plain
|
||||
/// name asked about could be in another case.
|
||||
lower: std::collections::HashSet<String>,
|
||||
/// Every entry was read, and every name is ASCII: a plain name that is not listed, in any
|
||||
/// case, is not there.
|
||||
sure: bool,
|
||||
}
|
||||
|
||||
/// Whether the disk is asked about `name` whatever the listing holds: a name Windows takes for
|
||||
/// a device (`NUL`, `aux.jpg`: there in every folder, or not openable where it is listed).
|
||||
fn asks_the_disk(name: &str) -> bool {
|
||||
let stem = name.split('.').next().unwrap_or_default().trim_end_matches(' ').to_ascii_lowercase();
|
||||
// (`COM` and `LPT` with any one digit, the superscript ones Windows also takes among them)
|
||||
let numbered = ["com", "lpt"].iter().any(|port| {
|
||||
let mut rest = stem.strip_prefix(port).unwrap_or_default().chars();
|
||||
matches!((rest.next(), rest.next()), (Some(d), None) if d.is_ascii_digit() || ['\u{b9}', '\u{b2}', '\u{b3}'].contains(&d))
|
||||
});
|
||||
numbered || ["con", "prn", "aux", "nul", "conin$", "conout$", "clock$"].contains(&stem.as_str())
|
||||
}
|
||||
|
||||
/// Whether `name` can only be found under itself in some case: ASCII, and nothing in it that a
|
||||
/// file system resolves another way (a short name with `~`, a stream after `:`, dots or spaces
|
||||
/// at its end, which Windows drops).
|
||||
fn plain_name(name: &str) -> bool {
|
||||
name.is_ascii() && !name.contains(['~', ':']) && !name.ends_with(['.', ' ']) && !name.is_empty()
|
||||
}
|
||||
|
||||
impl Listing {
|
||||
/// Take the listing of `folder` from the walk ([`crate::walk::files_in_noting`]).
|
||||
pub(crate) fn note(&mut self, folder: &Path, entries: Vec<(std::ffi::OsString, crate::walk::EntryKind)>, complete: bool) {
|
||||
let mut sure = complete;
|
||||
let mut names = HashMap::with_capacity(entries.len());
|
||||
let mut lower = std::collections::HashSet::new();
|
||||
for (name, kind) in entries {
|
||||
match name.into_string() {
|
||||
Ok(name) => {
|
||||
sure &= name.is_ascii();
|
||||
lower.insert(name.to_ascii_lowercase());
|
||||
names.insert(name, kind);
|
||||
}
|
||||
// (a name that is no text is not one a photo's path can be compared with)
|
||||
Err(_) => sure = false,
|
||||
}
|
||||
}
|
||||
if !sure {
|
||||
lower.clear();
|
||||
}
|
||||
self.folders.insert(folder.to_path_buf(), Listed { names, lower, sure });
|
||||
}
|
||||
|
||||
/// What the listing of `path`'s folder says of it: its kind when it is listed under exactly
|
||||
/// this name, `Some(None)` when it is surely not there, `None` when the listing can't say.
|
||||
fn kind(&self, path: &Path) -> Option<Option<crate::walk::EntryKind>> {
|
||||
let name = path.file_name()?.to_str()?;
|
||||
// (the path ends in that name: no `/.` or separator after it, which the disk would resolve)
|
||||
if !path.to_str()?.ends_with(name) {
|
||||
return None;
|
||||
}
|
||||
if asks_the_disk(name) {
|
||||
return None;
|
||||
}
|
||||
let folder = self.folders.get(path.parent()?)?;
|
||||
if let Some(kind) = folder.names.get(name) {
|
||||
return Some(Some(*kind));
|
||||
}
|
||||
(folder.sure && plain_name(name) && !folder.lower.contains(&name.to_ascii_lowercase())).then_some(None)
|
||||
}
|
||||
|
||||
/// `Path::exists`, where the listing can say.
|
||||
pub(crate) fn exists(&self, path: &Path) -> Option<bool> {
|
||||
match self.kind(path)? {
|
||||
Some(crate::walk::EntryKind::Other) => None,
|
||||
Some(_) => Some(true),
|
||||
None => Some(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// `Path::is_file`, where the listing can say.
|
||||
pub(crate) fn is_file(&self, path: &Path) -> Option<bool> {
|
||||
match self.kind(path)? {
|
||||
Some(crate::walk::EntryKind::Other) => None,
|
||||
Some(kind) => Some(kind == crate::walk::EntryKind::File),
|
||||
None => Some(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// The listing's answer, or the disk's where it has none.
|
||||
fn or_ask(&self, answer: Option<bool>, ask: impl FnOnce() -> bool) -> bool {
|
||||
answer.unwrap_or_else(|| {
|
||||
self.asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ask()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog, listing: &Listing) -> Check {
|
||||
let Some(file) = crate::cmd::missing::checked_path(p) else { return Check::Nothing };
|
||||
if !Path::new(file).exists() {
|
||||
let file = Path::new(file);
|
||||
if !listing.or_ask(listing.exists(file), || file.exists()) {
|
||||
Check::Missing
|
||||
} else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming) {
|
||||
} else if p.copy_of.is_none() && sidecar_has_news(labels, p, naming, listing) {
|
||||
Check::News
|
||||
} else {
|
||||
Check::Nothing
|
||||
@@ -348,12 +492,12 @@ fn check_photo(p: &Photo, naming: SidecarNaming, labels: &Catalog) -> Check {
|
||||
|
||||
/// [`check_photo`] for each photo, in order, on several threads (each file is a wait on the
|
||||
/// disk, not work for the CPU). Photos not reached before a cancel count as nothing found.
|
||||
fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, progress: &SyncProgress) -> Vec<Check> {
|
||||
fn check_photos(photos: &[(Arc<Photo>, SidecarNaming)], labels: &Catalog, listing: &Listing, progress: &SyncProgress) -> Vec<Check> {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
progress.to_check.store(photos.len(), Relaxed);
|
||||
let cancel = &progress.files.cancel;
|
||||
let check = |p: &Photo, naming: SidecarNaming| {
|
||||
let c = check_photo(p, naming, labels);
|
||||
let c = check_photo(p, naming, labels, listing);
|
||||
progress.checked.fetch_add(1, Relaxed);
|
||||
c
|
||||
};
|
||||
@@ -421,10 +565,17 @@ impl Session {
|
||||
|
||||
/// Whether `p`'s XMP sidecar file was saved after the library last had its say about the photo
|
||||
/// (imported, or edited here) and says something the library doesn't.
|
||||
fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming) -> bool {
|
||||
fn sidecar_has_news(labels: &Catalog, p: &Photo, naming: SidecarNaming, listing: &Listing) -> bool {
|
||||
let Source::File { path } = &p.source else { return false };
|
||||
let Some(file) = crate::sidecar::find_sidecar(path, naming) else { return false };
|
||||
let Ok(meta) = std::fs::metadata(&file) else { return false };
|
||||
// ([`crate::sidecar::find_sidecar`], each name looked up in the listing before the disk)
|
||||
// (a name the listing holds is read for its date next, and one that can't be read is passed
|
||||
// over for the next name, as a name that is no file was)
|
||||
let found = crate::sidecar::sidecar_candidates(path, naming).find_map(|c| {
|
||||
let there = listing.or_ask(listing.is_file(&c), || c.is_file());
|
||||
let meta = there.then(|| std::fs::metadata(&c).ok()).flatten().filter(std::fs::Metadata::is_file)?;
|
||||
Some((c, meta))
|
||||
});
|
||||
let Some((file, meta)) = found else { return false };
|
||||
if meta.len() > MAX_SIDECAR_BYTES {
|
||||
return false;
|
||||
}
|
||||
@@ -708,3 +859,7 @@ impl SyncCommit {
|
||||
self.report
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "sync/tests_listing.rs"]
|
||||
mod tests_listing;
|
||||
@@ -0,0 +1,192 @@
|
||||
//! Synchronize Folder answers "is the photo's file there" and "has it a sidecar" from the
|
||||
//! listing of the folder its scan has just made, instead of asking the disk about every photo.
|
||||
//!
|
||||
//! On a network share each question is a round trip (about 5 ms a file on a hard disk over SMB,
|
||||
//! measured), and a photo without a sidecar took five of them; the listing has every name.
|
||||
//! The listing answers only what it can be sure of on any file system: where a name could match
|
||||
//! another way (case, letters outside ASCII, a link), the disk is asked as before.
|
||||
use super::*;
|
||||
use crate::walk::EntryKind::{File, Folder, Other};
|
||||
|
||||
fn listing(folders: &[(&str, &[(&str, crate::walk::EntryKind)], bool)]) -> Listing {
|
||||
let mut l = Listing::default();
|
||||
for (dir, names, complete) in folders {
|
||||
l.note(Path::new(dir), names.iter().map(|(n, k)| (std::ffi::OsString::from(n), *k)).collect(), *complete);
|
||||
}
|
||||
l
|
||||
}
|
||||
|
||||
/// Given the listing of a folder, when it is asked about a name, then it says "there" for a
|
||||
/// name it lists, "not there" for one nothing in the folder could be taken for, and nothing
|
||||
/// (ask the disk) wherever a file system could see it otherwise.
|
||||
#[test]
|
||||
fn the_listing_answers_only_what_it_is_sure_of() {
|
||||
let l = listing(&[
|
||||
(
|
||||
"/pics/trip",
|
||||
&[("IMG_1.CR3", File), ("IMG_1.xmp", File), ("img_2.jpg", File), ("sub", Folder), ("link.jpg", Other), (".hidden.xmp", File)],
|
||||
true,
|
||||
),
|
||||
("/pics/cut", &[("IMG_1.CR3", File)], false),
|
||||
("/pics/names", &[("IMG_1.CR3", File), ("Größe.JPG", File)], true),
|
||||
// (the capital sharp s, which a file system that folds case takes for `SS`)
|
||||
("/pics/sharp", &[("IMG_1.CR3", File), ("STRA\u{1e9e}E.jpg", File)], true),
|
||||
("/pics/devices", &[("aux.jpg", File), ("COM\u{b9}.jpg", File), ("IMG_1.CR3", File)], true),
|
||||
]);
|
||||
let (there, gone, ask) = (Some(true), Some(false), None);
|
||||
for (path, exists, is_file) in [
|
||||
("/pics/trip/IMG_1.CR3", there, there),
|
||||
("/pics/trip/IMG_1.xmp", there, there),
|
||||
("/pics/trip/.hidden.xmp", there, there),
|
||||
("/pics/trip/sub", there, gone),
|
||||
("/pics/trip/IMG_9.CR3", gone, gone),
|
||||
("/pics/trip/IMG_1.XMP", ask, ask),
|
||||
("/pics/trip/IMG_2.JPG", ask, ask),
|
||||
("/pics/trip/link.jpg", ask, ask),
|
||||
("/pics/trip/SUB", ask, ask),
|
||||
// a folder the walk did not list to its end, or not at all
|
||||
("/pics/cut/IMG_1.CR3", there, there),
|
||||
("/pics/cut/IMG_2.CR3", ask, ask),
|
||||
("/pics/other/IMG_1.CR3", ask, ask),
|
||||
// a folder that holds a name outside ASCII: what it lists is there, and nothing is surely not there (file
|
||||
// systems hold such names equal in more ways than one)
|
||||
("/pics/names/IMG_1.CR3", there, there),
|
||||
("/pics/names/Größe.JPG", there, there),
|
||||
("/pics/names/IMG_2.CR3", ask, ask),
|
||||
("/pics/names/größe.JPG", ask, ask),
|
||||
("/pics/trip/Größe.JPG", ask, ask),
|
||||
("/pics/sharp/strasse.jpg", ask, ask),
|
||||
("/pics/sharp/STRASSE.xmp", ask, ask),
|
||||
("/pics/sharp/IMG_2.CR3", ask, ask),
|
||||
// names Windows resolves another way: a short name, a stream, dots or spaces at the end, a device
|
||||
("/pics/trip/IMG_10~1.CR3", ask, ask),
|
||||
("/pics/trip/IMG_1.CR3::$DATA", ask, ask),
|
||||
("/pics/trip/IMG_1.CR3.", ask, ask),
|
||||
("/pics/trip/IMG_1.CR3 ", ask, ask),
|
||||
("/pics/devices/aux.jpg", ask, ask),
|
||||
("/pics/devices/NUL", ask, ask),
|
||||
("/pics/devices/nul.xmp", ask, ask),
|
||||
("/pics/devices/COM1 .jpg", ask, ask),
|
||||
("/pics/devices/COM\u{b9}.jpg", ask, ask),
|
||||
("/pics/trip/con", ask, ask),
|
||||
("/pics/trip/PRN.txt", ask, ask),
|
||||
("/pics/trip/com2.jpg", ask, ask),
|
||||
("/pics/trip/LPT9.xmp", ask, ask),
|
||||
("/pics/trip/lpt0", ask, ask),
|
||||
("/pics/trip/nul.tar.gz", ask, ask),
|
||||
("/pics/trip/CONIN$", ask, ask),
|
||||
("/pics/trip/com10.jpg", gone, gone),
|
||||
("/pics/trip/computer.jpg", gone, gone),
|
||||
("/pics/devices/IMG_1.CR3", there, there),
|
||||
("/pics/devices/auxiliary.jpg", ask, ask),
|
||||
("/pics/trip/auxiliary.jpg", gone, gone),
|
||||
// a path that does not end in its file's name is the disk's to resolve
|
||||
("/pics/trip/IMG_1.CR3/", ask, ask),
|
||||
("/pics/trip/IMG_1.CR3/.", ask, ask),
|
||||
("/pics/trip/", ask, ask),
|
||||
("", ask, ask),
|
||||
] {
|
||||
assert_eq!((l.exists(Path::new(path)), l.is_file(Path::new(path))), (exists, is_file), "{path:?}");
|
||||
}
|
||||
// a name that is no text: nothing in its folder is surely not there
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
let mut l = Listing::default();
|
||||
l.note(
|
||||
Path::new("/pics/raw"),
|
||||
vec![(std::ffi::OsString::from("IMG_1.CR3"), File), (std::ffi::OsString::from_vec(vec![0x66, 0xff, 0x6f]), File)],
|
||||
true,
|
||||
);
|
||||
assert_eq!((l.exists(Path::new("/pics/raw/IMG_1.CR3")), l.exists(Path::new("/pics/raw/IMG_2.CR3"))), (there, ask));
|
||||
}
|
||||
}
|
||||
|
||||
fn photo(c: &mut Catalog, path: &Path) -> Arc<Photo> {
|
||||
let id = c.alloc_photo_id();
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
c.apply(lightcraft_catalog::Op::AddPhoto {
|
||||
photo: Box::new(Photo::new(id, Source::File { path: path.to_string_lossy().to_string() }, &name, "JPEG", 6, 4, "2020-01-01T00:00:00")),
|
||||
})
|
||||
.unwrap();
|
||||
Arc::clone(c.photo(id).unwrap())
|
||||
}
|
||||
|
||||
/// Given a folder on disk whose photos are there or gone, with sidecars under every name a
|
||||
/// sidecar can have, when each photo is checked with the folder's listing, then the answer is
|
||||
/// the one the disk gives; and for photos with plain names the disk was asked nothing but the
|
||||
/// sidecars' dates.
|
||||
#[test]
|
||||
fn a_check_from_the_listing_is_the_check_from_the_disk() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-listing-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(dir.join("sub")).unwrap();
|
||||
let news = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:xmp="http://ns.adobe.com/xap/1.0/" xmp:Rating="5"/></rdf:RDF></x:xmpmeta>"#;
|
||||
for name in [
|
||||
"plain.jpg",
|
||||
"stem.jpg",
|
||||
"stem.xmp",
|
||||
"full.jpg",
|
||||
"full.jpg.xmp",
|
||||
"upper.jpg",
|
||||
"upper.XMP",
|
||||
"sub/deep.jpg",
|
||||
"sub/deep.xmp",
|
||||
"Größe.jpg",
|
||||
"Größe.xmp",
|
||||
"dir.jpg.xmp",
|
||||
] {
|
||||
std::fs::write(dir.join(name), if name.ends_with("mp") || name.ends_with("MP") { news.as_bytes() } else { b"jpeg" }).unwrap();
|
||||
}
|
||||
let mut c = Catalog::new();
|
||||
let photos: Vec<Arc<Photo>> =
|
||||
["plain.jpg", "stem.jpg", "full.jpg", "upper.jpg", "sub/deep.jpg", "Größe.jpg", "gone.jpg", "sub/gone.jpg", "dir.jpg"]
|
||||
.iter()
|
||||
.map(|n| photo(&mut c, &dir.join(n)))
|
||||
.collect();
|
||||
// (`dir.jpg` is gone, though a sidecar of its name is there)
|
||||
let mut l = Listing::default();
|
||||
let walked =
|
||||
crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| l.note(d, names, complete));
|
||||
assert_eq!(walked.files.len(), 12);
|
||||
let nothing = Listing::default();
|
||||
for naming in [SidecarNaming::Stem, SidecarNaming::Full] {
|
||||
for p in &photos {
|
||||
let from_disk = check_photo(p, naming, &c, ¬hing);
|
||||
assert_eq!(check_photo(p, naming, &c, &l), from_disk, "{:?} {naming:?}", p.file_name);
|
||||
}
|
||||
}
|
||||
let expected = [Check::Nothing, Check::News, Check::News, Check::News, Check::News, Check::News, Check::Missing, Check::Missing, Check::Missing];
|
||||
assert_eq!(photos.iter().map(|p| check_photo(p, SidecarNaming::Stem, &c, &l)).collect::<Vec<_>>(), expected);
|
||||
|
||||
// what the disk was asked about the same photos, with the listing and without: fewer questions with it
|
||||
let asked = |l: &Listing| {
|
||||
for p in &photos {
|
||||
check_photo(p, SidecarNaming::Stem, &c, l);
|
||||
}
|
||||
l.asked.load(std::sync::atomic::Ordering::Relaxed)
|
||||
};
|
||||
let mut with = Listing::default();
|
||||
crate::walk::files_in_noting(&dir, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| with.note(d, names, complete));
|
||||
let (with, without) = (asked(&with), asked(&Listing::default()));
|
||||
// (without: each photo's file, and the names of its sidecar up to the one that is there: 20 questions where
|
||||
// names are found in any case, 21 where they are not)
|
||||
assert!(without >= 20, "{without}");
|
||||
// (with: this folder holds `Größe.*`, so what it does not list is still asked about: the nine names of the
|
||||
// sidecar `plain.jpg` has not (`.xmp`, `.XMP` and `.Xmp` of `plain`, `plain.jpg` and `plain.JPG`), the three
|
||||
// `full.jpg` has not, `upper.xmp`, and the two photos that are gone. What it lists is not asked about, nor
|
||||
// anything in `sub`, whose names are all ASCII)
|
||||
assert_eq!(with, 15, "asked {with} times with the listing, {without} without");
|
||||
|
||||
// a folder of plain names only: the disk is asked nothing at all
|
||||
let sub = dir.join("sub");
|
||||
let mut only_plain = Listing::default();
|
||||
crate::walk::files_in_noting(&sub, None, crate::walk::Limits::default(), |_| true, &mut |d, names, complete| only_plain.note(d, names, complete));
|
||||
let mut c2 = Catalog::new();
|
||||
for (name, expected) in [("deep.jpg", Check::News), ("gone.jpg", Check::Missing), ("gone.jpg.xmp", Check::Missing)] {
|
||||
let p = photo(&mut c2, &sub.join(name));
|
||||
assert_eq!(check_photo(&p, SidecarNaming::Stem, &c2, &only_plain), expected, "{name}");
|
||||
}
|
||||
assert_eq!(only_plain.asked.load(std::sync::atomic::Ordering::Relaxed), 0, "every answer came from the listing");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -1024,21 +1024,24 @@ fn same_folder_as_original_refuses_photos_without_a_folder() {
|
||||
let mut s = Session::with_demo();
|
||||
let ids: Vec<_> = s.catalog.photos().take(2).map(|p| p.id).collect();
|
||||
let mut written = Vec::new();
|
||||
let mut write = |p: &str, _: &[u8]| -> Result<(), String> {
|
||||
written.push(p.to_string());
|
||||
Ok(())
|
||||
// the writer lives in a block of its own, so its borrow of `written` has ended by the time
|
||||
// the assertion below reads it
|
||||
let files = {
|
||||
let mut write = |p: &str, _: &[u8]| -> Result<(), String> {
|
||||
written.push(p.to_string());
|
||||
Ok(())
|
||||
};
|
||||
let err = crate::export::export_batch(&mut s, &ids, &o, &to, &mut write, &|_| false).unwrap_err();
|
||||
assert!(err.contains("no folder on disk"), "{err}");
|
||||
// without stopping: one error per photo, nothing written
|
||||
let items = crate::export::prepare_batch(&mut s, &ids, &o).unwrap();
|
||||
crate::export::run_batch(items, &o, &to, &mut write, &|_| false, false, &mut |_, _| true).unwrap()
|
||||
};
|
||||
let err = crate::export::export_batch(&mut s, &ids, &o, &to, &mut write, &|_| false).unwrap_err();
|
||||
assert!(err.contains("no folder on disk"), "{err}");
|
||||
// without stopping: one error per photo, nothing written
|
||||
let items = crate::export::prepare_batch(&mut s, &ids, &o).unwrap();
|
||||
let files = crate::export::run_batch(items, &o, &to, &mut write, &|_| false, false, &mut |_, _| true).unwrap();
|
||||
assert_eq!(files.len(), 2);
|
||||
for (f, id) in files.iter().zip(&ids) {
|
||||
assert_eq!(f["photo"], id.0, "{f}");
|
||||
assert!(f["error"].as_str().unwrap().contains("no folder on disk"), "{f}");
|
||||
}
|
||||
drop(write);
|
||||
assert!(written.is_empty(), "{written:?}");
|
||||
|
||||
// a photo on disk whose folder is gone (moved, an unplugged drive)
|
||||
|
||||
@@ -627,3 +627,17 @@ fn recognition_cannot_be_turned_on_without_a_chosen_model() {
|
||||
assert!(s.execute("faces.enable", &json!({"enabled": true})).is_err());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cpu_only_switch_is_kept_and_bad_values_are_refused() {
|
||||
let d = temp("cpu-only");
|
||||
let mut s = session(&d);
|
||||
assert_eq!(s.execute("faces.gpu", &json!({})).unwrap()["cpuOnly"], false);
|
||||
assert_eq!(s.execute("faces.gpu", &json!({"cpuOnly": true})).unwrap()["cpuOnly"], true);
|
||||
// kept with the models, so a new session starts on the CPU too
|
||||
let mut again = session(&d);
|
||||
assert_eq!(again.execute("faces.models.list", &json!({})).unwrap()["gpu"]["cpuOnly"], true);
|
||||
assert!(again.execute("faces.gpu", &json!({"cpuOnly": "yes"})).is_err());
|
||||
assert_eq!(again.execute("faces.gpu", &json!({"cpuOnly": false})).unwrap()["cpuOnly"], false);
|
||||
let _ = std::fs::remove_dir_all(d);
|
||||
}
|
||||
@@ -778,6 +778,50 @@ fn assisted_culling_groups_bursts_and_scores_focus() {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Assisted culling: two look-alike shots one second apart across midnight form a burst at the end of
|
||||
/// every month, including those with fewer than 31 days.
|
||||
#[test]
|
||||
fn assisted_culling_groups_bursts_across_month_end() {
|
||||
let dir = temp_dir("cull-month-end");
|
||||
for (name, (w, h)) in [("a.png", (480, 320)), ("b.png", (482, 321))] {
|
||||
let img = lightcraft_scenes::demo_library()[0].render(w, h);
|
||||
let data: Vec<[u8; 4]> = img
|
||||
.data
|
||||
.iter()
|
||||
.map(|c| {
|
||||
let e = |v: f32| (lightcraft_color::transfer::linear_to_srgb(v.clamp(0.0, 1.0)) * 255.0).round() as u8;
|
||||
[e(c[0]), e(c[1]), e(c[2]), 255]
|
||||
})
|
||||
.collect();
|
||||
let png = lightcraft_codecs::encode_png(
|
||||
&lightcraft_codecs::EncodeImage::rgba8(&lightcraft_raster::Rgba8 { width: img.width, height: img.height, data }),
|
||||
&Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(dir.join(name), png).unwrap();
|
||||
}
|
||||
let mut s = Session::new().with_fs();
|
||||
let r = s.execute("library.import", &json!({"paths": [dir.to_string_lossy()]})).unwrap();
|
||||
let ids: Vec<u64> = r["imported"].as_array().unwrap().iter().filter_map(Value::as_u64).collect();
|
||||
let by_name = |s: &Session, n: &str| s.catalog.photos().find(|p| p.file_name == n).unwrap().id;
|
||||
let (a, b) = (by_name(&s, "a.png"), by_name(&s, "b.png"));
|
||||
for (before, after) in [
|
||||
("2026-01-31T23:59:59", "2026-02-01T00:00:00"),
|
||||
("2026-02-28T23:59:59", "2026-03-01T00:00:00"),
|
||||
("2026-04-30T23:59:59", "2026-05-01T00:00:00"),
|
||||
("2026-06-30T23:59:59", "2026-07-01T00:00:00"),
|
||||
("2026-09-30T23:59:59", "2026-10-01T00:00:00"),
|
||||
("2026-11-30T23:59:59", "2026-12-01T00:00:00"),
|
||||
("2026-12-31T23:59:59", "2027-01-01T00:00:00"),
|
||||
] {
|
||||
s.commit("t", lightcraft_catalog::Op::SetCaptured { id: a, captured: Some(before.into()) }).unwrap();
|
||||
s.commit("t", lightcraft_catalog::Op::SetCaptured { id: b, captured: Some(after.into()) }).unwrap();
|
||||
let r = s.execute("photo.analyze", &json!({"ids": ids})).unwrap();
|
||||
assert_eq!(r["groups"], 1, "shots 1 s apart ({before} / {after}) must form a burst: {r}");
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Find Dust Spots: soft dark spots on a smooth sky are found and healed (one undo step).
|
||||
#[test]
|
||||
fn dust_spots_are_found_and_healed() {
|
||||
|
||||
@@ -336,3 +336,23 @@ fn deleting_an_album_folder_preserves_photos_and_restores_its_subtree() {
|
||||
drop(reopened);
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
/// A huge `start` is a validation error, not an arithmetic overflow in the numbering.
|
||||
#[test]
|
||||
fn rename_huge_start_is_a_bad_param() {
|
||||
let mut s = Session::with_demo();
|
||||
let ids: Vec<u64> = s.catalog.photos().take(3).map(|p| p.id.0).collect();
|
||||
let names_before: Vec<String> = s.catalog.photos().map(|p| p.file_name.clone()).collect();
|
||||
for cmd in ["photo.renamePreview", "photo.rename"] {
|
||||
let e = s.execute(cmd, &json!({"template": "{seq}", "start": u64::MAX, "ids": ids})).unwrap_err();
|
||||
assert!(e.to_string().contains("start out of range"), "{cmd}: {e}");
|
||||
}
|
||||
let names_after: Vec<String> = s.catalog.photos().map(|p| p.file_name.clone()).collect();
|
||||
assert_eq!(names_before, names_after, "a rejected rename changes nothing");
|
||||
// the last start whose whole sequence fits is accepted, one more is not
|
||||
let fits = usize::MAX as u64 - 2;
|
||||
let r = s.execute("photo.renamePreview", &json!({"template": "{seq}", "start": fits, "ids": ids}));
|
||||
assert!(r.is_ok(), "{r:?}");
|
||||
let r = s.execute("photo.renamePreview", &json!({"template": "{seq}", "start": fits + 1, "ids": ids}));
|
||||
assert!(r.is_err(), "{r:?}");
|
||||
}
|
||||
@@ -142,6 +142,25 @@ fn preferences_persist_with_the_library() {
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn last_merge_settings_persist_with_the_library() {
|
||||
let lib = temp_dir("merge");
|
||||
let opts = json!({"align": false, "deghost": "high"});
|
||||
{
|
||||
let mut s = Session::new().with_fs();
|
||||
s.open_library(&lib, false).unwrap();
|
||||
s.last_merge.insert("merge.hdr".into(), opts.clone());
|
||||
s.save_prefs().unwrap();
|
||||
s.close_library().unwrap();
|
||||
}
|
||||
let mut s = Session::new().with_fs();
|
||||
s.open_library(&lib, false).unwrap();
|
||||
assert_eq!(s.last_merge.get("merge.hdr"), Some(&opts));
|
||||
assert!(!s.last_merge.contains_key("merge.panorama"));
|
||||
drop(s);
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_copyright_and_creator_fill_gaps_on_import() {
|
||||
let dir = temp_dir("copyright");
|
||||
|
||||
@@ -341,3 +341,23 @@ fn every_print_command_is_registered_with_a_description() {
|
||||
assert!(c.params.len() > 3, "{id}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_options_are_print_settings_and_template_fields() {
|
||||
let mut s = Session::with_demo();
|
||||
let g = run(&mut s, "print.get", json!({}));
|
||||
assert_eq!(g["settings"]["page"], json!({"background": [255, 255, 255], "pageNumbers": false, "pageInfo": false, "cropMarks": false}));
|
||||
let g = run(&mut s, "print.set", json!({"page": {"pageNumbers": true, "cropMarks": true}}));
|
||||
assert_eq!((g["settings"]["page"]["pageNumbers"].as_bool(), g["settings"]["page"]["pageInfo"].as_bool()), (Some(true), Some(false)));
|
||||
assert_eq!(g["settings"]["page"]["background"], json!([255, 255, 255]), "a patch merges");
|
||||
fails(&mut s, "print.set", json!({"page": {"pageInfo": "yes"}}));
|
||||
fails(&mut s, "print.set", json!({"page": {"pageNumber": true}}));
|
||||
// a template keeps them
|
||||
run(&mut s, "print.template.save", json!({"name": "Marked"}));
|
||||
run(&mut s, "print.reset", json!({}));
|
||||
assert!(!s.print.page.crop_marks);
|
||||
run(&mut s, "print.template.apply", json!({"name": "Marked"}));
|
||||
assert!(s.print.page.page_numbers && s.print.page.crop_marks && !s.print.page.page_info);
|
||||
let set = crate::find_command("print.set").unwrap();
|
||||
assert!(set.params.contains("pageNumbers") && set.params.contains("pageInfo") && set.params.contains("cropMarks"), "{}", set.params);
|
||||
}
|
||||
@@ -47,10 +47,35 @@ fn hidden(p: &Path) -> bool {
|
||||
p.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.'))
|
||||
}
|
||||
|
||||
/// What a folder's listing says an entry is.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EntryKind {
|
||||
File,
|
||||
Folder,
|
||||
/// A link, or an entry whose kind the listing did not say.
|
||||
Other,
|
||||
}
|
||||
|
||||
/// What [`files_in_noting`] hands over for each folder it lists: the folder, the name and kind
|
||||
/// of every entry in it (hidden ones too), and whether that is all of them (not when the walk
|
||||
/// stopped inside the folder, or an entry could not be read).
|
||||
pub type NoteFolder<'a> = &'a mut dyn FnMut(&Path, Vec<(std::ffi::OsString, EntryKind)>, bool);
|
||||
|
||||
/// The files below folder `root` that `keep` accepts. Hidden entries (names starting with `.`)
|
||||
/// are left out, the folder `skip` (e.g. the library's own) is never entered, and the walk stops
|
||||
/// at `limits` (see [`Walked::truncated`]). Unreadable folders are skipped.
|
||||
pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool) -> Walked {
|
||||
pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool) -> Walked {
|
||||
walk(root, skip, limits, keep, None)
|
||||
}
|
||||
|
||||
/// [`files_in`], handing the listing of every folder it reads to `note`: who walks a folder to
|
||||
/// find its files has, with that, the answer to "is this name there" for everything in it, and
|
||||
/// need not ask the disk again (a round trip for each name on a network share).
|
||||
pub fn files_in_noting(root: &Path, skip: Option<&Path>, limits: Limits, keep: impl FnMut(&Path) -> bool, note: NoteFolder) -> Walked {
|
||||
walk(root, skip, limits, keep, Some(note))
|
||||
}
|
||||
|
||||
fn walk(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl FnMut(&Path) -> bool, mut note: Option<NoteFolder>) -> Walked {
|
||||
let mut out = Walked::default();
|
||||
// canonical folders reached through a link, so a link loop is walked at most once
|
||||
let mut linked: HashSet<PathBuf> = HashSet::new();
|
||||
@@ -77,17 +102,41 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
|
||||
}
|
||||
let Ok(rd) = std::fs::read_dir(&path) else { continue };
|
||||
let mut children: Vec<(PathBuf, bool)> = Vec::new();
|
||||
for e in rd.flatten() {
|
||||
// (the folder's listing, for who asked for it; all of it unless something below says otherwise)
|
||||
let mut listed = note.is_some().then(Vec::new);
|
||||
let mut complete = true;
|
||||
for e in rd {
|
||||
let Ok(e) = e else {
|
||||
complete = false;
|
||||
continue;
|
||||
};
|
||||
if out.entries >= limits.max_entries {
|
||||
out.truncated = true;
|
||||
complete = false;
|
||||
break;
|
||||
}
|
||||
out.entries = out.entries.saturating_add(1);
|
||||
let p = e.path();
|
||||
if hidden(&p) {
|
||||
// (a hidden entry's kind is only asked for when the listing is wanted; every other
|
||||
// entry's is asked once)
|
||||
let is_hidden = hidden(&p);
|
||||
if is_hidden && listed.is_none() {
|
||||
continue;
|
||||
}
|
||||
let Ok(ft) = e.file_type() else { continue };
|
||||
let ft = e.file_type();
|
||||
if let Some(listed) = listed.as_mut() {
|
||||
let kind = match &ft {
|
||||
Ok(t) if t.is_symlink() => EntryKind::Other,
|
||||
Ok(t) if t.is_dir() => EntryKind::Folder,
|
||||
Ok(t) if t.is_file() => EntryKind::File,
|
||||
_ => EntryKind::Other,
|
||||
};
|
||||
listed.push((e.file_name(), kind));
|
||||
}
|
||||
if is_hidden {
|
||||
continue;
|
||||
}
|
||||
let Ok(ft) = ft else { continue };
|
||||
let dir = if ft.is_symlink() {
|
||||
// a link (or a Windows junction): a folder is entered once, a file is a file
|
||||
match std::fs::metadata(&p) {
|
||||
@@ -105,6 +154,9 @@ pub fn files_in(root: &Path, skip: Option<&Path>, limits: Limits, mut keep: impl
|
||||
};
|
||||
children.push((p, dir));
|
||||
}
|
||||
if let (Some(note), Some(listed)) = (note.as_mut(), listed) {
|
||||
note(&path, listed, complete);
|
||||
}
|
||||
children.sort();
|
||||
let below = depth.saturating_add(1);
|
||||
stack.extend(children.into_iter().rev().map(|(p, d)| (p, below, d)));
|
||||
@@ -123,6 +175,43 @@ mod tests {
|
||||
d
|
||||
}
|
||||
|
||||
/// Given a folder with files, a folder inside it, a hidden file and a link, when it is walked
|
||||
/// with its listings noted, then the walk finds what it finds without them, and each folder's
|
||||
/// listing holds every entry by name with its kind (hidden ones too, a link as neither file nor
|
||||
/// folder) and says it is all of them; a walk that stops inside a folder says it is not.
|
||||
#[test]
|
||||
fn a_walk_notes_each_folders_listing() {
|
||||
let root = tmp("noting");
|
||||
std::fs::create_dir_all(root.join("sub")).unwrap();
|
||||
for name in ["a.jpg", ".hidden", "sub/b.jpg"] {
|
||||
std::fs::write(root.join(name), b"x").unwrap();
|
||||
}
|
||||
#[cfg(unix)]
|
||||
std::os::unix::fs::symlink(root.join("a.jpg"), root.join("link.jpg")).unwrap();
|
||||
let mut noted: Vec<(PathBuf, Vec<(String, EntryKind)>, bool)> = Vec::new();
|
||||
let walked = files_in_noting(&root, None, Limits::default(), |_| true, &mut |dir, entries, complete| {
|
||||
let mut entries: Vec<(String, EntryKind)> = entries.into_iter().map(|(n, k)| (n.to_string_lossy().to_string(), k)).collect();
|
||||
entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
noted.push((dir.to_path_buf(), entries, complete));
|
||||
});
|
||||
assert_eq!(walked, files_in(&root, None, Limits::default(), |_| true), "the same walk, noted or not");
|
||||
let named = |n: &str, k: EntryKind| (n.to_string(), k);
|
||||
let mut top = vec![named(".hidden", EntryKind::File), named("a.jpg", EntryKind::File), named("sub", EntryKind::Folder)];
|
||||
if cfg!(unix) {
|
||||
top.insert(2, named("link.jpg", EntryKind::Other));
|
||||
}
|
||||
assert_eq!(noted, [(root.clone(), top, true), (root.join("sub"), vec![named("b.jpg", EntryKind::File)], true)]);
|
||||
|
||||
// stopped inside the first folder: its listing is not all of it
|
||||
let mut cut = Vec::new();
|
||||
let walked = files_in_noting(&root, None, Limits { max_depth: 8, max_entries: 2 }, |_| true, &mut |dir, entries, complete| {
|
||||
cut.push((dir.to_path_buf(), entries.len(), complete))
|
||||
});
|
||||
assert!(walked.truncated);
|
||||
assert_eq!(cut.first().map(|(dir, n, complete)| (dir.clone(), *n, *complete)), Some((root.clone(), 2, false)));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_come_in_path_order_without_hidden_ones_or_the_skipped_folder() {
|
||||
let d = tmp("order");
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
//!
|
||||
//! Everything here treats model files and manifests as hostile input: sizes are capped, numbers must be
|
||||
//! finite and sane, and a malformed file is an error, never a panic. Detection and recognition run
|
||||
//! on the same checked, single-thread pure-Rust CPU interpreter.
|
||||
//! on the same checked, single-thread pure-Rust CPU interpreter; an [`net::Accelerator`] (the GPU, in `lightcraft-gpu`) can
|
||||
//! take its convolutions once it agrees with it.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
|
||||
+451
-3
@@ -9,8 +9,10 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::rc::Rc;
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
|
||||
mod kernels;
|
||||
pub mod program;
|
||||
mod shape;
|
||||
|
||||
use crate::graph::{Data, Graph, Node};
|
||||
@@ -64,9 +66,64 @@ impl Tensor {
|
||||
}
|
||||
}
|
||||
|
||||
/// One convolution, as handed to an [`Accelerator`]: batch 1, planar (NCHW) float32, already checked.
|
||||
///
|
||||
/// `out[co][oy][ox] = bias[co] + Σ weights[co][ci][ky][kx] · input[g·cg + ci][oy·sh + ky − pt][ox·sw + kx − pl]`
|
||||
/// with `g = co / (cout / groups)` and zero outside the input.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ConvJob<'a> {
|
||||
/// The node's index in the graph: the same for every run of a net, so its weights can stay on the device.
|
||||
pub layer: usize,
|
||||
pub cin: usize,
|
||||
pub h: usize,
|
||||
pub w: usize,
|
||||
pub cout: usize,
|
||||
pub groups: usize,
|
||||
pub kh: usize,
|
||||
pub kw: usize,
|
||||
pub sh: usize,
|
||||
pub sw: usize,
|
||||
pub pt: usize,
|
||||
pub pl: usize,
|
||||
pub oh: usize,
|
||||
pub ow: usize,
|
||||
/// `[cout][cin / groups][kh][kw]`.
|
||||
pub weights: &'a [f32],
|
||||
/// `[cout]`, if the node has one.
|
||||
pub bias: Option<&'a [f32]>,
|
||||
/// `[cin][h][w]`.
|
||||
pub input: &'a [f32],
|
||||
}
|
||||
|
||||
impl ConvJob<'_> {
|
||||
/// Multiply-adds the convolution takes.
|
||||
pub fn macs(&self) -> usize {
|
||||
let per_group = (self.cin / self.groups.max(1)).saturating_mul(self.kh).saturating_mul(self.kw);
|
||||
self.cout.saturating_mul(per_group).saturating_mul(self.oh).saturating_mul(self.ow)
|
||||
}
|
||||
}
|
||||
|
||||
/// Something that can run a network's convolutions faster than the CPU kernels (the GPU, in `lightcraft-gpu`).
|
||||
///
|
||||
/// The CPU interpreter stays the reference: a net only uses an accelerator that agreed with it on a test input
|
||||
/// ([`Net::attach_checked`]), and the first error makes the net drop it and run on the CPU again.
|
||||
pub trait Accelerator: Send + Sync {
|
||||
/// What it runs on, for the user ("AMD Radeon RX 7900 (Vulkan)").
|
||||
fn device(&self) -> String;
|
||||
/// `[cout][oh][ow]`, or `Ok(None)` to leave this one to the CPU (too small to be worth a dispatch, say).
|
||||
fn conv(&self, job: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String>;
|
||||
/// The whole network on the device, keeping every value there between steps, for each of `inputs` (all of
|
||||
/// `program.lens[INPUT]` values) in one go: per input, each of `program.outputs` in order. `Ok(None)` leaves the
|
||||
/// run to the CPU path (which may still use [`Accelerator::conv`]).
|
||||
fn run_program(&self, _program: &program::Program, _inputs: &[&[f32]]) -> std::result::Result<Option<Vec<Vec<Vec<f32>>>>, String> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// A loaded network, ready to run.
|
||||
pub struct Net {
|
||||
graph: Graph,
|
||||
accelerator: RwLock<Option<Arc<dyn Accelerator>>>,
|
||||
}
|
||||
|
||||
/// Validate semantics rather than silently ignoring unknown or mistyped attributes.
|
||||
@@ -80,7 +137,75 @@ impl Net {
|
||||
for n in &graph.nodes {
|
||||
check(n, &graph)?;
|
||||
}
|
||||
Ok(Net { graph })
|
||||
Ok(Net { graph, accelerator: RwLock::new(None) })
|
||||
}
|
||||
|
||||
/// The accelerator in use, if any.
|
||||
pub fn accelerator(&self) -> Option<Arc<dyn Accelerator>> {
|
||||
self.accelerator.read().unwrap_or_else(PoisonError::into_inner).clone()
|
||||
}
|
||||
|
||||
/// Use `accelerator` for this net's convolutions, but only after it gave the CPU's outputs on `probe` (every
|
||||
/// output value within `tolerance` of the CPU's, relative to that output's largest magnitude) and, with
|
||||
/// `require_faster`, ran it faster than the CPU did (a GPU's per-layer round trip can cost more than a small
|
||||
/// model's layers). Returns why it was not used, so the caller can say so; the net then stays on the CPU.
|
||||
pub fn attach_checked(
|
||||
&self,
|
||||
accelerator: Arc<dyn Accelerator>,
|
||||
probe: &Tensor,
|
||||
tolerance: f32,
|
||||
require_faster: bool,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.detach();
|
||||
let started = web_time::Instant::now();
|
||||
let cpu = self.run(probe.clone()).map_err(|e| e.to_string())?;
|
||||
let cpu_time = started.elapsed();
|
||||
*self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = Some(accelerator);
|
||||
// the first run also uploads the weights and builds the kernels: time a second one
|
||||
let fast = self.run(probe.clone()).map_err(|e| e.to_string());
|
||||
let mut verdict = match fast {
|
||||
Err(e) => Err(e),
|
||||
Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
|
||||
Ok(fast) => agree(&cpu, &fast, tolerance),
|
||||
};
|
||||
if verdict.is_ok() && require_faster {
|
||||
let started = web_time::Instant::now();
|
||||
let again = self.run(probe.clone()).map_err(|e| e.to_string());
|
||||
let fast_time = started.elapsed();
|
||||
verdict = match again {
|
||||
Err(e) => Err(e),
|
||||
Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
|
||||
Ok(_) if fast_time >= cpu_time => Err(format!(
|
||||
"the device is not faster for this model ({:.0} ms against {:.0} ms on the CPU)",
|
||||
fast_time.as_secs_f64() * 1000.0,
|
||||
cpu_time.as_secs_f64() * 1000.0
|
||||
)),
|
||||
Ok(_) => Ok(()),
|
||||
};
|
||||
}
|
||||
if verdict.is_err() {
|
||||
self.detach();
|
||||
}
|
||||
verdict
|
||||
}
|
||||
|
||||
/// Run on the CPU only from now on.
|
||||
pub fn detach(&self) {
|
||||
*self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// A convolution on the accelerator, if one is attached and takes it. A failing accelerator is dropped.
|
||||
fn accelerated_conv(&self, layer: usize, node: &Node, x: &Tensor) -> Result<Option<Tensor>> {
|
||||
let Some(accelerator) = self.accelerator() else { return Ok(None) };
|
||||
let job = kernels::conv_job(layer, node, &x.shape, &x.data, &self.graph)?;
|
||||
match accelerator.conv(&job) {
|
||||
Ok(Some(out)) if out.len() == job.cout * job.oh * job.ow => Ok(Some(Tensor::new(vec![1, job.cout, job.oh, job.ow], out)?)),
|
||||
Ok(None) => Ok(None),
|
||||
Ok(Some(_)) | Err(_) => {
|
||||
self.detach();
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The shape the model declares for its input (0 for a dynamic size).
|
||||
@@ -96,7 +221,67 @@ impl Net {
|
||||
|
||||
/// Run the network on `input` (its single input); returns each output by name.
|
||||
pub fn run(&self, input: Tensor) -> Result<Vec<(String, Tensor)>> {
|
||||
shape::tensor(&input.shape, input.data.len(), &input.data)?;
|
||||
self.run_batch(vec![input])?.pop().ok_or_else(|| NetError::Shape("the run gave nothing".into()))
|
||||
}
|
||||
|
||||
/// Run the network on each of `inputs`; an accelerator that keeps the values on its device runs inputs of one shape
|
||||
/// together (one upload and one read-back for all of them).
|
||||
pub fn run_batch(&self, inputs: Vec<Tensor>) -> Result<Vec<Vec<(String, Tensor)>>> {
|
||||
for input in &inputs {
|
||||
shape::tensor(&input.shape, input.data.len(), &input.data)?;
|
||||
}
|
||||
if let Some(out) = self.run_on_device(&inputs)? {
|
||||
return Ok(out);
|
||||
}
|
||||
inputs.into_iter().map(|input| self.run_one(input)).collect()
|
||||
}
|
||||
|
||||
/// The whole batch on the attached accelerator, or `None` to run it on the CPU path. A failing or misbehaving
|
||||
/// accelerator is dropped.
|
||||
fn run_on_device(&self, inputs: &[Tensor]) -> Result<Option<Vec<Vec<(String, Tensor)>>>> {
|
||||
let (Some(accelerator), Some(first)) = (self.accelerator(), inputs.first()) else { return Ok(None) };
|
||||
if inputs.iter().any(|t| t.shape != first.shape) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(program) = self.program(&first.shape)? else { return Ok(None) };
|
||||
let slices: Vec<&[f32]> = inputs.iter().map(|t| t.data.as_slice()).collect();
|
||||
let started = web_time::Instant::now();
|
||||
let outs = match accelerator.run_program(&program, &slices) {
|
||||
Ok(Some(outs)) => outs,
|
||||
Ok(None) => return Ok(None),
|
||||
Err(_) => {
|
||||
self.detach();
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let gathered: Option<Vec<Vec<(String, Tensor)>>> = (outs.len() == inputs.len())
|
||||
.then(|| {
|
||||
outs.into_iter()
|
||||
.map(|values| {
|
||||
(values.len() == program.outputs.len())
|
||||
.then(|| {
|
||||
values
|
||||
.into_iter()
|
||||
.zip(&program.outputs)
|
||||
.map(|(data, (name, _, shape))| Tensor::new(shape.clone(), data).ok().map(|t| (name.clone(), t)))
|
||||
.collect::<Option<Vec<_>>>()
|
||||
})
|
||||
.flatten()
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.flatten();
|
||||
if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
|
||||
eprintln!("[faces-gpu] {} steps x {}: {:.3} ms", program.steps.len(), inputs.len(), started.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
if gathered.is_none() {
|
||||
// wrong sizes or values that are not finite: the device is not trusted any more
|
||||
self.detach();
|
||||
}
|
||||
Ok(gathered)
|
||||
}
|
||||
|
||||
fn run_one(&self, input: Tensor) -> Result<Vec<(String, Tensor)>> {
|
||||
self.validate_input(&input.shape)?;
|
||||
let mut scratch = kernels::Scratch::default();
|
||||
let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
|
||||
@@ -127,7 +312,13 @@ impl Net {
|
||||
};
|
||||
let started = web_time::Instant::now();
|
||||
let out = match node.op.as_str() {
|
||||
"Conv" => kernels::conv(node, &*get(0)?, g, &mut scratch)?,
|
||||
"Conv" => {
|
||||
let x = get(0)?;
|
||||
match self.accelerated_conv(i, node, &x)? {
|
||||
Some(out) => out,
|
||||
None => kernels::conv(node, &x, g, &mut scratch)?,
|
||||
}
|
||||
}
|
||||
"Relu" => map(&*get(0)?, |v| v.max(0.0)),
|
||||
"Sigmoid" => map(&*get(0)?, |v| 1.0 / (1.0 + (-v).exp())),
|
||||
"Add" | "Sub" | "Mul" => kernels::binary(node.op.as_str(), &*get(0)?, &*get(1)?)?,
|
||||
@@ -168,6 +359,25 @@ impl Net {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether two runs' outputs agree: same names and shapes, values within `tolerance` of each other relative to the
|
||||
/// output's largest magnitude.
|
||||
fn agree(a: &[(String, Tensor)], b: &[(String, Tensor)], tolerance: f32) -> std::result::Result<(), String> {
|
||||
if a.len() != b.len() {
|
||||
return Err("the device gave a different number of outputs".into());
|
||||
}
|
||||
for ((name, x), (other, y)) in a.iter().zip(b) {
|
||||
if name != other || x.shape != y.shape {
|
||||
return Err(format!("the device gave `{other}` a different shape"));
|
||||
}
|
||||
let scale = x.data.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-6);
|
||||
let worst = x.data.iter().zip(&y.data).fold(0f32, |m, (p, q)| m.max((p - q).abs()));
|
||||
if worst.is_nan() || worst > tolerance * scale {
|
||||
return Err(format!("the device's `{name}` differs from the CPU's by {:.2e} (of {scale:.2e})", worst));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map(x: &Tensor, f: impl Fn(f32) -> f32) -> Tensor {
|
||||
Tensor { shape: x.shape.clone(), data: x.data.iter().map(|v| f(*v)).collect() }
|
||||
}
|
||||
@@ -534,6 +744,244 @@ mod tests {
|
||||
assert_eq!(rs.shape, vec![1, 6, 2]);
|
||||
}
|
||||
|
||||
/// An accelerator built on the slow reference convolution, optionally wrong or failing.
|
||||
struct Fake {
|
||||
skew: f32,
|
||||
fail_after: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
impl Accelerator for Fake {
|
||||
fn device(&self) -> String {
|
||||
"test".into()
|
||||
}
|
||||
fn conv(&self, j: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String> {
|
||||
if self.fail_after.fetch_sub(1, std::sync::atomic::Ordering::Relaxed) == 0 {
|
||||
return Err("device lost".into());
|
||||
}
|
||||
assert!(j.pt == j.pl && j.sh == j.sw);
|
||||
let x = Tensor::new(vec![1, j.cin, j.h, j.w], j.input.to_vec()).unwrap();
|
||||
let zero = vec![0.0; j.cout];
|
||||
let dims = [j.cout, j.cin / j.groups, j.kh, j.kw];
|
||||
let out = reference_conv(&x, j.weights, dims, j.bias.unwrap_or(&zero), j.sh, j.pt, j.groups);
|
||||
Ok(Some(out.data.iter().map(|v| v + self.skew).collect()))
|
||||
}
|
||||
}
|
||||
|
||||
fn two_convs() -> (Graph, Tensor) {
|
||||
let attrs = [("strides", Attr::Ints(vec![2, 2])), ("pads", Attr::Ints(vec![1; 4]))];
|
||||
let mut a = node("Conv", &["x", "w1", "b1"], &attrs);
|
||||
a.outputs = vec!["h".into()];
|
||||
let b = node("Conv", &["h", "w2"], &[("group", Attr::Int(2)), ("pads", Attr::Ints(vec![1; 4]))]);
|
||||
let mut g = graph(
|
||||
vec![a, b],
|
||||
vec![
|
||||
("w1", w(&[4, 3, 3, 3], (0..108).map(|i| ((i * 7 % 13) as f32 - 6.0) * 0.05).collect())),
|
||||
("b1", w(&[4], vec![0.1, -0.2, 0.3, 0.0])),
|
||||
("w2", w(&[4, 2, 3, 3], (0..72).map(|i| ((i * 5 % 11) as f32 - 5.0) * 0.05).collect())),
|
||||
],
|
||||
);
|
||||
g.inputs = vec![("x".into(), vec![1, 3, 9, 8])];
|
||||
(g, input(3, 9, 8, |i| ((i * 13 % 17) as f32 - 8.0) * 0.1))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_accelerator_is_used_only_when_it_agrees_with_the_cpu() {
|
||||
let (g, x) = two_convs();
|
||||
let net = Net::new(g).unwrap();
|
||||
let cpu = net.run(x.clone()).unwrap();
|
||||
let good = Arc::new(Fake { skew: 0.0, fail_after: usize::MAX.into() });
|
||||
net.attach_checked(good, &x, 1e-4, false).unwrap();
|
||||
assert!(net.accelerator().is_some());
|
||||
assert_eq!(net.run(x.clone()).unwrap()[0].1.shape, cpu[0].1.shape);
|
||||
let wrong = Arc::new(Fake { skew: 0.5, fail_after: usize::MAX.into() });
|
||||
assert!(net.attach_checked(wrong, &x, 1e-4, false).is_err());
|
||||
assert!(net.accelerator().is_none(), "a device that disagrees is not used");
|
||||
assert_eq!(net.run(x).unwrap(), cpu);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failing_accelerator_falls_back_to_the_cpu_mid_run() {
|
||||
let (g, x) = two_convs();
|
||||
let net = Net::new(g).unwrap();
|
||||
let cpu = net.run(x.clone()).unwrap();
|
||||
// agrees on the probe (two convs), then fails on the second conv of the next run
|
||||
net.attach_checked(Arc::new(Fake { skew: 0.0, fail_after: 3.into() }), &x, 1e-4, false).unwrap();
|
||||
let out = net.run(x).unwrap();
|
||||
assert!(net.accelerator().is_none());
|
||||
for (a, b) in out[0].1.data.iter().zip(&cpu[0].1.data) {
|
||||
assert!((a - b).abs() < 1e-4);
|
||||
}
|
||||
}
|
||||
|
||||
/// An accelerator that runs whole programs with plain loops (the definition of each step), counting its calls;
|
||||
/// `poison` makes it give a value that is not finite.
|
||||
struct ProgramFake {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
poison: bool,
|
||||
}
|
||||
|
||||
fn run_steps(p: &program::Program, input: &[f32]) -> Vec<Vec<f32>> {
|
||||
use program::{Arg, Binary, Op, Unary};
|
||||
let mut values: Vec<Vec<f32>> = vec![Vec::new(); p.lens.len()];
|
||||
values[program::INPUT] = input.to_vec();
|
||||
for step in &p.steps {
|
||||
let arg = |i: usize| -> Vec<f32> {
|
||||
match &step.args[i] {
|
||||
Arg::Value(v) => values[*v].clone(),
|
||||
Arg::Const { data, .. } => data.to_vec(),
|
||||
}
|
||||
};
|
||||
let len = p.lens[step.out];
|
||||
let x = arg(0);
|
||||
let out: Vec<f32> = match &step.op {
|
||||
Op::Conv(j) => {
|
||||
let (m, cg) = (j.cout / j.groups, j.cin / j.groups);
|
||||
let mut out = vec![0f32; len];
|
||||
for co in 0..j.cout {
|
||||
for oy in 0..j.oh {
|
||||
for ox in 0..j.ow {
|
||||
let mut v = j.bias.map_or(0.0, |b| b[co]);
|
||||
for ci in 0..cg {
|
||||
for ky in 0..j.kh {
|
||||
for kx in 0..j.kw {
|
||||
let (iy, ix) = ((oy * j.sh + ky) as isize - j.pt as isize, (ox * j.sw + kx) as isize - j.pl as isize);
|
||||
if iy >= 0 && ix >= 0 && (iy as usize) < j.h && (ix as usize) < j.w {
|
||||
v += j.weights[((co * cg + ci) * j.kh + ky) * j.kw + kx]
|
||||
* x[((co / m * cg + ci) * j.h + iy as usize) * j.w + ix as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out[(co * j.oh + oy) * j.ow + ox] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
Op::Unary(Unary::Relu) => x.iter().map(|v| v.max(0.0)).collect(),
|
||||
Op::Unary(Unary::Sigmoid) => x.iter().map(|v| 1.0 / (1.0 + (-v).exp())).collect(),
|
||||
Op::ChannelAffine { channels, plane } => {
|
||||
let (s, o) = (arg(1), arg(2));
|
||||
x.iter().enumerate().map(|(i, v)| v * s[(i / plane) % channels] + o[(i / plane) % channels]).collect()
|
||||
}
|
||||
Op::ChannelPrelu { channels, plane } => {
|
||||
let a = arg(1);
|
||||
x.iter().enumerate().map(|(i, v)| if *v < 0.0 { v * a[(i / plane) % channels] } else { *v }).collect()
|
||||
}
|
||||
Op::Binary { op, dims, a, b } => {
|
||||
let y = arg(1);
|
||||
(0..len)
|
||||
.map(|i| {
|
||||
let (mut rest, mut ia, mut ib) = (i, 0, 0);
|
||||
for d in (0..4).rev() {
|
||||
let at = rest % dims[d];
|
||||
rest /= dims[d];
|
||||
ia += at * a[d];
|
||||
ib += at * b[d];
|
||||
}
|
||||
match op {
|
||||
Binary::Add => x[ia] + y[ib],
|
||||
Binary::Sub => x[ia] - y[ib],
|
||||
Binary::Mul => x[ia] * y[ib],
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
Op::Transpose { dims, strides } => (0..len)
|
||||
.map(|i| {
|
||||
let (mut rest, mut from) = (i, 0);
|
||||
for d in (0..4).rev() {
|
||||
from += (rest % dims[d]) * strides[d];
|
||||
rest /= dims[d];
|
||||
}
|
||||
x[from]
|
||||
})
|
||||
.collect(),
|
||||
Op::MaxPool { h, w, kh, kw, sh, sw, oh, ow, .. } => (0..len)
|
||||
.map(|i| {
|
||||
let (ox, oy, c) = (i % ow, (i / ow) % oh, i / (ow * oh));
|
||||
let mut m = f32::NEG_INFINITY;
|
||||
for ky in 0..*kh {
|
||||
for kx in 0..*kw {
|
||||
m = m.max(x[(c * h + oy * sh + ky) * w + ox * sw + kx]);
|
||||
}
|
||||
}
|
||||
m
|
||||
})
|
||||
.collect(),
|
||||
Op::Resize { h, w, sh, sw, .. } => (0..len)
|
||||
.map(|i| {
|
||||
let (ow, oh) = (w * sw, h * sh);
|
||||
let (ox, oy, c) = (i % ow, (i / ow) % oh, i / (ow * oh));
|
||||
x[(c * h + oy / sh) * w + ox / sw]
|
||||
})
|
||||
.collect(),
|
||||
Op::GlobalAverage { plane, .. } => x.chunks(*plane).map(|c| c.iter().sum::<f32>() / *plane as f32).collect(),
|
||||
Op::Gemv { k, n, trans_b, alpha } => {
|
||||
let (wt, bias) = (arg(1), arg(2));
|
||||
(0..*n).map(|o| alpha * (0..*k).map(|j| x[j] * wt[if *trans_b { o * k + j } else { j * n + o }]).sum::<f32>() + bias[o]).collect()
|
||||
}
|
||||
};
|
||||
assert_eq!(out.len(), len, "{:?}", step.op);
|
||||
values[step.out] = out;
|
||||
for f in &step.free {
|
||||
values[*f] = Vec::new();
|
||||
}
|
||||
}
|
||||
p.outputs.iter().map(|(_, v, _)| values[*v].clone()).collect()
|
||||
}
|
||||
|
||||
impl Accelerator for ProgramFake {
|
||||
fn device(&self) -> String {
|
||||
"program test".into()
|
||||
}
|
||||
fn conv(&self, _: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String> {
|
||||
Ok(None)
|
||||
}
|
||||
fn run_program(&self, p: &program::Program, inputs: &[&[f32]]) -> std::result::Result<Option<Vec<Vec<Vec<f32>>>>, String> {
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut out: Vec<Vec<Vec<f32>>> = inputs.iter().map(|x| run_steps(p, x)).collect();
|
||||
if self.poison {
|
||||
out[0][0][0] = f32::NAN;
|
||||
}
|
||||
Ok(Some(out))
|
||||
}
|
||||
}
|
||||
|
||||
fn every_operator_input(seed: usize) -> Tensor {
|
||||
input(4, 8, 8, |i| (((i * 31 + seed * 7) % 37) as f32 - 18.0) * 0.05)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_program_of_every_operator_gives_what_the_interpreter_gives() {
|
||||
let net = Net::new(crate::synthetic::every_operator_graph()).unwrap();
|
||||
let x = every_operator_input(0);
|
||||
let cpu = net.run(x.clone()).unwrap();
|
||||
let fake = Arc::new(ProgramFake { calls: 0.into(), poison: false });
|
||||
net.attach_checked(fake.clone(), &x, 1e-5, false).unwrap();
|
||||
let calls = fake.calls.load(std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(calls > 0, "the whole network ran as one program");
|
||||
// several inputs go to the device together, and each gets its own outputs
|
||||
let inputs: Vec<Tensor> = (0..3).map(every_operator_input).collect();
|
||||
let together = net.run_batch(inputs.clone()).unwrap();
|
||||
assert_eq!(fake.calls.load(std::sync::atomic::Ordering::Relaxed), calls + 1, "one dispatch for the batch");
|
||||
net.detach();
|
||||
for (x, got) in inputs.into_iter().zip(&together) {
|
||||
let want = net.run(x).unwrap();
|
||||
agree(&want, got, 1e-5).unwrap();
|
||||
}
|
||||
assert_eq!(cpu.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_device_that_gives_values_that_are_not_finite_is_dropped() {
|
||||
let net = Net::new(crate::synthetic::every_operator_graph()).unwrap();
|
||||
let x = every_operator_input(1);
|
||||
let cpu = net.run(x.clone()).unwrap();
|
||||
assert!(net.attach_checked(Arc::new(ProgramFake { calls: 0.into(), poison: true }), &x, 1e-5, false).is_err());
|
||||
assert!(net.accelerator().is_none());
|
||||
assert_eq!(net.run(x).unwrap(), cpu);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_and_inconsistent_models_are_errors() {
|
||||
let x = input(1, 3, 3, |i| i as f32);
|
||||
|
||||
@@ -59,6 +59,33 @@ fn shifted_plane(plane: &[f32], width: usize, ky: usize, kx: usize, pt: usize, p
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// The checked geometry of a convolution node, for an accelerator.
|
||||
pub(super) fn conv_job<'a>(layer: usize, n: &Node, shape: &[usize], input: &'a [f32], g: &'a Graph) -> Result<ConvJob<'a>> {
|
||||
let (cin, h, w) = program::chw_of(shape)?;
|
||||
let wt = shape::weight(n, g, 1)?;
|
||||
let weights = shape::floats(n, g, 1)?;
|
||||
let [cout, cg, kh, kw] = wt.dims.as_slice() else {
|
||||
return shape_err("invalid convolution weights");
|
||||
};
|
||||
let (cout, cg, kh, kw) = (*cout, *cg, *kh, *kw);
|
||||
let groups = usize::try_from(n.int("group").unwrap_or(1)).map_err(|_| NetError::Shape("invalid group".into()))?;
|
||||
let (sh, sw) = pair(n, "strides", 1)?;
|
||||
let (pt, pl, pb, pr) = pads(n)?;
|
||||
let ph = h.checked_add(pt).and_then(|v| v.checked_add(pb)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
|
||||
let pw = w.checked_add(pl).and_then(|v| v.checked_add(pr)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
|
||||
if ph < kh || pw < kw || groups == 0 || cg == 0 || kh == 0 || kw == 0 || cin != groups * cg || cout % groups != 0 {
|
||||
return shape_err("invalid convolution shape");
|
||||
}
|
||||
if cout.checked_mul(cg).and_then(|v| v.checked_mul(kh * kw)) != Some(weights.len()) {
|
||||
return shape_err("short convolution weights");
|
||||
}
|
||||
let bias = if n.inputs.get(2).is_some_and(|s| !s.is_empty()) { Some(shape::floats(n, g, 2)?) } else { None };
|
||||
if bias.is_some_and(|b| b.len() != cout) {
|
||||
return shape_err("the convolution bias does not match");
|
||||
}
|
||||
let (oh, ow) = ((ph - kh) / sh + 1, (pw - kw) / sw + 1);
|
||||
Ok(ConvJob { layer, cin, h, w, cout, groups, kh, kw, sh, sw, pt, pl, oh, ow, weights, bias, input })
|
||||
}
|
||||
pub(super) fn conv(n: &Node, x: &Tensor, g: &Graph, scratch: &mut Scratch) -> Result<Tensor> {
|
||||
let (cin, h, w) = chw(x)?;
|
||||
let wt = shape::weight(n, g, 1)?;
|
||||
@@ -181,7 +208,7 @@ pub(super) fn conv(n: &Node, x: &Tensor, g: &Graph, scratch: &mut Scratch) -> Re
|
||||
Ok(out)
|
||||
}
|
||||
/// Flat offset under ONNX's trailing-dimension broadcasting; shapes are checked before execution.
|
||||
fn broadcast_offset(mut at: usize, out: &[usize], input: &[usize]) -> usize {
|
||||
pub(super) fn broadcast_offset(mut at: usize, out: &[usize], input: &[usize]) -> usize {
|
||||
let mut offset = 0;
|
||||
let mut stride = 1;
|
||||
for (axis, &size) in out.iter().enumerate().rev() {
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
//! A network lowered to a flat list of steps over numbered values, for an [`Accelerator`] that keeps every activation
|
||||
//! on its device between layers (one upload, one read-back per run instead of one per convolution).
|
||||
//!
|
||||
//! Shapes are resolved for one input shape; every step names the values it reads and the one it writes, and which
|
||||
//! values nothing reads after it. Shape-only operators (`Reshape`, `Flatten`, `Identity`, `Dropout`) make no step: their
|
||||
//! output is the same value under another name. Anything this cannot express (a broadcast past four dimensions, a
|
||||
//! transposed or batched `Gemm`, a slope that is not per channel) makes the whole lowering `None`, and the net runs
|
||||
//! the CPU path instead.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Value 0 is the network's input.
|
||||
pub const INPUT: usize = 0;
|
||||
|
||||
/// An operand: a value computed on the device, or a constant (uploaded once and kept, by `id`).
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Arg<'a> {
|
||||
Value(usize),
|
||||
/// `id` is the same for every run of the same net, so the device can keep the data.
|
||||
Const {
|
||||
id: usize,
|
||||
data: Cow<'a, [f32]>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Unary {
|
||||
Relu,
|
||||
Sigmoid,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Binary {
|
||||
Add,
|
||||
Sub,
|
||||
Mul,
|
||||
}
|
||||
|
||||
/// What a step computes. Element counts come from the program's `lens` of the step's output.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Op<'a> {
|
||||
/// A convolution; its `input` is empty (it reads the step's first argument).
|
||||
Conv(ConvJob<'a>),
|
||||
/// `out[i] = f(x[i])`. Args: `[x]`.
|
||||
Unary(Unary),
|
||||
/// `out[i] = x[i] * scale[c] + offset[c]`, `c = (i / plane) % channels`. Args: `[x, scale, offset]`.
|
||||
ChannelAffine { channels: usize, plane: usize },
|
||||
/// `out[i] = x[i] < 0 ? x[i] * slope[c] : x[i]`. Args: `[x, slope]` (`channels` values).
|
||||
ChannelPrelu { channels: usize, plane: usize },
|
||||
/// Broadcasting over four dimensions: `dims` of the output, element strides of each operand (0 where it repeats).
|
||||
/// Args: `[a, b]`.
|
||||
Binary { op: Binary, dims: [usize; 4], a: [usize; 4], b: [usize; 4] },
|
||||
/// `out[n] = alpha · Σk x[k] · B(k, n) + bias[n]`, `B(k, n) = weights[n·k_len + k]` when `trans_b`, else
|
||||
/// `weights[k·n_len + n]`. Args: `[x, weights, bias]` (bias already times beta; zeros when the node has none).
|
||||
Gemv { k: usize, n: usize, trans_b: bool, alpha: f32 },
|
||||
/// The mean of each `plane`-sized run. Args: `[x]`.
|
||||
GlobalAverage { channels: usize, plane: usize },
|
||||
/// Unpadded max pooling. Args: `[x]`.
|
||||
MaxPool { c: usize, h: usize, w: usize, kh: usize, kw: usize, sh: usize, sw: usize, oh: usize, ow: usize },
|
||||
/// Nearest-neighbour upscale by whole numbers. Args: `[x]`.
|
||||
Resize { c: usize, h: usize, w: usize, sh: usize, sw: usize },
|
||||
/// `out` in its own order (`dims`), read from the input at `strides` (the input's strides, permuted). Args: `[x]`.
|
||||
Transpose { dims: [usize; 4], strides: [usize; 4] },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Step<'a> {
|
||||
pub op: Op<'a>,
|
||||
pub args: Vec<Arg<'a>>,
|
||||
pub out: usize,
|
||||
/// Values nothing reads after this step (never an output): their memory can be reused.
|
||||
pub free: Vec<usize>,
|
||||
}
|
||||
|
||||
/// A whole network for one input shape.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Program<'a> {
|
||||
/// Elements of each value; `lens[INPUT]` is the input's.
|
||||
pub lens: Vec<usize>,
|
||||
pub steps: Vec<Step<'a>>,
|
||||
/// Each graph output: its name, the value that holds it and its shape.
|
||||
pub outputs: Vec<(String, usize, Vec<usize>)>,
|
||||
}
|
||||
|
||||
/// Leading ones up to rank four, or `None` past four.
|
||||
fn rank4(s: &[usize]) -> Option<[usize; 4]> {
|
||||
let pad = 4usize.checked_sub(s.len())?;
|
||||
let mut out = [1; 4];
|
||||
for (i, d) in s.iter().enumerate() {
|
||||
*out.get_mut(pad + i)? = *d;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Element strides of `input` read as `out` under trailing-dimension broadcasting (0 along repeated axes).
|
||||
fn broadcast_strides(out: [usize; 4], input: &[usize]) -> Option<[usize; 4]> {
|
||||
let dims = rank4(input)?;
|
||||
let mut strides = [0; 4];
|
||||
let mut step = 1;
|
||||
for axis in (0..4).rev() {
|
||||
let (d, o) = (*dims.get(axis)?, *out.get(axis)?);
|
||||
if d == o && d != 1 {
|
||||
*strides.get_mut(axis)? = step;
|
||||
} else if d != 1 {
|
||||
return None;
|
||||
}
|
||||
step *= d;
|
||||
}
|
||||
Some(strides)
|
||||
}
|
||||
|
||||
impl Net {
|
||||
/// This network lowered for `input` (its shape), or `None` when a step has no device form.
|
||||
pub fn program(&self, input: &[usize]) -> Result<Option<Program<'_>>> {
|
||||
let g = &self.graph;
|
||||
let (shapes, _) = shape::infer_all(g, input)?;
|
||||
let Some((input_name, _)) = g.inputs.first() else { return shape_err("the model has no input") };
|
||||
let mut lens = vec![shape::elements(input)?];
|
||||
let mut names: HashMap<&str, (usize, Vec<usize>)> = HashMap::new();
|
||||
names.insert(input_name.as_str(), (INPUT, input.to_vec()));
|
||||
let mut steps: Vec<Step> = Vec::new();
|
||||
for (i, node) in g.nodes.iter().enumerate() {
|
||||
let Some(out_shape) = shapes.get(i) else { return shape_err("a node has no shape") };
|
||||
let Some(out_name) = node.outputs.first() else { return shape_err("a node has no output") };
|
||||
let value = |idx: usize| -> Option<&(usize, Vec<usize>)> { node.inputs.get(idx).and_then(|n| names.get(n.as_str())) };
|
||||
let constant = |idx: usize| -> Option<(Cow<'_, [f32]>, &[usize])> {
|
||||
let w = node.inputs.get(idx).and_then(|n| g.weights.get(n))?;
|
||||
match &w.data {
|
||||
Data::F32(v) => Some((Cow::Borrowed(v.as_slice()), w.dims.as_slice())),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
let id = |k: usize| i * 4 + k;
|
||||
let Some((x, x_shape)) = value(0).cloned() else { return Ok(None) };
|
||||
let (op, args) = match node.op.as_str() {
|
||||
"Reshape" | "Flatten" | "Identity" | "Dropout" => {
|
||||
names.insert(out_name.as_str(), (x, out_shape.clone()));
|
||||
continue;
|
||||
}
|
||||
"Conv" => (Op::Conv(kernels::conv_job(i, node, &x_shape, &[], g)?), vec![Arg::Value(x)]),
|
||||
"Relu" => (Op::Unary(Unary::Relu), vec![Arg::Value(x)]),
|
||||
"Sigmoid" => (Op::Unary(Unary::Sigmoid), vec![Arg::Value(x)]),
|
||||
"BatchNormalization" => {
|
||||
let channels = *x_shape.get(1).ok_or_else(|| NetError::Shape("BatchNormalization rank".into()))?;
|
||||
let plane = shape::elements(x_shape.get(2..).unwrap_or(&[]))?;
|
||||
let (s, b, m, v) =
|
||||
(shape::floats(node, g, 1)?, shape::floats(node, g, 2)?, shape::floats(node, g, 3)?, shape::floats(node, g, 4)?);
|
||||
let eps = shape::float(node, "epsilon", 1e-5);
|
||||
let mut scale = Vec::with_capacity(channels);
|
||||
let mut offset = Vec::with_capacity(channels);
|
||||
for c in 0..channels {
|
||||
let (Some(s), Some(b), Some(m), Some(v)) = (s.get(c), b.get(c), m.get(c), v.get(c)) else {
|
||||
return shape_err("short BatchNormalization parameters");
|
||||
};
|
||||
let k = s / (v + eps).sqrt();
|
||||
scale.push(k);
|
||||
offset.push(b - m * k);
|
||||
}
|
||||
(
|
||||
Op::ChannelAffine { channels, plane },
|
||||
vec![Arg::Value(x), Arg::Const { id: id(1), data: scale.into() }, Arg::Const { id: id(2), data: offset.into() }],
|
||||
)
|
||||
}
|
||||
"PRelu" => {
|
||||
let Some((slope, dims)) = constant(1) else { return Ok(None) };
|
||||
let channels = x_shape.get(1).copied().unwrap_or(1);
|
||||
let plane = shape::elements(x_shape.get(2..).unwrap_or(&[]))?;
|
||||
let mut expanded = vec![1; x_shape.len().saturating_sub(dims.len())];
|
||||
expanded.extend_from_slice(dims);
|
||||
let per_channel = expanded.iter().enumerate().all(|(i, &d)| d == 1 || i == 1) && slope.len() == channels;
|
||||
let slope: Cow<[f32]> = if per_channel {
|
||||
slope
|
||||
} else if let [one] = slope.as_ref() {
|
||||
vec![*one; channels].into()
|
||||
} else {
|
||||
return Ok(None);
|
||||
};
|
||||
(Op::ChannelPrelu { channels, plane }, vec![Arg::Value(x), Arg::Const { id: id(1), data: slope }])
|
||||
}
|
||||
"Add" | "Sub" | "Mul" => {
|
||||
let op = match node.op.as_str() {
|
||||
"Add" => Binary::Add,
|
||||
"Sub" => Binary::Sub,
|
||||
_ => Binary::Mul,
|
||||
};
|
||||
let (b, b_shape) = match (value(1), constant(1)) {
|
||||
(Some((v, s)), _) => (Arg::Value(*v), s.clone()),
|
||||
(None, Some((data, dims))) => (Arg::Const { id: id(1), data }, dims.to_vec()),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
let Some(dims) = rank4(out_shape) else { return Ok(None) };
|
||||
let (Some(sa), Some(sb)) = (broadcast_strides(dims, &x_shape), broadcast_strides(dims, &b_shape)) else { return Ok(None) };
|
||||
(Op::Binary { op, dims, a: sa, b: sb }, vec![Arg::Value(x), b])
|
||||
}
|
||||
"Gemm" => {
|
||||
let Some((weights, wdims)) = constant(1) else { return Ok(None) };
|
||||
let ([1, k], [n_out]) = (x_shape.as_slice(), out_shape.get(1..).unwrap_or(&[])) else { return Ok(None) };
|
||||
if node.int("transA").unwrap_or(0) != 0 || out_shape.first() != Some(&1) {
|
||||
return Ok(None);
|
||||
}
|
||||
let trans_b = node.int("transB").unwrap_or(0) != 0;
|
||||
if wdims.len() != 2 || weights.len() != k * n_out {
|
||||
return shape_err("Gemm weights do not match");
|
||||
}
|
||||
let mut bias = vec![0.0f32; *n_out];
|
||||
if node.inputs.get(2).is_some_and(|s| !s.is_empty()) {
|
||||
let wt = shape::weight(node, g, 2)?;
|
||||
let b = shape::floats(node, g, 2)?;
|
||||
let beta = shape::float(node, "beta", 1.0);
|
||||
for (j, to) in bias.iter_mut().enumerate() {
|
||||
*to = beta
|
||||
* b.get(kernels::broadcast_offset(j, out_shape, &wt.dims))
|
||||
.copied()
|
||||
.ok_or_else(|| NetError::Shape("short matrix bias".into()))?;
|
||||
}
|
||||
}
|
||||
(
|
||||
Op::Gemv { k: *k, n: *n_out, trans_b, alpha: shape::float(node, "alpha", 1.0) },
|
||||
vec![Arg::Value(x), Arg::Const { id: id(1), data: weights }, Arg::Const { id: id(2), data: bias.into() }],
|
||||
)
|
||||
}
|
||||
"GlobalAveragePool" => {
|
||||
let (c, h, w) = chw_of(&x_shape)?;
|
||||
(Op::GlobalAverage { channels: c, plane: h * w }, vec![Arg::Value(x)])
|
||||
}
|
||||
"MaxPool" => {
|
||||
let (c, h, w) = chw_of(&x_shape)?;
|
||||
let (kh, kw) = pair(node, "kernel_shape", 1)?;
|
||||
let (sh, sw) = pair(node, "strides", 1)?;
|
||||
if pads(node)? != (0, 0, 0, 0) {
|
||||
return Ok(None);
|
||||
}
|
||||
let (_, oh, ow) = chw_of(out_shape)?;
|
||||
(Op::MaxPool { c, h, w, kh, kw, sh, sw, oh, ow }, vec![Arg::Value(x)])
|
||||
}
|
||||
"Resize" => {
|
||||
let (c, h, w) = chw_of(&x_shape)?;
|
||||
let (_, oh, ow) = chw_of(out_shape)?;
|
||||
if oh % h != 0 || ow % w != 0 {
|
||||
return shape_err("a Resize that is not by whole numbers");
|
||||
}
|
||||
(Op::Resize { c, h, w, sh: oh / h, sw: ow / w }, vec![Arg::Value(x)])
|
||||
}
|
||||
"Transpose" => {
|
||||
let rank = x_shape.len();
|
||||
let default_perm: Vec<i64> = (0..rank as i64).rev().collect();
|
||||
let perm = node.ints("perm").unwrap_or(&default_perm);
|
||||
if rank > 4 || perm.len() != rank {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut in_strides = vec![1usize; rank];
|
||||
for a in (0..rank.saturating_sub(1)).rev() {
|
||||
let next = in_strides.get(a + 1).copied().unwrap_or(1) * x_shape.get(a + 1).copied().unwrap_or(1);
|
||||
if let Some(s) = in_strides.get_mut(a) {
|
||||
*s = next;
|
||||
}
|
||||
}
|
||||
let Some(dims) = rank4(out_shape) else { return Ok(None) };
|
||||
let mut strides = [0usize; 4];
|
||||
for (j, p) in perm.iter().enumerate() {
|
||||
let p = usize::try_from(*p).map_err(|_| NetError::Shape("a negative permutation".into()))?;
|
||||
let s = in_strides.get(p).copied().ok_or_else(|| NetError::Shape("a bad permutation".into()))?;
|
||||
*strides.get_mut(4 - rank + j).ok_or_else(|| NetError::Shape("a bad permutation".into()))? = s;
|
||||
}
|
||||
(Op::Transpose { dims, strides }, vec![Arg::Value(x)])
|
||||
}
|
||||
_ => return Ok(None),
|
||||
};
|
||||
let out = lens.len();
|
||||
lens.push(shape::elements(out_shape)?);
|
||||
names.insert(out_name.as_str(), (out, out_shape.clone()));
|
||||
steps.push(Step { op, args, out, free: Vec::new() });
|
||||
}
|
||||
let outputs = g
|
||||
.outputs
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let (v, s) = names.get(name.as_str()).ok_or_else(|| NetError::Shape(format!("output `{name}` was never computed")))?;
|
||||
Ok((name.clone(), *v, s.clone()))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
// the last step that reads each value; a value nothing reads is free right after it is made
|
||||
let mut last: Vec<Option<usize>> = vec![None; lens.len()];
|
||||
for (at, step) in steps.iter().enumerate() {
|
||||
if let Some(l) = last.get_mut(step.out) {
|
||||
*l = Some(at);
|
||||
}
|
||||
for a in &step.args {
|
||||
if let Arg::Value(v) = a
|
||||
&& let Some(l) = last.get_mut(*v)
|
||||
{
|
||||
*l = Some(at);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (v, at) in last.iter().enumerate() {
|
||||
if outputs.iter().any(|(_, o, _)| *o == v) {
|
||||
continue;
|
||||
}
|
||||
if let Some(step) = at.and_then(|at| steps.get_mut(at)) {
|
||||
step.free.push(v);
|
||||
}
|
||||
}
|
||||
Ok(Some(Program { lens, steps, outputs }))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn chw_of(s: &[usize]) -> Result<(usize, usize, usize)> {
|
||||
match s {
|
||||
[1, c, h, w] => Ok((*c, *h, *w)),
|
||||
s => unsupported(format!("a tensor of shape {s:?} (only [1, channels, height, width])")),
|
||||
}
|
||||
}
|
||||
@@ -257,7 +257,12 @@ fn image(s: &[usize]) -> Result<(usize, usize, usize)> {
|
||||
}
|
||||
}
|
||||
pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
|
||||
Ok(infer_all(g, input)?.1)
|
||||
}
|
||||
/// Each node's output shape, and the graph's outputs' shapes.
|
||||
pub(super) fn infer_all(g: &Graph, input: &[usize]) -> Result<(Vec<Vec<usize>>, Vec<Vec<usize>>)> {
|
||||
elements(input)?;
|
||||
let mut per_node = Vec::with_capacity(g.nodes.len());
|
||||
let mut shapes: HashMap<&str, Vec<usize>> = g.weights.iter().map(|(n, w)| (n.as_str(), w.dims.clone())).collect();
|
||||
shapes.insert(g.inputs.first().ok_or_else(|| NetError::Shape("no input".into()))?.0.as_str(), input.to_vec());
|
||||
let mut live = elements(input)?;
|
||||
@@ -480,6 +485,7 @@ pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
|
||||
.checked_add(out_count)
|
||||
.filter(|&n| n <= 128 * 1024 * 1024)
|
||||
.ok_or_else(|| NetError::Shape("live tensors exceed the 512 MiB working limit".into()))?;
|
||||
per_node.push(out.clone());
|
||||
shapes.insert(n.outputs.first().ok_or_else(|| NetError::Shape("no output".into()))?, out);
|
||||
let mut removed = std::collections::HashSet::new();
|
||||
for name in &n.inputs {
|
||||
@@ -488,5 +494,7 @@ pub(super) fn infer(g: &Graph, input: &[usize]) -> Result<Vec<Vec<usize>>> {
|
||||
}
|
||||
}
|
||||
}
|
||||
g.outputs.iter().map(|s| shapes.get(s.as_str()).cloned().ok_or_else(|| NetError::Shape("missing output".into()))).collect()
|
||||
let outputs =
|
||||
g.outputs.iter().map(|s| shapes.get(s.as_str()).cloned().ok_or_else(|| NetError::Shape("missing output".into()))).collect::<Result<_>>()?;
|
||||
Ok((per_node, outputs))
|
||||
}
|
||||
@@ -40,6 +40,9 @@ pub struct SelfTest {
|
||||
|
||||
type Plan = Net;
|
||||
|
||||
/// Most faces [`Embedder::embed_batch`] sends to a GPU in one go.
|
||||
pub const BATCH: usize = 16;
|
||||
|
||||
/// A loaded recognition model.
|
||||
#[derive(Clone)]
|
||||
pub struct Embedder {
|
||||
@@ -99,6 +102,31 @@ impl Embedder {
|
||||
})
|
||||
}
|
||||
|
||||
/// The network, e.g. to see which device it runs on.
|
||||
pub fn net(&self) -> &Net {
|
||||
&self.plan
|
||||
}
|
||||
|
||||
/// Run the model's convolutions on `accelerator` if it gives the CPU's embedding on a test face (and, with
|
||||
/// `require_faster`, runs it faster than the CPU); otherwise (or on
|
||||
/// its first error) recognition stays on the CPU. Returns why it was not used.
|
||||
pub fn accelerate(&self, accelerator: Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), RuntimeError> {
|
||||
let (mean, std) = (self.manifest.input.mean, self.manifest.input.std);
|
||||
let plane = self.width * self.height;
|
||||
let data: Vec<f32> = crate::yunet::probe_pixels(3 * plane)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| {
|
||||
let c = (i / plane.max(1)).min(2);
|
||||
(v - mean.get(c).copied().unwrap_or(0.0)) / std.get(c).copied().filter(|s| *s != 0.0).unwrap_or(1.0)
|
||||
})
|
||||
.collect();
|
||||
let probe = Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))?;
|
||||
guarded("checking the device", || {
|
||||
self.plan.attach_checked(accelerator, &probe, crate::yunet::ACCELERATOR_TOLERANCE, require_faster).map_err(RuntimeError::Run)
|
||||
})
|
||||
}
|
||||
|
||||
/// The size of the aligned face picture the model wants (width, height).
|
||||
pub fn input_size(&self) -> (usize, usize) {
|
||||
(self.width, self.height)
|
||||
@@ -110,6 +138,66 @@ impl Embedder {
|
||||
|
||||
/// The face as a unit-length vector. `rgb` is an aligned face at exactly [`Self::input_size`], 8-bit RGB.
|
||||
pub fn embed(&self, rgb: &[u8]) -> Result<Vec<f32>, RuntimeError> {
|
||||
let input = self.input(rgb)?;
|
||||
let out = guarded("running it", || {
|
||||
let mut result = self.plan.run(input).map_err(|e| RuntimeError::Run(e.to_string()))?;
|
||||
if result.len() != 1 {
|
||||
return Err(RuntimeError::Run("the model must give one embedding".into()));
|
||||
}
|
||||
Ok(result.pop().ok_or_else(|| RuntimeError::Run("no output".into()))?.1.data)
|
||||
})?;
|
||||
self.unit(out)
|
||||
}
|
||||
|
||||
/// Several faces at once, each as [`Self::embed`] gives it. On a GPU they run together, up to [`BATCH`] per
|
||||
/// dispatch (one upload and one read-back for all of them); on the CPU one after another.
|
||||
pub fn embed_batch(&self, faces: &[&[u8]]) -> Vec<Result<Vec<f32>, RuntimeError>> {
|
||||
if self.plan.accelerator().is_none() || faces.len() < 2 {
|
||||
return faces.iter().map(|f| self.embed(f)).collect();
|
||||
}
|
||||
let mut results: Vec<Option<Result<Vec<f32>, RuntimeError>>> = faces.iter().map(|_| None).collect();
|
||||
let mut ready: Vec<(usize, Tensor)> = Vec::new();
|
||||
for (i, face) in faces.iter().enumerate() {
|
||||
match self.input(face) {
|
||||
Ok(t) => ready.push((i, t)),
|
||||
Err(e) => {
|
||||
if let Some(r) = results.get_mut(i) {
|
||||
*r = Some(Err(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
while !ready.is_empty() {
|
||||
let chunk: Vec<(usize, Tensor)> = ready.drain(..ready.len().min(BATCH)).collect();
|
||||
let (at, inputs): (Vec<usize>, Vec<Tensor>) = chunk.into_iter().unzip();
|
||||
let ran = guarded("running it", || self.plan.run_batch(inputs).map_err(|e| RuntimeError::Run(e.to_string())));
|
||||
match ran {
|
||||
Ok(outs) if outs.len() == at.len() => {
|
||||
for (i, mut out) in at.into_iter().zip(outs) {
|
||||
let r = match (out.len(), out.pop()) {
|
||||
(1, Some((_, t))) => self.unit(t.data),
|
||||
_ => Err(RuntimeError::Run("the model must give one embedding".into())),
|
||||
};
|
||||
if let Some(slot) = results.get_mut(i) {
|
||||
*slot = Some(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
// one bad face spoils a batch: run them one by one so the others still get theirs
|
||||
_ => {
|
||||
for i in at {
|
||||
if let (Some(slot), Some(face)) = (results.get_mut(i), faces.get(i)) {
|
||||
*slot = Some(self.embed(face));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.into_iter().map(|r| r.unwrap_or_else(|| Err(RuntimeError::Run("the face was not embedded".into())))).collect()
|
||||
}
|
||||
|
||||
/// The model's input for an aligned face.
|
||||
fn input(&self, rgb: &[u8]) -> Result<Tensor, RuntimeError> {
|
||||
if rgb.len() != self.width * self.height * 3 {
|
||||
return Err(RuntimeError::Input("the face picture is not the size the model wants".into()));
|
||||
}
|
||||
@@ -124,14 +212,11 @@ impl Embedder {
|
||||
}
|
||||
}
|
||||
}
|
||||
let out = guarded("running it", || {
|
||||
let input = Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))?;
|
||||
let mut result = self.plan.run(input).map_err(|e| RuntimeError::Run(e.to_string()))?;
|
||||
if result.len() != 1 {
|
||||
return Err(RuntimeError::Run("the model must give one embedding".into()));
|
||||
}
|
||||
Ok(result.pop().ok_or_else(|| RuntimeError::Run("no output".into()))?.1.data)
|
||||
})?;
|
||||
Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))
|
||||
}
|
||||
|
||||
/// The model's output, checked and scaled to unit length.
|
||||
fn unit(&self, out: Vec<f32>) -> Result<Vec<f32>, RuntimeError> {
|
||||
if out.len() != self.dim {
|
||||
return Err(RuntimeError::Run(format!("the model gave {} numbers per face, not the {} its description says", out.len(), self.dim)));
|
||||
}
|
||||
|
||||
@@ -248,3 +248,63 @@ pub fn tiny_detector_manifest() -> crate::manifest::ModelManifest {
|
||||
m.size_bytes = Some(bytes.len() as u64);
|
||||
m
|
||||
}
|
||||
|
||||
/// A small network that uses every operator the interpreter runs (both kinds of convolution, batch norm, PRelu,
|
||||
/// sigmoid, broadcasting add/sub/mul with values and constants, max pooling, nearest upscaling, transposes, reshape,
|
||||
/// global pooling, flatten and a fully connected layer), with two outputs, for checking an accelerator's program
|
||||
/// against the CPU. Its input is `[1, 4, 8, 8]`.
|
||||
pub fn every_operator_graph() -> crate::graph::Graph {
|
||||
use crate::graph::{Attr, Data, Graph, Node, Weight};
|
||||
let node = |op: &str, inputs: &[&str], output: &str, attrs: &[(&str, Attr)]| Node {
|
||||
op: op.into(),
|
||||
inputs: inputs.iter().map(|s| s.to_string()).collect(),
|
||||
outputs: vec![output.into()],
|
||||
attrs: attrs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
|
||||
};
|
||||
let wave = |n: usize, k: usize| -> Vec<f32> { (0..n).map(|i| (((i * k) % 23) as f32 - 11.0) * 0.04).collect() };
|
||||
let w = |dims: &[usize], v: Vec<f32>| Weight { dims: dims.to_vec(), data: Data::F32(v) };
|
||||
let weights = [
|
||||
("w1", w(&[8, 4, 3, 3], wave(288, 7))),
|
||||
("b1", w(&[8], wave(8, 5))),
|
||||
("bn_s", w(&[8], (0..8).map(|i| 0.5 + i as f32 * 0.1).collect())),
|
||||
("bn_b", w(&[8], wave(8, 3))),
|
||||
("bn_m", w(&[8], wave(8, 11))),
|
||||
("bn_v", w(&[8], (0..8).map(|i| 0.2 + i as f32 * 0.05).collect())),
|
||||
("slope", w(&[8, 1, 1], (0..8).map(|i| 0.05 * i as f32).collect())),
|
||||
("w2", w(&[8, 1, 3, 3], wave(72, 13))),
|
||||
("scales", w(&[4], vec![1.0, 1.0, 2.0, 2.0])),
|
||||
("half", w(&[1], vec![0.5])),
|
||||
("per_channel", w(&[1, 8, 1, 1], wave(8, 17))),
|
||||
("wg", w(&[5, 8], wave(40, 19))),
|
||||
("bg", w(&[5], wave(5, 2))),
|
||||
("flat", Weight { dims: vec![2], data: Data::I64(vec![1, -1]) }),
|
||||
];
|
||||
let pads = |p: i64| ("pads", Attr::Ints(vec![p; 4]));
|
||||
let nodes = vec![
|
||||
node("Conv", &["x", "w1", "b1"], "c1", &[pads(1)]),
|
||||
node("BatchNormalization", &["c1", "bn_s", "bn_b", "bn_m", "bn_v"], "bn", &[]),
|
||||
node("PRelu", &["bn", "slope"], "pr", &[]),
|
||||
node("Conv", &["pr", "w2"], "c2", &[("group", Attr::Int(8)), ("strides", Attr::Ints(vec![2, 2])), pads(1)]),
|
||||
node("Sigmoid", &["c2"], "sg", &[]),
|
||||
node("Mul", &["c2", "sg"], "swish", &[]),
|
||||
node("Resize", &["swish", "", "scales"], "up", &[]),
|
||||
node("Add", &["up", "pr"], "sum", &[]),
|
||||
node("Sub", &["sum", "half"], "less", &[]),
|
||||
node("MaxPool", &["less"], "pool", &[("kernel_shape", Attr::Ints(vec![2, 2])), ("strides", Attr::Ints(vec![2, 2]))]),
|
||||
node("Relu", &["pool"], "relu", &[]),
|
||||
node("Transpose", &["relu"], "nhwc", &[("perm", Attr::Ints(vec![0, 2, 3, 1]))]),
|
||||
node("Transpose", &["nhwc"], "nchw", &[("perm", Attr::Ints(vec![0, 3, 1, 2]))]),
|
||||
node("Mul", &["nchw", "per_channel"], "scaled", &[]),
|
||||
node("Identity", &["scaled"], "same", &[]),
|
||||
node("GlobalAveragePool", &["same"], "avg", &[]),
|
||||
node("Flatten", &["avg"], "row", &[]),
|
||||
node("Reshape", &["row", "flat"], "row2", &[]),
|
||||
node("Gemm", &["row2", "wg", "bg"], "y", &[("transB", Attr::Int(1)), ("alpha", Attr::Float(0.5))]),
|
||||
];
|
||||
Graph {
|
||||
nodes,
|
||||
weights: weights.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
|
||||
inputs: vec![("x".into(), vec![1, 4, 8, 8])],
|
||||
outputs: vec!["y".into(), "nhwc".into()],
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,14 @@ pub struct Face {
|
||||
pub landmarks: [(f32, f32); 5],
|
||||
}
|
||||
|
||||
/// How far an accelerator's outputs may stray from the CPU's on the test input, relative to each output's largest value.
|
||||
pub const ACCELERATOR_TOLERANCE: f32 = 1e-3;
|
||||
|
||||
/// A deterministic, photo-like test input (smooth gradients plus texture, in 0..255).
|
||||
pub fn probe_pixels(n: usize) -> Vec<f32> {
|
||||
(0..n).map(|i| ((i % 640) as f32 * 0.3 + ((i * 7919) % 97) as f32 + (i / 640 % 480) as f32 * 0.2) % 255.0).collect()
|
||||
}
|
||||
|
||||
pub struct Detector {
|
||||
net: Net,
|
||||
/// Side of the square network input.
|
||||
@@ -90,6 +98,20 @@ impl Detector {
|
||||
Ok(Detector { net, side })
|
||||
}
|
||||
|
||||
/// The network, e.g. to see which device it runs on.
|
||||
pub fn net(&self) -> &Net {
|
||||
&self.net
|
||||
}
|
||||
|
||||
/// Run the network's convolutions on `accelerator` if it gives the CPU's outputs on a test picture (and, with `require_faster`,
|
||||
/// runs it faster than the CPU); otherwise
|
||||
/// (or on its first error) detection stays on the CPU. Returns why it was not used.
|
||||
pub fn accelerate(&self, accelerator: std::sync::Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), String> {
|
||||
let side = self.side;
|
||||
let probe = Tensor::new(vec![1, 3, side, side], probe_pixels(3 * side * side)).map_err(|e| e.to_string())?;
|
||||
self.net.attach_checked(accelerator, &probe, ACCELERATOR_TOLERANCE, require_faster)
|
||||
}
|
||||
|
||||
/// The square input side, used to check an installed manifest.
|
||||
pub fn input_side(&self) -> usize {
|
||||
self.side
|
||||
|
||||
+127
-20
@@ -10,7 +10,7 @@
|
||||
//! the downloading thread. No proxies (see docs/ai-masks.md).
|
||||
|
||||
use std::io::{ErrorKind, Read, Write};
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -88,6 +88,9 @@ impl Url {
|
||||
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
|
||||
let host = v6.get(..end).unwrap_or_default();
|
||||
let after = v6.get(end + 1..).unwrap_or_default();
|
||||
if !after.is_empty() && !after.starts_with(':') {
|
||||
return Err(bad("unexpected text after IPv6 address"));
|
||||
}
|
||||
(host.to_string(), after.strip_prefix(':'))
|
||||
} else {
|
||||
match authority.rsplit_once(':') {
|
||||
@@ -182,27 +185,68 @@ fn tls_config() -> Result<Arc<rustls::ClientConfig>, HttpError> {
|
||||
Ok(Arc::new(config))
|
||||
}
|
||||
|
||||
/// Resolve a host name within `deadline`, checking the cancel flag while it runs. The system resolver blocks with no
|
||||
/// timeout of its own, so it runs on a helper thread (which ends when the resolver returns); a lookup that takes too
|
||||
/// long is given up on, and Cancel works during it (issue #1532).
|
||||
fn lookup(
|
||||
host: &str,
|
||||
resolve: impl FnOnce() -> std::io::Result<Vec<SocketAddr>> + Send + 'static,
|
||||
deadline: Instant,
|
||||
limits: &Limits,
|
||||
) -> Result<Vec<SocketAddr>, HttpError> {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("lc-fetch-lookup".into())
|
||||
.spawn(move || {
|
||||
let _ = tx.send(resolve());
|
||||
})
|
||||
.map_err(|e| HttpError::Connect(format!("{host}: {e}")))?;
|
||||
loop {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
let left = deadline.saturating_duration_since(Instant::now());
|
||||
if left.is_zero() {
|
||||
return Err(HttpError::Connect(format!("{host}: the name lookup timed out")));
|
||||
}
|
||||
match rx.recv_timeout(left.min(TICK)) {
|
||||
Ok(r) => return r.map_err(|e| HttpError::Connect(format!("{host}: {e}"))),
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Err(HttpError::Connect(format!("{host}: the name lookup failed"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Connect to the first of `addrs` that answers, all of them together within `deadline` (each used to get the whole
|
||||
/// connect timeout of its own, issue #1532).
|
||||
fn connect_any(host: &str, addrs: &[SocketAddr], deadline: Instant, limits: &Limits) -> Result<TcpStream, HttpError> {
|
||||
let mut last = format!("{host}: no address");
|
||||
for a in addrs {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
let left = deadline.saturating_duration_since(Instant::now());
|
||||
if left.is_zero() {
|
||||
return Err(HttpError::Connect(format!("{host}: connecting timed out")));
|
||||
}
|
||||
match TcpStream::connect_timeout(a, left) {
|
||||
Ok(s) => return Ok(s),
|
||||
Err(e) => last = format!("{host}: {e}"),
|
||||
}
|
||||
}
|
||||
Err(HttpError::Connect(last))
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
fn connect(url: &Url, limits: &Limits) -> Result<Stream, HttpError> {
|
||||
let addrs: Vec<_> = (url.host.as_str(), url.port).to_socket_addrs().map_err(|e| HttpError::Connect(format!("{}: {e}", url.host)))?.collect();
|
||||
if addrs.is_empty() {
|
||||
return Err(HttpError::Connect(format!("{}: no address", url.host)));
|
||||
}
|
||||
let mut last = String::new();
|
||||
let mut sock = None;
|
||||
for a in addrs {
|
||||
if limits.cancel.load(Ordering::Relaxed) {
|
||||
return Err(HttpError::Cancelled);
|
||||
}
|
||||
match TcpStream::connect_timeout(&a, limits.connect) {
|
||||
Ok(s) => {
|
||||
sock = Some(s);
|
||||
break;
|
||||
}
|
||||
Err(e) => last = format!("{}: {e}", url.host),
|
||||
}
|
||||
}
|
||||
let sock = sock.ok_or(HttpError::Connect(last))?;
|
||||
// the lookup and the connection attempts share one connect timeout
|
||||
let deadline = Instant::now() + limits.connect;
|
||||
let (host, port) = (url.host.clone(), url.port);
|
||||
let addrs = lookup(&url.host, move || (host.as_str(), port).to_socket_addrs().map(Iterator::collect), deadline, limits)?;
|
||||
let sock = connect_any(&url.host, &addrs, deadline, limits)?;
|
||||
sock.set_read_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
|
||||
sock.set_write_timeout(Some(TICK)).map_err(|e| HttpError::Io(e.to_string()))?;
|
||||
let _ = sock.set_nodelay(true);
|
||||
@@ -512,6 +556,58 @@ pub fn get(url: &Url, headers: &[(&str, String)], limits: &Limits) -> Result<Res
|
||||
Ok(Response { status, headers, conn, body })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests_connect {
|
||||
use super::*;
|
||||
|
||||
fn limits(cancel: &AtomicBool) -> Limits<'_> {
|
||||
Limits { connect: Duration::from_millis(300), stall: Duration::from_secs(1), cancel }
|
||||
}
|
||||
|
||||
/// A name lookup that hangs is given up on at the connect timeout, and Cancel stops it at once (issue #1532).
|
||||
#[test]
|
||||
fn a_hung_name_lookup_times_out_and_can_be_cancelled() {
|
||||
let cancel = AtomicBool::new(false);
|
||||
let slow = || {
|
||||
std::thread::sleep(Duration::from_secs(10));
|
||||
Ok(Vec::new())
|
||||
};
|
||||
let t = Instant::now();
|
||||
let r = lookup("slow.example", slow, Instant::now() + Duration::from_millis(300), &limits(&cancel));
|
||||
assert!(matches!(&r, Err(HttpError::Connect(m)) if m.contains("timed out")), "{r:?}");
|
||||
assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed());
|
||||
// Cancel while the lookup is waiting on the resolver, not before it starts
|
||||
let t = Instant::now();
|
||||
let r = std::thread::scope(|scope| {
|
||||
scope.spawn(|| {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
});
|
||||
lookup("slow.example", slow, Instant::now() + Duration::from_secs(30), &limits(&cancel))
|
||||
});
|
||||
assert!(matches!(r, Err(HttpError::Cancelled)), "{r:?}");
|
||||
assert!(t.elapsed() >= Duration::from_millis(100) && t.elapsed() < Duration::from_secs(1), "{:?}", t.elapsed());
|
||||
// and a lookup asked for after Cancel doesn't start
|
||||
assert!(matches!(lookup("slow.example", slow, Instant::now() + Duration::from_secs(30), &limits(&cancel)), Err(HttpError::Cancelled)));
|
||||
// a lookup that answers is used
|
||||
let cancel = AtomicBool::new(false);
|
||||
let local: SocketAddr = "127.0.0.1:9".parse().unwrap();
|
||||
assert_eq!(lookup("local", move || Ok(vec![local]), Instant::now() + Duration::from_secs(5), &limits(&cancel)).unwrap(), [local]);
|
||||
}
|
||||
|
||||
/// Several addresses that don't answer share one deadline instead of each getting the full timeout.
|
||||
#[test]
|
||||
fn addresses_share_one_connect_deadline() {
|
||||
let cancel = AtomicBool::new(false);
|
||||
// non-routable addresses: each either hangs until the deadline or fails at once, never longer
|
||||
let addrs: Vec<SocketAddr> = (1..=5).map(|i| format!("10.255.255.{i}:81").parse().unwrap()).collect();
|
||||
let t = Instant::now();
|
||||
let r = connect_any("blackhole", &addrs, Instant::now() + Duration::from_millis(400), &limits(&cancel));
|
||||
assert!(r.is_err());
|
||||
assert!(t.elapsed() < Duration::from_millis(1500), "{:?}", t.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -537,6 +633,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_text_after_a_bracketed_ipv6_host() {
|
||||
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
|
||||
assert!(matches!(Url::parse(bad), Err(HttpError::BadUrl(_))), "{bad}");
|
||||
}
|
||||
let u = Url::parse("http://[::1]:8080/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
|
||||
let u = Url::parse("http://[::1]/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_redirects() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
|
||||
@@ -104,7 +104,8 @@ pub struct GeoNames {
|
||||
|
||||
fn cell(p: LatLon) -> (i32, i32) {
|
||||
// valid positions only: |lat| ≤ 90, |lon| ≤ 180, so the casts are in range
|
||||
(p.lat.floor() as i32, p.lon.floor() as i32)
|
||||
// longitude 180 is the same meridian as -180: key it as column -180, the wrap `reverse` probes
|
||||
(p.lat.floor() as i32, (p.lon.floor() as i32 + 180).rem_euclid(360) - 180)
|
||||
}
|
||||
|
||||
fn short(s: &str) -> String {
|
||||
@@ -253,6 +254,14 @@ mod tests {
|
||||
assert_eq!(fiji.reverse(LatLon::new(-17.0, -179.95)).unwrap().city, "X");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn city_on_the_antimeridian_is_found() {
|
||||
let g = GeoNames::parse("1\tTaveuni\tTaveuni\t\t0.0\t180.0\tP\tPPL\tFJ\t\t01\n", None, None);
|
||||
for lon in [179.9, -179.9, 180.0, -180.0] {
|
||||
assert_eq!(g.reverse(LatLon::new(0.0, lon)).map(|p| p.city).as_deref(), Some("Taveuni"), "from {lon}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_latitude_cities_two_cells_away() {
|
||||
let city = |lat: f64, lon: f64| GeoNames::parse(&format!("1\tX\tX\t\t{lat}\t{lon}\tP\tP\tNO\t\t\n"), None, None);
|
||||
|
||||
@@ -14,6 +14,7 @@ lightcraft-geom = { workspace = true }
|
||||
lightcraft-color = { workspace = true }
|
||||
lightcraft-raster = { workspace = true }
|
||||
lightcraft-develop = { workspace = true }
|
||||
lightcraft-faces = { workspace = true }
|
||||
lightcraft-pipeline = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -35,6 +35,8 @@ pub mod backend;
|
||||
mod ctx;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod exit;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod faces;
|
||||
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
|
||||
pub mod nn;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -239,9 +241,9 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the
|
||||
/// Why GPU compute that has a device of its own (denoise, faces) is switched off by the user's settings, or `None`: the
|
||||
/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
|
||||
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(crate) fn switched_off() -> Option<String> {
|
||||
if env_disabled() {
|
||||
return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
// Convolution for the face networks (lightcraft-faces), planar (NCHW) float32, batch 1, any stride, padding and grouping.
|
||||
//
|
||||
// `tiled` is an implicit matrix product per group: rows are the group's output channels (M), columns its output
|
||||
// pixels (N), and K runs over input channel x kernel tap. The input is gathered (im2col) straight into workgroup
|
||||
// memory, so the lowered matrix is never made. 256 threads make a 64 x 64 block, 4 x 4 values each.
|
||||
// `direct` gives one output value per thread, for depthwise convolutions (one output channel per group), where a
|
||||
// 64-row tile would be almost all waste.
|
||||
//
|
||||
// out[co][p] = bias[co] + sum_k W[co][k] * X[k][p] X[k][p] = input[g*cg + k/taps][oy*sh + ky - pt][ox*sw + kx - pl]
|
||||
|
||||
struct P {
|
||||
m: u32, // output channels per group
|
||||
n: u32, // output pixels (oh * ow)
|
||||
k: u32, // cg * kh * kw
|
||||
cg: u32, // input channels per group
|
||||
h: u32,
|
||||
w: u32,
|
||||
ow: u32,
|
||||
kh: u32,
|
||||
kw: u32,
|
||||
sh: u32,
|
||||
sw: u32,
|
||||
pt: u32,
|
||||
pl: u32,
|
||||
bias: u32, // 1 when there is a bias
|
||||
total: u32, // cout * n
|
||||
row: u32, // workgroups along x, for `direct` and `reduce` (so a large layer can use y too)
|
||||
splits: u32, // K is cut into this many slices (z = group * splits + slice), summed by `reduce`
|
||||
kchunk: u32, // K values per slice, a multiple of BK
|
||||
_a: u32,
|
||||
_b: u32,
|
||||
}
|
||||
|
||||
@group(0) @binding(0) var<uniform> p: P;
|
||||
@group(0) @binding(1) var<storage, read> src: array<f32>;
|
||||
@group(0) @binding(2) var<storage, read> wgt: array<f32>;
|
||||
@group(0) @binding(3) var<storage, read> bias: array<f32>;
|
||||
@group(0) @binding(4) var<storage, read_write> dst: array<f32>;
|
||||
|
||||
const BM: u32 = 64u;
|
||||
const BN: u32 = 64u;
|
||||
const BK: u32 = 16u;
|
||||
|
||||
var<workgroup> As: array<f32, 1024>; // [BK][BM]
|
||||
var<workgroup> Bs: array<f32, 1024>; // [BK][BN]
|
||||
|
||||
fn gather(g: u32, k: u32, pix: u32) -> f32 {
|
||||
let taps = p.kh * p.kw;
|
||||
let ci = g * p.cg + k / taps;
|
||||
let tap = k % taps;
|
||||
let oy = pix / p.ow;
|
||||
let ox = pix % p.ow;
|
||||
let iy = i32(oy * p.sh + tap / p.kw) - i32(p.pt);
|
||||
let ix = i32(ox * p.sw + tap % p.kw) - i32(p.pl);
|
||||
if (iy < 0 || ix < 0 || iy >= i32(p.h) || ix >= i32(p.w)) {
|
||||
return 0.0;
|
||||
}
|
||||
return src[(ci * p.h + u32(iy)) * p.w + u32(ix)];
|
||||
}
|
||||
|
||||
@compute @workgroup_size(256)
|
||||
fn tiled(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
|
||||
let n0 = wg.x * BN;
|
||||
let m0 = wg.y * BM;
|
||||
let g = wg.z / p.splits;
|
||||
let slice = wg.z % p.splits;
|
||||
let k_lo = slice * p.kchunk;
|
||||
let k_hi = min(p.k, k_lo + p.kchunk);
|
||||
let tx = tid % 16u;
|
||||
let ty = tid / 16u;
|
||||
var acc: array<array<f32, 4>, 4>;
|
||||
for (var i = 0u; i < 4u; i++) {
|
||||
for (var j = 0u; j < 4u; j++) {
|
||||
acc[i][j] = 0.0;
|
||||
}
|
||||
}
|
||||
for (var k0 = k_lo; k0 < k_hi; k0 += BK) {
|
||||
// 1024 values of each tile, 4 per thread; every slot has exactly one writer
|
||||
for (var e = tid; e < 1024u; e += 256u) {
|
||||
// weights: neighbouring threads read neighbouring k of one row
|
||||
let mm = e / BK;
|
||||
let ka = k0 + e % BK;
|
||||
let m = m0 + mm;
|
||||
var a = 0.0;
|
||||
if (ka < k_hi && m < p.m) {
|
||||
a = wgt[(g * p.m + m) * p.k + ka];
|
||||
}
|
||||
As[(e % BK) * BM + mm] = a;
|
||||
// input: neighbouring threads read neighbouring pixels
|
||||
let kk = e / BN;
|
||||
let k = k0 + kk;
|
||||
let pix = n0 + e % BN;
|
||||
var b = 0.0;
|
||||
if (k < k_hi && pix < p.n) {
|
||||
b = gather(g, k, pix);
|
||||
}
|
||||
Bs[e] = b;
|
||||
}
|
||||
workgroupBarrier();
|
||||
for (var kk = 0u; kk < BK; kk++) {
|
||||
var a: array<f32, 4>;
|
||||
var b: array<f32, 4>;
|
||||
for (var i = 0u; i < 4u; i++) {
|
||||
a[i] = As[kk * BM + ty * 4u + i];
|
||||
b[i] = Bs[kk * BN + tx + 16u * i];
|
||||
}
|
||||
for (var i = 0u; i < 4u; i++) {
|
||||
for (var j = 0u; j < 4u; j++) {
|
||||
acc[i][j] += a[i] * b[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
workgroupBarrier();
|
||||
}
|
||||
for (var i = 0u; i < 4u; i++) {
|
||||
let m = m0 + ty * 4u + i;
|
||||
if (m >= p.m) {
|
||||
continue;
|
||||
}
|
||||
let co = g * p.m + m;
|
||||
var b0 = 0.0;
|
||||
if (p.bias != 0u && p.splits == 1u) {
|
||||
b0 = bias[co];
|
||||
}
|
||||
// with slices, each writes a partial sum of its own: [slice][cout][n]
|
||||
let base = slice * p.total + co * p.n;
|
||||
for (var j = 0u; j < 4u; j++) {
|
||||
// columns tx, tx + 16, tx + 32, tx + 48: neighbouring threads write neighbouring pixels
|
||||
let pix = n0 + tx + 16u * j;
|
||||
if (pix < p.n) {
|
||||
dst[base + pix] = acc[i][j] + b0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@compute @workgroup_size(256)
|
||||
fn direct(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
|
||||
let at = (wg.y * p.row + wg.x) * 256u + tid;
|
||||
if (at >= p.total) {
|
||||
return;
|
||||
}
|
||||
let co = at / p.n;
|
||||
let pix = at % p.n;
|
||||
let g = co / p.m;
|
||||
var v = 0.0;
|
||||
if (p.bias != 0u) {
|
||||
v = bias[co];
|
||||
}
|
||||
for (var k = 0u; k < p.k; k++) {
|
||||
v += wgt[co * p.k + k] * gather(g, k, pix);
|
||||
}
|
||||
dst[at] = v;
|
||||
}
|
||||
|
||||
// The sum of the K slices' partial results, plus the bias. `src` holds [splits][cout][n].
|
||||
@compute @workgroup_size(256)
|
||||
fn reduce(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
|
||||
let at = (wg.y * p.row + wg.x) * 256u + tid;
|
||||
if (at >= p.total) {
|
||||
return;
|
||||
}
|
||||
var v = 0.0;
|
||||
if (p.bias != 0u) {
|
||||
v = bias[at / p.n];
|
||||
}
|
||||
for (var s = 0u; s < p.splits; s++) {
|
||||
v += src[s * p.total + at];
|
||||
}
|
||||
dst[at] = v;
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// The face networks' other operators (lightcraft-faces `net::program`), so activations stay on the device between layers.
|
||||
// Same bindings as faces_conv.wgsl: the parameters, three read-only operands and the output. Element kernels run one
|
||||
// thread per output value; `gavg` and `gemv` one workgroup per output value.
|
||||
//
|
||||
// Parameters are 24 numbers: q(0) = values (or workgroups) to make, q(1) = workgroups along x, then per kernel.
|
||||
|
||||
struct P {
|
||||
v: array<vec4<u32>, 6>,
|
||||
}
|
||||
|
||||
@group(0) @binding(0) var<uniform> p: P;
|
||||
@group(0) @binding(1) var<storage, read> a: array<f32>;
|
||||
@group(0) @binding(2) var<storage, read> b: array<f32>;
|
||||
@group(0) @binding(3) var<storage, read> c: array<f32>;
|
||||
@group(0) @binding(4) var<storage, read_write> dst: array<f32>;
|
||||
|
||||
fn q(i: u32) -> u32 {
|
||||
return p.v[i / 4u][i % 4u];
|
||||
}
|
||||
|
||||
fn index(wid: vec3<u32>, lid: u32) -> u32 {
|
||||
return (wid.y * q(1) + wid.x) * 256u + lid;
|
||||
}
|
||||
|
||||
// q(2): 0 relu, 1 sigmoid
|
||||
@compute @workgroup_size(256)
|
||||
fn unary(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
let x = a[i];
|
||||
if (q(2) == 0u) {
|
||||
dst[i] = max(x, 0.0);
|
||||
} else {
|
||||
dst[i] = 1.0 / (1.0 + exp(-x));
|
||||
}
|
||||
}
|
||||
|
||||
// q(2) plane, q(3) channels: x * b[c] + c[c]
|
||||
@compute @workgroup_size(256)
|
||||
fn affine(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
let ch = (i / q(2)) % q(3);
|
||||
dst[i] = a[i] * b[ch] + c[ch];
|
||||
}
|
||||
|
||||
// q(2) plane, q(3) channels: x < 0 ? x * b[c] : x
|
||||
@compute @workgroup_size(256)
|
||||
fn prelu(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
let x = a[i];
|
||||
let ch = (i / q(2)) % q(3);
|
||||
dst[i] = select(x, x * b[ch], x < 0.0);
|
||||
}
|
||||
|
||||
// q(2): 0 add, 1 sub, 2 mul; q(3..7) output dims; q(7..11) a's strides; q(11..15) b's strides
|
||||
@compute @workgroup_size(256)
|
||||
fn binary(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
var rest = i;
|
||||
var ia = 0u;
|
||||
var ib = 0u;
|
||||
for (var d = 3; d >= 0; d = d - 1) {
|
||||
let du = u32(d);
|
||||
let size = q(3u + du);
|
||||
let at = rest % size;
|
||||
rest = rest / size;
|
||||
ia = ia + at * q(7u + du);
|
||||
ib = ib + at * q(11u + du);
|
||||
}
|
||||
let x = a[ia];
|
||||
let y = b[ib];
|
||||
if (q(2) == 0u) {
|
||||
dst[i] = x + y;
|
||||
} else if (q(2) == 1u) {
|
||||
dst[i] = x - y;
|
||||
} else {
|
||||
dst[i] = x * y;
|
||||
}
|
||||
}
|
||||
|
||||
// q(3..7) output dims; q(7..11) the input's strides for each output axis
|
||||
@compute @workgroup_size(256)
|
||||
fn transpose(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
var rest = i;
|
||||
var at_in = 0u;
|
||||
for (var d = 3; d >= 0; d = d - 1) {
|
||||
let du = u32(d);
|
||||
let size = q(3u + du);
|
||||
at_in = at_in + (rest % size) * q(7u + du);
|
||||
rest = rest / size;
|
||||
}
|
||||
dst[i] = a[at_in];
|
||||
}
|
||||
|
||||
// q(2) h, q(3) w, q(4) kh, q(5) kw, q(6) sh, q(7) sw, q(8) oh, q(9) ow
|
||||
@compute @workgroup_size(256)
|
||||
fn maxpool(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
let h = q(2);
|
||||
let w = q(3);
|
||||
let oh = q(8);
|
||||
let ow = q(9);
|
||||
let ox = i % ow;
|
||||
let oy = (i / ow) % oh;
|
||||
let ch = i / (ow * oh);
|
||||
var m = -3.402823e38;
|
||||
for (var ky = 0u; ky < q(4); ky = ky + 1u) {
|
||||
for (var kx = 0u; kx < q(5); kx = kx + 1u) {
|
||||
let y = oy * q(6) + ky;
|
||||
let x = ox * q(7) + kx;
|
||||
if (y < h && x < w) {
|
||||
m = max(m, a[(ch * h + y) * w + x]);
|
||||
}
|
||||
}
|
||||
}
|
||||
dst[i] = m;
|
||||
}
|
||||
|
||||
// q(2) h, q(3) w, q(4) sh, q(5) sw: nearest upscale by whole numbers
|
||||
@compute @workgroup_size(256)
|
||||
fn resize(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let i = index(wid, lid);
|
||||
if (i >= q(0)) { return; }
|
||||
let h = q(2);
|
||||
let w = q(3);
|
||||
let ow = w * q(5);
|
||||
let oh = h * q(4);
|
||||
let ox = i % ow;
|
||||
let oy = (i / ow) % oh;
|
||||
let ch = i / (ow * oh);
|
||||
dst[i] = a[(ch * h + oy / q(4)) * w + ox / q(5)];
|
||||
}
|
||||
|
||||
var<workgroup> partial: array<f32, 256>;
|
||||
|
||||
fn reduce(lid: u32) {
|
||||
for (var s = 128u; s > 0u; s = s >> 1u) {
|
||||
workgroupBarrier();
|
||||
if (lid < s) {
|
||||
partial[lid] = partial[lid] + partial[lid + s];
|
||||
}
|
||||
}
|
||||
workgroupBarrier();
|
||||
}
|
||||
|
||||
// one workgroup per channel; q(2) plane
|
||||
@compute @workgroup_size(256)
|
||||
fn gavg(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let ch = wid.y * q(1) + wid.x;
|
||||
let plane = q(2);
|
||||
var sum = 0.0;
|
||||
if (ch < q(0)) {
|
||||
for (var j = lid; j < plane; j = j + 256u) {
|
||||
sum = sum + a[ch * plane + j];
|
||||
}
|
||||
}
|
||||
partial[lid] = sum;
|
||||
reduce(lid);
|
||||
if (lid == 0u && ch < q(0)) {
|
||||
dst[ch] = partial[0] / f32(plane);
|
||||
}
|
||||
}
|
||||
|
||||
// one workgroup per output; q(2) k, q(3) n, q(4) 1 when B is [n][k], q(5) alpha's bits
|
||||
@compute @workgroup_size(256)
|
||||
fn gemv(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) lid: u32) {
|
||||
let o = wid.y * q(1) + wid.x;
|
||||
let k = q(2);
|
||||
let n = q(3);
|
||||
var sum = 0.0;
|
||||
if (o < q(0)) {
|
||||
for (var j = lid; j < k; j = j + 256u) {
|
||||
let wi = select(j * n + o, o * k + j, q(4) == 1u);
|
||||
sum = sum + a[j] * b[wi];
|
||||
}
|
||||
}
|
||||
partial[lid] = sum;
|
||||
reduce(lid);
|
||||
if (lid == 0u && o < q(0)) {
|
||||
dst[o] = bitcast<f32>(q(5)) * partial[0] + c[o];
|
||||
}
|
||||
}
|
||||
@@ -277,6 +277,10 @@ fn decode_item(ctx: &Context<'_>, id: u32, options: &Options, with_alpha: bool)
|
||||
})
|
||||
}
|
||||
|
||||
/// How many times `max_pixels` the coded pictures of one item may hold together: a grid's tiles
|
||||
/// cover its canvas rounded up to whole tiles, so legitimate grids sit near 1.
|
||||
const GRID_BUDGET: u64 = 2;
|
||||
|
||||
/// The decoded pictures of an item: one, or a grid's tiles in `dimg` order.
|
||||
struct Coded<'a> {
|
||||
frames: Vec<Frame>,
|
||||
@@ -297,6 +301,24 @@ impl<'a> Coded<'a> {
|
||||
let first = *ids.first().ok_or_else(|| Error::Malformed("grid lists no tiles".into()))?;
|
||||
let first_props = if first == id { p.clone() } else { ctx.props(first).map_err(err)? };
|
||||
let chroma_format = first_props.hvcc.as_ref().map_or(1, |c| c.chroma_format);
|
||||
// #1518: the decoder allocates each picture at its SPS's uncropped size, which a
|
||||
// conformance window (or a slice picking another SPS) can make far larger than the
|
||||
// declared one, and a grid holds all its tiles at once. Both are refused before decoding.
|
||||
let mut total = 0u64;
|
||||
for t in &ids {
|
||||
let tp = ctx.props(*t).map_err(err)?;
|
||||
let hvcc = tp.hvcc.as_ref().ok_or(heic_rs::Error::MissingBox("hvcC")).map_err(err)?;
|
||||
let sizes = vui::coded_sizes(&hvcc.parameter_sets()).ok_or_else(|| Error::Malformed("unreadable sequence parameter set".into()))?;
|
||||
let mut largest = 0u64;
|
||||
for (w, h) in sizes {
|
||||
heic_rs::image::check_pixels(w, h, limit).map_err(err)?;
|
||||
largest = largest.max(u64::from(w) * u64::from(h));
|
||||
}
|
||||
total = total.saturating_add(largest);
|
||||
}
|
||||
if total > limit.saturating_mul(GRID_BUDGET) {
|
||||
return Err(Error::Limit(format!("the coded pictures hold {total} pixels, over the {limit}-pixel limit")));
|
||||
}
|
||||
let decode_one = |t: &u32| -> Result<Frame, Error> {
|
||||
let tp = ctx.props(*t).map_err(err)?;
|
||||
let hvcc = tp.hvcc.as_ref().ok_or(heic_rs::Error::MissingBox("hvcC")).map_err(err)?;
|
||||
|
||||
@@ -146,6 +146,17 @@ fn limits_are_checked_before_decoding() {
|
||||
assert!(matches!(r, Err(Error::Limit(_))), "{r:?}");
|
||||
}
|
||||
|
||||
/// #1518: a grid's tiles together are held to the pixel budget before any is decoded. A
|
||||
/// 33 × 33 canvas fits 1089 pixels, but its four 32 × 32 tiles hold 4096: over twice the limit.
|
||||
#[test]
|
||||
fn grid_tiles_are_held_to_the_pixel_budget_together() {
|
||||
let bytes = build(&Spec::new(33, 33, &grey));
|
||||
let r = decode(&bytes, &Options { max_pixels: 33 * 33, ..Default::default() });
|
||||
assert!(matches!(r, Err(Error::Limit(_))), "{r:?}");
|
||||
let d = decode(&bytes, &Options { max_pixels: 2 * 32 * 32, ..Default::default() }).unwrap();
|
||||
assert_eq!((d.width, d.height), (33, 33));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_and_corrupted_files_are_errors_not_panics() {
|
||||
let bytes = build(&Spec { props: vec![FULL_601, Prop::Irot(1)], thumbnail: Some(&grey), ..Spec::new(64, 64, &grey) });
|
||||
|
||||
+36
-13
@@ -33,6 +33,41 @@ pub fn signal(parameter_sets: &[&[u8]]) -> Option<Signal> {
|
||||
parse_sps(&unescape(sps.get(2..)?))
|
||||
}
|
||||
|
||||
/// The uncropped picture size (`pic_width/height_in_luma_samples`) of every SPS among
|
||||
/// `parameter_sets`: what the decoder allocates before it applies the conformance window, so the
|
||||
/// pixel budget is checked against it (heic-rs reports only the first SPS, cropped). `None`: an
|
||||
/// SPS this reader can't follow.
|
||||
pub fn coded_sizes(parameter_sets: &[&[u8]]) -> Option<Vec<(u32, u32)>> {
|
||||
parameter_sets
|
||||
.iter()
|
||||
.filter(|n| n.first().is_some_and(|b| (b >> 1) & 0x3f == SPS_NUT))
|
||||
.map(|sps| {
|
||||
let rbsp = unescape(sps.get(2..)?);
|
||||
let r = &mut Bits { data: &rbsp, pos: 0 };
|
||||
sps_size(r).map(|(w, h, _)| (w, h))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Reads an SPS up to and including the picture size, leaving `r` after it: width, height and
|
||||
/// `sps_max_sub_layers_minus1`.
|
||||
fn sps_size(r: &mut Bits) -> Option<(u32, u32, u32)> {
|
||||
r.skip(4)?; // sps_video_parameter_set_id
|
||||
let max_sub_layers_minus1 = r.u(3)?;
|
||||
if max_sub_layers_minus1 > 6 {
|
||||
return None;
|
||||
}
|
||||
r.skip(1)?; // sps_temporal_id_nesting_flag
|
||||
profile_tier_level(r, max_sub_layers_minus1)?;
|
||||
r.ue()?; // sps_seq_parameter_set_id
|
||||
if r.ue()? == 3 {
|
||||
r.skip(1)?; // separate_colour_plane_flag
|
||||
}
|
||||
let width = r.ue()?; // pic_width_in_luma_samples
|
||||
let height = r.ue()?; // pic_height_in_luma_samples
|
||||
Some((width, height, max_sub_layers_minus1))
|
||||
}
|
||||
|
||||
/// Removes emulation prevention bytes (`00 00 03` → `00 00`).
|
||||
fn unescape(nal: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(nal.len());
|
||||
@@ -165,19 +200,7 @@ fn st_ref_pic_set(r: &mut Bits, idx: usize, num_delta_pocs: &[u32]) -> Option<u3
|
||||
|
||||
fn parse_sps(rbsp: &[u8]) -> Option<Signal> {
|
||||
let r = &mut Bits { data: rbsp, pos: 0 };
|
||||
r.skip(4)?; // sps_video_parameter_set_id
|
||||
let max_sub_layers_minus1 = r.u(3)?;
|
||||
if max_sub_layers_minus1 > 6 {
|
||||
return None;
|
||||
}
|
||||
r.skip(1)?; // sps_temporal_id_nesting_flag
|
||||
profile_tier_level(r, max_sub_layers_minus1)?;
|
||||
r.ue()?; // sps_seq_parameter_set_id
|
||||
if r.ue()? == 3 {
|
||||
r.skip(1)?; // separate_colour_plane_flag
|
||||
}
|
||||
r.ue()?; // pic_width_in_luma_samples
|
||||
r.ue()?; // pic_height_in_luma_samples
|
||||
let (_, _, max_sub_layers_minus1) = sps_size(r)?;
|
||||
if r.flag()? {
|
||||
for _ in 0..4 {
|
||||
r.ue()?; // conf_win_*_offset
|
||||
|
||||
+75
-9
@@ -185,11 +185,22 @@ fn entity(name: &str) -> Option<char> {
|
||||
Some(h) => u32::from_str_radix(h, 16).ok()?,
|
||||
None => n.parse().ok()?,
|
||||
};
|
||||
char::from_u32(v)?
|
||||
char::from_u32(v).filter(|&c| is_xml_char(c))?
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// The XML 1.0 `Char` production (the surrogate range can't occur in a `char`).
|
||||
fn is_xml_char(c: char) -> bool {
|
||||
matches!(c, '\t' | '\n' | '\r') || (c >= '\u{20}' && c != '\u{FFFE}' && c != '\u{FFFF}')
|
||||
}
|
||||
|
||||
/// Drop the characters XML 1.0 forbids, which strict parsers (Adobe XMP Toolkit, expat, libxml2) reject the
|
||||
/// whole document for.
|
||||
fn xml_safe(s: &str) -> String {
|
||||
s.chars().filter(|&c| is_xml_char(c)).collect()
|
||||
}
|
||||
|
||||
/// Unescape attribute values (predefined + numeric references; unknown references kept verbatim).
|
||||
fn unescape_attr(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
@@ -506,10 +517,7 @@ pub fn parse_xmp(s: &str) -> Result<XmpData, XmpError> {
|
||||
label: first("xmp:Label"),
|
||||
..Default::default()
|
||||
};
|
||||
m.iso = num("exif:ISOSpeedRatings").or_else(|| num("exifEX:PhotographicSensitivity")).filter(|v| *v > 0.0).map(|v| v as u32);
|
||||
if let Some(v) = props.get("exif:ISOSpeedRatings").and_then(|v| v.first()).and_then(|s| parse_number(s)) {
|
||||
m.iso = Some(v as u32);
|
||||
}
|
||||
m.iso = [num("exif:ISOSpeedRatings"), num("exifEX:PhotographicSensitivity")].into_iter().flatten().find(|v| *v > 0.0).map(|v| v as u32);
|
||||
if let Some(spec) = props.get("exifEX:LensSpecification").filter(|v| v.len() == 4) {
|
||||
let v: Vec<f64> = spec.iter().filter_map(|s| parse_number(s)).collect();
|
||||
if v.len() == 4 {
|
||||
@@ -536,6 +544,8 @@ pub fn parse_xmp(s: &str) -> Result<XmpData, XmpError> {
|
||||
}
|
||||
if let (Some(lat), Some(lon)) =
|
||||
(first("exif:GPSLatitude").and_then(|s| parse_gps_coord(&s)), first("exif:GPSLongitude").and_then(|s| parse_gps_coord(&s)))
|
||||
&& (-90.0..=90.0).contains(&lat)
|
||||
&& (-180.0..=180.0).contains(&lon)
|
||||
{
|
||||
let altitude = num("exif:GPSAltitude").map(|a| if first("exif:GPSAltitudeRef").as_deref() == Some("1") { -a } else { a });
|
||||
m.gps = Some(Gps { latitude: lat, longitude: lon, altitude });
|
||||
@@ -623,12 +633,12 @@ pub fn write_xmp_lc(meta: &Metadata, lc: &[(&str, &str)]) -> String {
|
||||
}
|
||||
x.push_str(">\n");
|
||||
for (k, v) in &simple {
|
||||
x.push_str(&format!(" <{k}>{}</{k}>\n", escape(v.as_str())));
|
||||
x.push_str(&format!(" <{k}>{}</{k}>\n", escape(xml_safe(v))));
|
||||
}
|
||||
for (k, v) in lc {
|
||||
if !k.is_empty() && k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
|
||||
// element content: quotes needn't be escaped, which keeps the JSON readable
|
||||
x.push_str(&format!(" <lc:{k}>{}</lc:{k}>\n", partial_escape(*v)));
|
||||
x.push_str(&format!(" <lc:{k}>{}</lc:{k}>\n", partial_escape(xml_safe(v))));
|
||||
}
|
||||
}
|
||||
let array = |x: &mut String, k: &str, kind: &str, items: &[String], lang: bool| {
|
||||
@@ -638,9 +648,9 @@ pub fn write_xmp_lc(meta: &Metadata, lc: &[(&str, &str)]) -> String {
|
||||
x.push_str(&format!(" <{k}>\n <rdf:{kind}>\n"));
|
||||
for it in items {
|
||||
if lang {
|
||||
x.push_str(&format!(" <rdf:li xml:lang=\"x-default\">{}</rdf:li>\n", escape(it.as_str())));
|
||||
x.push_str(&format!(" <rdf:li xml:lang=\"x-default\">{}</rdf:li>\n", escape(xml_safe(it))));
|
||||
} else {
|
||||
x.push_str(&format!(" <rdf:li>{}</rdf:li>\n", escape(it.as_str())));
|
||||
x.push_str(&format!(" <rdf:li>{}</rdf:li>\n", escape(xml_safe(it))));
|
||||
}
|
||||
}
|
||||
x.push_str(&format!(" </rdf:{kind}>\n </{k}>\n"));
|
||||
@@ -802,6 +812,24 @@ mod tests {
|
||||
assert_eq!(read("True").copyright_url.as_deref(), Some("https://example.org/c"));
|
||||
}
|
||||
|
||||
/// A zero or negative `exif:ISOSpeedRatings` is "no ISO" (as in the EXIF reader), not `Some(0)`, and
|
||||
/// must not hide a valid `exifEX:PhotographicSensitivity`.
|
||||
#[test]
|
||||
fn non_positive_iso_is_ignored() {
|
||||
let read = |attrs: &str| {
|
||||
let x = format!(
|
||||
r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about="" xmlns:exif="http://ns.adobe.com/exif/1.0/" xmlns:exifEX="http://cipa.jp/exif/1.0/" {attrs}/>
|
||||
</rdf:RDF></x:xmpmeta>"#
|
||||
);
|
||||
parse_xmp(&x).unwrap().metadata.iso
|
||||
};
|
||||
assert_eq!(read(r#"exif:ISOSpeedRatings="0" exifEX:PhotographicSensitivity="400""#), Some(400));
|
||||
assert_eq!(read(r#"exif:ISOSpeedRatings="-100""#), None);
|
||||
assert_eq!(read(r#"exif:ISOSpeedRatings="0""#), None);
|
||||
assert_eq!(read(r#"exif:ISOSpeedRatings="200""#), Some(200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_inputs() {
|
||||
assert!(parse_xmp("<a><b></a>").is_ok() || parse_xmp("<a><b></a>").is_err());
|
||||
@@ -840,6 +868,24 @@ mod tests {
|
||||
assert_eq!(fmt_gps_coord(-45.51, 'N', 'S'), "45,30.600000S");
|
||||
}
|
||||
|
||||
/// Like the EXIF and GPX readers, XMP drops a position outside latitude ±90° / longitude ±180°.
|
||||
#[test]
|
||||
fn xmp_gps_out_of_range_rejected() {
|
||||
let gps = |lat: &str, lon: &str| {
|
||||
let x = format!(
|
||||
r#"<rdf:Description xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:e="http://ns.adobe.com/exif/1.0/" e:GPSLatitude="{lat}" e:GPSLongitude="{lon}"/>"#
|
||||
);
|
||||
parse_xmp(&x).unwrap().metadata.gps
|
||||
};
|
||||
assert_eq!(gps("95,0N", "200,30E"), None);
|
||||
assert_eq!(gps("95,0N", "10,0E"), None);
|
||||
assert_eq!(gps("10,0S", "200,30W"), None);
|
||||
assert_eq!(gps("90,0,1N", "0,0E"), None);
|
||||
assert_eq!(gps("90,0N", "180,0E"), Some(Gps { latitude: 90.0, longitude: 180.0, altitude: None }));
|
||||
assert_eq!(gps("90,0S", "180,0W"), Some(Gps { latitude: -90.0, longitude: -180.0, altitude: None }));
|
||||
}
|
||||
|
||||
/// MWG Region Guidelines v2.0, element form (`rdf:parseType="Resource"`, self-closing `Area`) — how
|
||||
/// Lightroom typically writes it. One named face, one untyped/unnamed region (still usable), one
|
||||
/// region with a description. `AppliedToDimensions` is present but irrelevant: normalized areas
|
||||
@@ -1039,4 +1085,24 @@ mod tests {
|
||||
a.overlay_user_fields(&b);
|
||||
assert_eq!(a.regions, vec![with_region]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writer_drops_characters_xml_1_0_forbids() {
|
||||
let bad = "a\u{1}b\u{0}c\u{FFFE}d\u{FFFF}e\u{1F}\tf\ng\rh";
|
||||
let m = Metadata { make: Some(bad.into()), caption: Some(bad.into()), keywords: vec![bad.into()], ..Default::default() };
|
||||
let x = write_xmp_lc(&m, &[("settings", bad)]);
|
||||
let illegal: Vec<char> =
|
||||
x.chars().filter(|&c| (c < '\u{20}' && !matches!(c, '\t' | '\n' | '\r')) || c == '\u{FFFE}' || c == '\u{FFFF}').collect();
|
||||
assert!(illegal.is_empty(), "illegal XML chars in output: {illegal:?}");
|
||||
assert!(x.contains(">abcde\tf\ng\rh<"), "allowed characters must survive: {x}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_references_to_forbidden_characters_are_not_decoded() {
|
||||
assert_eq!(entity("#1"), None);
|
||||
assert_eq!(entity("#0"), None);
|
||||
assert_eq!(entity("#xFFFE"), None);
|
||||
assert_eq!(entity("#x41"), Some('A'));
|
||||
assert_eq!(entity("#9"), Some('\t'));
|
||||
}
|
||||
}
|
||||
@@ -110,7 +110,7 @@ pub enum RawFormat {
|
||||
X3f,
|
||||
/// A TIFF-based raw that has no DNG version tag but describes itself the way DNG does (a
|
||||
/// full-resolution CFA image with black/white levels, a default crop and `AsShotNeutral`):
|
||||
/// Hasselblad 3FR and FFF. Read by the DNG reader.
|
||||
/// Hasselblad 3FR and FFF. Read by `vendor::hasselblad` (the DNG reader plus what Hasselblad adds).
|
||||
CfaTiff,
|
||||
/// Kodak DCS520C TIFF: a two-component lossless JPEG CFA sub-IFD with a response curve (`vendor::kodak`).
|
||||
KodakDcs,
|
||||
@@ -397,7 +397,7 @@ fn decode_with(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
RawFormat::Rw2 => vendor::rw2::decode(bytes, mode),
|
||||
RawFormat::Pef => vendor::pef::decode(bytes, mode),
|
||||
RawFormat::Orf => vendor::orf::decode(bytes, mode),
|
||||
RawFormat::CfaTiff => dng::decode_as(bytes, mode, RawFormat::CfaTiff).map(lift_clipped),
|
||||
RawFormat::CfaTiff => vendor::hasselblad::decode(bytes, mode).map(lift_clipped),
|
||||
RawFormat::Srw => vendor::srw::decode(bytes, mode),
|
||||
RawFormat::KodakDcs => vendor::kodak::decode(bytes, mode),
|
||||
RawFormat::Erf => vendor::erf::decode(bytes, mode),
|
||||
|
||||
@@ -148,8 +148,8 @@ fn candidates<'a>(data: &'a [u8], ifd: &Ifd, base: u64, out: &mut Vec<&'a [u8]>)
|
||||
}
|
||||
}
|
||||
|
||||
/// The largest embedded preview, if any: a JPEG, or (DNG 1.7) a JPEG XL file — both decode with
|
||||
/// `lightcraft_codecs::decode`.
|
||||
/// The largest embedded preview, if any: a JPEG, or (DNG 1.7) a JPEG XL file, or for a Hasselblad 3FR / FFF, which
|
||||
/// has no JPEG, its uncompressed RGB preview wrapped as a baseline TIFF — all decode with `lightcraft_codecs::decode`.
|
||||
pub fn embedded_preview(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
preview_covering(bytes, None)
|
||||
}
|
||||
@@ -220,6 +220,8 @@ fn preview_covering(bytes: &[u8], min_edge: Option<u32>) -> Option<Vec<u8>> {
|
||||
.map(|s| if is_jxl(s) { s.to_vec() } else { trim_eoi(s).to_vec() });
|
||||
// a TIFF raw whose preview no tag points to: one we can't decode (Leaf MOS), or Epson ERF (in its maker note)
|
||||
best.or_else(|| format.filter(|f| !f.is_supported() || *f == crate::RawFormat::Erf).and_then(|_| scan_for_jpeg(bytes)))
|
||||
// Hasselblad 3FR / FFF carry no JPEG, only uncompressed RGB images: one wrapped as a baseline TIFF
|
||||
.or_else(|| (format == Some(crate::RawFormat::CfaTiff)).then(|| crate::vendor::hasselblad::rgb_preview(&tiff, bytes)).flatten())
|
||||
}
|
||||
|
||||
/// The end of the DCT JPEG whose SOI is at `soi` (exclusive index after its EOI) and its component count,
|
||||
|
||||
@@ -75,6 +75,21 @@ impl Ranges {
|
||||
self.0 = out;
|
||||
}
|
||||
|
||||
/// Add every `[start, end)` of `spans` in one sort and merge (empty ones add nothing).
|
||||
pub fn extend(&mut self, spans: impl IntoIterator<Item = (u64, u64)>) {
|
||||
let mut all: Vec<(u64, u64)> = std::mem::take(&mut self.0);
|
||||
all.extend(spans.into_iter().filter(|&(a, b)| a < b));
|
||||
all.sort_unstable();
|
||||
let mut out: Vec<(u64, u64)> = Vec::with_capacity(all.len());
|
||||
for (a, b) in all {
|
||||
match out.last_mut() {
|
||||
Some(last) if a <= last.1 => last.1 = last.1.max(b),
|
||||
_ => out.push((a, b)),
|
||||
}
|
||||
}
|
||||
self.0 = out;
|
||||
}
|
||||
|
||||
/// Whether all of `[start, end)` is covered (an empty range always is).
|
||||
pub fn contains(&self, start: u64, end: u64) -> bool {
|
||||
start >= end || self.0.iter().any(|&(a, b)| a <= start && end <= b)
|
||||
@@ -232,6 +247,7 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
|
||||
}
|
||||
let Ok(tiff) = Tiff::parse(bytes) else { return HeaderPlan::Whole("TIFF structure") };
|
||||
let mut holes = Ranges::new();
|
||||
let mut spans = Vec::new();
|
||||
for ifd in tiff.all_ifds() {
|
||||
let chunks: Vec<(u64, u64)> = match ifd.image() {
|
||||
// a preview's strips are read when the preview is wanted: only the raw image is skipped
|
||||
@@ -249,11 +265,12 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
|
||||
if total < BULK {
|
||||
continue;
|
||||
}
|
||||
for (i, &(offset, size)) in chunks.iter().enumerate() {
|
||||
spans.extend(chunks.iter().enumerate().map(|(i, &(offset, size))| {
|
||||
let start = if i == 0 { offset.saturating_add(HEAD) } else { offset };
|
||||
holes.insert(start, offset.saturating_add(size).min(len));
|
||||
}
|
||||
(start, offset.saturating_add(size).min(len))
|
||||
}));
|
||||
}
|
||||
holes.extend(spans);
|
||||
// A Panasonic RW2 keeps its raw data from `RawDataOffset` to the end, outside any TIFF image.
|
||||
if holes.is_empty()
|
||||
&& bytes.starts_with(b"IIU\0")
|
||||
@@ -567,4 +584,40 @@ mod tests {
|
||||
assert!(crate::probe_info(&file).is_ok());
|
||||
assert_eq!(run(&file, 16, 0).0, HeaderPlan::Whole("an ARW6 header probe reads its tiles"));
|
||||
}
|
||||
|
||||
/// A TIFF with a million one-byte strips listed in descending order and all byte counts 0: the plan is
|
||||
/// built in one sort and merge, not one pass over all strips (or all holes) per strip.
|
||||
#[test]
|
||||
fn many_strips_without_byte_counts_plan_fast() {
|
||||
let n = 1_000_000u32;
|
||||
let (offsets_at, counts_at) = (8 + 2 + 5 * 12 + 4, 8 + 2 + 5 * 12 + 4 + 4 * n);
|
||||
let data = counts_at + 4 * n;
|
||||
let mut tiff = b"II*\0\x08\0\0\0".to_vec();
|
||||
tiff.extend_from_slice(&5u16.to_le_bytes());
|
||||
let entry = |tiff: &mut Vec<u8>, tag: u16, typ: u16, count: u32, value: u32| {
|
||||
tiff.extend_from_slice(&tag.to_le_bytes());
|
||||
tiff.extend_from_slice(&typ.to_le_bytes());
|
||||
tiff.extend_from_slice(&count.to_le_bytes());
|
||||
tiff.extend_from_slice(&value.to_le_bytes());
|
||||
};
|
||||
entry(&mut tiff, 256, 4, 1, 1);
|
||||
entry(&mut tiff, 257, 4, 1, n);
|
||||
entry(&mut tiff, 273, 4, n, offsets_at);
|
||||
entry(&mut tiff, 278, 4, 1, 1);
|
||||
entry(&mut tiff, 279, 4, n, counts_at);
|
||||
tiff.extend_from_slice(&0u32.to_le_bytes());
|
||||
for i in (0..n).rev() {
|
||||
tiff.extend_from_slice(&(data + i).to_le_bytes());
|
||||
}
|
||||
tiff.resize(data as usize + n as usize, 0);
|
||||
let len = tiff.len() as u64;
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut loaded = Ranges::new();
|
||||
loaded.insert(0, len);
|
||||
let _ = tx.send(header_plan(&tiff, &loaded));
|
||||
});
|
||||
let plan = rx.recv_timeout(std::time::Duration::from_secs(20)).expect("header_plan took longer than 20 s");
|
||||
assert!(!matches!(plan, HeaderPlan::Need(_)), "{plan:?}");
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,8 @@
|
||||
//! `tools/spectral_camera_table.py` from rawtoaces-data (Academy Software Foundation, Apache-2.0): per model, DNG
|
||||
//! `ColorMatrix1/2` and `ForwardMatrix1/2` for Standard Illuminant A and D65, fitted to the camera's measured spectral
|
||||
//! sensitivities. [`Camera::fill`] writes them into a file's [`ColorData`], and [`crate::color`]'s DNG model does the
|
||||
//! rest. Other names of a covered camera (Rebel T3i for EOS 600D) come from rawtoaces-data's alias list. No number
|
||||
//! rest. Other names of a covered camera (Rebel T3i for EOS 600D) come from rawtoaces-data's alias list, and other
|
||||
//! bodies built on a covered camera's sensor from [`SAME_SENSOR`]. No number
|
||||
//! comes from Adobe or from a GPL raw decoder. Which colour a raw starts from is the engine's choice
|
||||
//! (`docs/camera-preview-colour.md`).
|
||||
use crate::ColorData;
|
||||
@@ -40,6 +41,19 @@ impl Camera {
|
||||
}
|
||||
}
|
||||
|
||||
/// `(make, model as sold, table model)`: bodies sharing a table camera's sensor that rawtoaces-data's alias list
|
||||
/// doesn't name, kept here because the table is generated. Fujifilm's 26.1 MP X-Trans CMOS 4 bodies use the X-T3's
|
||||
/// sensor (X-T30 issue: a black-and-white Acros camera JPEG gives no colour to fit, and without matrices the photo
|
||||
/// opened in the muted neutral fallback).
|
||||
const SAME_SENSOR: &[(&str, &str, &str)] = &[
|
||||
("Fujifilm", "X-T30", "X-T3"),
|
||||
("Fujifilm", "X-T30 II", "X-T3"),
|
||||
("Fujifilm", "X-Pro3", "X-T3"),
|
||||
("Fujifilm", "X100V", "X-T3"),
|
||||
("Fujifilm", "X-S10", "X-T3"),
|
||||
("Fujifilm", "X-E4", "X-T3"),
|
||||
];
|
||||
|
||||
/// Model names compared without case, the brand, a leading `EOS` or a trailing `DIGITAL`
|
||||
/// (Exif "Canon EOS 400D DIGITAL" is the table's "EOS 400D").
|
||||
fn key(brand: &str, model: &str) -> String {
|
||||
@@ -70,7 +84,7 @@ pub fn find(make: Option<&str>, model: &str) -> Option<&'static Camera> {
|
||||
let wanted = key(&row_brand, model);
|
||||
make.as_ref().is_none_or(|m| *m == row_brand) && !wanted.is_empty() && key(&row_brand, row_model) == wanted
|
||||
};
|
||||
match table::ALIASES.iter().find(|(m, alias, _)| same(m, alias)) {
|
||||
match table::ALIASES.iter().chain(SAME_SENSOR).find(|(m, alias, _)| same(m, alias)) {
|
||||
Some((m, _, target)) => table::CAMERAS.iter().find(|c| brand(c.make) == brand(m) && c.model == *target),
|
||||
None => table::CAMERAS.iter().find(|c| same(c.make, c.model)),
|
||||
}
|
||||
@@ -142,10 +156,14 @@ mod tests {
|
||||
("Canon", "Canon EOS REBEL T3i", "EOS 600D"),
|
||||
("Canon", "Canon EOS Kiss X5", "EOS 600D"),
|
||||
("Canon", "Canon EOS R5 Mark II", "EOS R5m2"),
|
||||
("Canon", "Canon EOS R6 Mark II", "EOS R6m2"),
|
||||
("NIKON CORPORATION", "NIKON D800", "D800E"),
|
||||
("NIKON CORPORATION", "NIKON Z 6_2", "Z f"),
|
||||
("SONY", "ILCE-7RM4A", "ILCE-7RM4"),
|
||||
("FUJIFILM", "GFX100S", "GFX 100"),
|
||||
("FUJIFILM", "X-T30", "X-T3"),
|
||||
("FUJIFILM", "X-T30 II", "X-T3"),
|
||||
("FUJIFILM", "X100V", "X-T3"),
|
||||
] {
|
||||
assert_eq!(find(Some(make), model).map(|c| c.model), Some(row), "{make} {model}");
|
||||
}
|
||||
@@ -171,7 +189,7 @@ mod tests {
|
||||
assert_eq!(find_camera(&joined), Some(c), "{joined}");
|
||||
assert_eq!(find_camera(&joined), find(Some(c.make), c.model), "{joined}");
|
||||
}
|
||||
for (make, alias, _) in table::ALIASES {
|
||||
for (make, alias, _) in table::ALIASES.iter().chain(SAME_SENSOR) {
|
||||
let joined = format!("{make} {alias}");
|
||||
assert_eq!(find_camera(&joined), find(Some(make), alias), "{joined}");
|
||||
assert!(find_camera(&joined).is_some(), "{joined}");
|
||||
|
||||
@@ -335,6 +335,7 @@ pub(super) const ALIASES: &[(&str, &str, &str)] = &[
|
||||
("Canon", "EOS Rebel T3i", "EOS 600D"), // same model, alternative name
|
||||
("Canon", "Kiss X5", "EOS 600D"), // same model, alternative name
|
||||
("Canon", "EOS R5 Mark II", "EOS R5m2"), // same model, alternative name
|
||||
("Canon", "EOS R6 Mark II", "EOS R6m2"), // same model, alternative name
|
||||
("Canon", "EOS 5DS R", "EOS 5DS"), // different model, same sensor
|
||||
("Canon", "EOS 6D Mark II", "EOS RP"), // different model, same sensor
|
||||
("Canon", "EOS R5 C", "EOS R5"), // different model, same sensor
|
||||
|
||||
Vendored
+295
@@ -0,0 +1,295 @@
|
||||
//! Hasselblad 3FR and FFF raws ([`crate::RawFormat::CfaTiff`]).
|
||||
//!
|
||||
//! Sources: TIFF 6.0 (container), the DNG Specification (the tags these files borrow) and our own black-box
|
||||
//! analysis of a CC0 H4D-40 FFF from raw.pixls.us (931, "Hasselblad - H4D-40 - 16bit (4:3).fff"); every number
|
||||
//! below was measured on it.
|
||||
//!
|
||||
//! - Both are TIFFs without a DNG version tag that describe their raw image with DNG's tags: a full-resolution
|
||||
//! CFA sub-IFD with `BlackLevel`, `WhiteLevel`, `DefaultCropOrigin` / `DefaultCropSize`, and `AsShotNeutral`.
|
||||
//! [`crate::dng::decode_as`] reads all of that; this module adds what that reader does not know about.
|
||||
//! - The raw image is uncompressed (X1D, X1D II, X2D, CFV 100C) or Hasselblad's lossless JPEG with predictor
|
||||
//! selection value 8 ([`crate::ljpeg::decode_hasselblad`]); the H4D-40 FFF's 7410 x 5586 strip is the latter.
|
||||
//! - No `CFAPattern`. The layout is RGGB from the sensor origin: averaged over the 6 x 6 sensor blocks behind each
|
||||
//! pixel of the file's 1217 x 913 RGB IFD0 image (the default crop is exactly six times its size), the sites at
|
||||
//! (0, 0) correlate 0.976 with its red and 0.726 with its blue, the sites at (1, 1) 0.896 with blue and 0.847
|
||||
//! with red, and the two green sites' diagonal differences are half those of the other pair.
|
||||
//! - An FFF is a 3FR converted on the desktop. It has no colour matrix of its own: the camera's matrix stays in
|
||||
//! the original 3FR header, which the FFF carries whole as a little-endian TIFF in the private IFD0 tag
|
||||
//! `0xC51B` (4096 bytes in the sample: an IFD0 with Make, Model, `ColorMatrix1` and `AsShotNeutral`, a
|
||||
//! `SubIFDs` pair for the raw and preview images, whose data offsets point into the old 3FR and are not read).
|
||||
//! [`decode`] takes the matrices from there when the outer file has none. The white balance stays the outer
|
||||
//! file's, which is the one the converter last saved.
|
||||
//! - No JPEG preview. IFD0 is an uncompressed 8-bit RGB image (1217 x 913 in the H4D-40 FFF, IFD1 pointing at the
|
||||
//! same strip; 218 x 218 in the CFV 3FR, under the file's own `Orientation`); the FFF also has a 16-bit RGB
|
||||
//! copy in a `SubIFDs` child, with a `TransferFunction` whose meaning is not known, so it is not used.
|
||||
//! [`rgb_preview`] hands the largest 8-bit one on as a small baseline TIFF.
|
||||
use crate::dng;
|
||||
use crate::{ColorData, Mode, RawFormat, RawImage, Result};
|
||||
use lightcraft_tiff::image::{Layout, chunk_bytes};
|
||||
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, ParseOptions, Tiff, TiffWriter, Value, tags as t};
|
||||
|
||||
/// Private IFD0 tag holding the original 3FR header (a complete TIFF) inside an FFF.
|
||||
pub(crate) const ORIGINAL_HEADER: u16 = 0xc51b;
|
||||
|
||||
/// Upper bound on the embedded header's size: the sample's is 4 KiB; a larger block is not a header.
|
||||
const MAX_HEADER: usize = 1 << 20;
|
||||
|
||||
/// Decode a 3FR / FFF (or another TIFF raw in the DNG layout, see [`dng::is_plain_cfa_tiff`]).
|
||||
pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
let mut img = dng::decode_as(bytes, mode, RawFormat::CfaTiff)?;
|
||||
if !crate::color::has_matrix(&img.color)
|
||||
&& let Some(header) = original_header_color(bytes)
|
||||
{
|
||||
adopt_matrices(&mut img.color, &header);
|
||||
}
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// The colour tags of the original 3FR header an FFF embeds in [`ORIGINAL_HEADER`], when it has a colour matrix.
|
||||
fn original_header_color(bytes: &[u8]) -> Option<ColorData> {
|
||||
let outer = Tiff::parse(bytes).ok()?;
|
||||
let block = outer.ifds.first()?.bytes(ORIGINAL_HEADER)?;
|
||||
if block.len() > MAX_HEADER {
|
||||
return None;
|
||||
}
|
||||
let opts = ParseOptions { max_ifds: 16, max_depth: 2, ..Default::default() };
|
||||
let header = Tiff::parse_with(block, &opts).ok()?;
|
||||
let ifd0 = header.ifds.first()?;
|
||||
let raw = dng::raw_ifd(&header).unwrap_or(ifd0);
|
||||
let color = dng::color_data(ifd0, raw, header.order);
|
||||
crate::color::has_matrix(&color).then_some(color)
|
||||
}
|
||||
|
||||
/// Upper bound on a preview's pixel bytes (8-bit RGB): 64 MP, far above the 1.1 MP of the largest one seen.
|
||||
const MAX_PREVIEW_BYTES: usize = (64 << 20) * 3;
|
||||
|
||||
/// The file's largest uncompressed 8-bit RGB image (chunky, in strips), as a little-endian baseline TIFF holding
|
||||
/// just those pixels, which `lightcraft_codecs::decode` reads. No `Orientation` tag is written: the viewer
|
||||
/// applies the raw file's, which is the one that stands over this image in IFD0.
|
||||
pub(crate) fn rgb_preview(tiff: &Tiff, bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
let info = tiff
|
||||
.all_ifds()
|
||||
.into_iter()
|
||||
.filter_map(|i| i.image().ok())
|
||||
.filter(|i| {
|
||||
i.photometric == t::photometric::RGB
|
||||
&& i.compression == t::compression::NONE
|
||||
&& i.samples_per_pixel == 3
|
||||
&& i.bits_per_sample.iter().all(|&b| b == 8)
|
||||
&& i.planar == 1
|
||||
&& i.sample_format == 1
|
||||
&& matches!(i.layout, Layout::Strips { .. })
|
||||
&& i.width > 0
|
||||
&& i.height > 0
|
||||
})
|
||||
.max_by_key(|i| u64::from(i.width) * u64::from(i.height))?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let size = w.checked_mul(h)?.checked_mul(3)?;
|
||||
if size > MAX_PREVIEW_BYTES {
|
||||
return None;
|
||||
}
|
||||
let mut pixels = Vec::with_capacity(size);
|
||||
for chunk in info.chunks(bytes.len() as u64) {
|
||||
pixels.extend_from_slice(chunk_bytes(bytes, &chunk)?);
|
||||
if pixels.len() >= size {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// a short last strip is a truncated file: no preview rather than a picture with a black band
|
||||
if pixels.len() < size {
|
||||
return None;
|
||||
}
|
||||
pixels.truncate(size);
|
||||
let mut ifd = IfdBuilder::new();
|
||||
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![info.width]));
|
||||
ifd.set(t::IMAGE_LENGTH, Value::Long(vec![info.height]));
|
||||
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![8, 8, 8]));
|
||||
ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3]));
|
||||
ifd.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::RGB]));
|
||||
ifd.set(t::COMPRESSION, Value::Short(vec![t::compression::NONE]));
|
||||
ifd.set(t::PLANAR_CONFIGURATION, Value::Short(vec![1]));
|
||||
ifd.set_image(ImageData::Strips { rows_per_strip: info.height, strips: vec![pixels] });
|
||||
TiffWriter::new(ByteOrder::Little, false).write(&[ifd]).ok()
|
||||
}
|
||||
|
||||
/// Take the camera matrices (and the illuminants they belong to) of `from`; the rest of `color` stays.
|
||||
fn adopt_matrices(color: &mut ColorData, from: &ColorData) {
|
||||
color.illuminant = from.illuminant;
|
||||
color.color_matrix = from.color_matrix;
|
||||
color.forward_matrix = from.forward_matrix;
|
||||
color.camera_calibration = from.camera_calibration;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{RawData, RawError, decode as decode_file, probe, probe_info};
|
||||
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value, tags as t};
|
||||
|
||||
const MATRIX: [(i32, i32); 9] =
|
||||
[(686, 1000), (-196, 1000), (-32, 1000), (-437, 1000), (1228, 1000), (235, 1000), (-165, 1000), (317, 1000), (675, 1000)];
|
||||
|
||||
fn rgb_ifd() -> IfdBuilder {
|
||||
let mut ifd = IfdBuilder::new();
|
||||
ifd.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1]));
|
||||
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![4]));
|
||||
ifd.set(t::IMAGE_LENGTH, Value::Long(vec![3]));
|
||||
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![8, 8, 8]));
|
||||
ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3]));
|
||||
ifd.set(t::PHOTOMETRIC, Value::Short(vec![2]));
|
||||
ifd.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
ifd.set_image(ImageData::Strips { rows_per_strip: 3, strips: vec![(0..36).collect()] });
|
||||
ifd
|
||||
}
|
||||
|
||||
/// The camera's 3FR header as an FFF embeds it: a little-endian TIFF whose IFD0 has the colour matrix.
|
||||
fn original_header(matrix: bool) -> Vec<u8> {
|
||||
let mut ifd0 = rgb_ifd();
|
||||
ifd0.set(t::MAKE, Value::Ascii("Hasselblad".into()));
|
||||
ifd0.set(t::MODEL, Value::Ascii("Hasselblad H4D".into()));
|
||||
if matrix {
|
||||
ifd0.set(t::COLOR_MATRIX_1, Value::SRational(MATRIX.to_vec()));
|
||||
}
|
||||
ifd0.set(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(43, 100), (1, 1), (79, 100)]));
|
||||
TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
/// An FFF in miniature: big-endian, IFD0 an RGB image with `AsShotNeutral` and the embedded header `header`,
|
||||
/// a 16-bit uncompressed CFA sub-IFD of 8 x 6 samples, no colour matrix of its own unless `own_matrix`.
|
||||
fn fff(header: Option<Vec<u8>>, own_matrix: bool) -> Vec<u8> {
|
||||
fff_with(header, own_matrix, vec![])
|
||||
}
|
||||
|
||||
/// [`fff`] with more `SubIFDs` children of IFD0 (other preview images).
|
||||
fn fff_with(header: Option<Vec<u8>>, own_matrix: bool, extra: Vec<IfdBuilder>) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![8]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![6]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set(t::BLACK_LEVEL, Value::Rational(vec![(256, 1)]));
|
||||
raw.set(t::WHITE_LEVEL, Value::Short(vec![65535]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![(0..96u8).collect()] });
|
||||
let mut ifd0 = rgb_ifd();
|
||||
ifd0.set(t::MAKE, Value::Ascii("Hasselblad".into()));
|
||||
ifd0.set(t::MODEL, Value::Ascii("Hasselblad H4D-40".into()));
|
||||
ifd0.set(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(2, 5), (1, 1), (4, 5)]));
|
||||
if own_matrix {
|
||||
ifd0.set(t::COLOR_MATRIX_1, Value::SRational(vec![(1, 1), (0, 1), (0, 1), (0, 1), (1, 1), (0, 1), (0, 1), (0, 1), (1, 1)]));
|
||||
}
|
||||
if let Some(h) = header {
|
||||
ifd0.set(ORIGINAL_HEADER, Value::Undefined(h));
|
||||
}
|
||||
ifd0.add_sub_ifd(raw);
|
||||
for e in extra {
|
||||
ifd0.add_sub_ifd(e);
|
||||
}
|
||||
TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
fn matrix_row0(c: &ColorData) -> Option<[f64; 3]> {
|
||||
c.color_matrix[0].map(|m| m.0[0])
|
||||
}
|
||||
|
||||
/// The FFF's own tags give the samples and white balance; the colour matrix comes from the 3FR header it
|
||||
/// embeds, so the image is not developed with the "camera RGB ≈ sRGB" fallback.
|
||||
#[test]
|
||||
fn fff_takes_the_colour_matrix_from_the_embedded_3fr_header() {
|
||||
let bytes = fff(Some(original_header(true)), false);
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::CfaTiff));
|
||||
let r = decode_file(&bytes).unwrap();
|
||||
assert_eq!(r.data, RawData::U16((0..48u16).map(|i| u16::from_be_bytes([2 * i as u8, 2 * i as u8 + 1])).collect()));
|
||||
assert_eq!(r.cfa.as_ref().unwrap().name(), "RGGB");
|
||||
assert_eq!(matrix_row0(&r.color), Some([0.686, -0.196, -0.032]));
|
||||
assert_eq!(r.color.as_shot_neutral, Some([0.4, 1.0, 0.8]), "the outer file's white balance");
|
||||
assert!(!crate::color::camera_transform_of(&r.color, lightcraft_color::D65).matrix_is_fallback);
|
||||
// the import probe says the same
|
||||
assert_eq!(matrix_row0(&probe_info(&bytes).unwrap().color), Some([0.686, -0.196, -0.032]));
|
||||
}
|
||||
|
||||
/// A matrix of the file's own wins; without one anywhere (no header, or a header without a matrix) the
|
||||
/// file still decodes, with the fallback colour.
|
||||
#[test]
|
||||
fn own_matrix_wins_and_a_missing_one_is_not_an_error() {
|
||||
let own = decode_file(&fff(Some(original_header(true)), true)).unwrap();
|
||||
assert_eq!(matrix_row0(&own.color), Some([1.0, 0.0, 0.0]));
|
||||
for header in [None, Some(original_header(false)), Some(b"not a tiff".to_vec())] {
|
||||
let r = decode_file(&fff(header, false)).unwrap();
|
||||
assert!(!crate::color::has_matrix(&r.color));
|
||||
}
|
||||
}
|
||||
|
||||
/// A truncated or damaged file never panics: an error, or a decode.
|
||||
#[test]
|
||||
fn damaged_files_do_not_panic() {
|
||||
let bytes = fff(Some(original_header(true)), false);
|
||||
for n in (0..bytes.len()).step_by(7) {
|
||||
let _ = decode_file(&bytes[..n]);
|
||||
let _ = probe_info(&bytes[..n]);
|
||||
let _ = crate::embedded_preview(&bytes[..n]);
|
||||
}
|
||||
let mut header = original_header(true);
|
||||
for i in (8..header.len()).step_by(3) {
|
||||
header[i] ^= 0x5a;
|
||||
match decode_file(&fff(Some(header.clone()), false)) {
|
||||
Ok(_) | Err(RawError::Corrupt(_) | RawError::Tiff(_) | RawError::Unsupported(_) | RawError::Limit(_) | RawError::NotRaw) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `(width, height, pixels, has an Orientation tag)` of a wrapped preview.
|
||||
fn wrapped(tiff: &[u8]) -> (u32, u32, Vec<u8>, bool) {
|
||||
let t = Tiff::parse(tiff).unwrap();
|
||||
let ifd = &t.ifds[0];
|
||||
let info = ifd.image().unwrap();
|
||||
let chunks = info.chunks(tiff.len() as u64);
|
||||
let pixels = chunks.iter().flat_map(|c| chunk_bytes(tiff, c).unwrap().to_vec()).collect();
|
||||
(info.width, info.height, pixels, ifd.contains(t::ORIENTATION))
|
||||
}
|
||||
|
||||
/// No JPEG in the file: the 8-bit RGB IFD0 comes out as a baseline TIFF of the same pixels, without an
|
||||
/// orientation of its own (the raw file's applies).
|
||||
#[test]
|
||||
fn rgb_ifd0_is_the_embedded_preview() {
|
||||
let bytes = fff(Some(original_header(true)), false);
|
||||
let preview = crate::embedded_preview(&bytes).expect("a preview");
|
||||
assert_eq!(wrapped(&preview), (4, 3, (0..36).collect(), false));
|
||||
assert_eq!(crate::embedded_preview_covering(&bytes, 2000), Some(preview));
|
||||
}
|
||||
|
||||
/// The largest 8-bit RGB image wins; a bigger 16-bit one (the FFF's copy with a `TransferFunction`) is not
|
||||
/// used, and nor is a planar or compressed one.
|
||||
#[test]
|
||||
fn only_8_bit_chunky_uncompressed_rgb_is_taken() {
|
||||
let image = |w: u32, h: u32, bits: u16, planar: u16, compression: u16| {
|
||||
let mut ifd = IfdBuilder::new();
|
||||
ifd.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1]));
|
||||
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
|
||||
ifd.set(t::IMAGE_LENGTH, Value::Long(vec![h]));
|
||||
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits; 3]));
|
||||
ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3]));
|
||||
ifd.set(t::PHOTOMETRIC, Value::Short(vec![2]));
|
||||
ifd.set(t::COMPRESSION, Value::Short(vec![compression]));
|
||||
ifd.set(t::PLANAR_CONFIGURATION, Value::Short(vec![planar]));
|
||||
let n = (w * h * 3 * u32::from(bits / 8)) as usize;
|
||||
ifd.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![vec![7; n]] });
|
||||
ifd
|
||||
};
|
||||
let bytes = fff_with(None, false, vec![image(16, 12, 16, 1, 1), image(12, 9, 8, 2, 1), image(12, 9, 8, 1, 8)]);
|
||||
assert_eq!(wrapped(&crate::embedded_preview(&bytes).unwrap()).0, 4, "IFD0's 4 x 3");
|
||||
let bytes = fff_with(None, false, vec![image(6, 6, 8, 1, 1)]);
|
||||
assert_eq!(wrapped(&crate::embedded_preview(&bytes).unwrap()), (6, 6, vec![7; 108], false));
|
||||
}
|
||||
|
||||
/// A preview strip cut short by a truncated file gives no preview, not a partial picture.
|
||||
#[test]
|
||||
fn short_preview_strip_gives_none() {
|
||||
let mut ifd0 = rgb_ifd();
|
||||
ifd0.set_image(ImageData::Strips { rows_per_strip: 3, strips: vec![vec![1; 30]] });
|
||||
let tiff = TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap();
|
||||
assert_eq!(rgb_preview(&Tiff::parse(&tiff).unwrap(), &tiff), None);
|
||||
}
|
||||
}
|
||||
Vendored
+1
@@ -12,6 +12,7 @@ pub mod cr3;
|
||||
pub mod crx;
|
||||
pub(crate) mod crx_wavelet;
|
||||
pub mod erf;
|
||||
pub(crate) mod hasselblad;
|
||||
pub mod kodak;
|
||||
pub mod mef;
|
||||
pub mod nef;
|
||||
|
||||
@@ -293,7 +293,7 @@ pub fn parse_color(v: &Value) -> Option<[u8; 3]> {
|
||||
match v {
|
||||
Value::String(s) => {
|
||||
let h = s.trim().strip_prefix('#').unwrap_or(s.trim());
|
||||
if h.len() != 6 || !h.is_ascii() {
|
||||
if h.len() != 6 || !h.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return None;
|
||||
}
|
||||
let byte = |i: usize| h.get(i..i + 2).and_then(|x| u8::from_str_radix(x, 16).ok());
|
||||
@@ -356,6 +356,9 @@ mod tests {
|
||||
json!({"zoomToFill": 1}),
|
||||
json!({"strokeColor": "#12345"}),
|
||||
json!({"strokeColor": "#ééééé"}),
|
||||
json!({"strokeColor": "#+f+f+f"}),
|
||||
json!({"backdropColor": "+12345"}),
|
||||
json!({"washColor": "#-00000"}),
|
||||
json!({"washColor": [256, 0, 0]}),
|
||||
json!({"washColor": [-1, 0, 0]}),
|
||||
json!({"backdropColor": [1, 2]}),
|
||||
|
||||
@@ -127,6 +127,13 @@ impl ImageInfo {
|
||||
let (cw, ch) = self.chunk_size();
|
||||
let n = self.offsets.len().min(per_plane.saturating_mul(planes));
|
||||
let mut out = Vec::with_capacity(n);
|
||||
// the distinct offsets in order, for chunks without a byte count (sorted once, not scanned per chunk)
|
||||
let mut sorted: Vec<u64> = Vec::new();
|
||||
if (0..n).any(|i| self.byte_counts.get(i).is_none_or(|&l| l == 0)) {
|
||||
sorted = self.offsets.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
}
|
||||
for i in 0..n {
|
||||
let plane = i / per_plane.max(1);
|
||||
let j = i % per_plane.max(1);
|
||||
@@ -142,7 +149,7 @@ impl ImageInfo {
|
||||
Some(&l) if l > 0 => l,
|
||||
_ => {
|
||||
// infer: up to the next larger offset, else end of data
|
||||
let next = self.offsets.iter().copied().filter(|&o| o > offset).min().unwrap_or(data_len);
|
||||
let next = sorted.get(sorted.partition_point(|&o| o <= offset)).copied().unwrap_or(data_len);
|
||||
next.saturating_sub(offset)
|
||||
}
|
||||
};
|
||||
@@ -162,3 +169,48 @@ pub fn chunk_bytes<'a>(data: &'a [u8], c: &Chunk) -> Option<&'a [u8]> {
|
||||
let end = (c.offset.saturating_add(c.len)).min(data.len() as u64) as usize;
|
||||
Some(&data[start..end])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn strips(offsets: Vec<u64>, byte_counts: Vec<u64>) -> ImageInfo {
|
||||
ImageInfo {
|
||||
width: 1,
|
||||
height: offsets.len() as u32,
|
||||
bits_per_sample: vec![8],
|
||||
samples_per_pixel: 1,
|
||||
compression: 1,
|
||||
photometric: 1,
|
||||
planar: 1,
|
||||
predictor: 1,
|
||||
sample_format: 1,
|
||||
new_subfile_type: 0,
|
||||
layout: Layout::Strips { rows_per_strip: 1 },
|
||||
offsets,
|
||||
byte_counts,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_byte_counts_run_to_the_next_larger_offset() {
|
||||
let info = strips(vec![50, 10, 30, 10, 70], vec![0, 0, 5]);
|
||||
let lens: Vec<u64> = info.chunks(100).iter().map(|c| c.len).collect();
|
||||
assert_eq!(lens, vec![20, 20, 5, 20, 30]);
|
||||
}
|
||||
|
||||
/// A million strips without byte counts (offsets in descending order) are measured in one sort, not one
|
||||
/// scan of all offsets per strip.
|
||||
#[test]
|
||||
fn many_strips_without_byte_counts_are_fast() {
|
||||
let n = 1_000_000u64;
|
||||
let info = strips((0..n).rev().map(|i| 1000 + i * 2).collect(), Vec::new());
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let chunks = info.chunks(1000 + n * 2);
|
||||
let _ = tx.send(chunks.iter().map(|c| c.len).collect::<Vec<u64>>());
|
||||
});
|
||||
let lens = rx.recv_timeout(std::time::Duration::from_secs(20)).expect("chunks() took longer than 20 s");
|
||||
assert!(lens.iter().all(|&l| l == 2));
|
||||
}
|
||||
}
|
||||
@@ -2332,5 +2332,14 @@
|
||||
"Animal Eye Tracking": "Tieraugen-Tracking",
|
||||
"Human Eye Tracking": "Augen-Tracking (Mensch)",
|
||||
"Read Focus Points": "Fokuspunkte lesen",
|
||||
"Reading focus points": "Fokuspunkte werden gelesen"
|
||||
"Reading focus points": "Fokuspunkte werden gelesen",
|
||||
"Click to type a value": "Klicken, um einen Wert einzugeben",
|
||||
"Run face models on the CPU only": "Gesichtsmodelle nur auf der CPU ausführen",
|
||||
"Detection": "Erkennung",
|
||||
"Recognition": "Wiedererkennung",
|
||||
"Not loaded yet": "Noch nicht geladen",
|
||||
"Checking the GPU…": "GPU wird geprüft …",
|
||||
"CPU": "CPU",
|
||||
"A face model uses the GPU only when a check shows it gives the same results faster.": "Ein Gesichtsmodell nutzt die GPU nur, wenn eine Prüfung zeigt, dass sie dieselben Ergebnisse schneller liefert.",
|
||||
"Checking the GPU for faces": "GPU für Gesichter wird geprüft"
|
||||
}
|
||||
@@ -2331,5 +2331,14 @@
|
||||
"Animal Eye Tracking": "Seguimiento de ojos de animales",
|
||||
"Human Eye Tracking": "Seguimiento de ojos humanos",
|
||||
"Read Focus Points": "Leer puntos de enfoque",
|
||||
"Reading focus points": "Leyendo puntos de enfoque"
|
||||
"Reading focus points": "Leyendo puntos de enfoque",
|
||||
"Click to type a value": "Hacer clic para escribir un valor",
|
||||
"Run face models on the CPU only": "Ejecutar los modelos de caras solo en la CPU",
|
||||
"Detection": "Detección",
|
||||
"Recognition": "Reconocimiento",
|
||||
"Not loaded yet": "Aún no cargado",
|
||||
"Checking the GPU…": "Comprobando la GPU…",
|
||||
"CPU": "CPU",
|
||||
"A face model uses the GPU only when a check shows it gives the same results faster.": "Un modelo de caras usa la GPU solo cuando una comprobación muestra que da los mismos resultados más rápido.",
|
||||
"Checking the GPU for faces": "Comprobando la GPU para las caras"
|
||||
}
|
||||
@@ -2329,5 +2329,14 @@
|
||||
"Animal Eye Tracking": "Suivi des yeux d’animaux",
|
||||
"Human Eye Tracking": "Suivi des yeux humains",
|
||||
"Read Focus Points": "Lire les points de mise au point",
|
||||
"Reading focus points": "Lecture des points de mise au point"
|
||||
"Reading focus points": "Lecture des points de mise au point",
|
||||
"Click to type a value": "Cliquez pour saisir une valeur",
|
||||
"Run face models on the CPU only": "Exécuter les modèles de visages sur le processeur uniquement",
|
||||
"Detection": "Détection",
|
||||
"Recognition": "Reconnaissance",
|
||||
"Not loaded yet": "Pas encore chargé",
|
||||
"Checking the GPU…": "Vérification du GPU…",
|
||||
"CPU": "Processeur",
|
||||
"A face model uses the GPU only when a check shows it gives the same results faster.": "Un modèle de visages n’utilise le GPU que si une vérification montre qu’il donne les mêmes résultats plus vite.",
|
||||
"Checking the GPU for faces": "Vérification du GPU pour les visages"
|
||||
}
|
||||
Loaded 100 of 136 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user