Merge pull request #771 from vangel76/script-759-include

#759: #include / //@include / #includepath in scripts

crates/script/src/include.rs

@@ -0,0 +1,173 @@
+//! ExtendScript's preprocessor directives, from the JavaScript Tools Guide's description of their
+//! behaviour: `#include "file"` (and `//@include "file"`) inlines a file in place of the line,
+//! resolved against the including file's folder, then the folders the latest `#includepath "a;b"`
+//! (or `//@includepath`) named; `#target`, `#targetengine`, `#strict`, `#script` and `#engine`
+//! are dropped. Each file is inlined once (which also breaks include cycles), nesting is limited
+//! to [`MAX_DEPTH`], files and their total size are capped, and reads follow the scripts' file
+//! gate ([`crate::scriptfile::check_read`]). Directives are recognised on lines of their own.
+//!
+//! Errors in the expanded code are mapped back to the file and line they came from.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use effectcraft_engine::Session;
+
+use crate::runtime::ScriptError;
+use crate::scriptfile::{self, MAX_DEPTH, MAX_FILE_BYTES};
+
+/// The most text a script and the files it includes may add up to.
+const MAX_TOTAL_BYTES: usize = 4 * MAX_FILE_BYTES;
+
+/// A script with its includes inlined.
+pub(crate) struct Expanded {
+    pub(crate) code: String,
+    /// Per line of `code`: the file (index into `files`) and 1-based line it came from.
+    lines: Vec<(usize, u32)>,
+    /// The script itself, then each included file.
+    files: Vec<String>,
+}
+
+impl Expanded {
+    /// Point an error in the expanded code at the file and line it came from.
+    pub(crate) fn locate(&self, e: &mut ScriptError) {
+        if self.files.first() != Some(&e.file) {
+            return;
+        }
+        let Some(&(file, line)) = e.line.and_then(|l| l.checked_sub(1)).and_then(|l| self.lines.get(l as usize)) else { return };
+        if let Some(name) = self.files.get(file) {
+            e.file = name.clone();
+            e.line = Some(line);
+        }
+    }
+}
+
+enum Directive<'a> {
+    Include(&'a str),
+    IncludePath(&'a str),
+    Ignored,
+}
+
+/// The preprocessor directive on `line`, if it is one.
+fn directive(line: &str) -> Option<Directive<'_>> {
+    let t = line.trim();
+    let (rest, comment) = match t.strip_prefix("//@") {
+        Some(r) => (r, true),
+        None => (t.strip_prefix('#')?, false),
+    };
+    let end = rest.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(rest.len());
+    let (word, arg) = rest.split_at(end);
+    if !(arg.is_empty() || arg.starts_with(char::is_whitespace) || arg.starts_with(['"', '\''])) {
+        return None;
+    }
+    let arg = unquote(arg);
+    match word {
+        "include" => Some(Directive::Include(arg)),
+        "includepath" => Some(Directive::IncludePath(arg)),
+        // `//@target aftereffects` and friends are plain comments already.
+        "target" | "targetengine" | "strict" | "script" | "engine" if !comment => Some(Directive::Ignored),
+        _ => None,
+    }
+}
+
+/// A directive's argument: `"x"`, `'x'` or `x`, with an optional trailing `;`.
+fn unquote(arg: &str) -> &str {
+    let a = arg.trim();
+    let a = a.strip_suffix(';').map_or(a, str::trim_end);
+    for q in ['"', '\''] {
+        if let Some(inner) = a.strip_prefix(q).and_then(|r| r.split(q).next()) {
+            return inner;
+        }
+    }
+    a
+}
+
+struct Walk<'a> {
+    s: &'a Session,
+    /// The running script's folder: the file gate lets it load files below it.
+    root: Option<PathBuf>,
+    /// The latest `#includepath`.
+    search: Vec<PathBuf>,
+    seen: HashSet<PathBuf>,
+    total: usize,
+    out: Expanded,
+}
+
+impl Walk<'_> {
+    fn push(&mut self, line: &str, file: usize, n: usize) {
+        if !self.out.lines.is_empty() {
+            self.out.code.push('\n');
+        }
+        self.out.code.push_str(line);
+        self.out.lines.push((file, u32::try_from(n + 1).unwrap_or(u32::MAX)));
+    }
+
+    /// Inline `code` (file `file`, in folder `dir`), `depth` includes deep.
+    fn walk(&mut self, code: &str, file: usize, dir: Option<&Path>, depth: usize) -> Result<(), ScriptError> {
+        for (n, line) in code.split('\n').enumerate() {
+            let err = |w: &Self, message: String| ScriptError {
+                message,
+                line: Some(u32::try_from(n + 1).unwrap_or(u32::MAX)),
+                column: None,
+                file: w.out.files.get(file).cloned().unwrap_or_default(),
+            };
+            match directive(line) {
+                None => self.push(line, file, n),
+                Some(Directive::Ignored) => self.push("", file, n),
+                Some(Directive::IncludePath(paths)) => {
+                    self.search =
+                        paths.split(';').map(str::trim).filter(|p| !p.is_empty()).filter_map(|p| scriptfile::resolve(self.s, p, dir, &[]).ok()).collect();
+                    self.push("", file, n);
+                }
+                Some(Directive::Include(name)) => {
+                    if name.is_empty() {
+                        return Err(err(self, "#include needs a file name".into()));
+                    }
+                    let path = scriptfile::resolve(self.s, name, dir, &self.search).map_err(|e| err(self, format!("#include {e}")))?;
+                    if !self.seen.insert(path.clone()) {
+                        // Already inlined (or the file including it): once is enough.
+                        self.push("", file, n);
+                        continue;
+                    }
+                    if depth >= MAX_DEPTH {
+                        return Err(err(self, format!("#include \"{name}\": includes are nested too deeply (at most {MAX_DEPTH})")));
+                    }
+                    scriptfile::check_read(self.s, &path, self.root.as_deref()).map_err(|e| err(self, e))?;
+                    let text = scriptfile::read(self.s, &path).map_err(|e| err(self, e))?;
+                    self.total = self.total.saturating_add(text.len());
+                    if self.total > MAX_TOTAL_BYTES {
+                        return Err(err(self, format!("#include \"{name}\": the included files exceed {} MiB", MAX_TOTAL_BYTES >> 20)));
+                    }
+                    // The included lines take the directive's place.
+                    self.out.files.push(path.to_string_lossy().to_string());
+                    let idx = self.out.files.len().saturating_sub(1);
+                    self.walk(&text, idx, path.parent(), depth + 1)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+/// `code` (the script `name`) with its directives applied, or `None` when it has none.
+pub(crate) fn expand(s: &Session, code: &str, name: &str) -> Result<Option<Expanded>, ScriptError> {
+    if !code.split('\n').any(|l| directive(l).is_some()) {
+        return Ok(None);
+    }
+    let root = scriptfile::script_dir(s, name);
+    let mut w = Walk {
+        s,
+        root: root.clone(),
+        search: vec![],
+        seen: HashSet::new(),
+        total: code.len(),
+        out: Expanded { code: String::with_capacity(code.len()), lines: vec![], files: vec![if name.is_empty() { "script" } else { name }.to_string()] },
+    };
+    if root.is_some() {
+        w.seen.insert(crate::runtime::normalize(name));
+    }
+    // A script without a folder (the Script Console, code run by name) looks in the project's.
+    let dir = root.or_else(|| crate::runtime::project_dir(s).map(|d| crate::runtime::normalize(&d)));
+    w.walk(code, 0, dir.as_deref(), 0)?;
+    Ok(Some(w.out))
+}

crates/script/src/lib.rs

@@ -53,9 +53,11 @@
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 #![recursion_limit = "256"]
 
+mod include;
 pub mod matchnames;
 mod model;
 mod runtime;
+mod scriptfile;
 mod socket;
 mod ui;
 
@@ -87,6 +89,8 @@ mod tests_callable;
 #[cfg(test)]
 mod tests_ease_presets;
 #[cfg(test)]
+mod tests_include;
+#[cfg(test)]
 mod tests_panels;
 #[cfg(test)]
 mod tests_ui;

crates/script/src/runtime.rs

@@ -291,13 +291,13 @@ fn native_task(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult<JsV
     Ok(JsValue::from(id))
 }
 
-fn project_dir(s: &Session) -> Option<String> {
+pub(crate) fn project_dir(s: &Session) -> Option<String> {
     let p = s.path.as_deref()?;
     std::path::Path::new(p).parent().map(|d| d.to_string_lossy().to_string())
 }
 
 /// Lexically normalized absolute path (`a/./b/../c` → `a/c`).
-fn normalize(p: &str) -> std::path::PathBuf {
+pub(crate) fn normalize(p: &str) -> std::path::PathBuf {
     use std::path::{Component, Path, PathBuf};
     let path = Path::new(p);
     let abs = if path.is_absolute() { path.to_path_buf() } else { std::env::current_dir().unwrap_or_default().join(path) };
@@ -587,7 +587,23 @@ pub(crate) fn outcome(output: Vec<String>, result: Result<J, ScriptError>) -> Ou
 /// events reach the host through [`crate::dispatch_ui`]); palettes and panels keep their host
 /// alive for their handlers. The Script Console (and the web build) run inline, where dialogs
 /// don't block.
+///
+/// ExtendScript's `#include` / `#includepath` are inlined first, and errors point at the file
+/// and line they came from ([`crate::include`]).
 pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome {
+    let expanded = match crate::include::expand(session, req.code, req.name) {
+        Ok(x) => x,
+        Err(e) => return Outcome { error: Some(e), ..Default::default() },
+    };
+    let Some(x) = expanded else { return run_expanded(session, req) };
+    let mut out = run_expanded(session, &ScriptRequest { code: &x.code, name: req.name, console: req.console });
+    if let Some(e) = out.error.as_mut() {
+        x.locate(e);
+    }
+    out
+}
+
+fn run_expanded(session: &mut Session, req: &ScriptRequest) -> Outcome {
     if let Err(message) = check_nesting(req.code) {
         return Outcome { error: Some(ScriptError { message, file: req.name.into(), ..Default::default() }), ..Default::default() };
     }

crates/script/src/scriptfile.rs

@@ -0,0 +1,65 @@
+//! Script files a running script loads (`#include`, `$.evalFile`): where a name resolves, whether
+//! the script may read it, and the limits that keep hostile, huge or cyclic files from running
+//! away.
+
+use std::path::{Path, PathBuf};
+
+use effectcraft_engine::Session;
+
+use crate::runtime::{GATE, normalize, project_dir};
+
+/// How deep loaded files may nest (a loaded file loading another…).
+pub(crate) const MAX_DEPTH: usize = 32;
+/// The largest single file a script may load.
+pub(crate) const MAX_FILE_BYTES: usize = 16 << 20;
+
+/// The folder of the script file `name` runs from: `None` for the Script Console and for scripts
+/// run by name or from code, which have no file.
+pub(crate) fn script_dir(s: &Session, name: &str) -> Option<PathBuf> {
+    let parent = Path::new(name).parent().filter(|p| !p.as_os_str().is_empty())?;
+    s.services.exists(name).then(|| normalize(&parent.to_string_lossy()))
+}
+
+/// Resolve `name`: absolute paths as they are; relative ones against `dir` (the loading script's
+/// folder), then each of `search`. The first existing file wins; when none exists, the first
+/// candidate, so the read error names it.
+pub(crate) fn resolve(s: &Session, name: &str, dir: Option<&Path>, search: &[PathBuf]) -> Result<PathBuf, String> {
+    if Path::new(name).is_absolute() {
+        return Ok(normalize(name));
+    }
+    let candidates: Vec<PathBuf> = dir.into_iter().chain(search.iter().map(PathBuf::as_path)).map(|d| normalize(&d.join(name).to_string_lossy())).collect();
+    candidates
+        .iter()
+        .find(|c| s.services.exists(&c.to_string_lossy()))
+        .or(candidates.first())
+        .cloned()
+        .ok_or_else(|| format!("cannot find \"{name}\": the script has no folder to look in (use an absolute path or #includepath)"))
+}
+
+/// The read gate for loaded script files: files under the running script's own folder (`root`) or
+/// the project's folder; anywhere with the Allow Scripts to Write Files preference.
+pub(crate) fn check_read(s: &Session, path: &Path, root: Option<&Path>) -> Result<(), String> {
+    if s.prefs.scripting.allow_scripts_write_files {
+        return Ok(());
+    }
+    let project = project_dir(s).map(|d| normalize(&d));
+    if root.into_iter().chain(project.as_deref()).any(|d| path.starts_with(d)) {
+        return Ok(());
+    }
+    Err(format!("Unable to read {}: scripts can only load files from their own or the project's folder unless {GATE} is on", path.display()))
+}
+
+/// Read a script file as text (a UTF-8 byte order mark dropped), refusing files over
+/// [`MAX_FILE_BYTES`].
+pub(crate) fn read(s: &Session, path: &Path) -> Result<String, String> {
+    let too_big = || format!("Unable to read {}: script files are limited to {} MiB", path.display(), MAX_FILE_BYTES >> 20);
+    if std::fs::metadata(path).is_ok_and(|m| m.len() > MAX_FILE_BYTES as u64) {
+        return Err(too_big());
+    }
+    let bytes = s.services.read_file(&path.to_string_lossy()).map_err(|e| format!("Unable to read {}: {e}", path.display()))?;
+    if bytes.len() > MAX_FILE_BYTES {
+        return Err(too_big());
+    }
+    let text = String::from_utf8_lossy(&bytes);
+    Ok(text.strip_prefix('\u{feff}').unwrap_or(&text).to_string())
+}

crates/script/src/tests_include.rs

@@ -0,0 +1,94 @@
+//! #759: ExtendScript's `#include` / `//@include` / `#includepath` directives.
+
+use std::path::{Path, PathBuf};
+
+use effectcraft_engine::Session;
+use serde_json::json;
+
+use crate::{Outcome, run_code};
+
+fn session() -> Session {
+    let mut s = Session::default();
+    crate::install(&mut s);
+    s
+}
+
+fn tmp(tag: &str) -> PathBuf {
+    let dir = std::env::temp_dir().join(format!("ec-include-{tag}-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    dir
+}
+
+fn write(path: &Path, text: &str) {
+    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+    std::fs::write(path, text).unwrap();
+}
+
+/// Run the script file `main` (its code read from disk, as File ▸ Scripts ▸ Run Script File does).
+fn run_file(s: &mut Session, main: &Path) -> Outcome {
+    let code = std::fs::read_to_string(main).unwrap();
+    run_code(s, &code, &main.to_string_lossy())
+}
+
+#[test]
+fn includes_are_inlined_relative_to_the_script() {
+    let dir = tmp("ok");
+    write(&dir.join("lib/math.jsx"), "#include \"util.jsxinc\"\nfunction twice(x) { return 2 * x; }");
+    write(&dir.join("lib/util.jsxinc"), "// includes its includer back: inlined once only\n#include \"math.jsx\"\nvar unit = 'px';");
+    write(&dir.join("vendor/extra.jsx"), "var extra = 40;");
+    write(
+        &dir.join("main.jsx"),
+        "#target aftereffects\n#targetengine \"main\"\n#strict on\n//@include 'lib/math.jsx'\n#includepath \"vendor;lib\"\n#include \"extra.jsx\";\n#include \"util.jsxinc\"\n[twice(1) + extra, unit, $.fileName.split('/').pop()]",
+    );
+    let mut s = session();
+    let o = run_file(&mut s, &dir.join("main.jsx"));
+    assert!(o.error.is_none(), "{:?}", o.error);
+    assert_eq!(o.result, json!([42, "px", "main.jsx"]));
+    let _ = std::fs::remove_dir_all(&dir);
+}
+
+#[test]
+fn errors_point_at_the_file_and_line_they_came_from() {
+    let dir = tmp("err");
+    write(&dir.join("lib.jsx"), "var a = 1;\nvar b = 2;\nnotDefined();");
+    write(&dir.join("main.jsx"), "// main\n#include \"lib.jsx\"\nvar c = 3;");
+    let mut s = session();
+    let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");
+    assert!(e.file.ends_with("lib.jsx") && e.line == Some(3), "{e:?}");
+    // After the include, lines in the script itself keep their numbers.
+    write(&dir.join("lib.jsx"), "var a = 1;\nvar b = 2;");
+    write(&dir.join("main.jsx"), "// main\n#include \"lib.jsx\"\nvar c = 3;\nalsoNotDefined();");
+    let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");
+    assert!(e.file.ends_with("main.jsx") && e.line == Some(4), "{e:?}");
+    // A missing file is a clear error on the directive's line.
+    write(&dir.join("main.jsx"), "var x = 1;\n#include \"missing.jsx\"");
+    let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");
+    assert!(e.message.contains("missing.jsx") && e.line == Some(2), "{e:?}");
+    let _ = std::fs::remove_dir_all(&dir);
+}
+
+#[test]
+fn hostile_includes_are_bounded_and_gated() {
+    let dir = tmp("hostile");
+    // A chain deeper than the limit is an error, not a stack overflow.
+    for i in 0..100 {
+        write(&dir.join(format!("chain/{i}.jsx")), &format!("#include \"{}.jsx\"", i + 1));
+    }
+    write(&dir.join("chain/100.jsx"), "1");
+    let mut s = session();
+    let e = run_file(&mut s, &dir.join("chain/0.jsx")).error.expect("an error");
+    assert!(e.message.contains("nested too deeply"), "{e:?}");
+    // Files outside the script's (and the project's) folder need the file preference.
+    write(&dir.join("outside.jsx"), "var secret = 7;");
+    write(&dir.join("app/main.jsx"), "#include \"../outside.jsx\"\nsecret");
+    let e = run_file(&mut s, &dir.join("app/main.jsx")).error.expect("blocked");
+    assert!(e.message.contains("Allow Scripts to Write Files"), "{e:?}");
+    // Code run without a file can't include relative to anything but the project's folder.
+    let e = run_code(&mut s, "#include \"outside.jsx\"", "console").error.expect("no folder");
+    assert!(e.message.contains("outside.jsx"), "{e:?}");
+    s.prefs.scripting.allow_scripts_write_files = true;
+    let o = run_file(&mut s, &dir.join("app/main.jsx"));
+    assert_eq!(o.result, json!(7), "{:?}", o.error);
+    let _ = std::fs::remove_dir_all(&dir);
+}

docs/plugins.md

@@ -14,7 +14,7 @@ Guide):
 | Effect plug-ins (C/C++ SDK, `.aex` / `.plugin`) | [Effect plug-ins](#effect-plug-ins): WebAssembly modules (plug-in API v1) or Rust; the `Plug-ins` folder next to the settings loads at start-up | Different ABI: After Effects SDK plug-ins can't run (G8) |
 | Scripts folder: its scripts are listed in File ▸ Scripts | `Scripts/` in the settings folder (File ▸ Scripts ▸ Install Script File… copies a script there), listed in File ▸ Scripts with the bundled samples | Same |
 | File ▸ Scripts ▸ Run Script File… (`.jsx`, `.jsxbin`) | `.jsx` / `.js` | Gap: `.jsxbin` (an undocumented binary encoding) |
-| ExtendScript preprocessor directives (`#include`, `#target`, `#targetengine`) | Not understood: a script that starts with one doesn't parse | Gap |
+| ExtendScript preprocessor directives (`#include`, `#target`, `#targetengine`) | `#include` / `//@include` / `#includepath` inline files relative to the script (once each, depth-limited, file gate applies); `#target`, `#targetengine`, `#strict` are ignored | Same |
 | `Scripts/Startup` and `Scripts/Shutdown`: run in alphabetical order when the app starts and quits | Not run | Gap |
 | `Scripts/ScriptUI Panels`: listed at the bottom of the Window menu, dockable like the app's own panels, `this` is the panel | `Scripts/ScriptUI Panels/` in the settings folder (File ▸ Scripts ▸ Install ScriptUI Panel… copies one there), listed at the bottom of the Window menu, docked like any panel, `this` is the panel; the panels open when the app quits open again at the next launch | Same. Where a panel was docked isn't kept: workspace layouts aren't saved between launches |
 | Keyboard shortcuts for scripts (the Keyboard Shortcuts editor lists the scripts of the Scripts folder) | Shortcuts bind to commands and menu entries; the File ▸ Scripts and Window menu entries of scripts aren't offered | Gap |

docs/scripting-parity.md

@@ -22,7 +22,7 @@ our own scripts; no real-world script library has been run against both apps.
 | ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics | Same, dockable panels reopen at launch | good | `ui_scriptui`, `tests_ui*` |
 | Scripts menu, ScriptUI Panels folder | yes | yes | same | [plugins.md](plugins.md) |
 | Startup / Shutdown folders | yes | not run | missing | plugins.md |
-| `#include`, `#target` directives | yes | a script starting with one doesn't parse | missing | plugins.md; common in real scripts |
+| `#include`, `#target` directives | yes | `#include` / `//@include` / `#includepath` inline files relative to the including script (once each, nested at most 32 deep, under the scripts' file gate; errors name the included file and line); `#target`, `#targetengine`, `#strict` are ignored | good | `tests_include.rs` (#759); installed scripts and panels have no folder of their own, so they include from the project's folder or absolute paths |
 | `.jsxbin` | yes | no | out of scope | No public specification |
 | Keyboard shortcuts for scripts | yes | no | missing | plugins.md |
 | Command-line rendering `aerender` | yes | `effectcraft-cli render`, `script`, `exec` | same purpose | `apps/effectcraft-cli` |