Close the custom grader sandbox escape

actions/setup/js/trace_graders.cjs

@@ -880,13 +880,18 @@ function executeCustomGraderInSubprocess(id, script, trace, meta) {
     timeoutMs: SCRIPT_TIMEOUT_MS,
   };
   const safeEnv = {};
-  for (const key of ["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "SystemRoot", "ComSpec"]) {
+  for (const key of ["SystemRoot"]) {
     if (process.env[key]) {
       safeEnv[key] = process.env[key];
     }
   }
   const timeoutMs = SCRIPT_TIMEOUT_MS + SCRIPT_WORKER_OVERHEAD_MS;
-  const proc = cp.spawnSync(process.execPath, [SCRIPT_WORKER_PATH], {
+  const permissionFlag = process.allowedNodeEnvironmentFlags.has("--permission") ? "--permission" : process.allowedNodeEnvironmentFlags.has("--experimental-permission") ? "--experimental-permission" : null;
+  if (!permissionFlag) {
+    throw new Error("Node.js permission support is required to run custom graders; install Node.js 20 or newer");
+  }
+  const workerPath = fs.realpathSync(SCRIPT_WORKER_PATH);
+  const proc = cp.spawnSync(process.execPath, [permissionFlag, `--allow-fs-read=${workerPath}`, workerPath], {
     input: JSON.stringify(payload),
     encoding: "utf-8",
     timeout: timeoutMs,

actions/setup/js/trace_graders.test.cjs

@@ -549,20 +549,77 @@ printf '%s\\n' '[{"id":"goal-attained","value":0.75},{"id":"evidence-available",
       expect(result.value).toBeNull();
       expect(result.error).toContain("non-finite");
     });
+
+    it("rejects non-finite values in object results", () => {
+      const result = runCustomGrader("test", "return { value: NaN }", makeTrace(), { name: "test", unit: "", direction: "", source: "inline" });
+      expect(result.value).toBeNull();
+      expect(result.status).toBe("error");
+      expect(result.error).toContain("non-finite");
+    });
   });
 
   // --- runCustomGrader node:vm sandbox ---
   describe("runCustomGrader sandbox", () => {
     const meta = { name: "test", unit: "", direction: "", source: "inline" };
 
-    it("cannot access require", () => {
-      const result = runCustomGrader("test", "return typeof require", makeTrace(), meta);
-      expect(result.value).toBeNull(); // "undefined" is not a number
+    it("selects the supported permission flag", () => {
+      const cp = require("child_process");
+      const originalFlags = process.allowedNodeEnvironmentFlags;
+      for (const flag of ["--permission", "--experimental-permission"]) {
+        process.allowedNodeEnvironmentFlags = new Set([flag]);
+        const spawn = vi.spyOn(cp, "spawnSync").mockReturnValue({ status: 0, stdout: '{"ok":true,"value":1}' });
+        try {
+          expect(runCustomGrader("test", "return 1", makeTrace(), meta).value).toBe(1);
+          expect(spawn.mock.calls[0][1][0]).toBe(flag);
+          expect(spawn.mock.calls[0][1][1]).toBe(`--allow-fs-read=${spawn.mock.calls[0][1][2]}`);
+        } finally {
+          process.allowedNodeEnvironmentFlags = originalFlags;
+          spawn.mockRestore();
+        }
+      }
     });
 
-    it("cannot access process", () => {
-      const result = runCustomGrader("test", "return typeof process", makeTrace(), meta);
-      expect(result.value).toBeNull(); // "undefined" is not a number
+    it("fails closed when the permission model is unavailable", () => {
+      const originalFlags = process.allowedNodeEnvironmentFlags;
+      process.allowedNodeEnvironmentFlags = new Set();
+      try {
+        const result = runCustomGrader("test", "return 1", makeTrace(), meta);
+        expect(result.status).toBe("error");
+        expect(result.value).toBeNull();
+        expect(result.error).toContain("install Node.js 20 or newer");
+      } finally {
+        process.allowedNodeEnvironmentFlags = originalFlags;
+      }
+    });
+
+    it("cannot access process or require", () => {
+      const result = runCustomGrader("test", "return typeof process === 'undefined' && typeof require === 'undefined' ? 1 : 0", makeTrace(), meta);
+      expect(result.value).toBe(1);
+      expect(result.status).toBe("pass");
+    });
+
+    it("does not expose the bootstrap global's host constructor", () => {
+      const result = runCustomGrader("test", 'try { return typeof root.constructor.constructor("return process")() === "object" ? 1 : 0; } catch { return 0; }', makeTrace(), meta);
+      expect(result.value).toBe(0);
+      expect(result.status).toBe("pass");
+    });
+
+    it("does not let a host-object constructor escape execute a command", () => {
+      const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "grader-escape-"));
+      const markerPath = path.join(tempDir, "executed");
+      const script = `try {
+        const escapedProcess = trace.constructor.constructor("return process")();
+        escapedProcess.getBuiltinModule("node:child_process").execSync("touch ${markerPath}");
+      } catch {}
+      return 1;`;
+      try {
+        const result = runCustomGrader("test", script, makeTrace(), meta);
+        expect(result.value).toBe(1);
+        expect(result.status).toBe("pass");
+        expect(fs.existsSync(markerPath)).toBe(false);
+      } finally {
+        fs.rmSync(tempDir, { recursive: true, force: true });
+      }
     });
 
     it("cannot access fetch", () => {

actions/setup/js/trace_graders_worker.cjs

@@ -2,36 +2,6 @@
 
 const vm = require("vm");
 
-/**
- * @param {any} value
- * @param {string} label
- * @returns {any}
- */
-function tryStructuredCloneOrUndefined(value, label) {
-  try {
-    return structuredClone(value);
-  } catch {
-    process.stderr.write(`grader worker: failed to structuredClone ${label}; value will default to {}\n`);
-    return undefined;
-  }
-}
-
-/**
- * @param {any} obj
- * @returns {any}
- */
-function deepFreeze(obj) {
-  if (obj === null || typeof obj !== "object") return obj;
-  Object.freeze(obj);
-  for (const key of Object.getOwnPropertyNames(obj)) {
-    const value = obj[key];
-    if (value !== null && typeof value === "object" && !Object.isFrozen(value)) {
-      deepFreeze(value);
-    }
-  }
-  return obj;
-}
-
 function readStdin() {
   return new Promise(resolve => {
     let data = "";
@@ -55,35 +25,53 @@ async function main() {
   }
 
   try {
-    const trace = deepFreeze(tryStructuredCloneOrUndefined(payload.trace, "trace") ?? {});
-    const config = deepFreeze(tryStructuredCloneOrUndefined(payload.config, "config") ?? {});
-    const run = deepFreeze({ graderCount: Number(payload.graderCount) || 0 });
-    const workflow = deepFreeze({});
     const script = String(payload.script || "");
 
-    const sandbox = {
-      trace,
-      run,
-      workflow,
-      config,
+    const sandbox = Object.assign(Object.create(null), {
+      __payload: JSON.stringify({
+        trace: payload.trace || {},
+        run: { graderCount: Number(payload.graderCount) || 0 },
+        workflow: {},
+        config: payload.config || {},
+      }),
       Date: undefined,
       fetch: undefined,
       require: undefined,
       process: undefined,
       global: undefined,
-      globalThis: undefined,
       Function: undefined,
       eval: undefined,
       undefined,
       NaN,
       Infinity,
-    };
+    });
     const context = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } });
     const timeoutMs = Number(payload.timeoutMs) || 5000;
 
-    const runtimeBindings = vm.runInContext(
+    vm.runInContext(
       `
         "use strict";
+        (() => {
+        const root = globalThis;
+        const deepFreeze = value => {
+          if (value === null || typeof value !== "object") return value;
+          Object.freeze(value);
+          for (const key of Object.getOwnPropertyNames(value)) {
+            if (value[key] !== null && typeof value[key] === "object" && !Object.isFrozen(value[key])) {
+              deepFreeze(value[key]);
+            }
+          }
+          return value;
+        };
+        const data = JSON.parse(__payload);
+        for (const [key, value] of Object.entries(data)) {
+          Object.defineProperty(root, key, {
+            value: deepFreeze(value),
+            writable: false,
+            enumerable: true,
+            configurable: false
+          });
+        }
         (() => {
           const m = {};
           const descriptors = Object.getOwnPropertyDescriptors(Math);
@@ -97,44 +85,58 @@ async function main() {
             configurable: false
           });
           const safeMath = Object.freeze(m);
-          const helpers = Object.freeze({
+          Object.defineProperty(root, "__math", {
+            value: safeMath,
+            writable: false,
+            enumerable: false,
+            configurable: false
+          });
+          Object.defineProperty(root, "helpers", {
+            value: Object.freeze({
             clamp: (v, lo, hi) => safeMath.max(lo, safeMath.min(hi, v)),
             ratio: (num, den) => (den === 0 ? 0 : num / den),
             sum: arr => arr.reduce((a, b) => a + b, 0)
+            }),
+            writable: false,
+            enumerable: true,
+            configurable: false
           });
-          return { safeMath, helpers };
+        })();
+        delete root.__payload;
+        Object.defineProperty(root, "globalThis", {
+          value: undefined,
+          writable: false,
+          enumerable: false,
+          configurable: false
+        });
         })();
       `,
       context,
       { timeout: 1000, filename: "grader:bootstrap" }
     );
-    Object.defineProperty(context, "helpers", {
-      value: runtimeBindings.helpers,
-      writable: false,
-      enumerable: true,
-      configurable: false,
-    });
-    Object.defineProperty(context, "__math", {
-      value: runtimeBindings.safeMath,
-      writable: false,
-      enumerable: false,
-      configurable: false,
-    });
 
     const graderFn = vm.compileFunction(`"use strict";\n${script}`, ["trace", "run", "workflow", "config", "helpers", "Math"], {
       parsingContext: context,
       filename: `grader:${String(payload.id || "unknown")}`,
     });
     context.__grader = graderFn;
-    const value = vm.runInContext("__grader(trace, run, workflow, config, helpers, __math)", context, {
-      timeout: timeoutMs,
-      filename: `grader:${String(payload.id || "unknown")}:invoke`,
-    });
-    if (typeof value === "number" && !Number.isFinite(value)) {
-      throw new Error("custom grader returned non-finite numeric value");
-    }
-
-    process.stdout.write(JSON.stringify({ ok: true, value }));
+    const result = vm.runInContext(
+      `(() => {
+        try {
+          const value = __grader(trace, run, workflow, config, helpers, __math);
+          const metricValue = value !== null && typeof value === "object" && Object.hasOwn(value, "value") ? value.value : value;
+          if (typeof metricValue === "number" && !Number.isFinite(metricValue)) {
+            throw new Error("custom grader returned non-finite numeric value");
+          }
+          return JSON.stringify({ ok: true, value });
+        } catch (err) {
+          return JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) });
+        }
+      })()`,
+      context,
+      { timeout: timeoutMs, filename: `grader:${String(payload.id || "unknown")}:invoke` }
+    );
+    process.stdout.write(result);
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     process.stdout.write(JSON.stringify({ ok: false, error: message }));

docs/src/content/docs/experimental/drive-memory.md

@@ -24,7 +24,7 @@ Drive names are repository-wide and branch-aware according to the preview servic
 
 Drive-memory supports the same `validation.json-schemas` declarations as repo-memory and cache-memory. Each declaration has an inline `schema` and may specify a relative file path or glob (with `*` within a path segment and `**` across segments). If `file` is omitted, the schema applies to every eligible `.json` and `.jsonl` file in the memory directory, excluding `.git`; omit `format` too so the format is inferred for each file. Otherwise, each path must exist, each glob must match at least one file, and every matched file must pass validation before the drive accepts or persists the candidate. The format is inferred from each matched file's `.json` or `.jsonl` extension (case-insensitive). Optional `format: json` or `format: jsonl` overrides inference and is required for other extensions.
 
-JSON requires one non-empty document. JSONL validates each physical record independently, accepts LF/CRLF and an optional final newline, permits an existing empty file, and rejects blank or malformed records with a line number. Schema validation runs after filtering and configured normalization, before an optional `validation.script`, and gates drive persistence. The supported vocabulary is `type`, primitive `enum`, `required`, nested `properties`, `additionalProperties: false`, `items`, and standalone `oneOf`/`anyOf` (maximum depth 32); this is not full JSON Schema. Numeric enum integers must be exactly representable by JavaScript, and values checked against numeric enums must not lose precision during JSON parsing. Unsupported keywords are rejected at compile time. A script remains useful for cross-file or domain-specific rules. See [Cache Memory](../reference/cache-memory/) for a complete example and details.
+JSON requires one non-empty document. JSONL validates each physical record independently, accepts LF/CRLF and an optional final newline, permits an existing empty file, and rejects blank or malformed records with a line number. Schema validation runs after filtering and configured normalization, before an optional `validation.script`, and gates drive persistence. The supported vocabulary is `type`, primitive `enum`, `required`, nested `properties`, `additionalProperties: false`, `items`, and standalone `oneOf`/`anyOf` (maximum depth 32); this is not full JSON Schema. Numeric enum integers must be exactly representable by JavaScript, and values checked against numeric enums must not lose precision during JSON parsing. Unsupported keywords are rejected at compile time. A script remains useful for cross-file or domain-specific rules. See [Cache Memory](/gh-aw/reference/cache-memory/) for a complete example and details.
 
 ## Drive size
 

docs/src/content/docs/reference/glossary.md

@@ -1723,7 +1723,7 @@ An optional field under top-level `metadata:` frontmatter holding an absolute HT
 
 ### Graders (`graders:`)
 
-Deterministic, non-LLM checks that compute metrics from a workflow run's post-agent execution trace. Configured under the top-level `graders:` frontmatter field; an empty map (`graders: {}`) enables all built-in graders with default settings, and omitting the field disables grading entirely. Built-in graders cover tool success rate, retries, loop detection, trajectory efficiency, execution duration, and similar metrics. Custom inline graders run a trusted, sandboxed JavaScript expression against the preprocessed `trace` object. Graders are an experimental feature. See [Graders Reference](/gh-aw/experimental/trace-graders/).
+Deterministic, non-LLM checks that compute metrics from a workflow run's post-agent execution trace. Configured under the top-level `graders:` frontmatter field; an empty map (`graders: {}`) enables all built-in graders with default settings, and omitting the field disables grading entirely. Built-in graders cover tool success rate, retries, loop detection, trajectory efficiency, execution duration, and similar metrics. Custom inline graders run in a separate restricted Node.js process against the preprocessed `trace` object, without access to `process` or `require`. Graders are an experimental feature. See [Graders Reference](/gh-aw/experimental/trace-graders/).
 
 ### Operational Value Grader (`graders.operational-value`)
 

docs/src/content/docs/setup/cli.md

@@ -670,6 +670,11 @@ executes either Bash embedded in `graders.operational-value.script` or a Bash
 file referenced by `graders.operational-value.run`; operational-value graders
 require standard input because historical replay is not supported.
 
+Built-in and custom JavaScript graders require Node.js 20 or newer with the
+permission model available. The CLI selects `--permission` or
+`--experimental-permission` according to runtime support and never runs a
+JavaScript grader without permissions enabled.
+
 ```bash wrap
 gh aw graders run weekly-research loops 123456789
 cat payload.json | gh aw graders run weekly-research loops

docs/src/content/docs/specs/graders-specification.md

@@ -193,21 +193,6 @@ A custom grader MUST define `script`.
 - `script` MUST be non-empty.
 - `script` MUST NOT exceed 4096 characters.
 
-### 6.3 Forbidden Patterns
-
-Inline scripts MUST be rejected if they contain any forbidden pattern, including:
-
-- `require(`
-- `import(`
-- `import `
-- `fetch(`
-- `eval(`
-- `process.exit`
-- `child_process`
-- `execSync`
-- `spawnSync`
-- `Function(`
-
 ---
 
 ## 7. Operational Value Grader
@@ -288,7 +273,9 @@ semantic task correctness. The normative readiness, decision, and JSON contracts
 ## 10. Security and Isolation
 
 - Grading MUST operate on local run artifacts and MUST NOT require outbound network access for built-ins.
-- Custom inline graders MUST execute in a restricted context with blocked dangerous primitives.
+- Custom inline graders MUST execute in a separate Node.js process with the permission model enabled and a minimal environment.
+- Grader inputs MUST be reconstructed inside the JavaScript context; host-created objects and functions MUST NOT be exposed to the grader.
+- `process`, `require`, and other host capabilities MUST NOT be available in the grader context. Source-pattern blocklists MUST NOT be relied on as a security boundary.
 - Operational-value graders MAY access declared repository evidence using `GH_TOKEN`; implementations MUST NOT add agent-job permission scopes on behalf of the evaluator, and evaluators MUST NOT receive workflow secrets.
 - Implementations SHOULD enforce bounded execution time for inline scripts.
 - Implementations SHOULD redact grader outputs when custom scripts are enabled to reduce secret leakage risk.
@@ -364,5 +351,5 @@ semantic task correctness. The normative readiness, decision, and JSON contracts
 
 - Initial draft for gh-aw graders.
 - Defines `graders` configuration semantics and built-in grader set.
-- Defines custom inline grader constraints and forbidden patterns.
+- Defines custom inline grader constraints.
 - Defines grader artifact output contract and experiment metric references.