From 38e00dbc99343ff3417bdecd8611df58d6d31324 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:19:48 -0400 Subject: [PATCH 01/20] Browser: stop retaining stale immediate cancellations --- .../pages/network/network-demo-worker.ts | 39 +---- host/src/browser-immediate-polyfill.ts | 113 +++++++++++++++ host/src/browser-kernel-worker-entry.ts | 54 +------ host/test/browser-immediate-polyfill.test.ts | 134 ++++++++++++++++++ 4 files changed, 251 insertions(+), 89 deletions(-) create mode 100644 host/src/browser-immediate-polyfill.ts create mode 100644 host/test/browser-immediate-polyfill.test.ts diff --git a/apps/browser-demos/pages/network/network-demo-worker.ts b/apps/browser-demos/pages/network/network-demo-worker.ts index 89805b235d..e9be953c83 100644 --- a/apps/browser-demos/pages/network/network-demo-worker.ts +++ b/apps/browser-demos/pages/network/network-demo-worker.ts @@ -1,4 +1,5 @@ import { CAPTURED_STDIO, CentralizedKernelWorker } from "@host/kernel-worker"; +import { installBrowserSetImmediatePolyfill } from "@host/browser-immediate-polyfill"; import { BrowserWorkerAdapter } from "@host/worker-adapter-browser"; import { detectPtrWidth, extractHeapBase } from "@host/constants"; import { LocalVirtualNetwork } from "@host/networking/virtual-network"; @@ -16,43 +17,7 @@ import workerEntryUrl from "@host/worker-entry-browser.ts?worker&url"; import ncWasmUrl from "@binaries/programs/wasm32/nc.wasm?url"; import curlWasmUrl from "@binaries/programs/wasm32/curl.wasm?url"; -if (typeof (globalThis as typeof globalThis & { setImmediate?: unknown }).setImmediate === "undefined") { - const queue: Array<{ id: number; fn: (...args: unknown[]) => void; args: unknown[] }> = []; - const cancelled = new Set(); - const channel = new MessageChannel(); - let nextId = 0; - let scheduled = false; - let flushing = false; - - channel.port1.onmessage = () => { - scheduled = false; - flushing = true; - const count = queue.length; - for (let i = 0; i < count && queue.length > 0; i++) { - const entry = queue.shift()!; - if (cancelled.delete(entry.id)) continue; - entry.fn(...entry.args); - } - flushing = false; - if (queue.length > 0 && !scheduled) { - scheduled = true; - channel.port2.postMessage(null); - } - }; - - (globalThis as typeof globalThis & { setImmediate: (fn: (...args: unknown[]) => void, ...args: unknown[]) => number }).setImmediate = - (fn, ...args) => { - const id = ++nextId; - queue.push({ id, fn, args }); - if (!scheduled && !flushing) { - scheduled = true; - channel.port2.postMessage(null); - } - return id; - }; - (globalThis as typeof globalThis & { clearImmediate: (id: number) => void }).clearImmediate = - (id) => { cancelled.add(id); }; -} +installBrowserSetImmediatePolyfill(); const MAX_PAGES = 16384; const CH_TOTAL_SIZE = 72 + 65536; diff --git a/host/src/browser-immediate-polyfill.ts b/host/src/browser-immediate-polyfill.ts new file mode 100644 index 0000000000..6ccaeb16e9 --- /dev/null +++ b/host/src/browser-immediate-polyfill.ts @@ -0,0 +1,113 @@ +type BrowserImmediateCallback = (...args: any[]) => void; + +interface BrowserImmediateHandle { + readonly id: number; +} + +interface BrowserImmediateEntry { + handle: BrowserImmediateHandle; + fn: BrowserImmediateCallback; + args: any[]; + cancelled: boolean; +} + +export interface BrowserImmediatePolyfillTarget { + setImmediate?: unknown; + clearImmediate?: unknown; + MessageChannel: typeof MessageChannel; +} + +export interface BrowserImmediatePolyfillState { + pendingCount(): number; + queueLength(): number; +} + +/** + * Install the MessageChannel-backed setImmediate used by the browser kernel + * worker. Handles are opaque objects so clearImmediate cannot confuse a + * browser setTimeout's numeric handle with one of this polyfill's handles. + */ +export function installBrowserSetImmediatePolyfill( + target: BrowserImmediatePolyfillTarget = globalThis, +): BrowserImmediatePolyfillState | null { + if (typeof target.setImmediate !== "undefined") { + return null; + } + + const queue: BrowserImmediateEntry[] = []; + const pending = new Map(); + let nextId = 0; + let scheduled = false; + let flushing = false; + + const channel = new target.MessageChannel(); + channel.port1.onmessage = flush; + + function scheduleFlush(): void { + if (scheduled || flushing) { + return; + } + scheduled = true; + channel.port2.postMessage(null); + } + + function flush(): void { + scheduled = false; + flushing = true; + + // Process only items queued at flush start. Items added during this flush + // are deferred to a new macrotask so onmessage handlers can interleave. + const count = queue.length; + for (let i = 0; i < count && queue.length > 0; i++) { + const entry = queue.shift()!; + pending.delete(entry.handle); + if (entry.cancelled) { + continue; + } + + try { + entry.fn(...entry.args); + } catch (error) { + console.error("[setImmediate] callback threw:", error); + } + } + + flushing = false; + if (queue.length > 0) { + scheduleFlush(); + } + } + + (target as any).setImmediate = (fn: BrowserImmediateCallback, ...args: any[]) => { + const handle: BrowserImmediateHandle = { id: ++nextId }; + const entry: BrowserImmediateEntry = { + handle, + fn, + args, + cancelled: false, + }; + queue.push(entry); + pending.set(handle, entry); + scheduleFlush(); + return handle; + }; + + (target as any).clearImmediate = (handle: unknown) => { + if (typeof handle !== "object" || handle === null) { + return; + } + + const entry = pending.get(handle as BrowserImmediateHandle); + if (entry === undefined) { + return; + } + + entry.cancelled = true; + pending.delete(entry.handle); + }; + + return { + pendingCount: () => pending.size, + queueLength: () => queue.length, + }; +} diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 3567df1039..e89c350ce5 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -6,59 +6,9 @@ * instance, process spawning (fork/exec/clone), and the HTTP connection pump. */ -// Polyfill setImmediate for the web worker context. -// CentralizedKernelWorker uses setImmediate for yielding between syscall -// batches and waking blocked retries. In a dedicated worker there's no UI -// to starve, so we can use a simple MessageChannel polyfill. -if (typeof globalThis.setImmediate === "undefined") { - const _immQueue: Array<{ id: number; fn: (...args: any[]) => void; args: any[] }> = []; - let _immNextId = 0; - let _immScheduled = false; - let _immFlushing = false; - const _immCancelled = new Set(); - - const _immChannel = new MessageChannel(); - _immChannel.port1.onmessage = _immFlush; - - function _immFlush() { - _immScheduled = false; - _immFlushing = true; - // Process only items queued at flush start — items added during the flush - // are deferred to a new macrotask so onmessage handlers can interleave. - const count = _immQueue.length; - for (let i = 0; i < count && _immQueue.length > 0; i++) { - const entry = _immQueue.shift()!; - if (_immCancelled.has(entry.id)) { - _immCancelled.delete(entry.id); - continue; - } - try { - entry.fn(...entry.args); - } catch (e) { - console.error("[setImmediate] callback threw:", e); - } - } - _immFlushing = false; - // Schedule another flush if new items were added during processing - if (_immQueue.length > 0 && !_immScheduled) { - _immScheduled = true; - _immChannel.port2.postMessage(null); - } - } +import { installBrowserSetImmediatePolyfill } from "./browser-immediate-polyfill"; - (globalThis as any).setImmediate = (fn: (...args: any[]) => void, ...args: any[]) => { - const id = ++_immNextId; - _immQueue.push({ id, fn, args }); - if (!_immScheduled && !_immFlushing) { - _immScheduled = true; - _immChannel.port2.postMessage(null); - } - return id; - }; - (globalThis as any).clearImmediate = (id: number) => { - _immCancelled.add(id); - }; -} +installBrowserSetImmediatePolyfill(); import { CAPTURED_STDIO, diff --git a/host/test/browser-immediate-polyfill.test.ts b/host/test/browser-immediate-polyfill.test.ts new file mode 100644 index 0000000000..a9a1c937f6 --- /dev/null +++ b/host/test/browser-immediate-polyfill.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it, vi } from "vitest"; +import { + installBrowserSetImmediatePolyfill, + type BrowserImmediatePolyfillTarget, +} from "../src/browser-immediate-polyfill"; + +type MessageTask = () => void; + +class ManualPort { + onmessage: (() => void) | null = null; + peer: ManualPort | null = null; + + constructor(private readonly enqueue: (task: MessageTask) => void) {} + + postMessage(_value: unknown): void { + this.enqueue(() => this.peer?.onmessage?.()); + } +} + +class ManualMessageChannel { + static instances: ManualMessageChannel[] = []; + + readonly port1: ManualPort; + readonly port2: ManualPort; + private readonly tasks: MessageTask[] = []; + + constructor() { + const enqueue = (task: MessageTask) => this.tasks.push(task); + this.port1 = new ManualPort(enqueue); + this.port2 = new ManualPort(enqueue); + this.port1.peer = this.port2; + this.port2.peer = this.port1; + ManualMessageChannel.instances.push(this); + } + + flushNext(): void { + const task = this.tasks.shift(); + expect(task, "expected a queued MessageChannel task").toBeDefined(); + task!(); + } + + pendingTurns(): number { + return this.tasks.length; + } +} + +function makeTarget(): BrowserImmediatePolyfillTarget { + ManualMessageChannel.instances = []; + return { + MessageChannel: ManualMessageChannel as unknown as typeof MessageChannel, + }; +} + +function installedChannel(): ManualMessageChannel { + expect(ManualMessageChannel.instances).toHaveLength(1); + return ManualMessageChannel.instances[0]!; +} + +describe("browser setImmediate polyfill", () => { + it("keeps callbacks added during a flush for the next macrotask", () => { + const target = makeTarget(); + const state = installBrowserSetImmediatePolyfill(target)!; + const order: string[] = []; + + (target.setImmediate as any)(() => { + order.push("first"); + (target.setImmediate as any)(() => order.push("nested")); + }); + (target.setImmediate as any)((value: string) => order.push(value), "second"); + + const channel = installedChannel(); + expect(channel.pendingTurns()).toBe(1); + channel.flushNext(); + + expect(order).toEqual(["first", "second"]); + expect(state.pendingCount()).toBe(1); + expect(state.queueLength()).toBe(1); + expect(channel.pendingTurns()).toBe(1); + + channel.flushNext(); + expect(order).toEqual(["first", "second", "nested"]); + expect(state.pendingCount()).toBe(0); + expect(state.queueLength()).toBe(0); + }); + + it("cancels only a matching pending immediate", () => { + const target = makeTarget(); + const state = installBrowserSetImmediatePolyfill(target)!; + const kept = vi.fn(); + const cancelled = vi.fn(); + + (target.setImmediate as any)(kept); + (target.clearImmediate as any)(1); + const cancelledHandle = (target.setImmediate as any)(cancelled); + (target.clearImmediate as any)(cancelledHandle); + + installedChannel().flushNext(); + + expect(kept).toHaveBeenCalledOnce(); + expect(cancelled).not.toHaveBeenCalled(); + expect(state.pendingCount()).toBe(0); + expect(state.queueLength()).toBe(0); + }); + + it("does not retain unknown or already-delivered handles", () => { + const target = makeTarget(); + const state = installBrowserSetImmediatePolyfill(target)!; + + (target.clearImmediate as any)(1); + (target.clearImmediate as any)({ id: 1 }); + const delivered = (target.setImmediate as any)(() => {}); + installedChannel().flushNext(); + + for (let i = 0; i < 10_000; i++) { + (target.clearImmediate as any)(delivered); + (target.clearImmediate as any)(i); + (target.clearImmediate as any)({ id: i }); + } + + expect(state.pendingCount()).toBe(0); + expect(state.queueLength()).toBe(0); + }); + + it("does not replace a host-provided setImmediate", () => { + const setImmediate = vi.fn(); + const target = { + MessageChannel: ManualMessageChannel as unknown as typeof MessageChannel, + setImmediate, + }; + + expect(installBrowserSetImmediatePolyfill(target)).toBeNull(); + expect(target.setImmediate).toBe(setImmediate); + }); +}); From 321aa7bef8bbc7c2039d25d5f76d687a7f8c6f19 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 13:48:17 -0400 Subject: [PATCH 02/20] test(browser): preserve guest stderr on nonzero exit --- .../test/browser-nonzero-exit-stderr.spec.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 apps/browser-demos/test/browser-nonzero-exit-stderr.spec.ts diff --git a/apps/browser-demos/test/browser-nonzero-exit-stderr.spec.ts b/apps/browser-demos/test/browser-nonzero-exit-stderr.spec.ts new file mode 100644 index 0000000000..227ab3e345 --- /dev/null +++ b/apps/browser-demos/test/browser-nonzero-exit-stderr.spec.ts @@ -0,0 +1,61 @@ +import { expect, test } from "@playwright/test"; +import { readFile } from "node:fs/promises"; +import { resolveBinary } from "../../../host/src/binary-resolver"; + +const dashPath = resolveBinary("programs/dash.wasm"); + +type TestResult = { + exitCode: number; + stdout: string; + stderr: string; + combined: string; +}; + +type TestRunnerWindow = Window & { + __testRunnerReady: boolean; + __runTest( + wasmBytes: ArrayBuffer, + argv: string[], + timeoutMs: number, + ): Promise; +}; + +test("normal nonzero exits do not write host diagnostics to stderr", async ({ + page, +}) => { + const dashBytes = Array.from(await readFile(dashPath)); + + await page.goto("/pages/test-runner/", { waitUntil: "domcontentloaded" }); + await page.waitForFunction( + () => (window as unknown as TestRunnerWindow).__testRunnerReady === true, + ); + await page.evaluate(async () => { + await navigator.serviceWorker.register("/service-worker.js", { + scope: "/", + }); + await navigator.serviceWorker.ready; + }); + await page.reload({ waitUntil: "domcontentloaded" }); + await page.waitForFunction( + () => + (window as unknown as TestRunnerWindow).__testRunnerReady === true && + navigator.serviceWorker.controller !== null, + ); + + const result = await page.evaluate( + async (bytes) => + (window as unknown as TestRunnerWindow).__runTest( + new Uint8Array(bytes).buffer, + ["dash", "-c", "exit 7"], + 60_000, + ), + dashBytes, + ); + + expect(result).toEqual({ + exitCode: 7, + stdout: "", + stderr: "", + combined: "", + }); +}); From 297b547d7943c7c3c4df3319451e5a85fd941975 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:22:48 -0400 Subject: [PATCH 03/20] test(browser): cover guest HTTP through the configured CORS proxy Configure the browser test runner with its same-origin Vite proxy and keep a Chromium Wget regression for pages without a service-worker controller. Assert that BrowserKernel forwards the existing option and document the browser transport boundary. --- apps/browser-demos/pages/test-runner/main.ts | 6 ++ .../test/browser-cors-proxy.spec.ts | 82 +++++++++++++++++++ docs/architecture.md | 4 +- host/test/browser-kernel.test.ts | 6 +- 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 apps/browser-demos/test/browser-cors-proxy.spec.ts diff --git a/apps/browser-demos/pages/test-runner/main.ts b/apps/browser-demos/pages/test-runner/main.ts index 4babb8d7e3..6de5cf991e 100644 --- a/apps/browser-demos/pages/test-runner/main.ts +++ b/apps/browser-demos/pages/test-runner/main.ts @@ -50,6 +50,11 @@ let grepBytes: ArrayBuffer | null = null; let sedBytes: ArrayBuffer | null = null; let genCatBytes: ArrayBuffer | null = null; +const corsProxyUrl = new URL( + `${import.meta.env.BASE_URL}__kandelo_cors_proxy?url=`, + window.location.href, +).href; + const COREUTILS_NAMES = [ "arch", "b2sum", "base32", "base64", "basename", "basenc", "cat", "chcon", "chgrp", "chmod", "chown", "chroot", "cksum", "comm", "cp", @@ -182,6 +187,7 @@ async function init() { const kernel = new BrowserKernel({ kernelOwnedFs: true, + corsProxyUrl, onStdout: (data: Uint8Array) => { const text = new TextDecoder().decode(data); stdout += text; diff --git a/apps/browser-demos/test/browser-cors-proxy.spec.ts b/apps/browser-demos/test/browser-cors-proxy.spec.ts new file mode 100644 index 0000000000..1965e3634a --- /dev/null +++ b/apps/browser-demos/test/browser-cors-proxy.spec.ts @@ -0,0 +1,82 @@ +import { expect, test } from "@playwright/test"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { readFile } from "node:fs/promises"; +import { resolveBinary } from "../../../host/src/binary-resolver"; + +const wgetPath = resolveBinary("programs/wget.wasm"); + +type TestResult = { + exitCode: number; + stdout: string; + stderr: string; + combined: string; +}; + +type TestRunnerWindow = Window & { + __testRunnerReady: boolean; + __runTest( + wasmBytes: ArrayBuffer, + argv: string[], + timeoutMs: number, + ): Promise; +}; + +test("guest HTTP uses the test runner's same-origin CORS proxy", async ({ + page, +}) => { + const upstreamRequests: string[] = []; + const upstream = createServer((request, response) => { + upstreamRequests.push(request.url ?? ""); + response.writeHead(200, { "Content-Type": "text/plain" }); + response.end("Kandelo CORS proxy regression\n"); + }); + await new Promise((resolve, reject) => { + upstream.once("error", reject); + upstream.listen(0, "::1", () => { + upstream.off("error", reject); + resolve(); + }); + }); + + try { + const { port } = upstream.address() as AddressInfo; + // The trailing root dot avoids the guest's /etc/hosts localhost entry, so + // Kandelo delegates the connection to its browser backend. Node still + // resolves the proxy's upstream target to this test-only ::1 listener. + const targetUrl = `http://localhost.:${port}/probe`; + const wgetBytes = Array.from(await readFile(wgetPath)); + + await page.goto("/pages/test-runner/", { + waitUntil: "domcontentloaded", + }); + await page.waitForFunction( + () => (window as unknown as TestRunnerWindow).__testRunnerReady === true, + ); + expect( + await page.evaluate(() => navigator.serviceWorker.controller), + "the regression must exercise explicit BrowserKernel proxy configuration", + ).toBeNull(); + + const result = await page.evaluate( + async ({ bytes, url }) => + (window as unknown as TestRunnerWindow).__runTest( + new Uint8Array(bytes).buffer, + ["wget", "-qO-", url], + 60_000, + ), + { bytes: wgetBytes, url: targetUrl }, + ); + + expect( + result.exitCode, + JSON.stringify({ result, upstreamRequests }, null, 2), + ).toBe(0); + expect(result.stdout).toBe("Kandelo CORS proxy regression\n"); + expect(upstreamRequests).toEqual(["/probe"]); + } finally { + await new Promise((resolve, reject) => { + upstream.close((error) => (error ? reject(error) : resolve())); + }); + } +}); diff --git a/docs/architecture.md b/docs/architecture.md index 72fd8f43e7..1f6be3dcd5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -739,7 +739,9 @@ Browsers cannot create external raw TCP or UDP sockets. Local loopback and `Loca 1. **FetchNetworkBackend**: Buffers an entire HTTP request from the Wasm process, sends it via `fetch()`, and returns the raw HTTP response bytes. Works for simple HTTP clients. -2. **Service Worker HTTP Bridge**: For server demos (nginx, WordPress), a service worker intercepts browser `fetch()` requests to a configurable URL prefix (e.g., `/app/`) and forwards them to the kernel via a MessagePort connection pump. The kernel injects the request as a TCP connection to nginx's listening socket, and nginx's response flows back through the pipe to the service worker. +2. **TlsNetworkBackend**: Terminates the guest's TLS connection with a generated, in-VFS CA and sends the decoded HTTP request through browser `fetch()`. Service-worker-controlled apps may proxy cross-origin fetches transparently. Other embedders set `BrowserKernelOptions.corsProxyUrl`; the option crosses the main-thread/worker protocol and routes backend fetches through the application's CORS proxy. + +3. **Service Worker HTTP Bridge**: For server demos (nginx, WordPress), a service worker intercepts browser `fetch()` requests to a configurable URL prefix (e.g., `/app/`) and forwards them to the kernel via a MessagePort connection pump. The kernel injects the request as a TCP connection to nginx's listening socket, and nginx's response flows back through the pipe to the service worker. `TcpNetworkBackend`, `FetchNetworkBackend`, `TlsNetworkBackend`, and `LocalVirtualNetwork` share one numeric-address and hostname validator. It accepts decimal one-, two-, three-, and four-component IPv4 forms within their component widths, rejects malformed or overflowing numeric forms, enforces ASCII host-label syntax and DNS length limits, and preserves one trailing root dot. The Node TCP backend resolves validated names through the host resolver. The browser HTTP fetch/TLS bridges synthesize IPv4 mappings for syntactically acceptable DNS names; `LocalVirtualNetwork` resolves only aliases registered by attached machines. None of the browser paths adds browser DNS resolution or AF_INET6 transport. diff --git a/host/test/browser-kernel.test.ts b/host/test/browser-kernel.test.ts index 439b0fa4d1..ab22d026df 100644 --- a/host/test/browser-kernel.test.ts +++ b/host/test/browser-kernel.test.ts @@ -107,7 +107,10 @@ describe("BrowserKernel", () => { it("boot() spawns a worker, sends init, and resolves on `ready`", async () => { const BrowserKernel = await loadBrowserKernel(); - const kernel = new BrowserKernel({ kernelOwnedFs: true }); + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + corsProxyUrl: "https://proxy.example/?url=", + }); const bootPromise = kernel.boot({ kernelWasm: new ArrayBuffer(8), @@ -123,6 +126,7 @@ describe("BrowserKernel", () => { expect(init).toBeDefined(); expect(init.argv).toBeUndefined(); // argv goes in the spawn message expect(init.kernelWasmBytes).toBeInstanceOf(ArrayBuffer); + expect(init.config.corsProxyUrl).toBe("https://proxy.example/?url="); // Simulate the worker becoming ready, then reply to the spawn request. w.simulateMessage({ type: "ready" }); From 902e1f818d9ccd97648bfac427c51c4c2a0e4b3f Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 13:33:30 -0400 Subject: [PATCH 04/20] test(fork-instrument): verify reproducible output across processes The deterministic region ordering already landed through #907, but its in-process regression does not vary Rust HashMap random state. Run the CLI in twelve fresh processes against an alternating-type nested-region fixture and compare the emitted bytes. Document the cross-process byte-reproducibility contract and update the fork-instrumentation reference to the current ABI 39. This adds no ABI or package-artifact change. Validation: the full fork-instrument suite passed 187 tests; the focused fresh-process test passed; and the ABI snapshot check passed through scripts/dev-shell.sh. --- crates/fork-instrument/tests/determinism.rs | 81 +++++++++++++++++++ .../determinism/multiple_nested_regions.wat | 37 +++++++++ docs/fork-instrumentation.md | 6 +- 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 crates/fork-instrument/tests/determinism.rs create mode 100644 crates/fork-instrument/tests/fixtures/determinism/multiple_nested_regions.wat diff --git a/crates/fork-instrument/tests/determinism.rs b/crates/fork-instrument/tests/determinism.rs new file mode 100644 index 0000000000..3e5f3d8c4a --- /dev/null +++ b/crates/fork-instrument/tests/determinism.rs @@ -0,0 +1,81 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use wasmparser::Validator; + +struct TestDir(PathBuf); + +impl TestDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "fork-instrument-determinism-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).expect("create determinism test directory"); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn cli_output_is_byte_reproducible_across_processes() { + let dir = TestDir::new(); + let input_path = dir.path().join("input.wasm"); + let input = wat::parse_str(include_str!( + "fixtures/determinism/multiple_nested_regions.wat" + )) + .expect("compile determinism fixture"); + fs::write(&input_path, &input).expect("write determinism fixture"); + + let mut expected: Option> = None; + for run in 0..12 { + // Each CLI invocation is a fresh process with a fresh randomized + // HashMap state. In-process repetition cannot exercise that boundary. + let output_path = dir.path().join(format!("output-{run}.wasm")); + let output = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg(&input_path) + .arg("--output") + .arg(&output_path) + .output() + .expect("run wasm-fork-instrument"); + assert!( + output.status.success(), + "instrumentation run {run} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let bytes = fs::read(&output_path).expect("read instrumented output"); + if let Some(expected) = &expected { + if bytes != *expected { + let first_difference = bytes + .iter() + .zip(expected) + .position(|(actual, wanted)| actual != wanted) + .unwrap_or(bytes.len().min(expected.len())); + panic!( + "instrumentation run {run} differed at byte {first_difference} \ + (baseline {} bytes, actual {} bytes)", + expected.len(), + bytes.len() + ); + } + } else { + assert_ne!(bytes, input, "instrumentation unexpectedly changed no bytes"); + Validator::new() + .validate_all(&bytes) + .expect("instrumented baseline validates"); + expected = Some(bytes); + } + } +} diff --git a/crates/fork-instrument/tests/fixtures/determinism/multiple_nested_regions.wat b/crates/fork-instrument/tests/fixtures/determinism/multiple_nested_regions.wat new file mode 100644 index 0000000000..a6c2fda297 --- /dev/null +++ b/crates/fork-instrument/tests/fixtures/determinism/multiple_nested_regions.wat @@ -0,0 +1,37 @@ +(module + (type $pass-i32 (func (param i32) (result i32))) + (type $pass-i64 (func (param i64) (result i64))) + + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + + (memory (export "memory") 1) + + ;; Each typed block is a distinct fork-bearing region whose body parameter + ;; receives a synthetic local. Alternating parameter types makes a changed + ;; allocation order visible in the emitted local declarations. + (func $main (export "_start") + (i32.const 11) + (block (type $pass-i32) + drop + (call $kernel_fork)) + drop + + (i64.const 22) + (block (type $pass-i64) + drop + (call $kernel_fork) + i64.extend_i32_s) + drop + + (i32.const 33) + (block (type $pass-i32) + drop + (call $kernel_fork)) + drop + + (i64.const 44) + (block (type $pass-i64) + drop + (call $kernel_fork) + i64.extend_i32_s) + drop)) diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index f93ae0cbfa..809423d37e 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -20,7 +20,7 @@ For motivation, tradeoffs, and the rollout plan that led here, read for the post-rollout switch-dispatch redesign and non-fork-path-call gating that fix the kernel-side-effect re-fire bug, read [`plans/2026-04-22-fork-instrument-switch-dispatch-redesign.md`](plans/2026-04-22-fork-instrument-switch-dispatch-redesign.md). -ABI version: `12` (see +ABI version: `39` (see [`crates/shared/src/lib.rs`](../crates/shared/src/lib.rs) — see [abi-versioning.md](abi-versioning.md) for the policy). @@ -941,6 +941,10 @@ K-04, and K-07 cover the current behavior. - **Ref-typed user locals.** funcref, externref, and exnref locals are spilled to aux tables at unwind and restored at rewind. Slot assignments are deterministic per module. +- **Byte-reproducible instrumentation.** Given the same input bytes, CLI + options, and built tool, separate processes emit byte-identical Wasm. + Synthetic locals and nested regions are assigned in canonical sequence-ID + order rather than randomized hash-map iteration order. - **Mutable scalar globals.** Snapshotted in `wpk_fork_unwind_begin` and restored in `wpk_fork_rewind_begin`. Includes `__stack_pointer`, `__tls_base`, and any program-declared From b26b750e584c13b12da5f4c5e889b637a6988203 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:22:01 -0400 Subject: [PATCH 05/20] MariaDB: share writable-directory setup across VFS images --- apps/browser-demos/lib/init/index.ts | 3 --- apps/browser-demos/lib/init/mariadb-config.ts | 24 ------------------- images/vfs/scripts/build-lamp-vfs-image.ts | 18 ++------------ packages/registry/lamp/build.toml | 1 + 4 files changed, 3 insertions(+), 43 deletions(-) delete mode 100644 apps/browser-demos/lib/init/mariadb-config.ts diff --git a/apps/browser-demos/lib/init/index.ts b/apps/browser-demos/lib/init/index.ts index 166579e3f0..45d42b07ef 100644 --- a/apps/browser-demos/lib/init/index.ts +++ b/apps/browser-demos/lib/init/index.ts @@ -23,6 +23,3 @@ export { COREUTILS_NAMES } from "./shell-binaries"; // Service worker bridge export { initServiceWorkerBridge } from "./service-worker-bridge"; - -// MariaDB directory setup -export { populateMariadbDirs } from "./mariadb-config"; diff --git a/apps/browser-demos/lib/init/mariadb-config.ts b/apps/browser-demos/lib/init/mariadb-config.ts deleted file mode 100644 index e19b8ab109..0000000000 --- a/apps/browser-demos/lib/init/mariadb-config.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * MariaDB directory setup for the kandelo browser environment. - * - * Creates the data directory structure MariaDB needs for bootstrap and - * server operation. - */ -import type { MemoryFileSystem } from "../../../../host/src/vfs/memory-fs"; -import { ensureDir } from "./vfs-utils"; - -/** - * Create the MariaDB data directory structure. - * - * Directories: - * /data — main data directory (--datadir) - * /data/mysql — system tables - * /data/tmp — temporary files (--tmpdir) - * /data/test — default test database - */ -export function populateMariadbDirs(fs: MemoryFileSystem): void { - ensureDir(fs, "/data"); - ensureDir(fs, "/data/mysql"); - ensureDir(fs, "/data/tmp"); - ensureDir(fs, "/data/test"); -} diff --git a/images/vfs/scripts/build-lamp-vfs-image.ts b/images/vfs/scripts/build-lamp-vfs-image.ts index 57881a9e41..f543d97aa9 100644 --- a/images/vfs/scripts/build-lamp-vfs-image.ts +++ b/images/vfs/scripts/build-lamp-vfs-image.ts @@ -49,6 +49,7 @@ import { import { MYSQL_BENCHMARK_PHP } from "../../../apps/browser-demos/lib/init/mysql-benchmark"; import { loadShellBaseFileSystem } from "./shell-vfs-build"; import { preinstallWordPressMariaDb } from "./wordpress-preinstall"; +import { prepareMariadbWritableDirectories } from "./mariadb-image-helpers"; const REPO_ROOT = findRepoRoot(); const BROWSER_DIR = join(REPO_ROOT, "apps", "browser-demos"); @@ -77,8 +78,6 @@ const OPCACHE_SO_PATH = resolveBinary("programs/php/opcache.so"); const MSMTPD_PATH = resolveBinary("programs/msmtpd.wasm"); const OUT_FILE = join(BROWSER_DIR, "public", "lamp.vfs.zst"); const PHP_FPM_WORKERS = 6; -const MYSQL_UID = 101; -const MYSQL_GID = 101; const MARIADB_SOCKET_PATH = "/tmp/mysql.sock"; const LAMP_IMAGE_MAX_BYTES = 768 * 1024 * 1024; const MARIADB_ARIA_LOG_FILE_SIZE = 16 * 1024 * 1024; @@ -87,19 +86,6 @@ const MARIADB_INNODB_LOG_FILE_SIZE = 16 * 1024 * 1024; const MARIADB_INNODB_LOG_BUFFER_SIZE = 1024 * 1024; const MARIADB_INNODB_BUFFER_POOL_SIZE = 8 * 1024 * 1024; -// LAMP-specific data dirs that mariadbd writes to at runtime. The image -// starts from the full shell demo VFS, so the bootstrap script gets the same -// /bin/sh and utility layout users see in the interactive terminal. -function populateMariadbDataDirs(fs: MemoryFileSystem): void { - for (const dir of ["/data", "/data/mysql", "/data/tmp", "/data/test"]) { - ensureDirRecursive(fs, dir); - fs.chown(dir, MYSQL_UID, MYSQL_GID); - fs.chmod(dir, 0o775); - } - ensureDirRecursive(fs, "/tmp"); - fs.chmod("/tmp", 0o1777); -} - function populateMariadb(fs: MemoryFileSystem): void { ensureDirRecursive(fs, "/usr/sbin"); writeVfsBinary(fs, "/usr/sbin/mariadbd", new Uint8Array(readFileSync(MARIADB_PATH))); @@ -430,7 +416,7 @@ async function main() { console.log("Loading shell base image..."); const fs = loadShellBaseFileSystem(LAMP_IMAGE_MAX_BYTES); - populateMariadbDataDirs(fs); + prepareMariadbWritableDirectories(fs); console.log("Writing nginx + php-fpm + msmtpd binaries..."); ensureDirRecursive(fs, "/usr/sbin"); diff --git a/packages/registry/lamp/build.toml b/packages/registry/lamp/build.toml index 3153c80868..5e14698f88 100644 --- a/packages/registry/lamp/build.toml +++ b/packages/registry/lamp/build.toml @@ -11,6 +11,7 @@ inputs = [ "images/rootfs/etc/services", "images/vfs/scripts/kandelo-demo-config.ts", "images/vfs/scripts/kandelo-demo-guides.ts", + "images/vfs/scripts/mariadb-image-helpers.ts", "images/vfs/scripts/opcache-prewarm.ts", "images/vfs/scripts/shell-vfs-build.ts", "images/vfs/scripts/smtp-capture-helpers.ts", From 8d9e1df28957ed8069c70c0343fdf551727dce01 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 13:50:00 -0400 Subject: [PATCH 06/20] VFS: share canonical rootfs overlay assembly Move the recursive /etc merge landed with #907 out of the browser app and into the shared host VFS layer. Preserve caller-owned entries and source metadata, and keep missing canonical state, short reads, and capacity failures visible during image assembly. --- apps/browser-demos/lib/kernel-owned-boot.ts | 113 +----------- docs/architecture.md | 11 +- host/src/vfs/index.ts | 1 + host/src/vfs/rootfs-overlay.ts | 171 ++++++++++++++++++ host/test/kernel-owned-boot.test.ts | 74 -------- host/test/vfs/rootfs-overlay.test.ts | 183 ++++++++++++++++++++ 6 files changed, 368 insertions(+), 185 deletions(-) create mode 100644 host/src/vfs/rootfs-overlay.ts delete mode 100644 host/test/kernel-owned-boot.test.ts create mode 100644 host/test/vfs/rootfs-overlay.test.ts diff --git a/apps/browser-demos/lib/kernel-owned-boot.ts b/apps/browser-demos/lib/kernel-owned-boot.ts index bd08b08616..a85f8d5f81 100644 --- a/apps/browser-demos/lib/kernel-owned-boot.ts +++ b/apps/browser-demos/lib/kernel-owned-boot.ts @@ -9,9 +9,12 @@ // left is the small, transient per-boot image-build FS; these helpers track it // and nudge WebKit's collector to reclaim it between boots. import { MemoryFileSystem } from "@host/vfs/memory-fs"; +import { overlayEtcFromRootfs } from "@host/vfs/rootfs-overlay"; // @ts-expect-error — vite ?url virtual module (resolved by the kernel-artifacts plugin) import rootfsVfsUrl from "@rootfs-vfs?url"; +export { overlayEtcFromRootfs }; + const WEBKIT_RECLAIM_TIMEOUT_MS = 1_500; const WEBKIT_RECLAIM_STEP_MS = 150; const WEBKIT_RECLAIM_PRESSURE_BYTES = 32 * 1024 * 1024; @@ -91,116 +94,6 @@ export function createEmptyBuildFs(maxByteLength = 64 * 1024 * 1024): MemoryFile return MemoryFileSystem.create(sab, maxByteLength); } -const S_IFMT = 0xf000; -const S_IFREG = 0x8000; -const S_IFDIR = 0x4000; -const S_IFLNK = 0xa000; - -function copyMissingRootfsPath( - source: MemoryFileSystem, - target: MemoryFileSystem, - path: string, -): void { - const sourceStat = source.lstat(path); - const sourceKind = sourceStat.mode & S_IFMT; - let targetKind: number | null = null; - try { - targetKind = target.lstat(path).mode & S_IFMT; - } catch { - // Missing in the caller's image: copy it from the canonical rootfs. - } - - // Existing caller-owned leaves always win. Existing directories merge with - // canonical directories so a demo can override one file without losing the - // rest of that subtree. - if (targetKind !== null && (sourceKind !== S_IFDIR || targetKind !== S_IFDIR)) { - return; - } - - if (sourceKind === S_IFDIR) { - if (targetKind === null) { - target.mkdirWithOwner( - path, - sourceStat.mode & 0o7777, - sourceStat.uid, - sourceStat.gid, - ); - } - const dh = source.opendir(path); - try { - for (;;) { - const entry = source.readdir(dh); - if (entry === null) break; - if (entry.name === "." || entry.name === "..") continue; - copyMissingRootfsPath( - source, - target, - path === "/" ? `/${entry.name}` : `${path}/${entry.name}`, - ); - } - } finally { - source.closedir(dh); - } - return; - } - - if (sourceKind === S_IFLNK) { - target.symlinkWithOwner( - source.readlink(path), - path, - sourceStat.uid, - sourceStat.gid, - ); - return; - } - - if (sourceKind !== S_IFREG) { - throw new Error(`Unsupported canonical /etc file type at ${path}`); - } - - const bytes = new Uint8Array(sourceStat.size); - const fd = source.open(path, 0, 0); - let offset = 0; - try { - while (offset < bytes.length) { - const count = source.read( - fd, - bytes.subarray(offset), - null, - bytes.length - offset, - ); - if (count <= 0) { - throw new Error( - `Short read while copying canonical rootfs path ${path}: ` + - `${offset}/${bytes.length} bytes`, - ); - } - offset += count; - } - } finally { - source.close(fd); - } - target.createFileWithOwner( - path, - sourceStat.mode & 0o7777, - sourceStat.uid, - sourceStat.gid, - bytes, - ); -} - -/** - * Recursively merge canonical `/etc` state into `target`, without overwriting - * files or symlinks the caller already wrote. This replaces the legacy - * worker-side `/etc` overlay that `kernel.init()` performed: demos that start - * from a small custom image still inherit rootfs-owned account, resolver, TLS, - * and other configuration through the normal VFS path. - */ -export function overlayEtcFromRootfs(target: MemoryFileSystem, rootfsImage: Uint8Array): void { - const source = MemoryFileSystem.fromImage(rootfsImage); - copyMissingRootfsPath(source, target, "/etc"); -} - /** * Convenience: an empty build FS pre-seeded with `/etc` from the canonical * rootfs — the kernel-owned equivalent of the legacy empty-FS + init()-overlay diff --git a/docs/architecture.md b/docs/architecture.md index 1f6be3dcd5..72de3c810e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -555,7 +555,16 @@ VFS images can also carry image-level metadata outside the guest file tree. The `BrowserKernel.boot({ vfsImage, ... })` is the kernel-owned VFS path. The worker restores the supplied image (per-demo `.vfs.zst`, typically built on top of the canonical rootfs as a base layer) into a `MemoryFileSystem`, applies `DEFAULT_MOUNT_SPEC` via `resolveForBrowser` (the image becomes the `/` mount; the seven scratch mounts come up empty), and layers `/dev/shm` + `/dev` on top. Browser networking then replaces `/etc/ssl/certs/ca-certificates.crt` with its generated per-session MITM root; the image-owned OpenSSL configuration and compiled-in `/etc/ssl/cert.pem` trust path remain unchanged. -The legacy `kernel.spawn(programBytes, argv, { fsSab })` path is still supported for demos that own a single `MemoryFileSystem` SAB at `/` (used by `benchmark`, `erlang`, `shell`). To keep NSS and other static system policy available on that path, the browser kernel worker recursively merges `/etc/**` from `rootfs.vfs` into the demo SAB at boot (`overlayEtcFromRootfs` in `host/src/vfs/rootfs-overlay.ts`). Existing leaf files and symlinks remain demo-owned, while existing directories are traversed so missing canonical descendants such as `/etc/ssl/openssl.cnf` are still installed. This is a temporary bridge until those demos move to the `vfsImage` boot path. +The browser test runner and Git test assemble small kernel-owned VFS images with +`createBuildFsWithEtc` in `apps/browser-demos/lib/kernel-owned-boot.ts`, then +serialize them with `finalizeKernelOwnedImage` and boot them through +`BrowserKernel.boot`. Before serialization, the shared host helper +`overlayEtcFromRootfs` in `host/src/vfs/rootfs-overlay.ts` recursively merges +`/etc/**` from the canonical `rootfs.vfs`. Existing leaves and directory +metadata remain caller-owned, while missing canonical descendants such as +`/etc/ssl/openssl.cnf` retain their source modes and ownership. Missing +canonical `/etc` state, short reads, and target capacity failures abort image +assembly instead of producing an incomplete filesystem. ### Lazy Files diff --git a/host/src/vfs/index.ts b/host/src/vfs/index.ts index a1aaaffba7..92c764a0b7 100644 --- a/host/src/vfs/index.ts +++ b/host/src/vfs/index.ts @@ -27,3 +27,4 @@ export { } from "./default-mounts"; export type { MountSpec, BrowserResolverOptions } from "./default-mounts"; export { resolveForNode } from "./default-mounts-node"; +export { overlayEtcFromRootfs } from "./rootfs-overlay"; diff --git a/host/src/vfs/rootfs-overlay.ts b/host/src/vfs/rootfs-overlay.ts new file mode 100644 index 0000000000..9c922bef01 --- /dev/null +++ b/host/src/vfs/rootfs-overlay.ts @@ -0,0 +1,171 @@ +import { MemoryFileSystem } from "./memory-fs"; +import { + ENOENT, + ENOSPC, + O_CREAT, + O_RDONLY, + O_TRUNC, + O_WRONLY, + S_IFDIR, + S_IFLNK, + S_IFMT, + S_IFREG, + SFSError, +} from "./sharedfs-vendor"; + +function lstatIfPresent(fs: MemoryFileSystem, path: string) { + try { + return fs.lstat(path); + } catch (error) { + if (error instanceof SFSError && error.code === ENOENT) return null; + throw error; + } +} + +function readFile( + fs: MemoryFileSystem, + path: string, + size: number, +): Uint8Array { + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + let offset = 0; + try { + while (offset < bytes.length) { + const count = fs.read( + fd, + bytes.subarray(offset), + null, + bytes.length - offset, + ); + if (count <= 0) break; + offset += count; + } + } finally { + fs.close(fd); + } + + if (offset !== bytes.length) { + throw new Error( + `Short read while copying canonical rootfs path ${path}: ` + + `${offset}/${bytes.length} bytes`, + ); + } + return bytes; +} + +function writeFile( + fs: MemoryFileSystem, + path: string, + bytes: Uint8Array, + mode: number, + uid: number, + gid: number, +): void { + const fd = fs.open(path, O_WRONLY | O_CREAT | O_TRUNC, mode); + let offset = 0; + try { + while (offset < bytes.length) { + const count = fs.write( + fd, + bytes.subarray(offset), + null, + bytes.length - offset, + ); + if (count <= 0) { + throw new SFSError( + ENOSPC, + `No space left on device while copying canonical rootfs path ${path}: ` + + `${offset}/${bytes.length} bytes`, + ); + } + offset += count; + } + } finally { + fs.close(fd); + } + fs.chown(path, uid, gid); + fs.chmod(path, mode); +} + +/** + * Merge one canonical rootfs path into a caller-owned filesystem without + * overwriting an existing leaf. Existing directories are traversed so missing + * canonical descendants can still be added below caller-owned directory trees. + */ +function copyMissingRootfsPath( + source: MemoryFileSystem, + target: MemoryFileSystem, + path: string, +): void { + const sourceStat = source.lstat(path); + const sourceKind = sourceStat.mode & S_IFMT; + const targetStat = lstatIfPresent(target, path); + + if (sourceKind === S_IFDIR) { + if (targetStat) { + if ((targetStat.mode & S_IFMT) !== S_IFDIR) return; + } else { + target.mkdirWithOwner( + path, + sourceStat.mode & 0o7777, + sourceStat.uid, + sourceStat.gid, + ); + } + + const dh = source.opendir(path); + try { + for (;;) { + const entry = source.readdir(dh); + if (entry === null) break; + if (entry.name === "." || entry.name === "..") continue; + const child = path === "/" ? `/${entry.name}` : `${path}/${entry.name}`; + copyMissingRootfsPath(source, target, child); + } + } finally { + source.closedir(dh); + } + return; + } + + // A caller-owned file or symlink is authoritative for that exact leaf. + if (targetStat) return; + + if (sourceKind === S_IFLNK) { + target.symlinkWithOwner( + source.readlink(path), + path, + sourceStat.uid, + sourceStat.gid, + ); + return; + } + + if (sourceKind !== S_IFREG) { + throw new Error(`Unsupported canonical /etc file type at ${path}`); + } + + writeFile( + target, + path, + readFile(source, path, sourceStat.size), + sourceStat.mode & 0o7777, + sourceStat.uid, + sourceStat.gid, + ); +} + +/** + * Recursively merge canonical `/etc` image state into an image under + * construction. Existing leaves and directory metadata remain caller-owned; + * missing canonical directories, regular files, and symlinks retain their + * source ownership and modes. + */ +export function overlayEtcFromRootfs( + target: MemoryFileSystem, + rootfsImage: Uint8Array, +): void { + const source = MemoryFileSystem.fromImage(rootfsImage); + copyMissingRootfsPath(source, target, "/etc"); +} diff --git a/host/test/kernel-owned-boot.test.ts b/host/test/kernel-owned-boot.test.ts deleted file mode 100644 index 0747c6cb26..0000000000 --- a/host/test/kernel-owned-boot.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - createEmptyBuildFs, - overlayEtcFromRootfs, -} from "../../apps/browser-demos/lib/kernel-owned-boot"; -import type { MemoryFileSystem } from "../src/vfs/memory-fs"; - -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -function addFile( - fs: MemoryFileSystem, - path: string, - contents: string, - mode = 0o644, - uid = 0, - gid = 0, -): void { - fs.createFileWithOwner(path, mode, uid, gid, encoder.encode(contents)); -} - -function readFile(fs: MemoryFileSystem, path: string): string { - const stat = fs.stat(path); - const bytes = new Uint8Array(stat.size); - const fd = fs.open(path, 0, 0); - try { - expect(fs.read(fd, bytes, null, bytes.length)).toBe(bytes.length); - } finally { - fs.close(fd); - } - return decoder.decode(bytes); -} - -describe("kernel-owned browser image assembly", () => { - it("recursively merges canonical /etc while preserving caller leaves", async () => { - const source = createEmptyBuildFs(); - source.mkdirWithOwner("/etc", 0o755, 0, 0); - source.mkdirWithOwner("/etc/ssl", 0o750, 12, 34); - addFile(source, "/etc/hosts", "canonical hosts\n"); - addFile(source, "/etc/ssl/openssl.cnf", "canonical config\n"); - addFile(source, "/etc/ssl/cert.pem", "canonical cert\n", 0o640, 12, 34); - source.symlinkWithOwner("cert.pem", "/etc/ssl/current.pem", 12, 34); - - const target = createEmptyBuildFs(); - target.mkdirWithOwner("/etc", 0o755, 0, 0); - target.mkdirWithOwner("/etc/ssl", 0o755, 1000, 1000); - addFile(target, "/etc/ssl/openssl.cnf", "demo config\n", 0o600, 1000, 1000); - - overlayEtcFromRootfs(target, await source.saveImage()); - - expect(readFile(target, "/etc/hosts")).toBe("canonical hosts\n"); - expect(readFile(target, "/etc/ssl/cert.pem")).toBe("canonical cert\n"); - expect(readFile(target, "/etc/ssl/openssl.cnf")).toBe("demo config\n"); - expect(target.readlink("/etc/ssl/current.pem")).toBe("cert.pem"); - expect(target.stat("/etc/ssl/cert.pem")).toMatchObject({ - mode: expect.any(Number), - uid: 12, - gid: 34, - }); - expect(target.stat("/etc/ssl/cert.pem").mode & 0o7777).toBe(0o640); - expect(target.stat("/etc/ssl/openssl.cnf")).toMatchObject({ - uid: 1000, - gid: 1000, - }); - }); - - it("fails loudly when the canonical image has no /etc tree", async () => { - const source = createEmptyBuildFs(); - const target = createEmptyBuildFs(); - const image = await source.saveImage(); - - expect(() => overlayEtcFromRootfs(target, image)).toThrow(); - }); -}); diff --git a/host/test/vfs/rootfs-overlay.test.ts b/host/test/vfs/rootfs-overlay.test.ts new file mode 100644 index 0000000000..8bb603f078 --- /dev/null +++ b/host/test/vfs/rootfs-overlay.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; +import { overlayEtcFromRootfs } from "../../src/vfs/rootfs-overlay"; +import { + ENOENT, + ENOSPC, + S_IFDIR, + S_IFLNK, + S_IFMT, + S_IFREG, + SFSError, +} from "../../src/vfs/sharedfs-vendor"; + +function createFs(bytes = 2 * 1024 * 1024): MemoryFileSystem { + return MemoryFileSystem.create(new SharedArrayBuffer(bytes)); +} + +function writeText( + fs: MemoryFileSystem, + path: string, + text: string, + mode = 0o644, + uid = 0, + gid = 0, +): void { + fs.createFileWithOwner( + path, + mode, + uid, + gid, + new TextEncoder().encode(text), + ); +} + +function readText(fs: MemoryFileSystem, path: string): string { + const stat = fs.stat(path); + const bytes = new Uint8Array(stat.size); + const fd = fs.open(path, 0, 0); + try { + expect(fs.read(fd, bytes, null, bytes.length)).toBe(bytes.length); + } finally { + fs.close(fd); + } + return new TextDecoder().decode(bytes); +} + +function captureError(operation: () => void): unknown { + try { + operation(); + } catch (error) { + return error; + } + throw new Error("Expected operation to fail"); +} + +describe("canonical rootfs /etc overlay", () => { + it("copies nested state and metadata while preserving caller-owned entries", async () => { + const source = createFs(); + source.mkdirWithOwner("/etc", 0o755, 0, 0); + source.mkdirWithOwner("/etc/ssl", 0o755, 0, 0); + source.mkdirWithOwner("/etc/ssl/certs", 0o750, 12, 34); + writeText(source, "/etc/ssl/openssl.cnf", "canonical\n"); + writeText(source, "/etc/ssl/cert.pem", "root bundle\n", 0o640, 12, 34); + source.symlinkWithOwner("../cert.pem", "/etc/ssl/certs/default.pem", 12, 34); + source.symlinkWithOwner("../cert.pem", "/etc/ssl/certs/copied.pem", 12, 34); + + const target = createFs(); + target.mkdirWithOwner("/etc", 0o755, 0, 0); + target.mkdirWithOwner("/etc/ssl", 0o700, 1000, 1000); + target.mkdirWithOwner("/etc/ssl/certs", 0o700, 1000, 1000); + writeText(target, "/etc/ssl/openssl.cnf", "caller policy\n", 0o600, 1000, 1000); + target.symlinkWithOwner( + "/caller/trust.pem", + "/etc/ssl/certs/default.pem", + 1000, + 1000, + ); + + overlayEtcFromRootfs(target, await source.saveImage()); + + expect(readText(target, "/etc/ssl/openssl.cnf")).toBe("caller policy\n"); + expect(target.stat("/etc/ssl/openssl.cnf")).toMatchObject({ + mode: S_IFREG | 0o600, + uid: 1000, + gid: 1000, + }); + expect(readText(target, "/etc/ssl/cert.pem")).toBe("root bundle\n"); + expect(target.stat("/etc/ssl/cert.pem")).toMatchObject({ + mode: S_IFREG | 0o640, + uid: 12, + gid: 34, + }); + expect(target.readlink("/etc/ssl/certs/default.pem")).toBe( + "/caller/trust.pem", + ); + expect(target.readlink("/etc/ssl/certs/copied.pem")).toBe("../cert.pem"); + expect(target.lstat("/etc/ssl/certs/copied.pem")).toMatchObject({ + mode: S_IFLNK | 0o777, + uid: 12, + gid: 34, + }); + + expect(target.lstat("/etc").mode & S_IFMT).toBe(S_IFDIR); + expect(target.lstat("/etc/ssl/cert.pem").mode & S_IFMT).toBe(S_IFREG); + expect(target.lstat("/etc/ssl/certs/copied.pem").mode & S_IFMT).toBe( + S_IFLNK, + ); + expect(target.stat("/etc/ssl")).toMatchObject({ + mode: S_IFDIR | 0o700, + uid: 1000, + gid: 1000, + }); + expect(target.stat("/etc/ssl/certs")).toMatchObject({ + mode: S_IFDIR | 0o700, + uid: 1000, + gid: 1000, + }); + }); + + it("preserves metadata on canonical directories created in the target", async () => { + const source = createFs(); + source.mkdirWithOwner("/etc", 0o751, 12, 34); + source.mkdirWithOwner("/etc/ssl", 0o750, 56, 78); + const target = createFs(); + + overlayEtcFromRootfs(target, await source.saveImage()); + + expect(target.stat("/etc")).toMatchObject({ + mode: S_IFDIR | 0o751, + uid: 12, + gid: 34, + }); + expect(target.stat("/etc/ssl")).toMatchObject({ + mode: S_IFDIR | 0o750, + uid: 56, + gid: 78, + }); + }); + + it("propagates ENOENT when the canonical image has no /etc tree", async () => { + const source = createFs(); + const target = createFs(); + const image = await source.saveImage(); + + const error = captureError(() => overlayEtcFromRootfs(target, image)); + + expect(error).toBeInstanceOf(SFSError); + expect((error as SFSError).code).toBe(ENOENT); + }); + + it("rejects a short source read instead of copying a truncated file", async () => { + const source = createFs(); + source.mkdirWithOwner("/etc", 0o755, 0, 0); + writeText(source, "/etc/hosts", "127.0.0.1 localhost\n"); + const image = await source.saveImage(); + const target = createFs(); + const readSpy = vi + .spyOn(MemoryFileSystem.prototype, "read") + .mockReturnValueOnce(0); + + try { + expect(() => overlayEtcFromRootfs(target, image)).toThrow( + "Short read while copying canonical rootfs path /etc/hosts: 0/20 bytes", + ); + } finally { + readSpy.mockRestore(); + } + }); + + it("propagates target capacity failures instead of accepting a partial overlay", async () => { + const source = createFs(); + source.mkdirWithOwner("/etc", 0o755, 0, 0); + source.mkdirWithOwner("/etc/ssl", 0o755, 0, 0); + writeText(source, "/etc/ssl/cert.pem", "x".repeat(128 * 1024)); + const target = createFs(64 * 1024); + const image = await source.saveImage(); + + const error = captureError(() => overlayEtcFromRootfs(target, image)); + + expect(error).toBeInstanceOf(SFSError); + expect((error as SFSError).code).toBe(ENOSPC); + }); +}); From 1d8085e8e6fd9c1f67befd8e86e890420c21323a Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:18:00 -0400 Subject: [PATCH 07/20] Test SjLj across C++ noexcept boundaries Add raw wasm32/wasm64 and fork-instrumented controls for issue #918, plus a positive SIGCHLD child-reaping fixture. Cover the behavior in Node and Chromium and document the pinned LLVM 21 limitation without replaying Dinit production or package changes from #911. --- .../test/sjlj-noexcept-boundary.spec.ts | 117 ++++++++++++++++++ docs/sdk-guide.md | 17 +++ host/test/sjlj-noexcept-boundary.test.ts | 83 +++++++++++++ programs/sigchld_sjlj.c | 81 ++++++++++++ programs/sjlj_noexcept_boundary.cpp | 89 +++++++++++++ scripts/build-programs.sh | 67 ++++++++-- 6 files changed, 442 insertions(+), 12 deletions(-) create mode 100644 apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts create mode 100644 host/test/sjlj-noexcept-boundary.test.ts create mode 100644 programs/sigchld_sjlj.c create mode 100644 programs/sjlj_noexcept_boundary.cpp diff --git a/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts b/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts new file mode 100644 index 0000000000..580f572f52 --- /dev/null +++ b/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts @@ -0,0 +1,117 @@ +import { expect, test } from "@playwright/test"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { findRepoRoot, resolveBinary } from "../../../host/src/binary-resolver"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const browserKernelModulePath = resolve( + __dirname, + "../../../host/src/browser-kernel-host.ts", +); +const repoRoot = findRepoRoot(); + +const fixturePaths = { + rawWasm32: join( + repoRoot, + "local-binaries/test-fixtures/wasm32/sjlj_noexcept_boundary.raw.wasm", + ), + rawWasm64: join( + repoRoot, + "local-binaries/test-fixtures/wasm64/sjlj_noexcept_boundary.raw.wasm", + ), + instrumented: resolveBinary("programs/sjlj_noexcept_boundary.wasm"), + sigchld: resolveBinary("programs/sigchld_sjlj.wasm"), +}; + +test("Chromium preserves the SjLj controls and positive SIGCHLD path", async ({ + page, + baseURL, +}) => { + test.setTimeout(180_000); + expect(baseURL).toBeTruthy(); + + const asViteFsUrl = (path: string) => + new URL(`/@fs/${path}`, baseURL).href; + const browserKernelModuleUrl = asViteFsUrl(browserKernelModulePath); + const fixtureUrls = Object.fromEntries( + Object.entries(fixturePaths).map(([name, path]) => [name, asViteFsUrl(path)]), + ); + + await page.goto(new URL("/trap-signal-test.html", baseURL).href); + const results = await page.evaluate( + async ({ browserKernelModuleUrl, fixtureUrls }) => { + const { BrowserKernel } = await import( + /* @vite-ignore */ browserKernelModuleUrl + ); + const decoder = new TextDecoder(); + let stdout = ""; + let stderr = ""; + const kernel = new BrowserKernel({ + maxWorkers: 4, + onStdout: (data: Uint8Array) => { + stdout += decoder.decode(data); + }, + onStderr: (data: Uint8Array) => { + stderr += decoder.decode(data); + }, + }); + let initialized = false; + + const run = async (url: string, argv: string[]) => { + stdout = ""; + stderr = ""; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`fixture fetch failed: ${response.status} ${url}`); + } + const exitCode = await kernel.spawn(await response.arrayBuffer(), argv); + return { exitCode, stdout, stderr }; + }; + + try { + await kernel.initFromImage({ vfsImage: "default" }); + initialized = true; + return { + rawWasm32: await run(fixtureUrls.rawWasm32, [ + "sjlj_noexcept_boundary", + "--noexcept", + ]), + instrumented: await run(fixtureUrls.instrumented, [ + "sjlj_noexcept_boundary", + "--noexcept", + ]), + permissive: await run(fixtureUrls.instrumented, [ + "sjlj_noexcept_boundary", + "--permissive", + ]), + sigchld: await run(fixtureUrls.sigchld, ["sigchld_sjlj"]), + rawWasm64: await run(fixtureUrls.rawWasm64, [ + "sjlj_noexcept_boundary", + "--noexcept", + ]), + }; + } finally { + if (initialized) await kernel.destroy(); + } + }, + { browserKernelModuleUrl, fixtureUrls }, + ); + + for (const control of [ + results.rawWasm32, + results.instrumented, + results.rawWasm64, + ]) { + expect(control.exitCode).toBe(128 + 6); + expect(control.stderr).toContain("HANDLER: siglongjmp"); + expect(control.stderr).toContain("libc++abi: terminating"); + expect(control.stdout).not.toContain("LANDING: siglongjmp resumed"); + } + + expect(results.permissive).toMatchObject({ exitCode: 0 }); + expect(results.permissive.stdout).toContain("LANDING: siglongjmp resumed"); + expect(results.sigchld).toMatchObject({ exitCode: 0 }); + expect(results.sigchld.stdout).toContain( + "PASS: SIGCHLD siglongjmp resumed at pselect landing pad", + ); +}); diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md index f17eaac35f..7e1798c83d 100644 --- a/docs/sdk-guide.md +++ b/docs/sdk-guide.md @@ -118,6 +118,23 @@ separate `-lunwind`. to wasm-EH `try_table` / `catch_ref` instructions. Without it, catch handlers are dead-code-eliminated and `throw` hangs at runtime. +#### LLVM 21 SjLj and `noexcept` limitation + +Kandelo's pinned LLVM 21.1.7 toolchain lowers `longjmp` and `siglongjmp` to an +internal Wasm exception. If that transfer crosses a C++ `noexcept` frame, +Clang's generated termination handler can intercept the internal tag before +the matching `setjmp` or `sigsetjmp` landing consumes it. The process then +calls `std::terminate()` even when the C control transfer itself is valid. + +This is a known SDK/toolchain limitation tracked in +[issue #918](https://github.com/Automattic/kandelo/issues/918), not a change to +POSIX signal or `longjmp` semantics. It is present in raw clang-linked wasm32 +and wasm64 modules and remains present after Kandelo's wasm32 fork +instrumentation. Until the pinned compiler is fixed, code that establishes a +jump landing and calls work that can jump back across the current frame must +not mark that crossed frame `noexcept`. Keep the workaround scoped to that +boundary; do not disable C++ exceptions, signal delivery, or child reaping. + ### Building static libraries ```bash diff --git a/host/test/sjlj-noexcept-boundary.test.ts b/host/test/sjlj-noexcept-boundary.test.ts new file mode 100644 index 0000000000..53fe0be083 --- /dev/null +++ b/host/test/sjlj-noexcept-boundary.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { findRepoRoot, resolveBinary } from "../src/binary-resolver"; +import { runCentralizedProgram } from "./centralized-test-helper"; + +const repoRoot = findRepoRoot(); +const rawWasm32Fixture = join( + repoRoot, + "local-binaries/test-fixtures/wasm32/sjlj_noexcept_boundary.raw.wasm", +); +const rawWasm64Fixture = join( + repoRoot, + "local-binaries/test-fixtures/wasm64/sjlj_noexcept_boundary.raw.wasm", +); +const instrumentedFixture = resolveBinary( + "programs/sjlj_noexcept_boundary.wasm", +); +const sigchldFixture = resolveBinary("programs/sigchld_sjlj.wasm"); +const TERMINATED_BY_SIGABRT = 128 + 6; + +describe("LLVM Wasm SjLj across a noexcept boundary", () => { + it("keeps the raw wasm32 control independent of fork instrumentation", () => { + const rawModule = new WebAssembly.Module(readFileSync(rawWasm32Fixture)); + const instrumentedModule = new WebAssembly.Module( + readFileSync(instrumentedFixture), + ); + const exportNames = (module: WebAssembly.Module) => + WebAssembly.Module.exports(module).map(({ name }) => name); + + expect(exportNames(rawModule)).not.toContain("wpk_fork_state"); + expect(exportNames(instrumentedModule)).toContain("wpk_fork_state"); + }); + + it.each([ + ["raw wasm32", rawWasm32Fixture], + ["fork-instrumented wasm32", instrumentedFixture], + ["raw wasm64", rawWasm64Fixture], + ])("documents the pinned LLVM failure in the %s control", async (_, path) => { + const result = await runCentralizedProgram({ + programPath: path, + argv: ["sjlj_noexcept_boundary", "--noexcept"], + timeout: 10_000, + useDefaultRootfs: false, + }); + + expect(result.exitCode).toBe(TERMINATED_BY_SIGABRT); + expect(result.stderr).toContain("HANDLER: siglongjmp"); + expect(result.stderr).toContain("libc++abi: terminating"); + expect(result.stdout).not.toContain("LANDING: siglongjmp resumed"); + }); + + it("resumes the same SjLj tag when it does not cross noexcept", async () => { + const result = await runCentralizedProgram({ + programPath: instrumentedFixture, + argv: ["sjlj_noexcept_boundary", "--permissive"], + timeout: 10_000, + useDefaultRootfs: false, + }); + + expect(result.exitCode).toBe(0); + expect(result.stderr).toContain("HANDLER: siglongjmp"); + expect(result.stdout).toContain("LANDING: siglongjmp resumed"); + expect(result.stderr).not.toContain("libc++abi: terminating"); + }); +}); + +describe("SIGCHLD SjLj control", () => { + it("resumes pselect and reaps the child after SIGCHLD", async () => { + const result = await runCentralizedProgram({ + programPath: sigchldFixture, + argv: ["sigchld_sjlj"], + timeout: 10_000, + useDefaultRootfs: false, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain( + "PASS: SIGCHLD siglongjmp resumed at pselect landing pad", + ); + expect(result.stderr).not.toContain("libc++abi: terminating"); + }); +}); diff --git a/programs/sigchld_sjlj.c b/programs/sigchld_sjlj.c new file mode 100644 index 0000000000..27cbd8dfe7 --- /dev/null +++ b/programs/sigchld_sjlj.c @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static sigjmp_buf signal_landing; + +static void sigchld_handler(int signo, siginfo_t *info, void *context) +{ + (void)info; + (void)context; + if (signo == SIGCHLD) { + siglongjmp(signal_landing, 1); + } +} + +static int wait_for_sigchld(pid_t child, const sigset_t *original_mask) +{ + if (sigsetjmp(signal_landing, 1) == 0) { + sigset_t wait_mask = *original_mask; + sigdelset(&wait_mask, SIGCHLD); + int result = pselect(0, NULL, NULL, NULL, NULL, &wait_mask); + fprintf(stderr, "pselect returned without siglongjmp: %d (%s)\n", + result, strerror(errno)); + return 1; + } + + int status = 0; + pid_t waited; + do { + waited = waitpid(child, &status, 0); + } while (waited == -1 && errno == EINTR); + + if (sigprocmask(SIG_SETMASK, original_mask, NULL) != 0) { + fprintf(stderr, "sigprocmask restore: %s\n", strerror(errno)); + return 1; + } + if (waited != child || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fputs("waitpid did not reap the expected clean child\n", stderr); + return 1; + } + + puts("PASS: SIGCHLD siglongjmp resumed at pselect landing pad"); + return 0; +} + +int main(void) +{ + sigset_t blocked_mask; + sigset_t original_mask; + sigemptyset(&blocked_mask); + sigaddset(&blocked_mask, SIGCHLD); + if (sigprocmask(SIG_BLOCK, &blocked_mask, &original_mask) != 0) { + fprintf(stderr, "sigprocmask block: %s\n", strerror(errno)); + return 1; + } + + struct sigaction action = {0}; + action.sa_sigaction = sigchld_handler; + action.sa_flags = SA_SIGINFO; + sigfillset(&action.sa_mask); + if (sigaction(SIGCHLD, &action, NULL) != 0) { + fprintf(stderr, "sigaction: %s\n", strerror(errno)); + return 1; + } + + pid_t child = fork(); + if (child == -1) { + fprintf(stderr, "fork: %s\n", strerror(errno)); + return 1; + } + if (child == 0) { + _exit(0); + } + + return wait_for_sigchld(child, &original_mask); +} diff --git a/programs/sjlj_noexcept_boundary.cpp b/programs/sjlj_noexcept_boundary.cpp new file mode 100644 index 0000000000..a7b63d757c --- /dev/null +++ b/programs/sjlj_noexcept_boundary.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include + +static sigjmp_buf signal_landing; + +static void signal_handler(int signo) +{ + static const char marker[] = "HANDLER: siglongjmp\n"; + if (signo == SIGUSR1) { + (void)write(STDERR_FILENO, marker, sizeof(marker) - 1); + siglongjmp(signal_landing, 1); + } +} + +// LLVM 21 lowers noexcept to a catch-all termination region. With Wasm SjLj, +// that region intercepts the internal longjmp exception before the enclosing +// sigsetjmp landing can consume it. See issue #918. +__attribute__((noinline)) static void raise_from_noexcept() noexcept +{ + if (raise(SIGUSR1) != 0) { + std::fprintf(stderr, "raise: %s\n", std::strerror(errno)); + } +} + +__attribute__((noinline)) static void raise_from_permissive_boundary() +{ + if (raise(SIGUSR1) != 0) { + std::fprintf(stderr, "raise: %s\n", std::strerror(errno)); + } +} + +#ifndef KANDELO_SJLJ_NO_FORK_ANCHOR +// The test never selects this branch. Its kernel_fork import makes the wasm32 +// program a real input to fork-instrument, so the saved raw module and normal +// program exercise distinct pre- and post-instrumentation artifacts. +__attribute__((noinline)) static int fork_instrumentation_anchor() +{ + pid_t child = fork(); + if (child == -1) { + return 1; + } + if (child == 0) { + _exit(0); + } + + int status = 0; + return waitpid(child, &status, 0) == child && WIFEXITED(status) + && WEXITSTATUS(status) == 0 + ? 0 + : 1; +} +#endif + +int main(int argc, char **argv) +{ +#ifndef KANDELO_SJLJ_NO_FORK_ANCHOR + if (argc == 2 && std::strcmp(argv[1], "--fork-instrumentation-anchor") == 0) { + return fork_instrumentation_anchor(); + } +#endif + + struct sigaction action = {}; + action.sa_handler = signal_handler; + sigfillset(&action.sa_mask); + if (sigaction(SIGUSR1, &action, nullptr) != 0) { + std::fprintf(stderr, "sigaction: %s\n", std::strerror(errno)); + return 1; + } + + if (sigsetjmp(signal_landing, 1) == 0) { + if (argc == 2 && std::strcmp(argv[1], "--permissive") == 0) { + raise_from_permissive_boundary(); + } else { + raise_from_noexcept(); + } + static const char unexpected[] = "FAIL: raise returned past signal handler\n"; + (void)write(STDERR_FILENO, unexpected, sizeof(unexpected) - 1); + return 2; + } + + static const char landed[] = "LANDING: siglongjmp resumed\n"; + (void)write(STDOUT_FILENO, landed, sizeof(landed) - 1); + return 0; +} diff --git a/scripts/build-programs.sh b/scripts/build-programs.sh index 68bbf342a2..ee5253d474 100755 --- a/scripts/build-programs.sh +++ b/scripts/build-programs.sh @@ -18,7 +18,8 @@ GLUE_DIR="$REPO_ROOT/libc/glue" # last-write-wins across arches. OUT_DIR_32="$REPO_ROOT/local-binaries/programs/wasm32" OUT_DIR_64="$REPO_ROOT/local-binaries/programs/wasm64" -mkdir -p "$OUT_DIR_32" "$OUT_DIR_64" +TEST_FIXTURE_DIR="$REPO_ROOT/local-binaries/test-fixtures" +mkdir -p "$OUT_DIR_32" "$OUT_DIR_64" "$TEST_FIXTURE_DIR" find_llvm_bin() { if [ -n "${LLVM_BIN:-}" ] && [ -x "$LLVM_BIN/clang" ]; then @@ -161,6 +162,16 @@ build_cpp_program() { -lc++ -lc++abi \ -o "$wasm" + # Preserve a real pre-instrumentation control for issue #918. The source + # contains an unreachable-at-test-time fork branch solely so the normal + # output is transformed below. A raw module with kernel_fork but without + # wpk_fork_* exports is test evidence, not a distributable program, so it + # lives outside the resolver's programs tree. + if [ "$name" = "sjlj_noexcept_boundary" ]; then + mkdir -p "$TEST_FIXTURE_DIR/wasm32" + cp "$wasm" "$TEST_FIXTURE_DIR/wasm32/${name}.raw.wasm" + fi + # Phase 7: fork support comes from wasm-fork-instrument. The tool is # a no-op for modules without `kernel.kernel_fork`, so it's safe to # run unconditionally — programs without fork stay byte-identical @@ -169,21 +180,35 @@ build_cpp_program() { mv "$wasm.instr" "$wasm" } +ensure_libcxx_in_sysroot() { + local arch="$1" + local sysroot="$2" + if [ -f "$sysroot/lib/libc++.a" ] && \ + [ -f "$sysroot/lib/libc++abi.a" ] && \ + [ -d "$sysroot/include/c++/v1" ]; then + return + fi + + echo "==> Resolving libcxx for $arch C++ programs..." + local host_triple + local libcxx_prefix + host_triple="$(rustc -vV | awk '/^host/ {print $2}')" + (cd "$REPO_ROOT" && cargo run -p xtask --target "$host_triple" --quiet -- \ + build-deps --arch "$arch" resolve libcxx >/dev/null) + libcxx_prefix="$(cd "$REPO_ROOT" && cargo run -p xtask \ + --target "$host_triple" --quiet -- build-deps --arch "$arch" path libcxx)" + ln -sf "$libcxx_prefix/lib/libc++.a" "$sysroot/lib/libc++.a" + ln -sf "$libcxx_prefix/lib/libc++abi.a" "$sysroot/lib/libc++abi.a" + mkdir -p "$sysroot/include/c++" + rm -rf "$sysroot/include/c++/v1" + ln -sfn "$libcxx_prefix/include/c++/v1" "$sysroot/include/c++/v1" +} + # Resolve libcxx and symlink its outputs into the sysroot if there are # any .cpp programs to build. Skip the resolver entirely when libc++.a # is already present so repeat runs are fast. if ls "$REPO_ROOT/programs/"*.cpp >/dev/null 2>&1; then - if [ ! -f "$SYSROOT/lib/libc++.a" ]; then - echo "==> Resolving libcxx for C++ programs..." - HOST_TRIPLE="$(rustc -vV | awk '/^host/ {print $2}')" - (cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TRIPLE" --quiet -- build-deps resolve libcxx >/dev/null) - LIBCXX_PREFIX="$(cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TRIPLE" --quiet -- build-deps path libcxx)" - ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libc++.a" - ln -sf "$LIBCXX_PREFIX/lib/libc++abi.a" "$SYSROOT/lib/libc++abi.a" - mkdir -p "$SYSROOT/include/c++" - rm -rf "$SYSROOT/include/c++/v1" - ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" - fi + ensure_libcxx_in_sysroot wasm32 "$SYSROOT" fi echo "Building user programs..." @@ -301,6 +326,24 @@ if [ -f "$SYSROOT64/lib/libc.a" ]; then "$CC" "${CFLAGS64[@]}" "$wait_lifecycle_src" "${LINK_FLAGS64[@]}" \ -o "$REPO_ROOT/examples/wait_lifecycle_test.wasm64.wasm" fi + + # Fork continuation instrumentation is currently a wasm32 artifact + # contract. Still cover the compiler's architecture-independent SjLj / + # noexcept ordering on wasm64 with a raw fixture that omits the dormant + # fork anchor. Keep it in the test-only tree for symmetry with wasm32. + sjlj_noexcept_src="$REPO_ROOT/programs/sjlj_noexcept_boundary.cpp" + if [ -f "$sjlj_noexcept_src" ]; then + ensure_libcxx_in_sysroot wasm64 "$SYSROOT64" + mkdir -p "$TEST_FIXTURE_DIR/wasm64" + echo " Compiling sjlj_noexcept_boundary (raw wasm64 test fixture)..." + wasm64posix-c++ \ + -O2 \ + -fwasm-exceptions \ + -DKANDELO_SJLJ_NO_FORK_ANCHOR \ + "$sjlj_noexcept_src" \ + -lc++ -lc++abi \ + -o "$TEST_FIXTURE_DIR/wasm64/sjlj_noexcept_boundary.raw.wasm" + fi fi echo "Programs built." From 19cb17d861afe29df03ee48cd48066c128e4b212 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:14:08 -0400 Subject: [PATCH 08/20] VFS: keep positioned I/O off shared file offsets --- host/src/vfs/memory-fs.ts | 13 +--- host/src/vfs/sharedfs-vendor.ts | 36 ++++++++++ host/test/vfs/sharedfs-positioned-io.test.ts | 75 ++++++++++++++++++++ 3 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 host/test/vfs/sharedfs-positioned-io.test.ts diff --git a/host/src/vfs/memory-fs.ts b/host/src/vfs/memory-fs.ts index 03f850fd6e..698e5d0e62 100644 --- a/host/src/vfs/memory-fs.ts +++ b/host/src/vfs/memory-fs.ts @@ -1539,12 +1539,7 @@ export class MemoryFileSystem implements FileSystemBackend { length: number, ): number { if (offset !== null) { - // pread semantics: read at offset without changing file position - const savedPos = this.fs.lseek(handle, 0, 1); // SEEK_CUR - this.fs.lseek(handle, offset, 0); // SEEK_SET - const n = this.fs.read(handle, buffer.subarray(0, length)); - this.fs.lseek(handle, savedPos, 0); // restore position - return n; + return this.fs.readAt(handle, buffer.subarray(0, length), offset); } return this.fs.read(handle, buffer.subarray(0, length)); } @@ -1556,11 +1551,7 @@ export class MemoryFileSystem implements FileSystemBackend { length: number, ): number { if (offset !== null) { - // pwrite semantics: write at offset without changing file position - const savedPos = this.fs.lseek(handle, 0, 1); // SEEK_CUR - this.fs.lseek(handle, offset, 0); // SEEK_SET - const n = this.fs.write(handle, buffer.subarray(0, length)); - this.fs.lseek(handle, savedPos, 0); // restore position + const n = this.fs.writeAt(handle, buffer.subarray(0, length), offset); if (n > 0) this.invalidateLazyData(this.fs.fstat(handle)); return n; } diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 46a41f3b1f..205a7ba5fb 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -2444,6 +2444,22 @@ export class SharedFS { } } + readAt(fd: number, buffer: Uint8Array, offset: number): number { + const entry = this.fdGet(fd); + if (!entry) throw new SFSError(EBADF); + const inoOff = this.inodeOffset(entry.ino); + const mode = this.r32(inoOff + INO_MODE); + if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); + this.validateSeekPosition(offset); + + this.inodeReadLock(entry.ino); + try { + return this.inodeReadData(entry.ino, offset, buffer, buffer.length); + } finally { + this.inodeReadUnlock(entry.ino); + } + } + write(fd: number, data: Uint8Array): number { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); @@ -2481,6 +2497,26 @@ export class SharedFS { } } + writeAt(fd: number, data: Uint8Array, offset: number): number { + const entry = this.fdGet(fd); + if (!entry) throw new SFSError(EBADF); + + const accMode = entry.flags & O_ACCMODE; + if (accMode === O_RDONLY) throw new SFSError(EBADF); + this.validateSeekPosition(offset); + + this.inodeWriteLock(entry.ino); + try { + // Positioned writes use their explicit offset even on an O_APPEND fd. + if (offset > MAX_FILE_SIZE || data.length > MAX_FILE_SIZE - offset) { + throw new SFSError(EFBIG); + } + return this.inodeWriteData(entry.ino, offset, data, data.length); + } finally { + this.inodeWriteUnlock(entry.ino); + } + } + lseek(fd: number, offset: number, whence: number): number { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); diff --git a/host/test/vfs/sharedfs-positioned-io.test.ts b/host/test/vfs/sharedfs-positioned-io.test.ts new file mode 100644 index 0000000000..6579fc96f0 --- /dev/null +++ b/host/test/vfs/sharedfs-positioned-io.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; +import { + O_APPEND, + O_CREAT, + O_RDWR, + O_TRUNC, + SEEK_SET, + SharedFS, +} from "../../src/vfs/sharedfs-vendor"; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function text(bytes: Uint8Array): string { + return decoder.decode(bytes); +} + +describe("SharedFS positioned I/O", () => { + it("readAt and writeAt do not mutate the shared fd offset", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const fs = SharedFS.mkfs(sab); + const fd = fs.open( + "/sorter.tmp", + O_RDWR | O_CREAT | O_TRUNC | O_APPEND, + 0o600, + ); + + expect(fs.write(fd, encoder.encode("0123456789abcdef"))).toBe(16); + expect(fs.lseek(fd, 10, SEEK_SET)).toBe(10); + + const positionedRead = new Uint8Array(4); + expect(fs.readAt(fd, positionedRead, 2)).toBe(4); + expect(text(positionedRead)).toBe("2345"); + + expect(fs.writeAt(fd, encoder.encode("XY"), 4)).toBe(2); + + const sequentialRead = new Uint8Array(3); + expect(fs.read(fd, sequentialRead)).toBe(3); + expect(text(sequentialRead)).toBe("abc"); + + expect(fs.lseek(fd, 0, SEEK_SET)).toBe(0); + const full = new Uint8Array(16); + expect(fs.read(fd, full)).toBe(16); + expect(text(full)).toBe("0123XY6789abcdef"); + }); + + it("MemoryFileSystem pread and pwrite keep the shared offset stable", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const fs = MemoryFileSystem.create(sab); + const fd = fs.open( + "/sorter.tmp", + O_RDWR | O_CREAT | O_TRUNC | O_APPEND, + 0o600, + ); + + expect(fs.write(fd, encoder.encode("0123456789abcdef"), null, 16)).toBe(16); + expect(fs.seek(fd, 10, SEEK_SET)).toBe(10); + + const positionedRead = new Uint8Array(4); + expect(fs.read(fd, positionedRead, 2, 4)).toBe(4); + expect(text(positionedRead)).toBe("2345"); + + expect(fs.write(fd, encoder.encode("XY"), 4, 2)).toBe(2); + + const sequentialRead = new Uint8Array(3); + expect(fs.read(fd, sequentialRead, null, 3)).toBe(3); + expect(text(sequentialRead)).toBe("abc"); + + expect(fs.seek(fd, 0, SEEK_SET)).toBe(0); + const full = new Uint8Array(16); + expect(fs.read(fd, full, null, 16)).toBe(16); + expect(text(full)).toBe("0123XY6789abcdef"); + }); +}); From 1092a4759cb6a314b5298eb6be56d6dfe88b5bb9 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 13:54:28 -0400 Subject: [PATCH 09/20] host: preserve output buffers on zero-byte syscalls --- host/src/kernel-worker.ts | 9 +- host/test/kernel-worker-copyback.test.ts | 164 +++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 host/test/kernel-worker-copyback.test.ts diff --git a/host/src/kernel-worker.ts b/host/src/kernel-worker.ts index 31709f3363..4492213997 100644 --- a/host/src/kernel-worker.ts +++ b/host/src/kernel-worker.ts @@ -4532,8 +4532,15 @@ export class CentralizedKernelWorker { if (!(desc.direction === "out" && retVal < 0)) { let copySize = size; if (desc.direction === "out" && desc.size.type === "arg") { + // For read/recv/getdents-like syscalls, retVal is the number of + // bytes produced. Successful EOF must not copy the zero-filled + // scratch buffer over bytes the caller already owns. Some + // descriptors prepend fixed metadata that is still produced when + // the variable-length result is empty. const copyRetvalAdd = desc.copyRetvalAdd ?? 0; - if (retVal > 0 && retVal + copyRetvalAdd < size) { + if (retVal === 0) { + copySize = Math.min(copyRetvalAdd, size); + } else if (retVal + copyRetvalAdd < size) { copySize = retVal + copyRetvalAdd; } } diff --git a/host/test/kernel-worker-copyback.test.ts b/host/test/kernel-worker-copyback.test.ts new file mode 100644 index 0000000000..c8eb166852 --- /dev/null +++ b/host/test/kernel-worker-copyback.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from "vitest"; +import { CentralizedKernelWorker } from "../src/kernel-worker"; +import { + ABI_SYSCALLS, + CHANNEL_STATUS_COMPLETE, + CH_DATA, + CH_ERRNO, + CH_RETURN, + CH_STATUS, + type SyscallArgDesc, + SYSCALL_ARGS, +} from "../src/generated/abi"; + +interface TestChannel { + pid: number; + memory: WebAssembly.Memory; + channelOffset: number; + i32View: Int32Array; + consecutiveSyscalls: number; + handling: boolean; +} + +interface CopybackHarnessWorker { + completeChannel( + channel: TestChannel, + syscallNr: number, + origArgs: number[], + argDescs: SyscallArgDesc[] | undefined, + retVal: number, + errVal: number, + ): void; +} + +function makeCopybackHarness() { + const pid = 1; + const kernelMemory = new WebAssembly.Memory({ initial: 2 }); + const processMemory = new WebAssembly.Memory({ + initial: 2, + maximum: 2, + shared: true, + }); + const channel: TestChannel = { + pid, + memory: processMemory, + channelOffset: 0, + i32View: new Int32Array(processMemory.buffer), + consecutiveSyscalls: 0, + handling: true, + }; + const worker = Object.assign( + Object.create(CentralizedKernelWorker.prototype), + { + kernelMemory, + scratchOffset: 0, + cachedKernelMem: null, + cachedKernelBuffer: null, + processes: new Map([ + [ + pid, + { + pid, + memory: processMemory, + channels: [channel], + ptrWidth: 4, + explicitMaxAddr: false, + }, + ], + ]), + clearSocketTimeout: () => {}, + clearReadinessWait: () => {}, + drainAllPtyOutputs: () => {}, + flushTcpSendPipes: () => {}, + drainAndProcessWakeupEvents: () => {}, + synchronizeSharedMemoryForBoundary: () => {}, + relistenChannel: () => {}, + }, + ) as CopybackHarnessWorker; + + return { + worker, + channel, + kernelMem: new Uint8Array(kernelMemory.buffer), + processMem: new Uint8Array(processMemory.buffer), + }; +} + +describe("CentralizedKernelWorker syscall copy-back", () => { + it("leaves the destination unchanged when read reports EOF", () => { + const { worker, channel, kernelMem, processMem } = makeCopybackHarness(); + const dest = 1024; + const original = Uint8Array.from({ length: 16 }, (_, i) => 0xa0 + i); + + processMem.set(original, dest); + kernelMem.fill(0, CH_DATA, CH_DATA + original.length); + + worker.completeChannel( + channel, + ABI_SYSCALLS.Read, + [0, dest, original.length], + SYSCALL_ARGS[ABI_SYSCALLS.Read], + 0, + 0, + ); + + expect(processMem.slice(dest, dest + original.length)).toEqual(original); + const channelView = new DataView(processMem.buffer); + expect(channelView.getBigInt64(CH_RETURN, true)).toBe(0n); + expect(channelView.getUint32(CH_ERRNO, true)).toBe(0); + expect(Atomics.load(channel.i32View, CH_STATUS / 4)).toBe( + CHANNEL_STATUS_COMPLETE, + ); + }); + + it("copies only the byte count reported by read", () => { + const { worker, channel, kernelMem, processMem } = makeCopybackHarness(); + const dest = 2048; + const original = Uint8Array.from({ length: 8 }, (_, i) => 0xc0 + i); + + processMem.set(original, dest); + kernelMem.set([1, 2, 3, 0, 0, 0, 0, 0], CH_DATA); + + worker.completeChannel( + channel, + ABI_SYSCALLS.Read, + [0, dest, original.length], + SYSCALL_ARGS[ABI_SYSCALLS.Read], + 3, + 0, + ); + + expect(Array.from(processMem.slice(dest, dest + original.length))).toEqual([ + 1, + 2, + 3, + ...original.slice(3), + ]); + }); + + it("copies fixed prefix metadata when a zero-length msgrcv succeeds", () => { + const { worker, channel, kernelMem, processMem } = makeCopybackHarness(); + const dest = 3072; + const original = Uint8Array.from({ length: 12 }, (_, i) => 0xd0 + i); + + processMem.set(original, dest); + kernelMem.set([0x11, 0x22, 0x33, 0x44, 0, 0, 0, 0], CH_DATA); + + worker.completeChannel( + channel, + ABI_SYSCALLS.Msgrcv, + [0, dest, 8], + SYSCALL_ARGS[ABI_SYSCALLS.Msgrcv], + 0, + 0, + ); + + expect(Array.from(processMem.slice(dest, dest + original.length))).toEqual([ + 0x11, + 0x22, + 0x33, + 0x44, + ...original.slice(4), + ]); + }); +}); From a757932294a16ba9af5af2ca3076bc76f781dae8 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:33:03 -0400 Subject: [PATCH 10/20] Runner: start examples with requested user and group IDs --- examples/README.md | 5 ++ examples/initial-credentials-test.c | 12 ++++ examples/run-example.ts | 28 ++++++++ host/test/global-setup.ts | 1 + host/test/run-example-credentials.test.ts | 78 +++++++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 examples/initial-credentials-test.c create mode 100644 host/test/run-example-credentials.test.ts diff --git a/examples/README.md b/examples/README.md index 7bb2c7160c..840aa47088 100644 --- a/examples/README.md +++ b/examples/README.md @@ -18,6 +18,11 @@ wasm32posix-cc examples/hello.c -o hello.wasm npx tsx examples/run-example.ts hello ``` +`run-example.ts` starts guests as root by default. Set `KERNEL_UID` and +`KERNEL_GID` to decimal values from 0 through 4294967294 when a test needs a +different initial user or group. The maximum unsigned 32-bit value is reserved +by the host protocol and is rejected rather than being mistaken for an ID. + See [docs/sdk-guide.md](../docs/sdk-guide.md) for full SDK documentation. ## Programs diff --git a/examples/initial-credentials-test.c b/examples/initial-credentials-test.c new file mode 100644 index 0000000000..dd5469fea0 --- /dev/null +++ b/examples/initial-credentials-test.c @@ -0,0 +1,12 @@ +/* Report the initial process credentials supplied by the host. */ +#include +#include + +int main(void) { + printf("uid=%lu euid=%lu gid=%lu egid=%lu\n", + (unsigned long) getuid(), + (unsigned long) geteuid(), + (unsigned long) getgid(), + (unsigned long) getegid()); + return 0; +} diff --git a/examples/run-example.ts b/examples/run-example.ts index 93fbbcdbf4..36bb5a49cb 100644 --- a/examples/run-example.ts +++ b/examples/run-example.ts @@ -10,6 +10,7 @@ * Example: * npx tsx examples/run-example.ts hello * npx tsx examples/run-example.ts /path/to/test.wasm + * KERNEL_UID=1000 KERNEL_GID=1000 npx tsx examples/run-example.ts hello */ import { closeSync, existsSync, openSync, readFileSync, statSync, writeSync } from "fs"; @@ -20,6 +21,29 @@ import { isWithinRealDirectory } from "./run-example-paths"; const repoRoot = resolve(dirname(new URL(import.meta.url).pathname), ".."); +const MAX_CONFIGURABLE_CREDENTIAL = 0xfffffffe; + +function parseKernelCredential(name: "KERNEL_UID" | "KERNEL_GID"): number | undefined { + const raw = process.env[name]; + if (raw === undefined || raw === "") return undefined; + + // u32::MAX is the host/kernel protocol's "leave unchanged" sentinel. If + // it were accepted here, a request for that ID would silently leave the + // new process running as root. + if (!/^[0-9]+$/.test(raw)) { + throw new Error( + `${name} must be a decimal integer from 0 to ${MAX_CONFIGURABLE_CREDENTIAL}`, + ); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value > MAX_CONFIGURABLE_CREDENTIAL) { + throw new Error( + `${name} must be a decimal integer from 0 to ${MAX_CONFIGURABLE_CREDENTIAL}`, + ); + } + return value; +} + // Built-in program resolution via the binary-resolver. Resolver returns // null for programs that aren't fetched or locally built; callers that // need the path must handle null explicitly. @@ -312,6 +336,8 @@ async function main() { console.error("Usage: npx tsx examples/run-example.ts "); process.exit(1); } + const uid = parseKernelCredential("KERNEL_UID"); + const gid = parseKernelCredential("KERNEL_GID"); let programPath: string; if (name.endsWith(".wasm")) { @@ -386,6 +412,8 @@ async function main() { ...gitEnv, ], cwd: process.env.KERNEL_CWD || process.cwd(), + uid, + gid, stdin: stdinData, }); const timeoutPromise = new Promise((_, reject) => { diff --git a/host/test/global-setup.ts b/host/test/global-setup.ts index 300adea434..2604ccd70d 100644 --- a/host/test/global-setup.ts +++ b/host/test/global-setup.ts @@ -51,6 +51,7 @@ const TEST_PROGRAMS = [ "spawn-pause.c", "mount_probe_test.c", "getpwent_smoke.c", + "initial-credentials-test.c", "thread-exit-group.c", ]; diff --git a/host/test/run-example-credentials.test.ts b/host/test/run-example-credentials.test.ts new file mode 100644 index 0000000000..b7f826a93c --- /dev/null +++ b/host/test/run-example-credentials.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(__dirname, "..", ".."); +const runExample = join(repoRoot, "examples", "run-example.ts"); +const credentialProbe = join(repoRoot, "examples", "initial-credentials-test.wasm"); + +function runCredentialProbe(overrides: Record) { + const env = { ...process.env }; + delete env.KERNEL_UID; + delete env.KERNEL_GID; + for (const [name, value] of Object.entries(overrides)) { + if (value === undefined) delete env[name]; + else env[name] = value; + } + + return spawnSync( + process.execPath, + [ + "--experimental-wasm-exnref", + "--import", + "tsx/esm", + runExample, + credentialProbe, + ], + { + cwd: repoRoot, + // The probe only inspects credentials. Keep its guest cwd independent + // of checkout ownership and group modes on the CI host. + env: { ...env, KERNEL_CWD: "/tmp", TIMEOUT: "30000" }, + encoding: "utf8", + timeout: 45_000, + }, + ); +} + +describe("run-example initial credentials", () => { + it("starts the guest with the requested real and effective IDs", () => { + const result = runCredentialProbe({ KERNEL_UID: "1000", KERNEL_GID: "1001" }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("uid=1000 euid=1000 gid=1001 egid=1001"); + }); + + it("leaves an omitted credential at the kernel default", () => { + const result = runCredentialProbe({ KERNEL_UID: "2000" }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("uid=2000 euid=2000 gid=0 egid=0"); + }); + + it("accepts the largest ID that is not the unchanged sentinel", () => { + const result = runCredentialProbe({ + KERNEL_UID: "4294967294", + KERNEL_GID: "4294967294", + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain( + "uid=4294967294 euid=4294967294 gid=4294967294 egid=4294967294", + ); + }); + + it.each(["-1", "1.5", "0x10", " 1000 ", "4294967295", "4294967296"])( + "rejects an invalid KERNEL_UID value (%s)", + (value) => { + const result = runCredentialProbe({ KERNEL_UID: value }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "KERNEL_UID must be a decimal integer from 0 to 4294967294", + ); + }, + ); +}); From 4d9c74f44de51deabc9ae7f895308c330b3c6e31 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:39:45 -0400 Subject: [PATCH 11/20] mremap: reject growth without matching mapping metadata --- crates/kernel/src/memory.rs | 21 +++++++++++++++---- crates/kernel/src/syscalls.rs | 39 ++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/crates/kernel/src/memory.rs b/crates/kernel/src/memory.rs index b7808b0a5e..dc13ec36ec 100644 --- a/crates/kernel/src/memory.rs +++ b/crates/kernel/src/memory.rs @@ -570,14 +570,16 @@ impl MemoryManager { } /// Extend an existing mapping at `addr` from `old_len` to `new_len`. - /// The caller must ensure the space is free (via `can_grow_at`). - pub fn extend_mapping(&mut self, addr: usize, old_len: usize, new_len: usize) { + /// The caller must ensure the space is free (via `can_grow_at`). Returns + /// whether an exact mapping was found and updated. + pub fn extend_mapping(&mut self, addr: usize, old_len: usize, new_len: usize) -> bool { for m in &mut self.mappings { if m.addr == addr && m.len == old_len { m.len = new_len; - return; + return true; } } + false } } @@ -1276,10 +1278,21 @@ mod tests { let rw = PROT_READ | PROT_WRITE; let anon = MAP_PRIVATE | MAP_ANONYMOUS; let addr = mm.mmap_anonymous(0, 0x10000, rw, anon); - mm.extend_mapping(addr, 0x10000, 0x20000); + assert!(mm.extend_mapping(addr, 0x10000, 0x20000)); assert!(mm.is_mapped(addr + 0x10000)); // extended area is now mapped } + #[test] + fn test_extend_mapping_reports_mismatched_metadata() { + let mut mm = MemoryManager::new(); + let rw = PROT_READ | PROT_WRITE; + let anon = MAP_PRIVATE | MAP_ANONYMOUS; + let addr = mm.mmap_anonymous(0, 0x10000, rw, anon); + + assert!(!mm.extend_mapping(addr, 0x20000, 0x30000)); + assert_eq!(mm.mappings()[0].len, 0x10000); + } + #[test] fn test_host_reserved_region_blocks_mmap_and_reuses_next_gap() { let mut mm = MemoryManager::new(); diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index 1057d45874..bc346080e3 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -6471,9 +6471,13 @@ pub fn sys_mremap( let extra = aligned_new - aligned_old; let grow_start = old_addr + aligned_old; if proc.memory.can_grow_at(grow_start, extra) { - proc.memory - .extend_mapping(old_addr, aligned_old, aligned_new); - return Ok(old_addr); + if proc + .memory + .extend_mapping(old_addr, aligned_old, aligned_new) + { + return Ok(old_addr); + } + return Err(Errno::EFAULT); } // MREMAP_MAYMOVE: allocate a new mapping and free the old one. @@ -27879,6 +27883,35 @@ mod tests { assert!(proc.memory.is_mapped(addr + 0x10000)); } + #[test] + fn test_mremap_grow_requires_matching_mapping_metadata() { + let mut proc = Process::new(1); + use wasm_posix_shared::mmap::{MAP_ANONYMOUS, MAP_PRIVATE, PROT_READ, PROT_WRITE}; + let addr = proc.memory.mmap_anonymous( + 0, + 0x10000, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + ); + + // The claimed two-page old range does not match the one-page mapping. + // A free third page must not turn the missing metadata update into a + // successful in-place grow. + assert_eq!( + sys_mremap(&mut proc, addr, 0x20000, 0x30000, 0).unwrap_err(), + Errno::EFAULT + ); + assert_eq!(proc.memory.mappings()[0].len, 0x10000); + + let next = proc.memory.mmap_anonymous( + 0, + 0x10000, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + ); + assert_eq!(next, addr + 0x10000); + } + #[test] fn test_mremap_maymove() { let mut proc = Process::new(1); From 06551eb0957eef488c2b77c895a3d35e05ee4f4e Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 14:58:11 -0400 Subject: [PATCH 12/20] Locks: report shared lock-table exhaustion as ENOLCK --- docs/posix-status.md | 4 +-- host/src/kernel.ts | 14 +++++++--- host/src/shared-lock-table.ts | 40 ++++++++++++++++++++++------- host/test/kernel-fcntl-lock.test.ts | 29 ++++++++++++++++----- host/test/shared-lock-table.test.ts | 18 +++++++++++++ 5 files changed, 84 insertions(+), 21 deletions(-) diff --git a/docs/posix-status.md b/docs/posix-status.md index 1f31926b6f..1c9d3e951b 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -93,8 +93,8 @@ Kandelo uses a single kernel Wasm instance that holds a `ProcessTable` and serve | `F_GETFL` | Full | Returns status flags + access mode. Use O_ACCMODE mask. | | `F_SETFL` | Full | Only O_APPEND, O_NONBLOCK modifiable. Access mode bits preserved. | | `F_GETLK` | Full | Advisory record locking. Returns blocking lock info or F_UNLCK if no conflict. Locks released on close() and exit() per POSIX. | -| `F_SETLK` | Full | Non-blocking lock acquisition. Returns EAGAIN on conflict. Read/write access mode validated. Locks released on close() and exit() per POSIX. | -| `F_SETLKW` | Partial | Blocking lock acquisition. Host-backed locks and in-kernel fallback locks are coordinated across processes; blocking conflicts use an internal EAGAIN retry path in the host worker until the lock is available. No deadlock detection. | +| `F_SETLK` | Full | Non-blocking lock acquisition. Returns EAGAIN on conflict and ENOLCK when the fixed-size shared host lock table is full. Read/write access mode validated. Locks released on close() and exit() per POSIX. | +| `F_SETLKW` | Partial | Blocking lock acquisition. Host-backed locks and in-kernel fallback locks are coordinated across processes; blocking conflicts use an internal EAGAIN retry path in the host worker until the lock is available. Shared host lock-table exhaustion returns ENOLCK instead of retrying. No deadlock detection. | | `F_GETOWN` | Full | Returns async I/O owner PID from OFD. Default 0. | | `F_SETOWN` | Full | Sets async I/O owner PID on OFD. SIGIO delivery deferred to signal delivery phase. | diff --git a/host/src/kernel.ts b/host/src/kernel.ts index fcc2cbaac6..c1e31e9524 100644 --- a/host/src/kernel.ts +++ b/host/src/kernel.ts @@ -2706,12 +2706,18 @@ export class WasmPosixKernel { return 0; } case WasmPosixKernel.F_SETLK: { - const ok = this.sharedLockTable.setLock(pathHash, pid, lockType, start, len); - return ok ? 0 : -11; // -EAGAIN + const result = this.sharedLockTable.setLockResult(pathHash, pid, lockType, start, len); + if (result === "acquired") return 0; + if (result === "blocked") return -11; // -EAGAIN + return -37; // -ENOLCK } case WasmPosixKernel.F_SETLKW: { - const ok = this.sharedLockTable.setLock(pathHash, pid, lockType, start, len); - return ok ? 0 : -11; // -EAGAIN, kernel-worker retries blocking fcntl + const result = this.sharedLockTable.setLockResult(pathHash, pid, lockType, start, len); + // The worker retries EAGAIN for a blocking lock. ENOLCK is a real + // capacity failure and must be returned instead of spinning. + if (result === "acquired") return 0; + if (result === "blocked") return -11; // -EAGAIN + return -37; // -ENOLCK } default: return -22; // -EINVAL diff --git a/host/src/shared-lock-table.ts b/host/src/shared-lock-table.ts index 0d16362fff..8b9d88a75d 100644 --- a/host/src/shared-lock-table.ts +++ b/host/src/shared-lock-table.ts @@ -55,6 +55,8 @@ export interface LockInfo { len: bigint; } +export type LockSetResult = "acquired" | "blocked" | "no-space"; + export class SharedLockTable { private view: Int32Array; private sab: SharedArrayBuffer; @@ -231,7 +233,8 @@ export class SharedLockTable { /** * Set a lock (non-blocking). For F_UNLCK, removes matching locks. - * Returns true on success, false if conflicting lock exists (EAGAIN). + * Returns true on success, false if the lock conflicts or the table is full. + * Errno-producing callers must use setLockResult() to preserve the cause. */ setLock( pathHash: number, @@ -240,6 +243,23 @@ export class SharedLockTable { start: bigint, len: bigint, ): boolean { + return ( + this.setLockResult(pathHash, pid, lockType, start, len) === "acquired" + ); + } + + /** + * Set a lock and preserve the reason it could not be installed. Callers + * that translate the result to an errno must distinguish a conflicting + * lock (EAGAIN) from an exhausted system lock table (ENOLCK). + */ + setLockResult( + pathHash: number, + pid: number, + lockType: number, + start: bigint, + len: bigint, + ): LockSetResult { this.acquire(); try { return this._setLockUnsafe(pathHash, pid, lockType, start, len); @@ -254,7 +274,7 @@ export class SharedLockTable { lockType: number, start: bigint, len: bigint, - ): boolean { + ): LockSetResult { // For unlock: remove overlapping locks from same pid on same path, then wake waiters if (lockType === F_UNLCK) { let i = 0; @@ -274,12 +294,12 @@ export class SharedLockTable { // Wake any F_SETLKW waiters Atomics.add(this.view, WAKE_COUNTER, 1); Atomics.notify(this.view, WAKE_COUNTER); - return true; + return "acquired"; } // Check for conflicts if (this._getBlockingLockUnsafe(pathHash, lockType, start, len, pid)) { - return false; // caller should return EAGAIN + return "blocked"; } // Remove overlapping locks from same pid on same path (upgrade/replace) @@ -301,16 +321,17 @@ export class SharedLockTable { const count = this.view[COUNT]; const capacity = this.view[CAPACITY]; if (count >= capacity) { - return false; // table full — treat as EAGAIN + return "no-space"; } this.writeEntry(count, { pathHash, pid, lockType, start, len }); this.view[COUNT] = count + 1; - return true; + return "acquired"; } /** * Set a lock, blocking until it can be acquired (F_SETLKW). * Uses Atomics.wait on wake_counter to sleep between retries. + * Returns no-space instead of waiting when the fixed-size table is full. */ setLockWait( pathHash: number, @@ -318,7 +339,7 @@ export class SharedLockTable { lockType: number, start: bigint, len: bigint, - ): void { + ): "acquired" | "no-space" { while (true) { this.acquire(); const blocker = this._getBlockingLockUnsafe( @@ -329,9 +350,10 @@ export class SharedLockTable { pid, ); if (!blocker) { - this._setLockUnsafe(pathHash, pid, lockType, start, len); + const result = this._setLockUnsafe(pathHash, pid, lockType, start, len); this.release(); - return; + if (result !== "blocked") return result; + continue; } const wakeCount = Atomics.load(this.view, WAKE_COUNTER); this.release(); diff --git a/host/test/kernel-fcntl-lock.test.ts b/host/test/kernel-fcntl-lock.test.ts index e23bb13f7a..3e0eb891b8 100644 --- a/host/test/kernel-fcntl-lock.test.ts +++ b/host/test/kernel-fcntl-lock.test.ts @@ -6,6 +6,7 @@ const F_SETLK = 13; const F_SETLKW = 14; const F_WRLCK = 1; const EAGAIN = 11; +const ENOLCK = 37; type LockCall = [number, number, number, bigint, bigint]; @@ -54,9 +55,9 @@ describe("WasmPosixKernel fcntl locking import", () => { const setLockCalls: LockCall[] = []; let setLockWaitCalled = false; const { kernel, path } = makeKernel({ - setLock: (...args: LockCall) => { + setLockResult: (...args: LockCall) => { setLockCalls.push(args); - return false; + return "blocked"; }, setLockWait: () => { setLockWaitCalled = true; @@ -75,9 +76,9 @@ describe("WasmPosixKernel fcntl locking import", () => { const setLockCalls: LockCall[] = []; let setLockWaitCalled = false; const { kernel, path } = makeKernel({ - setLock: (...args: LockCall) => { + setLockResult: (...args: LockCall) => { setLockCalls.push(args); - return false; + return "blocked"; }, setLockWait: () => { setLockWaitCalled = true; @@ -96,9 +97,9 @@ describe("WasmPosixKernel fcntl locking import", () => { const setLockCalls: LockCall[] = []; let setLockWaitCalled = false; const { kernel, path } = makeKernel({ - setLock: (...args: LockCall) => { + setLockResult: (...args: LockCall) => { setLockCalls.push(args); - return true; + return "acquired"; }, setLockWait: () => { setLockWaitCalled = true; @@ -112,4 +113,20 @@ describe("WasmPosixKernel fcntl locking import", () => { expect(setLockCalls[0].slice(1)).toEqual([2, F_WRLCK, 32n, 64n]); expect(setLockWaitCalled).toBe(false); }); + + it.each([F_SETLK, F_SETLKW])( + "returns ENOLCK when command %i exhausts the shared table", + (cmd) => { + const setLockCalls: LockCall[] = []; + const { kernel, path } = makeKernel({ + setLockResult: (...args: LockCall) => { + setLockCalls.push(args); + return "no-space"; + }, + }); + + expect(hostFcntlLock(kernel, path, cmd)).toBe(-ENOLCK); + expect(setLockCalls).toHaveLength(1); + }, + ); }); diff --git a/host/test/shared-lock-table.test.ts b/host/test/shared-lock-table.test.ts index 0f4b54de70..b421697484 100644 --- a/host/test/shared-lock-table.test.ts +++ b/host/test/shared-lock-table.test.ts @@ -98,6 +98,24 @@ describe("SharedLockTable", () => { expect(result).toBe(false); }); + it("distinguishes table exhaustion from a conflicting lock", () => { + const table = SharedLockTable.create(1); + + expect(table.setLockResult(100, 1, 1, 0n, 1n)).toBe("acquired"); + expect(table.setLockResult(100, 2, 1, 0n, 1n)).toBe("blocked"); + expect(table.setLockResult(200, 2, 1, 0n, 1n)).toBe("no-space"); + // Keep the existing boolean API compatible for callers that only need a + // success/failure answer. + expect(table.setLock(200, 2, 1, 0n, 1n)).toBe(false); + }); + + it("does not wait when a blocking request exhausts the table", () => { + const table = SharedLockTable.create(1); + + expect(table.setLock(100, 1, 1, 0n, 1n)).toBe(true); + expect(table.setLockWait(200, 2, 1, 0n, 1n)).toBe("no-space"); + }); + it("should removeLocksByPid", () => { const table = SharedLockTable.create(); table.setLock(100, 1, 1, 0n, 50n); From 5fc238783bc244c18f4908ed2f3174ed998c6820 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:04:44 -0400 Subject: [PATCH 13/20] VFS: synchronize directory file descriptors Delegate fsync on directory descriptors to the active filesystem backend so Node can issue its native durability barrier. Keep memory-backed and OPFS behavior truthful at their host boundaries, and cover the kernel, host backends, Chromium OPFS, and POSIX regression path. --- .../opfs-directory-fsync-client-worker.ts | 41 +++++++++ .../test/opfs-directory-fsync.spec.ts | 92 +++++++++++++++++++ crates/kernel/src/syscalls.rs | 26 ++++-- docs/browser-support.md | 2 +- docs/posix-status.md | 2 +- host/src/vfs/opfs-worker.ts | 10 +- host/test/vfs/directory-fsync.test.ts | 45 +++++++++ .../basic/unistd/fsync-directory.c | 26 ++++++ 8 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 apps/browser-demos/test/fixtures/opfs-directory-fsync-client-worker.ts create mode 100644 apps/browser-demos/test/opfs-directory-fsync.spec.ts create mode 100644 host/test/vfs/directory-fsync.test.ts create mode 100644 tests/sortix/os-test-local/basic/unistd/fsync-directory.c diff --git a/apps/browser-demos/test/fixtures/opfs-directory-fsync-client-worker.ts b/apps/browser-demos/test/fixtures/opfs-directory-fsync-client-worker.ts new file mode 100644 index 0000000000..f61db29217 --- /dev/null +++ b/apps/browser-demos/test/fixtures/opfs-directory-fsync-client-worker.ts @@ -0,0 +1,41 @@ +import { OpfsFileSystem } from "../../../../host/src/vfs/opfs"; + +const O_RDONLY = 0; +const O_DIRECTORY = 0x010000; + +self.onmessage = ( + event: MessageEvent<{ buffer: SharedArrayBuffer; path: string }>, +) => { + const { buffer, path } = event.data; + const fs = OpfsFileSystem.create(buffer); + let fd = -1; + + try { + fs.mkdir(path, 0o700); + fd = fs.open(path, O_RDONLY | O_DIRECTORY, 0); + fs.fsync(fd); + fs.close(fd); + fd = -1; + fs.rmdir(path); + self.postMessage({ type: "result" }); + } catch (error) { + if (fd >= 0) { + try { + fs.close(fd); + } catch { + // Preserve the original failure. + } + } + try { + fs.rmdir(path); + } catch { + // Preserve the original failure. + } + self.postMessage({ + type: "error", + error: error instanceof Error ? error.message : String(error), + }); + } finally { + self.close(); + } +}; diff --git a/apps/browser-demos/test/opfs-directory-fsync.spec.ts b/apps/browser-demos/test/opfs-directory-fsync.spec.ts new file mode 100644 index 0000000000..c3460ce934 --- /dev/null +++ b/apps/browser-demos/test/opfs-directory-fsync.spec.ts @@ -0,0 +1,92 @@ +import { expect, test } from "@playwright/test"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const proxyWorkerPath = resolve( + __dirname, + "../../../host/src/vfs/opfs-worker.ts", +); +const clientWorkerPath = resolve( + __dirname, + "fixtures/opfs-directory-fsync-client-worker.ts", +); + +test("OPFS accepts fsync on an open directory", async ({ + page, + baseURL, + browserName, +}) => { + test.skip( + browserName !== "chromium", + "OPFS sync access handles are Chromium-only here", + ); + expect(baseURL).toBeTruthy(); + + const proxyWorkerUrl = new URL(`/@fs/${proxyWorkerPath}`, baseURL).href; + const clientWorkerUrl = new URL(`/@fs/${clientWorkerPath}`, baseURL).href; + await page.goto(new URL("/trap-signal-test.html", baseURL).href); + + const result = await page.evaluate( + async ({ proxyWorkerUrl, clientWorkerUrl }) => { + const buffer = new SharedArrayBuffer(4 * 1024 * 1024); + const proxy = new Worker(proxyWorkerUrl, { type: "module" }); + const client = new Worker(clientWorkerUrl, { type: "module" }); + + const receive = (worker: Worker, expectedType: string): Promise => + new Promise((resolvePromise, reject) => { + const timeout = setTimeout( + () => reject(new Error(`timed out waiting for ${expectedType}`)), + 15_000, + ); + worker.addEventListener( + "message", + (event) => { + if (event.data?.type === "error") { + clearTimeout(timeout); + reject(new Error(event.data.error)); + return; + } + if (event.data?.type === expectedType) { + clearTimeout(timeout); + resolvePromise(event.data as T); + } + }, + { once: false }, + ); + worker.addEventListener( + "error", + (event) => { + clearTimeout(timeout); + reject( + new Error( + `${expectedType}: ${event.message || "worker module failed to load"} ` + + `(${event.filename}:${event.lineno}:${event.colno})`, + ), + ); + }, + { once: true }, + ); + }); + + try { + const ready = receive<{ type: "ready" }>(proxy, "ready"); + proxy.postMessage({ type: "init", buffer }); + await ready; + + const pending = receive<{ type: "result" }>(client, "result"); + client.postMessage({ + buffer, + path: `/kandelo-opfs-directory-fsync-${crypto.randomUUID()}`, + }); + return await pending; + } finally { + client.terminate(); + proxy.terminate(); + } + }, + { proxyWorkerUrl, clientWorkerUrl }, + ); + + expect(result).toEqual({ type: "result" }); +}); diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index bc346080e3..0cc29c0711 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -12713,12 +12713,10 @@ pub fn sys_fsync(proc: &mut Process, host: &mut dyn HostIO, fd: i32) -> Result<( let ofd_idx = entry.ofd_ref.0; let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; - // Must be a regular file - if ofd.file_type != FileType::Regular { - return Err(Errno::EINVAL); + match ofd.file_type { + FileType::Regular | FileType::Directory => host.host_fsync(ofd.host_handle), + _ => Err(Errno::EINVAL), } - - host.host_fsync(ofd.host_handle) } /// truncate -- truncate a file to a specified length (path-based). @@ -13824,6 +13822,7 @@ mod tests { symlink_targets: std::collections::HashMap, Vec>, lstat_paths: Vec>, statfs_by_path: std::collections::HashMap, WasmStatfs>, + fsync_calls: Vec, /// Recorded `(pid, bo_id, addr, len)` for every `gbm_bo_bind` call so /// the DRI mmap path can be asserted against. gbm_bo_bind_calls: Vec<(i32, u32, usize, usize)>, @@ -13875,6 +13874,7 @@ mod tests { symlink_targets: std::collections::HashMap::new(), lstat_paths: Vec::new(), statfs_by_path: std::collections::HashMap::new(), + fsync_calls: Vec::new(), gbm_bo_bind_calls: Vec::new(), gbm_bo_unbind_calls: Vec::new(), gl_unbind_calls: Vec::new(), @@ -14219,7 +14219,8 @@ mod tests { Ok(()) } - fn host_fsync(&mut self, _handle: i64) -> Result<(), Errno> { + fn host_fsync(&mut self, handle: i64) -> Result<(), Errno> { + self.fsync_calls.push(handle); Ok(()) } @@ -20698,6 +20699,19 @@ mod tests { .unwrap(); let result = sys_fsync(&mut proc, &mut host, fd); assert!(result.is_ok()); + assert_eq!(host.fsync_calls, vec![100]); + } + + #[test] + fn test_fsync_directory_delegates_to_host() { + let mut proc = Process::new(1); + let mut host = MockHostIO::new(); + let fd = sys_open(&mut proc, &mut host, b"/tmp", O_RDONLY | O_DIRECTORY, 0).unwrap(); + + let result = sys_fsync(&mut proc, &mut host, fd); + + assert!(result.is_ok()); + assert_eq!(host.fsync_calls, vec![100]); } #[test] diff --git a/docs/browser-support.md b/docs/browser-support.md index 057f5c4d68..9b3f217525 100644 --- a/docs/browser-support.md +++ b/docs/browser-support.md @@ -115,7 +115,7 @@ pipe pair. ### Filesystem - `MemoryFileSystem` — SharedArrayBuffer-based VFS shared between main thread and kernel worker -- `OpfsFileSystem` — Origin Private File System for browser persistence. Its current stat metadata has no stable inode identity, so regular-file `MAP_SHARED` returns `ENOTSUP` instead of using unsafe pathname identity; `MAP_PRIVATE` is unaffected. +- `OpfsFileSystem` — Origin Private File System for browser persistence. Regular-file `fsync()` calls the browser's file-handle `flush()` operation. Directory `fsync()` succeeds after already-completed directory operations because the File System API exposes no directory flush primitive; it is not an additional crash-durability barrier. Its current stat metadata has no stable inode identity, so regular-file `MAP_SHARED` returns `ENOTSUP` instead of using unsafe pathname identity; `MAP_PRIVATE` is unaffected. - `DeviceFileSystem` — `/dev/null`, `/dev/zero`, `/dev/urandom`, `/dev/ptmx` - Stable-identity regular files can be shared across process memories through the host mapping cache, but updates become visible at syscall boundaries rather than immediately on direct loads/stores. Cross-process futex waits/wakes remain unsupported; see [architecture.md](architecture.md#shared-mapping-coherence). diff --git a/docs/posix-status.md b/docs/posix-status.md index 1c9d3e951b..fbb188939e 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -56,7 +56,7 @@ Kandelo uses a single kernel Wasm instance that holds a `ProcessTable` and serve | `writev()` | Full | Gather write. Enforces aggregate count and RLIMIT_FSIZE once across the full iovec operation, including host scratch-buffer decomposition, then stops on a short underlying write. | | `fstat()` | Partial | Host-delegated for regular files. Pipe returns S_IFIFO | 0o600. Full struct stat populated. | | `ftruncate()` | Partial | Host-delegated for regular files, with in-kernel memfd support. Requires write access, validates length >= 0, rejects non-regular fds, and enforces RLIMIT_FSIZE before changing either backing. | -| `fsync()` | Partial | Host-delegated for regular files. Rejects non-regular fds (pipes, sockets). | +| `fsync()` | Partial | Host-delegated for regular files and directories. Node-backed directories use the native durability barrier; memory-backed filesystems have no queued writes. Browser OPFS flushes regular-file access handles, but its API exposes no separate directory durability barrier. Rejects pipes and sockets. | | `fdatasync()` | Partial | Alias for fsync(). No metadata distinction in Wasm environment. | | `truncate()` | Partial | Path-based. Opens file O_WRONLY, calls ftruncate, closes. | | `fchmod()` | Partial | Regular files and directories update VFS metadata. Rejects pipes/sockets. Node host-backed files never receive native mode changes after creation. | diff --git a/host/src/vfs/opfs-worker.ts b/host/src/vfs/opfs-worker.ts index 67d98c963a..038fea2a37 100644 --- a/host/src/vfs/opfs-worker.ts +++ b/host/src/vfs/opfs-worker.ts @@ -589,13 +589,19 @@ async function handleFtruncate(): Promise { async function handleFsync(): Promise { const handle = channel.getArg(0); const entry = fileHandles.get(handle); - if (!entry || !entry.handle) { + if (!entry) { channel.notifyError(EBADF); return; } try { - entry.handle.flush(); + if (entry.handle) { + entry.handle.flush(); + } + // The File System API exposes flush() for file access handles but no + // equivalent durability barrier for directories. Directory mutations are + // already complete before their OPFS promises resolve, so there is no + // additional browser operation to issue for an O_DIRECTORY handle. channel.result = 0; channel.notifyComplete(); } catch (err) { diff --git a/host/test/vfs/directory-fsync.test.ts b/host/test/vfs/directory-fsync.test.ts new file mode 100644 index 0000000000..d82cd0eb41 --- /dev/null +++ b/host/test/vfs/directory-fsync.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { HostFileSystem } from "../../src/vfs/host-fs"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; + +const O_RDONLY = 0; +const O_DIRECTORY = 0o200000; + +describe("directory fsync", () => { + const roots: string[] = []; + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("uses the native durability barrier for Node-backed directories", () => { + const root = mkdtempSync(join(tmpdir(), "kandelo-directory-fsync-")); + roots.push(root); + mkdirSync(join(root, "journal")); + const fs = new HostFileSystem(root); + const fd = fs.open("/journal", O_RDONLY | O_DIRECTORY, 0); + + try { + expect(() => fs.fsync(fd)).not.toThrow(); + } finally { + fs.close(fd); + } + }); + + it("accepts directory fsync when memory writes are already synchronous", () => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(1024 * 1024)); + fs.mkdir("/journal", 0o700); + const fd = fs.open("/journal", O_RDONLY | O_DIRECTORY, 0); + + try { + expect(() => fs.fsync(fd)).not.toThrow(); + } finally { + fs.close(fd); + } + }); +}); diff --git a/tests/sortix/os-test-local/basic/unistd/fsync-directory.c b/tests/sortix/os-test-local/basic/unistd/fsync-directory.c new file mode 100644 index 0000000000..04fce4d88c --- /dev/null +++ b/tests/sortix/os-test-local/basic/unistd/fsync-directory.c @@ -0,0 +1,26 @@ +#include + +#include +#include + +#include "../basic.h" + +int main(void) +{ + const char path[] = "fsync-directory.tmp"; + if ( mkdir(path, 0700) < 0 ) + err(1, "mkdir"); + + int fd = open(path, O_RDONLY | O_DIRECTORY); + if ( fd < 0 ) + err(1, "open"); + + if ( fsync(fd) < 0 ) + err(1, "fsync"); + + if ( close(fd) < 0 ) + err(1, "close"); + if ( rmdir(path) < 0 ) + err(1, "rmdir"); + return 0; +} From 75d9d3cb7e3afbcfb3cbbe02fc8d587c54649b52 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:13:59 -0400 Subject: [PATCH 14/20] libc: zero stat fields the kernel does not report --- docs/posix-status.md | 6 +- libc/musl-overlay/arch/wasm32posix/kstat.h | 11 +- libc/musl-overlay/arch/wasm64posix/kstat.h | 11 +- libc/musl-overlay/src/stat/fstatat.c | 154 ++++++++++++++++++ .../basic/sys_stat/stat-unreported-fields.c | 61 +++++++ 5 files changed, 230 insertions(+), 13 deletions(-) create mode 100644 libc/musl-overlay/src/stat/fstatat.c create mode 100644 tests/sortix/os-test-local/basic/sys_stat/stat-unreported-fields.c diff --git a/docs/posix-status.md b/docs/posix-status.md index fbb188939e..13cebaca6d 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -54,7 +54,7 @@ Kandelo uses a single kernel Wasm instance that holds a `ProcessTable` and serve | `pipe2()` | Full | Like pipe with O_NONBLOCK and O_CLOEXEC flag support. | | `readv()` | Full | Scatter read. Iterates over iovec array calling sys_read for each buffer. Stops on short read or EOF. | | `writev()` | Full | Gather write. Enforces aggregate count and RLIMIT_FSIZE once across the full iovec operation, including host scratch-buffer decomposition, then stops on a short underlying write. | -| `fstat()` | Partial | Host-delegated for regular files. Pipe returns S_IFIFO | 0o600. Full struct stat populated. | +| `fstat()` | Partial | Host-delegated for regular files. Pipe returns S_IFIFO | 0o600. ABI 39 does not report `st_rdev`, `st_blksize`, or `st_blocks`; libc initializes those fields to zero instead of exposing uninitialized memory. Truthful backend metadata is tracked in [issue #928](https://github.com/Automattic/kandelo/issues/928). | | `ftruncate()` | Partial | Host-delegated for regular files, with in-kernel memfd support. Requires write access, validates length >= 0, rejects non-regular fds, and enforces RLIMIT_FSIZE before changing either backing. | | `fsync()` | Partial | Host-delegated for regular files and directories. Node-backed directories use the native durability barrier; memory-backed filesystems have no queued writes. Browser OPFS flushes regular-file access handles, but its API exposes no separate directory durability barrier. Rejects pipes and sockets. | | `fdatasync()` | Partial | Alias for fsync(). No metadata distinction in Wasm environment. | @@ -70,8 +70,8 @@ Kandelo uses a single kernel Wasm instance that holds a `ProcessTable` and serve | `splice()` | Full | Emulated through the copy loop with optional offsets. The output RLIMIT_FSIZE budget is fixed before input is consumed. | | `tee()` / `vmsplice()` | Stub | Returns ENOSYS. | | `readahead()` | Stub | Returns 0 (no-op advisory). | -| `fstatat()` | Full | AT_FDCWD delegates to stat/lstat. AT_SYMLINK_NOFOLLOW supported. Real dirfd supported via stored OFD paths. | -| `statx()` | Full | Delegates to fstatat, fills statx struct (256 bytes) from WasmStat. STATX_BASIC_STATS mask. | +| `fstatat()` | Partial | AT_FDCWD delegates to stat/lstat. AT_SYMLINK_NOFOLLOW and real dirfds are supported. ABI 39 omits `st_rdev`, `st_blksize`, and `st_blocks`; libc reports zero for those fields pending [issue #928](https://github.com/Automattic/kandelo/issues/928). | +| `statx()` | Partial | Delegates to fstatat and fills the 256-byte statx structure from WasmStat. Basic identity, mode, ownership, size, and timestamp fields are reported, but block and device metadata are incomplete pending [issue #928](https://github.com/Automattic/kandelo/issues/928). | | `unlinkat()` | Full | AT_FDCWD delegates to unlink/rmdir. AT_REMOVEDIR flag supported. Real dirfd supported. | | `mkdirat()` | Full | AT_FDCWD delegates to mkdir. umask applied. Real dirfd supported. | | `renameat()` | Full | Both dirfds supported (AT_FDCWD, absolute, or real dirfd). | diff --git a/libc/musl-overlay/arch/wasm32posix/kstat.h b/libc/musl-overlay/arch/wasm32posix/kstat.h index e5be1a8353..8c78961340 100644 --- a/libc/musl-overlay/arch/wasm32posix/kstat.h +++ b/libc/musl-overlay/arch/wasm32posix/kstat.h @@ -3,8 +3,9 @@ * This matches the kernel's WasmStat layout (88 bytes) exactly. * musl's fstatat.c copies from kstat fields to struct stat fields. * - * The kernel fills all 88 bytes. The rdev/blksize/blocks fields - * are appended for musl compatibility but the kernel doesn't fill them. + * The kernel fills all 88 bytes. The rdev/blksize/blocks fields are appended + * for musl compatibility, initialized to zero by libc, and not filled by the + * kernel. See #928 for adding truthful filesystem-provided values. */ struct kstat { unsigned long long st_dev; /* offset 0, 8 bytes */ @@ -24,7 +25,7 @@ struct kstat { unsigned int st_ctime_nsec; /* offset 80, 4 bytes */ unsigned int __ctime_pad; /* offset 84, 4 bytes */ /* --- end of 88-byte WasmStat --- */ - unsigned long long st_rdev; /* not from kernel; stays 0 */ - int st_blksize; /* not from kernel; stays 0 */ - int st_blocks; /* not from kernel; stays 0 */ + unsigned long long st_rdev; /* zero until kernel reports it */ + int st_blksize; /* zero until kernel reports it */ + int st_blocks; /* zero until kernel reports it */ }; diff --git a/libc/musl-overlay/arch/wasm64posix/kstat.h b/libc/musl-overlay/arch/wasm64posix/kstat.h index f6d10de79c..cb6c45ef37 100644 --- a/libc/musl-overlay/arch/wasm64posix/kstat.h +++ b/libc/musl-overlay/arch/wasm64posix/kstat.h @@ -3,8 +3,9 @@ * This matches the kernel's WasmStat layout (88 bytes) exactly. * musl's fstatat.c copies from kstat fields to struct stat fields. * - * The kernel fills all 88 bytes. The rdev/blksize/blocks fields - * are appended for musl compatibility but the kernel doesn't fill them. + * The kernel fills all 88 bytes. The rdev/blksize/blocks fields are appended + * for musl compatibility, initialized to zero by libc, and not filled by the + * kernel. See #928 for adding truthful filesystem-provided values. */ struct kstat { unsigned long long st_dev; /* offset 0, 8 bytes */ @@ -24,7 +25,7 @@ struct kstat { unsigned int st_ctime_nsec; /* offset 80, 4 bytes */ unsigned int __ctime_pad; /* offset 84, 4 bytes */ /* --- end of 88-byte WasmStat --- */ - unsigned long long st_rdev; /* not from kernel; stays 0 */ - int st_blksize; /* not from kernel; stays 0 */ - int st_blocks; /* not from kernel; stays 0 */ + unsigned long long st_rdev; /* zero until kernel reports it */ + int st_blksize; /* zero until kernel reports it */ + int st_blocks; /* zero until kernel reports it */ }; diff --git a/libc/musl-overlay/src/stat/fstatat.c b/libc/musl-overlay/src/stat/fstatat.c new file mode 100644 index 0000000000..723576eca9 --- /dev/null +++ b/libc/musl-overlay/src/stat/fstatat.c @@ -0,0 +1,154 @@ +#define _BSD_SOURCE +#include +#include +#include +#include +#include +#include +#include "syscall.h" + +struct statx { + uint32_t stx_mask; + uint32_t stx_blksize; + uint64_t stx_attributes; + uint32_t stx_nlink; + uint32_t stx_uid; + uint32_t stx_gid; + uint16_t stx_mode; + uint16_t pad1; + uint64_t stx_ino; + uint64_t stx_size; + uint64_t stx_blocks; + uint64_t stx_attributes_mask; + struct { + int64_t tv_sec; + uint32_t tv_nsec; + int32_t pad; + } stx_atime, stx_btime, stx_ctime, stx_mtime; + uint32_t stx_rdev_major; + uint32_t stx_rdev_minor; + uint32_t stx_dev_major; + uint32_t stx_dev_minor; + uint64_t spare[14]; +}; + +static int fstatat_statx(int fd, const char *restrict path, struct stat *restrict st, int flag) +{ + struct statx stx; + + flag |= AT_NO_AUTOMOUNT; + int ret = __syscall(SYS_statx, fd, path, flag, 0x7ff, &stx); + if (ret) return ret; + + *st = (struct stat){ + .st_dev = makedev(stx.stx_dev_major, stx.stx_dev_minor), + .st_ino = stx.stx_ino, + .st_mode = stx.stx_mode, + .st_nlink = stx.stx_nlink, + .st_uid = stx.stx_uid, + .st_gid = stx.stx_gid, + .st_rdev = makedev(stx.stx_rdev_major, stx.stx_rdev_minor), + .st_size = stx.stx_size, + .st_blksize = stx.stx_blksize, + .st_blocks = stx.stx_blocks, + .st_atim.tv_sec = stx.stx_atime.tv_sec, + .st_atim.tv_nsec = stx.stx_atime.tv_nsec, + .st_mtim.tv_sec = stx.stx_mtime.tv_sec, + .st_mtim.tv_nsec = stx.stx_mtime.tv_nsec, + .st_ctim.tv_sec = stx.stx_ctime.tv_sec, + .st_ctim.tv_nsec = stx.stx_ctime.tv_nsec, +#if _REDIR_TIME64 + .__st_atim32.tv_sec = stx.stx_atime.tv_sec, + .__st_atim32.tv_nsec = stx.stx_atime.tv_nsec, + .__st_mtim32.tv_sec = stx.stx_mtime.tv_sec, + .__st_mtim32.tv_nsec = stx.stx_mtime.tv_nsec, + .__st_ctim32.tv_sec = stx.stx_ctime.tv_sec, + .__st_ctim32.tv_nsec = stx.stx_ctime.tv_nsec, +#endif + }; + return 0; +} + +#ifdef SYS_fstatat + +#include "kstat.h" + +static int fstatat_kstat(int fd, const char *restrict path, struct stat *restrict st, int flag) +{ + int ret; + struct kstat kst = {0}; + + if (flag==AT_EMPTY_PATH && fd>=0 && !*path) { + ret = __syscall(SYS_fstat, fd, &kst); + if (ret==-EBADF && __syscall(SYS_fcntl, fd, F_GETFD)>=0) { + ret = __syscall(SYS_fstatat, fd, path, &kst, flag); + if (ret==-EINVAL) { + char buf[15+3*sizeof(int)]; + __procfdname(buf, fd); +#ifdef SYS_stat + ret = __syscall(SYS_stat, buf, &kst); +#else + ret = __syscall(SYS_fstatat, AT_FDCWD, buf, &kst, 0); +#endif + } + } + } +#ifdef SYS_lstat + else if ((fd == AT_FDCWD || *path=='/') && flag==AT_SYMLINK_NOFOLLOW) + ret = __syscall(SYS_lstat, path, &kst); +#endif +#ifdef SYS_stat + else if ((fd == AT_FDCWD || *path=='/') && !flag) + ret = __syscall(SYS_stat, path, &kst); +#endif + else ret = __syscall(SYS_fstatat, fd, path, &kst, flag); + + if (ret) return ret; + + *st = (struct stat){ + .st_dev = kst.st_dev, + .st_ino = kst.st_ino, + .st_mode = kst.st_mode, + .st_nlink = kst.st_nlink, + .st_uid = kst.st_uid, + .st_gid = kst.st_gid, + .st_rdev = kst.st_rdev, + .st_size = kst.st_size, + .st_blksize = kst.st_blksize, + .st_blocks = kst.st_blocks, + .st_atim.tv_sec = kst.st_atime_sec, + .st_atim.tv_nsec = kst.st_atime_nsec, + .st_mtim.tv_sec = kst.st_mtime_sec, + .st_mtim.tv_nsec = kst.st_mtime_nsec, + .st_ctim.tv_sec = kst.st_ctime_sec, + .st_ctim.tv_nsec = kst.st_ctime_nsec, +#if _REDIR_TIME64 + .__st_atim32.tv_sec = kst.st_atime_sec, + .__st_atim32.tv_nsec = kst.st_atime_nsec, + .__st_mtim32.tv_sec = kst.st_mtime_sec, + .__st_mtim32.tv_nsec = kst.st_mtime_nsec, + .__st_ctim32.tv_sec = kst.st_ctime_sec, + .__st_ctim32.tv_nsec = kst.st_ctime_nsec, +#endif + }; + + return 0; +} +#endif + +int __fstatat(int fd, const char *restrict path, struct stat *restrict st, int flag) +{ + int ret; +#ifdef SYS_fstatat + if (sizeof((struct kstat){0}.st_atime_sec) < sizeof(time_t)) { + ret = fstatat_statx(fd, path, st, flag); + if (ret!=-ENOSYS) return __syscall_ret(ret); + } + ret = fstatat_kstat(fd, path, st, flag); +#else + ret = fstatat_statx(fd, path, st, flag); +#endif + return __syscall_ret(ret); +} + +weak_alias(__fstatat, fstatat); diff --git a/tests/sortix/os-test-local/basic/sys_stat/stat-unreported-fields.c b/tests/sortix/os-test-local/basic/sys_stat/stat-unreported-fields.c new file mode 100644 index 0000000000..6e71b5de72 --- /dev/null +++ b/tests/sortix/os-test-local/basic/sys_stat/stat-unreported-fields.c @@ -0,0 +1,61 @@ +#include + +#include +#include +#include + +#include "../basic.h" + +static __attribute__((noinline)) void dirty_stack(void) +{ + volatile unsigned char scratch[4096]; + for ( size_t i = 0; i < sizeof(scratch); i++ ) + scratch[i] = 0x6a; +} + +static void check_unreported_fields(const char* label, const struct stat* st) +{ + if ( st->st_rdev != 0 ) + errx(1, "%s: st_rdev was %ju, expected 0", label, + (uintmax_t) st->st_rdev); + if ( st->st_blksize != 0 ) + errx(1, "%s: st_blksize was %jd, expected 0", label, + (intmax_t) st->st_blksize); + if ( st->st_blocks != 0 ) + errx(1, "%s: st_blocks was %jd, expected 0", label, + (intmax_t) st->st_blocks); +} + +int main(void) +{ + const char path[] = "stat-unreported-fields.tmp"; + int fd = open(path, O_CREAT | O_TRUNC | O_RDWR, 0600); + if ( fd < 0 ) + err(1, "open"); + + char byte = 0; + if ( write(fd, &byte, 1) != 1 ) + err(1, "write"); + + struct stat st; + dirty_stack(); + if ( fstat(fd, &st) < 0 ) + err(1, "fstat"); + check_unreported_fields("fstat", &st); + + dirty_stack(); + if ( stat(path, &st) < 0 ) + err(1, "stat"); + check_unreported_fields("stat", &st); + + dirty_stack(); + if ( lstat(path, &st) < 0 ) + err(1, "lstat"); + check_unreported_fields("lstat", &st); + + if ( close(fd) < 0 ) + err(1, "close"); + if ( unlink(path) < 0 ) + err(1, "unlink"); + return 0; +} From b82a32dd0d18c9ce8ecd2462056799d258187a38 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 22:47:27 -0400 Subject: [PATCH 15/20] SQLite: keep recursive tests within WebAssembly stack limits Apply one supported recursion policy to both the shipped library and official testfixture: compound-select, expression, JSON, and trigger recursion stay within current browser and Node WebAssembly stacks. Keep the test patches aligned with those compiled limits and preserve the existing package revision recorded by the reviewed batch tree. --- docs/porting-guide.md | 8 ++ packages/registry/sqlite/build-sqlite.sh | 11 ++ packages/registry/sqlite/build-testfixture.sh | 36 ++++++- packages/registry/sqlite/build.toml | 2 +- .../0001-misc5-respect-expr-depth-limit.patch | 63 +++++++++++ ...002-json101-respect-json-depth-limit.patch | 56 ++++++++++ ...deep-and-chains-for-expr-depth-limit.patch | 102 ++++++++++++++++++ ...0004-randexpr1-omit-expr-depth-cases.patch | 61 +++++++++++ ...-keep-sql-length-filler-comment-only.patch | 22 ++++ 9 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 packages/registry/sqlite/patches/0001-misc5-respect-expr-depth-limit.patch create mode 100644 packages/registry/sqlite/patches/0002-json101-respect-json-depth-limit.patch create mode 100644 packages/registry/sqlite/patches/0003-misc1-omit-deep-and-chains-for-expr-depth-limit.patch create mode 100644 packages/registry/sqlite/patches/0004-randexpr1-omit-expr-depth-cases.patch create mode 100644 packages/registry/sqlite/patches/0005-sqllimits1-keep-sql-length-filler-comment-only.patch diff --git a/docs/porting-guide.md b/docs/porting-guide.md index 871b441467..b8d1eaf8b9 100644 --- a/docs/porting-guide.md +++ b/docs/porting-guide.md @@ -789,6 +789,14 @@ bash packages/registry/tcl/build-tcl.sh bash packages/registry/sqlite/build-testfixture.sh ``` +Kandelo builds both the shipped SQLite library and the official testfixture +with compound-select, expression, JSON, and trigger recursion limits that fit +current browser and Node WebAssembly host stacks. The testfixture patch set +reads those compiled limits and omits only upstream stress cases that +deliberately exceed them; it does not turn platform failures into successful +SQLite results. The `sqllimits1.test` SQL-length filler stays comment-only so +the length-limit check does not hit the lower expression-depth limit first. + Then run the harness: ```bash diff --git a/packages/registry/sqlite/build-sqlite.sh b/packages/registry/sqlite/build-sqlite.sh index c3337a1eeb..cb3fdb7635 100755 --- a/packages/registry/sqlite/build-sqlite.sh +++ b/packages/registry/sqlite/build-sqlite.sh @@ -22,6 +22,10 @@ INSTALL_DIR="${WASM_POSIX_DEP_OUT_DIR:-$SCRIPT_DIR/sqlite-install}" # Legacy default URL uses the packed version form (3.49.1 → 3490100). SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://www.sqlite.org/2025/sqlite-amalgamation-3490100.zip}" SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-}" +SQLITE_MAX_COMPOUND_SELECT="${SQLITE_MAX_COMPOUND_SELECT:-50}" +SQLITE_MAX_EXPR_DEPTH="${SQLITE_MAX_EXPR_DEPTH:-100}" +SQLITE_JSON_MAX_DEPTH="${SQLITE_JSON_MAX_DEPTH:-100}" +SQLITE_MAX_TRIGGER_DEPTH="${SQLITE_MAX_TRIGGER_DEPTH:-50}" # CLI is a consumer artifact, not a library. Skip it when invoked via # the resolver — it would waste cache space and the consumer-side @@ -54,11 +58,18 @@ if [ ! -d "$SRC_DIR/sqlite3.c" ] && [ ! -f "$SRC_DIR/sqlite3.c" ]; then rm "$TARBALL" fi +# Browser and Node Wasm engines exhaust their host stacks before SQLite's +# default recursive SQL limits are reached. Keep the shipped library and the +# official testfixture on the same supported limits. SQLITE_CFLAGS="-O2 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_THREADSAFE=1 \ -DSQLITE_DEFAULT_SYNCHRONOUS=0 \ -DSQLITE_ENABLE_SETLK_TIMEOUT=2 \ + -DSQLITE_MAX_COMPOUND_SELECT=$SQLITE_MAX_COMPOUND_SELECT \ + -DSQLITE_MAX_EXPR_DEPTH=$SQLITE_MAX_EXPR_DEPTH \ + -DSQLITE_JSON_MAX_DEPTH=$SQLITE_JSON_MAX_DEPTH \ + -DSQLITE_MAX_TRIGGER_DEPTH=$SQLITE_MAX_TRIGGER_DEPTH \ -DHAVE_PREAD=1 \ -DHAVE_PWRITE=1 \ -DSQLITE_ENABLE_FTS5 \ diff --git a/packages/registry/sqlite/build-testfixture.sh b/packages/registry/sqlite/build-testfixture.sh index 34e9a03bac..3b2fe3e4ee 100755 --- a/packages/registry/sqlite/build-testfixture.sh +++ b/packages/registry/sqlite/build-testfixture.sh @@ -21,6 +21,10 @@ SQLITE_FULL="$SCRIPT_DIR/sqlite-full-src" ZLIB_INSTALL="$SCRIPT_DIR/../zlib/zlib-install" BUILD_DIR="$SCRIPT_DIR/testfixture-build" SQLITE_VERSION="${SQLITE_VERSION:-3.49.1}" +SQLITE_MAX_COMPOUND_SELECT="${SQLITE_MAX_COMPOUND_SELECT:-50}" +SQLITE_MAX_EXPR_DEPTH="${SQLITE_MAX_EXPR_DEPTH:-100}" +SQLITE_JSON_MAX_DEPTH="${SQLITE_JSON_MAX_DEPTH:-100}" +SQLITE_MAX_TRIGGER_DEPTH="${SQLITE_MAX_TRIGGER_DEPTH:-50}" sqlite_packed_version() { local major minor patch @@ -64,6 +68,24 @@ if [ ! -d "$SQLITE_FULL/src" ]; then rm -rf "$TMP_DIR" "$TMP_ZIP" fi +PATCH_DIR="$SCRIPT_DIR/patches" +if [ -d "$PATCH_DIR" ]; then + echo "==> Applying SQLite testfixture patches..." + for patch_file in "$PATCH_DIR"/*.patch; do + [ -f "$patch_file" ] || continue + patch_name="$(basename "$patch_file")" + if (cd "$SQLITE_FULL" && git apply -p0 --check "$patch_file") >/dev/null 2>&1; then + echo " Applying $patch_name..." + (cd "$SQLITE_FULL" && git apply -p0 "$patch_file") + elif (cd "$SQLITE_FULL" && git apply -p0 --reverse --check "$patch_file") >/dev/null 2>&1; then + echo " $patch_name already applied" + else + echo "ERROR: $patch_name does not apply cleanly" >&2 + exit 1 + fi + done +fi + export WASM_POSIX_SYSROOT="$SYSROOT" # --- Generate required headers --- @@ -86,7 +108,8 @@ echo "/* Generated stub */" > "$BUILD_DIR/sqlite_cfg.h" cd "$BUILD_DIR" -# Common CFLAGS for the testfixture build +# Common CFLAGS for the testfixture build. Keep recursive SQL limits aligned +# with the shipped library so the tests observe the supported configuration. CFLAGS=( -O2 -DSQLITE_TEST=1 @@ -95,6 +118,10 @@ CFLAGS=( -DSQLITE_THREADSAFE=1 -DSQLITE_NO_SYNC=1 -DSQLITE_ENABLE_SETLK_TIMEOUT=2 + -DSQLITE_MAX_COMPOUND_SELECT="$SQLITE_MAX_COMPOUND_SELECT" + -DSQLITE_MAX_EXPR_DEPTH="$SQLITE_MAX_EXPR_DEPTH" + -DSQLITE_JSON_MAX_DEPTH="$SQLITE_JSON_MAX_DEPTH" + -DSQLITE_MAX_TRIGGER_DEPTH="$SQLITE_MAX_TRIGGER_DEPTH" -DHAVE_PREAD=1 -DHAVE_PWRITE=1 -DSQLITE_OMIT_LOAD_EXTENSION @@ -133,6 +160,12 @@ CFLAGS=( -I"$ZLIB_INSTALL/include" ) +# SQLite's recursive test cases need more than wasm-ld's default 64 KiB shadow +# stack. Use 1 MiB without reviving the old 2 GiB maximum-memory workaround. +TESTFIXTURE_LDFLAGS=( + -Wl,-z,stack-size=1048576 +) + # TESTSRC — test C files (excluding test_thread.c) TESTSRC_FILES=( "$SQLITE_FULL/src/test1.c" @@ -265,6 +298,7 @@ wasm32posix-cc "${CFLAGS[@]}" \ "${OBJ_FILES[@]}" \ -L"$TCL_INSTALL/lib" -ltcl8.6 \ -L"$ZLIB_INSTALL/lib" -lz \ + "${TESTFIXTURE_LDFLAGS[@]}" \ -o testfixture if [ ! -f testfixture ]; then diff --git a/packages/registry/sqlite/build.toml b/packages/registry/sqlite/build.toml index 7df6c5f1c9..a072372d5c 100644 --- a/packages/registry/sqlite/build.toml +++ b/packages/registry/sqlite/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/sqlite/build-sqlite.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 4 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/sqlite/patches/0001-misc5-respect-expr-depth-limit.patch b/packages/registry/sqlite/patches/0001-misc5-respect-expr-depth-limit.patch new file mode 100644 index 0000000000..581d62613a --- /dev/null +++ b/packages/registry/sqlite/patches/0001-misc5-respect-expr-depth-limit.patch @@ -0,0 +1,63 @@ +--- test/misc5.test ++++ test/misc5.test +@@ -574,28 +574,36 @@ + # stack is grown automatically such that the application calling + # SQLite never notices. + # +-do_test misc5-7.1.1 { +- execsql {CREATE TABLE t1(x)} +- set sql "INSERT INTO t1 VALUES(" +- set tail "" +- for {set i 0} {$i<200} {incr i} { +- append sql "(1+" +- append tail ")" +- } +- append sql "0$tail); SELECT * FROM t1;" +- catchsql $sql +-} {0 200} +-do_test misc5-7.1.2 { +- execsql {DELETE FROM t1} +- set sql "INSERT INTO t1 VALUES(" +- set tail "" +- for {set i 0} {$i<900} {incr i} { +- append sql "(1+" +- append tail ")" +- } +- append sql "0$tail); SELECT * FROM t1;" +- catchsql $sql +-} {0 900} ++if {$SQLITE_MAX_EXPR_DEPTH>0 && $SQLITE_MAX_EXPR_DEPTH<=200} { ++ omit_test misc5-7.1.1 "requires a 200-deep expression tree; Kandelo builds SQLite with SQLITE_MAX_EXPR_DEPTH=$SQLITE_MAX_EXPR_DEPTH to stay within browser wasm engine call stacks" ++} else { ++ do_test misc5-7.1.1 { ++ execsql {CREATE TABLE t1(x)} ++ set sql "INSERT INTO t1 VALUES(" ++ set tail "" ++ for {set i 0} {$i<200} {incr i} { ++ append sql "(1+" ++ append tail ")" ++ } ++ append sql "0$tail); SELECT * FROM t1;" ++ catchsql $sql ++ } {0 200} ++} ++if {$SQLITE_MAX_EXPR_DEPTH>0 && $SQLITE_MAX_EXPR_DEPTH<=900} { ++ omit_test misc5-7.1.2 "requires a 900-deep expression tree; Kandelo builds SQLite with SQLITE_MAX_EXPR_DEPTH=$SQLITE_MAX_EXPR_DEPTH to stay within browser wasm engine call stacks" ++} else { ++ do_test misc5-7.1.2 { ++ execsql {DELETE FROM t1} ++ set sql "INSERT INTO t1 VALUES(" ++ set tail "" ++ for {set i 0} {$i<900} {incr i} { ++ append sql "(1+" ++ append tail ")" ++ } ++ append sql "0$tail); SELECT * FROM t1;" ++ catchsql $sql ++ } {0 900} ++} +- +- ++ ++ + # Parser stack overflow is silently ignored when it occurs while parsing the diff --git a/packages/registry/sqlite/patches/0002-json101-respect-json-depth-limit.patch b/packages/registry/sqlite/patches/0002-json101-respect-json-depth-limit.patch new file mode 100644 index 0000000000..7c012b608a --- /dev/null +++ b/packages/registry/sqlite/patches/0002-json101-respect-json-depth-limit.patch @@ -0,0 +1,56 @@ +--- src/test_config.c ++++ src/test_config.c +@@ -785,13 +785,17 @@ + static const int cv_ ## x = SQLITE_ ## x; \ + Tcl_LinkVar(interp, "SQLITE_" #x, (char *)&(cv_ ## x), \ + TCL_LINK_INT | TCL_LINK_READ_ONLY); } +- ++#ifndef SQLITE_JSON_MAX_DEPTH ++# define SQLITE_JSON_MAX_DEPTH 1000 ++#endif ++ + LINKVAR( MAX_LENGTH ); + LINKVAR( MAX_COLUMN ); + LINKVAR( MAX_SQL_LENGTH ); + LINKVAR( MAX_EXPR_DEPTH ); + LINKVAR( MAX_COMPOUND_SELECT ); ++ LINKVAR( JSON_MAX_DEPTH ); + LINKVAR( MAX_VDBE_OP ); + LINKVAR( MAX_FUNCTION_ARG ); + LINKVAR( MAX_VARIABLE_NUMBER ); + LINKVAR( MAX_PAGE_SIZE ); +--- test/json101.test ++++ test/json101.test +@@ -813,23 +813,23 @@ + # The following tests confirm that deeply nested JSON is considered invalid. + # + do_execsql_test json101-11.0 { +- /* Shallow enough to be parsed */ ++ /* Valid when the compiled JSON depth limit is at least 1000 */ + SELECT json_valid(printf('%.1000c0%.1000c','[',']')); +-} {1} ++} [expr {$SQLITE_JSON_MAX_DEPTH>=1000 ? 1 : 0}] + do_execsql_test json101-11.1 { +- /* Too deep by one */ ++ /* Valid when the compiled JSON depth limit is at least 1001 */ + SELECT json_valid(printf('%.1001c0%.1001c','[',']')); +-} {0} ++} [expr {$SQLITE_JSON_MAX_DEPTH>=1001 ? 1 : 0}] + do_execsql_test json101-11.2 { +- /* Shallow enough to be parsed { */ ++ /* Valid when the compiled JSON depth limit is at least 1000 { */ + SELECT json_valid(replace(printf('%.1000c0%.1000c','[','}'),'[','{"a":')); + /* } */ +-} {1} ++} [expr {$SQLITE_JSON_MAX_DEPTH>=1000 ? 1 : 0}] + do_execsql_test json101-11.3 { +- /* Too deep by one { */ ++ /* Valid when the compiled JSON depth limit is at least 1001 { */ + SELECT json_valid(replace(printf('%.1001c0%.1001c','[','}'),'[','{"a":')); + /* } */ +-} {0} ++} [expr {$SQLITE_JSON_MAX_DEPTH>=1001 ? 1 : 0}] +- ++ + # 2017-10-27. Demonstrate the ability to access an element from + # a json structure even though the element name constains a "." diff --git a/packages/registry/sqlite/patches/0003-misc1-omit-deep-and-chains-for-expr-depth-limit.patch b/packages/registry/sqlite/patches/0003-misc1-omit-deep-and-chains-for-expr-depth-limit.patch new file mode 100644 index 0000000000..6ddc356144 --- /dev/null +++ b/packages/registry/sqlite/patches/0003-misc1-omit-deep-and-chains-for-expr-depth-limit.patch @@ -0,0 +1,102 @@ +--- test/misc1.test ++++ test/misc1.test +@@ -286,42 +286,71 @@ + do_test misc1-10.0 { + execsql {SELECT count(*) FROM manycol} + } {9} +-do_test misc1-10.1 { +- set ::where {WHERE x0>=0} +- for {set i 1} {$i<=99} {incr i} { +- append ::where " AND x$i<>0" +- } +- catchsql "SELECT count(*) FROM manycol $::where" +-} {0 9} +-do_test misc1-10.2 { +- catchsql "SELECT count(*) FROM manycol $::where AND rowid>0" +-} {0 9} +-do_test misc1-10.3 { +- regsub "x0>=0" $::where "x0=0" ::where +- catchsql "DELETE FROM manycol $::where" +-} {0 {}} ++set misc1_expr_depth_limited [expr { ++ $SQLITE_MAX_EXPR_DEPTH>0 && $SQLITE_MAX_EXPR_DEPTH<=100 ++}] ++set misc1_expr_depth_reason "requires a 100-term AND expression above the compiled SQLITE_MAX_EXPR_DEPTH=$SQLITE_MAX_EXPR_DEPTH limit" ++set ::where {WHERE x0>=0} ++for {set i 1} {$i<=99} {incr i} { ++ append ::where " AND x$i<>0" ++} ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.1 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.1 { ++ catchsql "SELECT count(*) FROM manycol $::where" ++ } {0 9} ++} ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.2 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.2 { ++ catchsql "SELECT count(*) FROM manycol $::where AND rowid>0" ++ } {0 9} ++} ++regsub "x0>=0" $::where "x0=0" ::where ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.3 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.3 { ++ catchsql "DELETE FROM manycol $::where" ++ } {0 {}} ++} + do_test misc1-10.4 { + execsql {SELECT count(*) FROM manycol} +-} {8} +-do_test misc1-10.5 { +- catchsql "DELETE FROM manycol $::where AND rowid>0" +-} {0 {}} ++} [expr {$misc1_expr_depth_limited ? 9 : 8}] ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.5 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.5 { ++ catchsql "DELETE FROM manycol $::where AND rowid>0" ++ } {0 {}} ++} + do_test misc1-10.6 { + execsql {SELECT x1 FROM manycol WHERE x0=100} + } {101} +-do_test misc1-10.7 { +- regsub "x0=0" $::where "x0=100" ::where +- catchsql "UPDATE manycol SET x1=x1+1 $::where" +-} {0 {}} ++regsub "x0=0" $::where "x0=100" ::where ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.7 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.7 { ++ catchsql "UPDATE manycol SET x1=x1+1 $::where" ++ } {0 {}} ++} + do_test misc1-10.8 { + execsql {SELECT x1 FROM manycol WHERE x0=100} +-} {102} +-do_test misc1-10.9 { +- catchsql "UPDATE manycol SET x1=x1+1 $::where AND rowid>0" +-} {0 {}} ++} [expr {$misc1_expr_depth_limited ? 101 : 102}] ++if {$misc1_expr_depth_limited} { ++ omit_test misc1-10.9 $misc1_expr_depth_reason ++} else { ++ do_test misc1-10.9 { ++ catchsql "UPDATE manycol SET x1=x1+1 $::where AND rowid>0" ++ } {0 {}} ++} + do_test misc1-10.10 { + execsql {SELECT x1 FROM manycol WHERE x0=100} +-} {103} ++} [expr {$misc1_expr_depth_limited ? 101 : 103}] ++unset misc1_expr_depth_limited misc1_expr_depth_reason +- ++ + # Make sure the initialization works even if a database is opened while + # another process has the database locked. diff --git a/packages/registry/sqlite/patches/0004-randexpr1-omit-expr-depth-cases.patch b/packages/registry/sqlite/patches/0004-randexpr1-omit-expr-depth-cases.patch new file mode 100644 index 0000000000..99916a98a4 --- /dev/null +++ b/packages/registry/sqlite/patches/0004-randexpr1-omit-expr-depth-cases.patch @@ -0,0 +1,61 @@ +--- test/randexpr1.test ++++ test/randexpr1.test +@@ -25,8 +25,49 @@ + ifcapable !compound { + finish_test + return + } +- ++if {$SQLITE_MAX_EXPR_DEPTH>0 && $SQLITE_MAX_EXPR_DEPTH<=100} { ++ set randexpr1_expr_depth_omits { ++ randexpr-2.284 ++ randexpr-2.285 ++ randexpr-2.497 ++ randexpr-2.498 ++ randexpr-2.499 ++ randexpr-2.590 ++ randexpr-2.591 ++ randexpr-2.1101 ++ randexpr-2.1102 ++ randexpr-2.1103 ++ randexpr-2.1315 ++ randexpr-2.1316 ++ randexpr-2.1317 ++ randexpr-2.1969 ++ randexpr-2.1970 ++ randexpr-2.1971 ++ randexpr-2.2078 ++ randexpr-2.2079 ++ randexpr-2.2082 ++ randexpr-2.2083 ++ randexpr-2.2597 ++ randexpr-2.2598 ++ randexpr-2.2599 ++ } ++ ++ rename do_test randexpr1_orig_do_test ++ proc do_test {name cmd expected} { ++ fix_testname name ++ if {[lsearch -exact $::randexpr1_expr_depth_omits $name]>=0} { ++ set omittedName $name ++ if {[info exists ::G(perm:prefix)]} { ++ set omittedName "$::G(perm:prefix)$omittedName" ++ } ++ omit_test $omittedName "requires generated expression trees above Kandelo's SQLITE_MAX_EXPR_DEPTH=$::SQLITE_MAX_EXPR_DEPTH browser-safe build limit" ++ return ++ } ++ uplevel 1 [list randexpr1_orig_do_test $name $cmd $expected] ++ } ++} ++ + # Create test data + # + do_test randexpr1-1.1 { +@@ -7837,1 +7878,7 @@ ++if {[info commands randexpr1_orig_do_test] ne ""} { ++ rename do_test {} ++ rename randexpr1_orig_do_test do_test ++ unset randexpr1_expr_depth_omits ++} ++ + finish_test diff --git a/packages/registry/sqlite/patches/0005-sqllimits1-keep-sql-length-filler-comment-only.patch b/packages/registry/sqlite/patches/0005-sqllimits1-keep-sql-length-filler-comment-only.patch new file mode 100644 index 0000000000..407ebd21a5 --- /dev/null +++ b/packages/registry/sqlite/patches/0005-sqllimits1-keep-sql-length-filler-comment-only.patch @@ -0,0 +1,22 @@ +--- test/sqllimits1.test ++++ test/sqllimits1.test +@@ -429,8 +429,7 @@ do_test sqllimits1-6.1 { + sqlite3_limit db SQLITE_LIMIT_SQL_LENGTH 50000 + set sql "SELECT 1 WHERE 1==1" + set tail " /* A comment to take up space in order to make the string\ +- longer without increasing the expression depth */\ +- AND 1 == 1" ++ longer without increasing the expression depth */ " + set N [expr {(50000 / [string length $tail])+1}] + append sql [string repeat $tail $N] + catchsql $sql +@@ -439,8 +438,7 @@ do_test sqllimits1-6.3 { + sqlite3_limit db SQLITE_LIMIT_SQL_LENGTH 50000 + set sql "SELECT 1 WHERE 1==1" + set tail " /* A comment to take up space in order to make the string\ +- longer without increasing the expression depth */\ +- AND 1 == 1" ++ longer without increasing the expression depth */ " + set N [expr {(50000 / [string length $tail])+1}] + append sql [string repeat $tail $N] + set nbytes [string length $sql] From 0ba71dd0b08840fa1751e2a888428b2525f977eb Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:19:30 -0400 Subject: [PATCH 16/20] SQLite tests: run all-mode child jobs on Kandelo --- apps/browser-demos/pages/sqlite-test/main.ts | 26 +- .../pages/sqlite-test/testrunner-patch.ts | 189 ++++++++++++ host/test/sqlite-testrunner-patch.test.ts | 48 +++ scripts/run-browser-sqlite-official-tests.sh | 80 +++++ scripts/run-sqlite-official-tests.sh | 279 +++++++++++++++++- 5 files changed, 607 insertions(+), 15 deletions(-) create mode 100644 apps/browser-demos/pages/sqlite-test/testrunner-patch.ts create mode 100644 host/test/sqlite-testrunner-patch.test.ts diff --git a/apps/browser-demos/pages/sqlite-test/main.ts b/apps/browser-demos/pages/sqlite-test/main.ts index 85bdab7f88..3105cc4a87 100644 --- a/apps/browser-demos/pages/sqlite-test/main.ts +++ b/apps/browser-demos/pages/sqlite-test/main.ts @@ -7,6 +7,10 @@ import { BrowserKernel } from "@host/browser-kernel-host"; import { MemoryFileSystem } from "@host/vfs/memory-fs"; import { writeVfsFile } from "@host/vfs/image-helpers"; import { finalizeKernelOwnedImage, settleWebKitReclaim } from "../../lib/kernel-owned-boot"; +import { + patchTestrunnerForKandelo, + testrunnerPlatformShim, +} from "./testrunner-patch"; import kernelWasmUrl from "@kernel-wasm?url"; declare global { @@ -91,6 +95,20 @@ async function collectArtifactsFromKernel( return artifacts.length > 0 ? artifacts : undefined; } +function installTestrunnerPatches(fs: MemoryFileSystem): void { + const runnerPath = "/sqlite/test/testrunner.tcl"; + const decoder = new TextDecoder(); + const runner = decoder.decode(readVfsFile(fs, runnerPath)); + writeVfsFile(fs, runnerPath, patchTestrunnerForKandelo(runner), 0o644); + + writeVfsFile(fs, "/sqlite/kandelo-testrunner.tcl", [ + testrunnerPlatformShim, + "set argv0 test/testrunner.tcl", + "source $argv0", + "", + ].join("\n"), 0o644); +} + function createFs(): MemoryFileSystem { if (!vfsImageBytes) throw new Error("SQLite test VFS image not loaded"); const fs = MemoryFileSystem.fromImage(vfsImageBytes, { @@ -153,13 +171,7 @@ async function init() { // SharedArrayBuffer across the per-test loop (Safari OOM fix). const buildFs = createFs(); if (argv[1] === "kandelo-testrunner.tcl") { - writeVfsFile(buildFs, "/sqlite/kandelo-testrunner.tcl", [ - "set ::tcl_platform(os) OpenBSD", - "set ::tcl_platform(platform) unix", - "set argv0 test/testrunner.tcl", - "source $argv0", - "", - ].join("\n"), 0o644); + installTestrunnerPatches(buildFs); } const vfsImage = await finalizeKernelOwnedImage(buildFs); const kernel = new BrowserKernel({ diff --git a/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts new file mode 100644 index 0000000000..6a7ff853ba --- /dev/null +++ b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts @@ -0,0 +1,189 @@ +export const testrunnerPlatformShim = [ + "# Kandelo testrunner host selection for child jobs.", + "# This controls only SQLite's helper-script launcher. Keep tcl_platform", + "# unchanged so the tests continue to observe the real Tcl target.", + "set ::kandelo_testrunner_host Kandelo", +].join("\n"); + +const testrunnerGuestPathShim = [ + "# Kandelo guest path shim for all-mode child jobs.", + "# testrunner.tcl builds child run.sh files from host-normalized paths;", + "# convert workdir-local paths back to paths relative to each testdirN", + "# directory, because SQLite runs the script after cd-ing into it.", + "proc kandelo_guest_path {path} {", + " if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {", + " return $path", + " }", + " set normalized [file normalize $path]", + " set topdir [file normalize [file dirname $::testdir]]", + " set script [file normalize [info script]]", + " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }", + " if {[string equal $normalized $topdir]} { return \"..\" }", + " set prefix \"${topdir}/\"", + " if {[string first $prefix $normalized] == 0} {", + " return \"../[string range $normalized [string length $prefix] end]\"", + " }", + " return $path", + "}", + "set ::kandelo_inline_run_sh 1", + "set ::kandelo_chunk_pipe_output 1", +].join("\n"); + +function replaceRequired(source: string, search: string, replacement: string, label: string): string { + if (!source.includes(search)) { + throw new Error(`SQLite testrunner patch is incompatible: missing ${label}`); + } + return source.replace(search, replacement); +} + +export function patchTestrunnerForKandelo(runner: string): string { + let patched = runner; + + if (!patched.includes("Kandelo testrunner host selection for child jobs")) { + const lines = patched.split("\n"); + if (lines.length < 4) { + throw new Error("SQLite testrunner patch is incompatible: file is shorter than four lines"); + } + lines.splice(3, 0, "", testrunnerPlatformShim); + patched = lines.join("\n"); + patched = replaceRequired( + patched, + "switch -nocase -glob -- $tcl_platform(os) {", + [ + "set testrunner_host $tcl_platform(os)", + "if {[info exists ::kandelo_testrunner_host]} {", + " set testrunner_host $::kandelo_testrunner_host", + "}", + "switch -nocase -glob -- $testrunner_host {", + ].join("\n"), + "host-selection switch", + ); + patched = replaceRequired( + patched, + " *openbsd* {\n", + [ + " *kandelo* {", + " set TRG(platform) linux", + " set TRG(make) make.sh", + " set TRG(makecmd) \"sh make.sh\"", + " set TRG(testfixture) testfixture", + " set TRG(shell) sqlite3", + " set TRG(run) run.sh", + " set TRG(runcmd) \"sh run.sh\"", + " }", + " *openbsd* {", + "", + ].join("\n"), + "OpenBSD host branch", + ); + } + + if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { + patched = replaceRequired( + patched, + "cd $dir\n", + `cd $dir\n\n${testrunnerGuestPathShim}\n`, + "child work-directory anchor", + ); + patched = replaceRequired( + patched, + " set displayname [string map [list $topdir/ {}] $f]\n", + [ + " set displayname [string map [list $topdir/ {}] $f]", + " set testfixture_guest [kandelo_guest_path $testfixture]", + " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]", + " set f_guest [kandelo_guest_path $f]", + "", + ].join("\n"), + "job display-name anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture_guest $f_guest\"", + "direct test command anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "configured test command anchor", + ); + patched = replaceRequired( + patched, + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "temporary-directory anchor", + ); + patched = replaceRequired( + patched, + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + [ + " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {", + " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"", + " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + " } else {", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " }", + ].join("\n"), + "child shell command anchor", + ); + patched = replaceRequired( + patched, + " set rc [catch { gets $fd line } res]", + [ + " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {", + " set rc [catch { read $fd 4096 } res]", + " if {$rc} {", + " puts \"ERROR $res\"", + " }", + " if {!$rc && [string length $res] > 0} {", + " append O($iJob) $res", + " }", + " } else {", + " set rc [catch { gets $fd line } res]", + ].join("\n"), + "pipe read anchor", + ); + patched = replaceRequired( + patched, + " if {$res>=0} {", + [ + " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {", + " if {$res>=0} {", + ].join("\n"), + "line-result anchor", + ); + patched = replaceRequired( + patched, + " append O($iJob) \"$line\\n\"", + [ + " append O($iJob) \"$line\\n\"", + " }", + " }", + ].join("\n"), + "line append anchor", + ); + } + + for (const required of [ + "Kandelo testrunner host selection for child jobs", + "set ::kandelo_testrunner_host Kandelo", + "switch -nocase -glob -- $testrunner_host", + "*kandelo* {", + "Kandelo guest path shim for all-mode child jobs", + "set testfixture_guest [kandelo_guest_path $testfixture]", + "set cmd \"$testfixture_guest $f_guest\"", + "set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + "set ::kandelo_chunk_pipe_output 1", + "set rc [catch { read $fd 4096 } res]", + ]) { + if (!patched.includes(required)) { + throw new Error(`SQLite testrunner patch is incomplete: missing ${required}`); + } + } + + return patched; +} diff --git a/host/test/sqlite-testrunner-patch.test.ts b/host/test/sqlite-testrunner-patch.test.ts new file mode 100644 index 0000000000..6bd2eeb28d --- /dev/null +++ b/host/test/sqlite-testrunner-patch.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { patchTestrunnerForKandelo } from "../../apps/browser-demos/pages/sqlite-test/testrunner-patch"; + +const upstreamFixture = [ + "#!/usr/bin/env tclsh", + "# SQLite test runner fixture", + "set TRG(nJob) 1", + "set dir testdir1", + "switch -nocase -glob -- $tcl_platform(os) {", + " *openbsd* {", + " }", + "}", + "cd $dir", + " set displayname [string map [list $topdir/ {}] $f]", + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " set rc [catch { gets $fd line } res]", + " if {$res>=0} {", + " append O($iJob) \"$line\\n\"", + " }", + "", +].join("\n"); + +describe("SQLite browser testrunner patch", () => { + it("rewrites all-mode child commands and is idempotent", () => { + const patched = patchTestrunnerForKandelo(upstreamFixture); + + expect(patched).toContain("set ::kandelo_testrunner_host Kandelo"); + expect(patched).not.toContain("set ::tcl_platform(os)"); + expect(patched).toContain("switch -nocase -glob -- $testrunner_host"); + expect(patched).toContain("*kandelo* {"); + expect(patched).toContain("set testfixture_guest [kandelo_guest_path $testfixture]"); + expect(patched).toContain("set cmd \"$testfixture_guest $f_guest\""); + expect(patched).toContain("set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\""); + expect(patched).toContain("set set_tmp_dir \"export SQLITE_TMPDIR=.\""); + expect(patched).toContain("set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]"); + expect(patched).toContain("set rc [catch { read $fd 4096 } res]"); + expect(patchTestrunnerForKandelo(patched)).toBe(patched); + }); + + it("fails loudly when an upstream anchor changes", () => { + expect(() => patchTestrunnerForKandelo("one\ntwo\nthree\nfour\n")).toThrow( + "missing host-selection switch", + ); + }); +}); diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index 1103e76d70..9e35b17317 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -69,12 +69,89 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state IN ('running', 'ready') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT sum(state='done') AS passed_jobs, + sum(state='failed') AS failed_jobs, + sum(state='omit') AS skipped_jobs, + sum(state IN ('running','ready')) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$RESULTS_DIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" + write_unavailable_outcome_lists "No testrunner.db was created at $db." return fi @@ -93,11 +170,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" return fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=browser" diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index a27544ad4d..c364f0a638 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -12,6 +12,7 @@ SQLITE_FULL="$REPO_ROOT/packages/registry/sqlite/sqlite-full-src" TCL_INSTALL="$REPO_ROOT/packages/registry/tcl/tcl-install" TESTFIXTURE="$REPO_ROOT/packages/registry/sqlite/bin/testfixture.wasm" SQLITE3="$REPO_ROOT/packages/registry/sqlite/sqlite-install/bin/sqlite3.wasm" +GUEST_SHELL="${SQLITE_TEST_SHELL:-}" HOST="node" PERMUTATION="full" @@ -131,6 +132,14 @@ if [ ! -f "$TESTFIXTURE" ] || [ ! -f "$SQLITE3" ] || [ ! -d "$SQLITE_FULL/test" exit 1 fi +if [ -z "$GUEST_SHELL" ]; then + if ! GUEST_SHELL="$("$REPO_ROOT/scripts/resolve-binary.sh" programs/dash.wasm)"; then + echo "ERROR: SQLite testrunner child jobs require a current guest /bin/sh-compatible shell." >&2 + echo "Fetch/build dash, or set SQLITE_TEST_SHELL=/path/to/sh.wasm." >&2 + exit 1 + fi +fi + if [ -z "$WORKDIR" ]; then WORKDIR="$(mktemp -d "${SQLITE_OFFICIAL_TMPDIR:-/tmp}/kandelo-sqlite-official.XXXXXX")" else @@ -143,17 +152,99 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' AND coalesce(nerr, 0) = 0 + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' OR (state = 'done' AND coalesce(nerr, 0) > 0) + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state IN ('running', 'ready') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT coalesce(sum(state='done' AND coalesce(nerr, 0)=0), 0) AS passed_jobs, + coalesce(sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)), 0) AS failed_jobs, + coalesce(sum(state='omit'), 0) AS skipped_jobs, + coalesce(sum(state IN ('running','ready')), 0) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$WORKDIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" + write_unavailable_outcome_lists "No testrunner.db was created at $db." return fi mkdir -p "$RESULTS_DIR" - for artifact in testrunner.db testrunner.log testrunner_build.log; do + + # SQLite's testrunner keeps its control database in WAL mode. Checkpoint + # before copying so timeout artifacts remain self-contained after cleanup. + sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true + + for artifact in testrunner.db testrunner.db-wal testrunner.db-shm testrunner.log testrunner_build.log; do if [ -f "$WORKDIR/$artifact" ]; then cp "$WORKDIR/$artifact" "$RESULTS_DIR/$artifact" fi @@ -175,11 +266,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" return fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=$HOST" @@ -234,6 +328,7 @@ write_sqlite_report() { coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs WHERE state IN ('failed', 'running', 'omit') + OR (state='done' AND coalesce(nerr, 0)>0) ORDER BY state, jobid;" } > "$report" @@ -242,12 +337,175 @@ write_sqlite_report() { coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs WHERE state IN ('failed', 'running', 'omit') + OR (state='done' AND coalesce(nerr, 0)>0) ORDER BY state, jobid;" > "$failures" echo "===== SQLite official testrunner database summary =====" cat "$report" } +patch_sqlite_testrunner_platform() { + local runner="$1" + local tmp + + if grep -q "Kandelo testrunner host selection for child jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-platform.$$" + awk ' + NR == 4 { + print "" + print "# Kandelo testrunner host selection for child jobs." + print "# This controls only the SQLite helper-script launcher. Keep tcl_platform" + print "# unchanged so the tests continue to observe the real Tcl target." + print "set ::kandelo_testrunner_host Kandelo" + } + $0 == "switch -nocase -glob -- $tcl_platform(os) {" { + print "set testrunner_host $tcl_platform(os)" + print "if {[info exists ::kandelo_testrunner_host]} {" + print " set testrunner_host $::kandelo_testrunner_host" + print "}" + print "switch -nocase -glob -- $testrunner_host {" + next + } + $0 == " *openbsd* {" { + print " *kandelo* {" + print " set TRG(platform) linux" + print " set TRG(make) make.sh" + print " set TRG(makecmd) \"sh make.sh\"" + print " set TRG(testfixture) testfixture" + print " set TRG(shell) sqlite3" + print " set TRG(run) run.sh" + print " set TRG(runcmd) \"sh run.sh\"" + print " }" + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" + + for required in \ + 'set ::kandelo_testrunner_host Kandelo' \ + 'switch -nocase -glob -- $testrunner_host {' \ + '*kandelo* {' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl host selection: missing $required" >&2 + exit 1 + fi + done +} + +patch_sqlite_testrunner_guest_paths() { + local runner="$1" + local tmp + + if grep -q "Kandelo guest path shim for all-mode child jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-paths.$$" + awk ' + { + print + if (!inserted && $0 == "cd $dir") { + print "" + print "# Kandelo guest path shim for all-mode child jobs." + print "# testrunner.tcl builds child run.sh files from host-normalized paths;" + print "# convert workdir-local paths back to paths relative to each testdirN" + print "# directory, because SQLite runs the script after cd-ing into it." + print "proc kandelo_guest_path {path} {" + print " set normalized [file normalize $path]" + print " set topdir [file normalize [file dirname $::testdir]]" + print " set exe [file normalize [info nameofexec]]" + print " set script [file normalize [info script]]" + print " if {[string equal $normalized $exe]} { return \"../testfixture.wasm\" }" + print " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }" + print " if {[string equal $normalized $topdir]} { return \"..\" }" + print " set prefix \"${topdir}/\"" + print " if {[string first $prefix $normalized] == 0} {" + print " return \"../[string range $normalized [string length $prefix] end]\"" + print " }" + print " return $path" + print "}" + print "set ::kandelo_inline_run_sh 1" + print "set ::kandelo_chunk_pipe_output 1" + inserted = 1 + } else if ($0 == " set displayname [string map [list $topdir/ {}] $f]") { + print " set testfixture_guest [kandelo_guest_path $testfixture]" + print " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]" + print " set f_guest [kandelo_guest_path $f]" + } + } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + + tmp="${runner}.kandelo-paths-subst.$$" + awk ' + $0 == " set cmd \"$testfixture $f\"" { + print " set cmd \"$testfixture_guest $f_guest\"" + next + } + $0 == " set cmd \"$testfixture $testrunner_tcl $config $f\"" { + print " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"" + next + } + $0 == " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"" { + print " set set_tmp_dir \"export SQLITE_TMPDIR=.\"" + next + } + $0 == " set fd [open \"|$TRG(runcmd) 2>@1\" r]" { + print " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {" + print " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"" + print " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]" + print " } else {" + print " set fd [open \"|$TRG(runcmd) 2>@1\" r]" + print " }" + next + } + $0 == " set rc [catch { gets $fd line } res]" { + print " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {" + print " set rc [catch { read $fd 4096 } res]" + print " if {$rc} {" + print " puts \"ERROR $res\"" + print " }" + print " if {!$rc && [string length $res] > 0} {" + print " append O($iJob) $res" + print " }" + print " } else {" + print " set rc [catch { gets $fd line } res]" + next + } + $0 == " if {$res>=0} {" { + print " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {" + print " if {$res>=0} {" + next + } + $0 == " append O($iJob) \"$line\\n\"" { + print + print " }" + print " }" + next + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" + + for required in \ + 'set ::kandelo_inline_run_sh 1' \ + 'set ::kandelo_chunk_pipe_output 1' \ + 'set fd [open "|sh -c [list $inline_cmd] 2>@1" r]' \ + 'set rc [catch { read $fd 4096 } res]' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl for Kandelo all-mode jobs: missing $required" >&2 + exit 1 + fi + done +} + cleanup() { if [ "$KEEP_WORKDIR" = "1" ]; then echo "Keeping SQLite official workdir: $WORKDIR" @@ -272,15 +530,19 @@ cp "$TESTFIXTURE" "$WORKDIR/testfixture.wasm" cp "$SQLITE3" "$WORKDIR/sqlite3" cp "$SQLITE3" "$WORKDIR/sqlite3.wasm" chmod a+rx "$WORKDIR/testfixture" "$WORKDIR/testfixture.wasm" "$WORKDIR/sqlite3" "$WORKDIR/sqlite3.wasm" +if [ -n "$GUEST_SHELL" ]; then + cp "$GUEST_SHELL" "$WORKDIR/sh" + cp "$GUEST_SHELL" "$WORKDIR/sh.wasm" + chmod a+rx "$WORKDIR/sh" "$WORKDIR/sh.wasm" +fi +patch_sqlite_testrunner_platform "$WORKDIR/test/testrunner.tcl" +patch_sqlite_testrunner_guest_paths "$WORKDIR/test/testrunner.tcl" RUNNER_TCL="$WORKDIR/kandelo-testrunner.tcl" cat > "$RUNNER_TCL" <<'TCL' -# Kandelo's Tcl build reports a target OS name that SQLite's testrunner.tcl -# does not classify. Present a Unix-like platform to the upstream runner and -# use its OpenBSD branch so generated helper scripts run with sh instead of -# bash. -set ::tcl_platform(os) OpenBSD -set ::tcl_platform(platform) unix +# Select Kandelo's helper-script launcher without changing Tcl target metadata +# observed by SQLite's tests. +set ::kandelo_testrunner_host Kandelo set argv0 test/testrunner.tcl source $argv0 TCL @@ -302,12 +564,13 @@ echo "Results dir: $RESULTS_DIR" set +e TCL_LIBRARY="$TCL_INSTALL/lib/tcl8.6" \ KERNEL_CWD="$WORKDIR" \ +KERNEL_PATH="$WORKDIR:${KERNEL_PATH:-/usr/local/bin:/usr/bin:/bin}" \ KERNEL_UID="${SQLITE_TEST_UID:-1000}" \ KERNEL_GID="${SQLITE_TEST_GID:-1000}" \ TIMEOUT="$TIMEOUT_MS" \ node --experimental-wasm-exnref --import tsx/esm \ "$REPO_ROOT/examples/run-example.ts" \ - "$TESTFIXTURE" \ + "$WORKDIR/testfixture.wasm" \ "${ARGS[@]}" status=$? set -e From c7eb1242078f480c0742bd4cb0310acb223490fe Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:20:29 -0400 Subject: [PATCH 17/20] SQLite tests: preserve truthful outcome evidence --- scripts/browser-sqlite-official-runner.ts | 39 +- scripts/run-browser-sqlite-official-tests.sh | 68 ++- scripts/run-sqlite-official-tests.sh | 67 ++- scripts/sqlite-case-outcomes.py | 508 +++++++++++++++++++ tests/scripts/sqlite-case-outcomes.sh | 176 +++++++ 5 files changed, 829 insertions(+), 29 deletions(-) create mode 100755 scripts/sqlite-case-outcomes.py create mode 100755 tests/scripts/sqlite-case-outcomes.sh diff --git a/scripts/browser-sqlite-official-runner.ts b/scripts/browser-sqlite-official-runner.ts index 2715c11f99..f3ebac0129 100755 --- a/scripts/browser-sqlite-official-runner.ts +++ b/scripts/browser-sqlite-official-runner.ts @@ -1,5 +1,5 @@ #!/usr/bin/env tsx -import { spawn, type ChildProcess } from "node:child_process"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { resolve } from "node:path"; @@ -108,6 +108,39 @@ function writeArtifacts(resultsDir: string, artifacts: BrowserArtifact[] | undef } } +function inferDisplayName(command: string[]): string { + const tests = command.filter((arg) => arg.endsWith(".test")); + if (tests.length > 0) return tests[tests.length - 1]; + return command.join(" "); +} + +function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): boolean { + if (!resultsDir) return true; + mkdirSync(resultsDir, { recursive: true }); + const stdoutPath = resolve(resultsDir, "runner.stdout"); + const stderrPath = resolve(resultsDir, "runner.stderr"); + const dbPath = resolve(resultsDir, "testrunner.db"); + writeFileSync(stdoutPath, stdout); + writeFileSync(stderrPath, stderr); + const sourceArgs = existsSync(dbPath) + ? ["--db", dbPath] + : ["--stdout-file", stdoutPath, "--display-name", inferDisplayName(command)]; + const outcome = spawnSync( + "python3", + [ + resolve(REPO_ROOT, "scripts/sqlite-case-outcomes.py"), + ...sourceArgs, + "--results-dir", resultsDir, + "--host", "browser", + ], + { cwd: REPO_ROOT, stdio: "inherit" }, + ); + if (outcome.status !== 0) { + console.error(`[sqlite-outcomes] case outcome extraction failed with status ${outcome.status}`); + } + return outcome.status === 0; +} + async function main() { const argv = process.argv.slice(2); let timeoutMs = 600_000; @@ -189,10 +222,12 @@ async function main() { } if (!result.artifacts && latestArtifacts) result.artifacts = latestArtifacts; writeArtifacts(resultsDir, result.artifacts); + const outcomesOk = writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.error) process.stderr.write(`${result.error}\n`); - process.exit(result.exitCode === 0 ? 0 : 1); + process.exitCode = result.exitCode === 0 && outcomesOk ? 0 : 1; + return; } finally { await browser?.close().catch(() => {}); if (vite) { diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index 9e35b17317..daa1e9d6d0 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -97,7 +97,7 @@ write_outcome_lists() { coalesce(span, 0) AS ms, 'testrunner.db' AS source FROM jobs - WHERE state = 'done' + WHERE state = 'done' AND coalesce(nerr, 0) = 0 ORDER BY jobid;" > "$out/passed-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ @@ -108,6 +108,7 @@ write_outcome_lists() { 'testrunner.db' AS source FROM jobs WHERE state = 'failed' + OR (state = 'done' AND coalesce(nerr, 0) > 0) ORDER BY jobid;" > "$out/failed-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ @@ -133,14 +134,14 @@ write_outcome_lists() { END AS reason, 'testrunner.db' AS source FROM jobs - WHERE state IN ('running', 'ready') + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ - "SELECT sum(state='done') AS passed_jobs, - sum(state='failed') AS failed_jobs, + "SELECT sum(state='done' AND coalesce(nerr, 0)=0) AS passed_jobs, + sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)) AS failed_jobs, sum(state='omit') AS skipped_jobs, - sum(state IN ('running','ready')) AS incomplete_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, 'testrunner.db' AS source FROM jobs;" > "$out/counts.tsv" } @@ -152,7 +153,7 @@ write_sqlite_report() { if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" write_unavailable_outcome_lists "No testrunner.db was created at $db." - return + return 1 fi if ! sqlite3 "$db" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1;" | grep -qx 1; then @@ -173,7 +174,7 @@ write_sqlite_report() { write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi write_outcome_lists "$db" @@ -225,12 +226,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -238,11 +240,46 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$db" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi + echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi } ARGS=(testfixture kandelo-testrunner.tcl --jobs "$JOBS") @@ -266,5 +303,12 @@ node --import tsx/esm "$REPO_ROOT/scripts/browser-sqlite-official-runner.ts" \ status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index c364f0a638..0e5adc84e3 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -216,14 +216,14 @@ write_outcome_lists() { END AS reason, 'testrunner.db' AS source FROM jobs - WHERE state IN ('running', 'ready') + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ "SELECT coalesce(sum(state='done' AND coalesce(nerr, 0)=0), 0) AS passed_jobs, coalesce(sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)), 0) AS failed_jobs, coalesce(sum(state='omit'), 0) AS skipped_jobs, - coalesce(sum(state IN ('running','ready')), 0) AS incomplete_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, 'testrunner.db' AS source FROM jobs;" > "$out/counts.tsv" } @@ -235,15 +235,11 @@ write_sqlite_report() { if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" write_unavailable_outcome_lists "No testrunner.db was created at $db." - return + return 1 fi mkdir -p "$RESULTS_DIR" - # SQLite's testrunner keeps its control database in WAL mode. Checkpoint - # before copying so timeout artifacts remain self-contained after cleanup. - sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true - for artifact in testrunner.db testrunner.db-wal testrunner.db-shm testrunner.log testrunner_build.log; do if [ -f "$WORKDIR/$artifact" ]; then cp "$WORKDIR/$artifact" "$RESULTS_DIR/$artifact" @@ -269,7 +265,7 @@ write_sqlite_report() { write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi write_outcome_lists "$db" @@ -322,13 +318,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') - OR (state='done' AND coalesce(nerr, 0)>0) + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -336,12 +332,46 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') - OR (state='done' AND coalesce(nerr, 0)>0) + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$RESULTS_DIR/testrunner.db" \ + --results-dir "$RESULTS_DIR" \ + --host "$HOST" \ + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi + echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi } patch_sqlite_testrunner_platform() { @@ -575,5 +605,12 @@ node --experimental-wasm-exnref --import tsx/esm \ status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py new file mode 100755 index 0000000000..cc5fbe2873 --- /dev/null +++ b/scripts/sqlite-case-outcomes.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Emit durable SQLite testrunner case and job outcome lists.""" + +from __future__ import annotations + +import argparse +import json +import re +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +OK_RE = re.compile(r"^(?P.+?)\.\.\. Ok$") +OMITTED_RE = re.compile(r"^(?P.+?)\.\.\. Omitted$") +FAIL_RE = re.compile(r"^! (?P\S+) expected:") +FAILURES_RE = re.compile(r"^!Failures on these tests:\s*(?P.*)$") +SUMMARY_RE = re.compile(r"\b(?P\d+) errors out of (?P\d+) tests\b") +OMITTED_DETAIL_RE = re.compile(r"^\.\s+(?P\S+)\s+(?P.+)$") + +@dataclass +class ParsedOutput: + passed: list[str] = field(default_factory=list) + failed: list[str] = field(default_factory=list) + skipped_counted: list[tuple[str, str]] = field(default_factory=list) + skipped_detail: list[tuple[str, str]] = field(default_factory=list) + summary_tests: int | None = None + summary_errors: int | None = None + + +@dataclass +class Job: + jobid: int | None + state: str + displaytype: str + displayname: str + ntest: int | None + nerr: int | None + span: int | None + output: str + + +def normalize_output(text: str) -> str: + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def append_unique(items: list[str], value: str) -> None: + if value not in items: + items.append(value) + + +def append_unique_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + if value not in items: + items.append(value) + + +def upsert_named_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + for index, (name, _) in enumerate(items): + if name == value[0]: + items[index] = value + return + items.append(value) + + +def parse_output(text: str) -> ParsedOutput: + parsed = ParsedOutput() + in_omitted_detail = False + failure_summary: list[str] = [] + + for line in normalize_output(text).splitlines(): + line = line.strip() + if not line: + continue + + summary = SUMMARY_RE.search(line) + if summary: + parsed.summary_errors = int(summary.group("errors")) + parsed.summary_tests = int(summary.group("tests")) + + if line == "Omitted test cases:": + in_omitted_detail = True + continue + + if in_omitted_detail: + omitted_detail = OMITTED_DETAIL_RE.match(line) + if omitted_detail: + append_unique_pair( + parsed.skipped_detail, + (omitted_detail.group("name"), omitted_detail.group("reason")), + ) + continue + in_omitted_detail = False + + ok = OK_RE.match(line) + if ok: + parsed.passed.append(ok.group("name")) + continue + + omitted = OMITTED_RE.match(line) + if omitted: + append_unique_pair(parsed.skipped_counted, (omitted.group("name"), "omitted by SQLite test harness")) + continue + + failures = FAILURES_RE.match(line) + if failures: + failure_summary.extend(name for name in failures.group("names").split() if name) + continue + + fail = FAIL_RE.match(line) + if fail: + append_unique(parsed.failed, fail.group("name")) + + if failure_summary: + parsed.failed = failure_summary + + return parsed + + +def decode_db_text(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if value is None: + return "" + return str(value) + + +def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: + if not db_path.exists(): + return [], f"testrunner database is missing: {db_path}" + + db_uri = f"{db_path.resolve().as_uri()}?mode=ro" + con = sqlite3.connect(db_uri, uri=True) + # Some archived browser runs contain non-UTF-8 bytes in Tcl output. Read + # TEXT as bytes so one diagnostic byte cannot make the whole report fail. + con.text_factory = bytes + try: + has_jobs = con.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1" + ).fetchone() + if not has_jobs: + return [], f"testrunner database has no jobs table: {db_path}" + + rows = con.execute( + """ + SELECT jobid, state, displaytype, displayname, + ntest, nerr, span, coalesce(output, '') + FROM jobs + ORDER BY jobid + """ + ).fetchall() + finally: + con.close() + + jobs = [ + Job( + jobid=row[0], + state=decode_db_text(row[1]), + displaytype=decode_db_text(row[2]), + displayname=decode_db_text(row[3]), + ntest=row[4], + nerr=row[5], + span=row[6], + output=decode_db_text(row[7]), + ) + for row in rows + ] + return jobs, None + + +def read_stdout_job(stdout_path: Path, display_name: str) -> tuple[list[Job], str | None]: + if not stdout_path.exists(): + return [], f"stdout file is missing: {stdout_path}" + output = stdout_path.read_text(encoding="utf-8", errors="replace") + parsed = parse_output(output) + ntest = parsed.summary_tests + nerr = parsed.summary_errors + state = "failed" if nerr and nerr > 0 else "done" + return [ + Job( + jobid=1, + state=state, + displaytype="direct", + displayname=display_name or "direct stdout", + ntest=ntest, + nerr=nerr, + span=None, + output=output, + ) + ], None + + +def job_row(job: Job) -> list[Any]: + return [ + "" if job.jobid is None else job.jobid, + job.state, + job.displaytype, + job.displayname, + "" if job.ntest is None else job.ntest, + "" if job.nerr is None else job.nerr, + "" if job.span is None else job.span, + ] + + +def add_unavailable(unavailable: list[dict[str, Any]], category: str, reason: str, job: Job | None = None) -> None: + entry: dict[str, Any] = {"category": category, "reason": reason} + if job is not None: + entry["job"] = { + "jobid": job.jobid, + "state": job.state, + "displaytype": job.displaytype, + "displayname": job.displayname, + "ntest": job.ntest, + "nerr": job.nerr, + } + unavailable.append(entry) + + +def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any]: + passed_cases: list[str] = [] + failed_cases: list[str] = [] + skipped_cases: list[tuple[str, str]] = [] + passed_jobs: list[list[Any]] = [] + failed_jobs: list[list[Any]] = [] + skipped_jobs: list[list[Any]] = [] + incomplete_jobs: list[list[Any]] = [] + unattributed_passed_cases: list[list[Any]] = [] + unavailable: list[dict[str, Any]] = [] + + if source_reason is not None: + add_unavailable(unavailable, "all", source_reason) + + reported_executed_cases = 0 + reported_case_errors = 0 + counted_skipped_cases = 0 + detail_only_skipped_cases = 0 + + for job in jobs: + parsed = parse_output(job.output) if job.output else None + effective_errors = job.nerr + if effective_errors is None and parsed is not None: + effective_errors = parsed.summary_errors + + if job.state == "done" and not effective_errors: + passed_jobs.append(job_row(job)) + elif job.state == "failed" or (job.state == "done" and effective_errors): + failed_jobs.append(job_row(job)) + elif job.state == "omit": + skipped_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "skipped_cases", + "SQLite testrunner marked this job omitted; no case-level skip names are available.", + job, + ) + continue + elif job.state in {"ready", "running", "halt", ""}: + incomplete_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "all_cases", + f"SQLite testrunner job is incomplete with state {job.state or ''}.", + job, + ) + continue + + if not job.output: + add_unavailable(unavailable, "all_cases", "SQLite job output is empty.", job) + continue + + assert parsed is not None + ntest = job.ntest if job.ntest is not None else parsed.summary_tests + nerr = effective_errors + if ntest is None or nerr is None: + add_unavailable(unavailable, "all_cases", "Could not find SQLite 'errors out of tests' summary.", job) + continue + + reported_executed_cases += ntest + reported_case_errors += nerr + + job_skipped: list[tuple[str, str]] = [] + counted_skipped_names = {name for name, _reason in parsed.skipped_counted} + for skipped in parsed.skipped_counted: + upsert_named_pair(job_skipped, skipped) + for skipped in parsed.skipped_detail: + # SQLite prints some omissions twice: once as `... Omitted` and + # again in its detailed reason list. Keep one row and prefer the + # specific reason. + upsert_named_pair(job_skipped, skipped) + for skipped in job_skipped: + upsert_named_pair(skipped_cases, skipped) + counted_skipped_cases += len(counted_skipped_names) + detail_only_skipped_cases += sum(1 for name, _reason in job_skipped if name not in counted_skipped_names) + + if len(parsed.failed) == nerr: + failed_cases.extend(parsed.failed) + elif nerr == 0 and not parsed.failed: + pass + else: + failed_cases.extend(parsed.failed) + add_unavailable( + unavailable, + "failed_cases", + f"Parsed {len(parsed.failed)} failed case names, but SQLite reported {nerr} case errors.", + job, + ) + + expected_passed = ntest - nerr - len(parsed.skipped_counted) + missing_passed = expected_passed - len(parsed.passed) + if missing_passed > 0: + unattributed_passed_cases.append([ + "" if job.jobid is None else job.jobid, + job.displayname, + missing_passed, + "SQLite reported passed cases without corresponding named Ok lines", + ]) + add_unavailable( + unavailable, + "passed_cases", + f"SQLite reported {expected_passed} passed cases, but only {len(parsed.passed)} names were present in output.", + job, + ) + elif missing_passed < 0: + add_unavailable( + unavailable, + "passed_cases", + f"Parsed {len(parsed.passed)} passed case names, but SQLite reported only {expected_passed} passed cases.", + job, + ) + passed_cases.extend(parsed.passed) + + # SQLite includes `... Omitted` cases in ntest, but some harness skips are + # only named in the detailed omission list. Add only those detail-only + # cases so selected_cases does not count ordinary omissions twice. + selected_cases = reported_executed_cases + detail_only_skipped_cases + categories = { + "passed_cases": { + "count": len(passed_cases), + "unattributed_count": sum(row[2] for row in unattributed_passed_cases), + "status": "available" + if not any(entry["category"] in {"passed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "failed_cases": { + "count": len(failed_cases), + "status": "available" + if not any(entry["category"] in {"failed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "skipped_cases": { + "count": len(skipped_cases), + "status": "available" + if not any(entry["category"] in {"skipped_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + } + + return { + "schema_version": 1, + "summary": { + "reported_executed_cases": reported_executed_cases, + "reported_case_errors": reported_case_errors, + "selected_cases": selected_cases, + "passed_cases": len(passed_cases), + "failed_cases": len(failed_cases), + "skipped_cases": len(skipped_cases), + "counted_skipped_cases": counted_skipped_cases, + "detail_only_skipped_cases": detail_only_skipped_cases, + "unattributed_passed_cases": sum(row[2] for row in unattributed_passed_cases), + "jobs": { + "passed": len(passed_jobs), + "failed": len(failed_jobs), + "skipped": len(skipped_jobs), + "incomplete": len(incomplete_jobs), + }, + }, + "categories": categories, + "unavailable": unavailable, + "lists": { + "passed_cases": passed_cases, + "failed_cases": failed_cases, + "skipped_cases": skipped_cases, + "unattributed_passed_cases": unattributed_passed_cases, + "passed_jobs": passed_jobs, + "failed_jobs": failed_jobs, + "skipped_jobs": skipped_jobs, + "incomplete_jobs": incomplete_jobs, + }, + } + + +def write_lines(path: Path, lines: list[str]) -> None: + path.write_text("".join(f"{line}\n" for line in lines), encoding="utf-8") + + +def write_tsv(path: Path, header: list[str], rows: list[list[Any] | tuple[Any, ...]]) -> None: + out = ["\t".join(header)] + for row in rows: + out.append("\t".join(str(value) for value in row)) + path.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, Any]) -> None: + out_dir = results_dir / "outcome-lists" + out_dir.mkdir(parents=True, exist_ok=True) + lists = outcomes["lists"] + + write_lines(out_dir / "passed-cases.txt", lists["passed_cases"]) + write_lines(out_dir / "failed-cases.txt", lists["failed_cases"]) + write_tsv(out_dir / "skipped-cases.tsv", ["case", "reason"], lists["skipped_cases"]) + write_tsv( + out_dir / "unattributed-passed-cases.tsv", + ["jobid", "displayname", "count", "reason"], + lists["unattributed_passed_cases"], + ) + job_header = ["jobid", "state", "displaytype", "displayname", "cases", "errors", "ms"] + write_tsv(out_dir / "passed-jobs.tsv", job_header, lists["passed_jobs"]) + write_tsv(out_dir / "failed-jobs.tsv", job_header, lists["failed_jobs"]) + write_tsv(out_dir / "skipped-jobs.tsv", job_header, lists["skipped_jobs"]) + write_tsv(out_dir / "incomplete-jobs.tsv", job_header, lists["incomplete_jobs"]) + + serializable = { + "schema_version": outcomes["schema_version"], + "source": source, + "summary": outcomes["summary"], + "categories": outcomes["categories"], + "unavailable": outcomes["unavailable"], + "artifacts": { + "passed_cases": str(out_dir / "passed-cases.txt"), + "failed_cases": str(out_dir / "failed-cases.txt"), + "skipped_cases": str(out_dir / "skipped-cases.tsv"), + "unattributed_passed_cases": str(out_dir / "unattributed-passed-cases.tsv"), + "passed_jobs": str(out_dir / "passed-jobs.tsv"), + "failed_jobs": str(out_dir / "failed-jobs.tsv"), + "skipped_jobs": str(out_dir / "skipped-jobs.tsv"), + "incomplete_jobs": str(out_dir / "incomplete-jobs.tsv"), + }, + } + (out_dir / "case-outcomes.json").write_text( + json.dumps(serializable, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (out_dir / "unavailable-categories.json").write_text( + json.dumps(outcomes["unavailable"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + summary = outcomes["summary"] + lines = [ + "# SQLite Case Outcomes", + "", + f"Executed cases reported by SQLite: {summary['reported_executed_cases']}", + f"Case errors reported by SQLite: {summary['reported_case_errors']}", + f"Passed case list entries: {summary['passed_cases']}", + f"Failed case list entries: {summary['failed_cases']}", + f"Skipped case list entries: {summary['skipped_cases']}", + f"Passed cases without names in SQLite output: {summary['unattributed_passed_cases']}", + "", + ] + if outcomes["unavailable"]: + lines.append("Unavailable or partial categories:") + for entry in outcomes["unavailable"]: + lines.append(f"- {entry['category']}: {entry['reason']}") + else: + lines.append("All emitted case categories are complete for the reported harness totals.") + lines.append("") + (results_dir / "summary-case-outcomes.md").write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--db", type=Path, help="SQLite testrunner.db to inspect") + parser.add_argument("--stdout-file", type=Path, help="Direct-run stdout file to inspect") + parser.add_argument("--results-dir", type=Path, required=True, help="Directory for outcome-list artifacts") + parser.add_argument("--host", default="", help="Host label for metadata") + parser.add_argument("--permutation", default="", help="SQLite permutation label for metadata") + parser.add_argument("--display-name", default="", help="Display name for stdout-only direct runs") + args = parser.parse_args() + + jobs: list[Job] + reason: str | None + source: dict[str, Any] = { + "host": args.host, + "permutation": args.permutation, + } + + if args.db is not None and args.db.exists(): + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + elif args.stdout_file is not None: + jobs, reason = read_stdout_job(args.stdout_file, args.display_name) + source.update({"kind": "direct-stdout", "path": str(args.stdout_file)}) + elif args.db is not None: + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + else: + jobs, reason = [], "no --db or --stdout-file source was provided" + source.update({"kind": "missing"}) + + outcomes = build_outcomes(jobs, reason) + write_outputs(args.results_dir, outcomes, source) + print(f"===== SQLite case outcome lists: {args.results_dir / 'outcome-lists'} =====") + print((args.results_dir / "summary-case-outcomes.md").read_text(encoding="utf-8")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh new file mode 100755 index 0000000000..88c766917c --- /dev/null +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +STDOUT_FILE="$TMP_ROOT/direct.stdout" +RESULTS_DIR="$TMP_ROOT/results" + +cat > "$STDOUT_FILE" <<'EOF' +utf16.misc1-1.1... Ok +utf16.misc1-1.2... Ok +SQLite 2025-02-18 13:38:58 abcdef +0 errors out of 3 tests on OpenBSD 32-bit +Omitted test cases: +. misc1-10.1 skipped by focused test +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$STDOUT_FILE" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --display-name "test/misc1.test" >/dev/null + +if [ "$(wc -l < "$RESULTS_DIR/outcome-lists/passed-cases.txt" | tr -d ' ')" != "2" ]; then + echo "expected two named passed case entries" >&2 + exit 1 +fi + +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/unattributed-passed-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one unattributed passed-case row" >&2 + exit 1 +fi + +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one skipped case entry" >&2 + exit 1 +fi + +python3 - "$RESULTS_DIR/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +summary = data["summary"] +assert summary["reported_executed_cases"] == 3, summary +assert summary["passed_cases"] == 2, summary +assert summary["skipped_cases"] == 1, summary +assert summary["unattributed_passed_cases"] == 1, summary +assert data["categories"]["passed_cases"]["status"] == "partial", data["categories"] +assert data["unavailable"][0]["category"] == "passed_cases", data["unavailable"] +PY + +OMITTED_STDOUT="$TMP_ROOT/omitted.stdout" +OMITTED_RESULTS="$TMP_ROOT/omitted-results" +cat > "$OMITTED_STDOUT" <<'EOF' +omitted-1.1... Omitted +0 errors out of 1 tests on OpenBSD 32-bit +Omitted test cases: +. omitted-1.1 requires an unavailable optional feature +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$OMITTED_STDOUT" \ + --results-dir "$OMITTED_RESULTS" \ + --host browser \ + --display-name "test/omitted.test" >/dev/null + +if [ "$(tail -n +2 "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected duplicate omitted output to produce one skipped row" >&2 + exit 1 +fi +if ! grep -q 'requires an unavailable optional feature' "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv"; then + echo "expected the detailed omission reason" >&2 + exit 1 +fi + +python3 - "$OMITTED_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +summary = json.loads(Path(sys.argv[1]).read_text())["summary"] +assert summary["reported_executed_cases"] == 1, summary +assert summary["selected_cases"] == 1, summary +assert summary["counted_skipped_cases"] == 1, summary +assert summary["detail_only_skipped_cases"] == 0, summary +PY + +DB="$TMP_ROOT/testrunner.db" +python3 - "$DB" "$STDOUT_FILE" <<'PY' +import sqlite3 +import sys +from pathlib import Path + +db_path = Path(sys.argv[1]) +stdout = Path(sys.argv[2]).read_text() +con = sqlite3.connect(db_path) +con.executescript(""" +CREATE TABLE jobs( + jobid INTEGER PRIMARY KEY, + displaytype TEXT NOT NULL, + displayname TEXT NOT NULL, + build TEXT NOT NULL DEFAULT '', + dirname TEXT NOT NULL DEFAULT '', + cmd TEXT NOT NULL, + depid INTEGER, + priority INTEGER NOT NULL, + starttime INTEGER, + endtime INTEGER, + span INTEGER, + estwork INTEGER, + state TEXT, + ntest INT, + nerr INT, + svers TEXT, + pltfm TEXT, + output TEXT +); +""") +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(1, 'tcl', 'test/misc1.test', '', 1, 'done', 3, 0, ?)", + (stdout,), +) +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(2, 'tcl', 'test/failing.test', '', 1, 'done', 1, 1, ?)", + ("! failing-1.1 expected: value\n1 errors out of 1 tests on OpenBSD 32-bit\n",), +) +invalid_output = b"invalid-\x80-1.1... Ok\n0 errors out of 1 tests on OpenBSD 32-bit\n" +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(3, 'tcl', 'test/nonutf8.test', '', 1, 'done', 1, 0, CAST(? AS TEXT))", + (sqlite3.Binary(invalid_output),), +) +con.commit() +con.close() +PY + +# Reporting must not mutate or clean up the evidence it reads. +: > "$DB-wal" +: > "$DB-shm" +SIDECAR_HASHES_BEFORE="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" + +DB_RESULTS="$TMP_ROOT/db-results" +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$DB" \ + --results-dir "$DB_RESULTS" \ + --host node >/dev/null + +SIDECAR_HASHES_AFTER="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" +if [ "$SIDECAR_HASHES_BEFORE" != "$SIDECAR_HASHES_AFTER" ]; then + echo "outcome extraction changed its source database or sidecars" >&2 + exit 1 +fi + +python3 - "$DB_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +assert data["summary"]["jobs"] == { + "passed": 2, + "failed": 1, + "skipped": 0, + "incomplete": 0, +}, data["summary"]["jobs"] +assert data["summary"]["failed_cases"] == 1, data["summary"] +assert data["summary"]["unattributed_passed_cases"] == 1, data["summary"] +assert "invalid-\ufffd-1.1" in Path( + data["artifacts"]["passed_cases"] +).read_text().splitlines() +PY + +echo "sqlite-case-outcomes ok" From 590f48df7cae659583b33213dee897c9d5e88f15 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:48:26 -0400 Subject: [PATCH 18/20] SQLite tests: account for UTF-16 STAT4 sample bytes Keep analyze9's memory-accounting assertion, but use the encoding-correct allocation delta for SQLite's utf16 permutation. --- ...analyze9-account-for-utf16-stat4-sample-bytes.patch | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 packages/registry/sqlite/patches/0006-analyze9-account-for-utf16-stat4-sample-bytes.patch diff --git a/packages/registry/sqlite/patches/0006-analyze9-account-for-utf16-stat4-sample-bytes.patch b/packages/registry/sqlite/patches/0006-analyze9-account-for-utf16-stat4-sample-bytes.patch new file mode 100644 index 0000000000..126ab442d8 --- /dev/null +++ b/packages/registry/sqlite/patches/0006-analyze9-account-for-utf16-stat4-sample-bytes.patch @@ -0,0 +1,10 @@ +--- test/analyze9.test ++++ test/analyze9.test +@@ -813,2 +813,6 @@ do_test 16.1 { +- expr {$nByte2 > $nByte+900 && $nByte2 < $nByte+1100} ++ set minDelta 900 ++ set maxDelta 1100 ++ if {[permutation]=="utf16"} { set minDelta 1900; set maxDelta 2100 } ++ ++ expr {$nByte2 > $nByte+$minDelta && $nByte2 < $nByte+$maxDelta} + } {1} From 224faa7c855afefacbddfa79f9e62955346d89f8 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:59:44 -0400 Subject: [PATCH 19/20] SQLite: clean up TestRecover Tcl commands --- packages/registry/sqlite/build-testfixture.sh | 24 +++ ...testfixture-recover-command-lifetime.patch | 147 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 packages/registry/sqlite/patches/0007-testfixture-recover-command-lifetime.patch diff --git a/packages/registry/sqlite/build-testfixture.sh b/packages/registry/sqlite/build-testfixture.sh index 3b2fe3e4ee..1548ef44de 100755 --- a/packages/registry/sqlite/build-testfixture.sh +++ b/packages/registry/sqlite/build-testfixture.sh @@ -74,6 +74,9 @@ if [ -d "$PATCH_DIR" ]; then for patch_file in "$PATCH_DIR"/*.patch; do [ -f "$patch_file" ] || continue patch_name="$(basename "$patch_file")" + if [ "$patch_name" = "0007-testfixture-recover-command-lifetime.patch" ]; then + continue + fi if (cd "$SQLITE_FULL" && git apply -p0 --check "$patch_file") >/dev/null 2>&1; then echo " Applying $patch_name..." (cd "$SQLITE_FULL" && git apply -p0 "$patch_file") @@ -86,6 +89,27 @@ if [ -d "$PATCH_DIR" ]; then done fi +# The upstream TestRecover Tcl binding must own the sqlite3_recover handle for +# exactly as long as its generated command exists. Keep this testfixture-only +# patch out of build-sqlite.sh so it cannot affect the package's declared +# library, header, or pkg-config outputs. +RECOVER_LIFETIME_PATCH="$PATCH_DIR/0007-testfixture-recover-command-lifetime.patch" +SQLITE_FULL_REL="${SQLITE_FULL#"$REPO_ROOT/"}" +echo "==> Applying testfixture recover-command lifetime patch..." +if git -C "$REPO_ROOT" apply --directory="$SQLITE_FULL_REL" \ + --check "$RECOVER_LIFETIME_PATCH" >/dev/null 2>&1; then + git -C "$REPO_ROOT" apply --directory="$SQLITE_FULL_REL" \ + "$RECOVER_LIFETIME_PATCH" +elif git -C "$REPO_ROOT" apply --directory="$SQLITE_FULL_REL" \ + --reverse --check "$RECOVER_LIFETIME_PATCH" >/dev/null 2>&1 \ + && grep -q 'static void testRecoverDelete' "$SQLITE_FULL/ext/recover/test_recover.c" \ + && [ -f "$SQLITE_FULL/ext/recover/recoverlifetime.test" ]; then + echo " $(basename "$RECOVER_LIFETIME_PATCH") already applied" +else + echo "ERROR: $(basename "$RECOVER_LIFETIME_PATCH") does not apply cleanly" >&2 + exit 1 +fi + export WASM_POSIX_SYSROOT="$SYSROOT" # --- Generate required headers --- diff --git a/packages/registry/sqlite/patches/0007-testfixture-recover-command-lifetime.patch b/packages/registry/sqlite/patches/0007-testfixture-recover-command-lifetime.patch new file mode 100644 index 0000000000..3d3b07c11e --- /dev/null +++ b/packages/registry/sqlite/patches/0007-testfixture-recover-command-lifetime.patch @@ -0,0 +1,147 @@ +diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c +index 1f485a5..9c70915 100644 +--- a/ext/recover/test_recover.c ++++ b/ext/recover/test_recover.c +@@ -22,9 +22,25 @@ typedef struct TestRecover TestRecover; + struct TestRecover { + sqlite3_recover *p; + Tcl_Interp *interp; ++ Tcl_Command cmd; + Tcl_Obj *pScript; + }; + ++/* ++** Destroy the recovery handle and the Tcl wrapper state when its generated ++** command is deleted, including by [rename CMD {}] or interpreter teardown. ++*/ ++static void testRecoverDelete(void *clientData){ ++ TestRecover *pTest = (TestRecover*)clientData; ++ if( pTest->p ){ ++ sqlite3_recover_finish(pTest->p); ++ } ++ if( pTest->pScript ){ ++ Tcl_DecrRefCount(pTest->pScript); ++ } ++ ckfree((char*)pTest); ++} ++ + static int xSqlCallback(void *pSqlArg, const char *zSql){ + TestRecover *p = (TestRecover*)pSqlArg; + Tcl_Obj *pEval = 0; +@@ -194,7 +210,9 @@ static int testRecoverCmd( + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + } + res2 = sqlite3_recover_finish(pTest->p); ++ pTest->p = 0; + assert( res2==res ); ++ Tcl_DeleteCommandFromToken(interp, pTest->cmd); + if( res ) return TCL_ERROR; + break; + } +@@ -236,6 +254,7 @@ static int test_sqlite3_recover_init( + if( zDb[0]=='\0' ) zDb = 0; + + pNew = (TestRecover*)ckalloc(sizeof(TestRecover)); ++ memset(pNew, 0, sizeof(TestRecover)); + if( bSql==0 ){ + zUri = Tcl_GetString(objv[3]); + pNew->p = sqlite3_recover_init(db, zDb, zUri); +@@ -247,7 +266,9 @@ static int test_sqlite3_recover_init( + } + + sprintf(zCmd, "sqlite_recover%d", iTestRecoverCmd++); +- Tcl_CreateObjCommand(interp, zCmd, testRecoverCmd, (void*)pNew, 0); ++ pNew->cmd = Tcl_CreateObjCommand( ++ interp, zCmd, testRecoverCmd, (void*)pNew, testRecoverDelete ++ ); + + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + return TCL_OK; +diff --git a/ext/recover/recoverlifetime.test b/ext/recover/recoverlifetime.test +new file mode 100644 +index 0000000..59ab834 +--- /dev/null ++++ b/ext/recover/recoverlifetime.test +@@ -0,0 +1,82 @@ ++# 2026 July 13 ++# ++# The author disclaims copyright to this source code. In place of ++# a legal notice, here is a blessing: ++# ++# May you do good and not evil. ++# May you find forgiveness for yourself and forgive others. ++# May you share freely, never taking more than you give. ++# ++#*********************************************************************** ++# ++# Verify the lifetime of Tcl commands created by sqlite3_recover_init() ++# and sqlite3_recover_init_sql(). ++# ++ ++source [file join [file dirname [info script]] recover_common.tcl] ++set testprefix recoverlifetime ++ ++reset_db ++ ++proc recover_lifetime_sql {tag sql} { ++ return 0 ++} ++ ++# Finishing a recovery destroys its generated Tcl command. ++do_test 1.1 { ++ set R [sqlite3_recover_init db main test.db2] ++ set zCmd $R ++ $R finish ++ expr {[info commands $zCmd] eq ""} ++} 1 ++ ++# Deleting an unfinished command releases its sqlite3_recover handle. If ++# SQLite memory accounting is disabled, both memory readings are zero and ++# only the command-lifetime assertion is available. ++do_test 1.2 { ++ set nBefore [sqlite3_memory_used] ++ set R [sqlite3_recover_init db main test.db2] ++ set nDuring [sqlite3_memory_used] ++ rename $R {} ++ set nAfter [sqlite3_memory_used] ++ list \ ++ [expr {[info commands $R] eq ""}] \ ++ [expr {$nDuring==0 || $nDuring>$nBefore}] \ ++ [expr {$nAfter==$nBefore}] ++} {1 1 1} ++ ++# The command token remains valid if the Tcl command is renamed. Finishing ++# it under the new name must still destroy the renamed command and handle. ++do_test 1.3 { ++ set nBefore [sqlite3_memory_used] ++ set R [sqlite3_recover_init_sql db main [list recover_lifetime_sql rename]] ++ rename $R recover_lifetime_renamed ++ recover_lifetime_renamed finish ++ list \ ++ [expr {[info commands recover_lifetime_renamed] eq ""}] \ ++ [expr {[sqlite3_memory_used]==$nBefore}] ++} {1 1} ++ ++# Repeated direct-output and SQL-callback initialization must not leave Tcl ++# commands or recovery handles behind. ++do_test 1.4 { ++ set nBefore [sqlite3_memory_used] ++ set aCmd [list] ++ for {set i 0} {$i<20} {incr i} { ++ if {$i%2} { ++ set R [sqlite3_recover_init_sql \ ++ db main [list recover_lifetime_sql repeat-$i]] ++ } else { ++ set R [sqlite3_recover_init db main test.db2] ++ } ++ lappend aCmd $R ++ $R finish ++ } ++ list \ ++ [llength [lsort -unique $aCmd]] \ ++ [llength [info commands sqlite_recover*]] \ ++ [expr {[sqlite3_memory_used]==$nBefore}] ++} {20 0 1} ++ ++forcedelete test.db2 ++finish_test From 3e3070b5ead7a42ad0e3821c7e96cc3131d9d513 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 16:00:19 -0400 Subject: [PATCH 20/20] libc: align wasm32 pthread entry stacks Align the stack passed to kernel_clone to the 16-byte boundary required by Wasm code generation. This keeps stack-based variadic arguments intact when a new pthread begins. Add a local Sortix regression that formats the SQLite temporary-file pattern from fresh pthreads. The kernel_clone interface and ABI layout are unchanged. --- .../src/thread/wasm32posix/clone.c | 10 ++- .../pthread/pthread_stack_snprintf_varargs.c | 67 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/sortix/os-test-local/basic/pthread/pthread_stack_snprintf_varargs.c diff --git a/libc/musl-overlay/src/thread/wasm32posix/clone.c b/libc/musl-overlay/src/thread/wasm32posix/clone.c index bd6c45f58d..6e1475e48a 100644 --- a/libc/musl-overlay/src/thread/wasm32posix/clone.c +++ b/libc/musl-overlay/src/thread/wasm32posix/clone.c @@ -27,9 +27,17 @@ int __clone(int (*fn)(void *), void *stack, int flags, void *arg, ...) int *ctid = __builtin_va_arg(ap, int *); __builtin_va_end(ap); + /* + * pthread_create places its start_args object on the child stack, but only + * realigns the resulting stack pointer to pointer width. Wasm codegen + * assumes a 16-byte-aligned __stack_pointer at function entry; starting a + * thread at a 4-byte residue corrupts stack-based varargs such as %llx%c. + */ + uintptr_t stack_ptr = (uintptr_t)stack & ~(uintptr_t)15; + return kernel_clone( (uint32_t)(uintptr_t)fn, - (uint32_t)(uintptr_t)stack, + (uint32_t)stack_ptr, (uint32_t)flags, (uint32_t)(uintptr_t)arg, (uint32_t)(uintptr_t)ptid, diff --git a/tests/sortix/os-test-local/basic/pthread/pthread_stack_snprintf_varargs.c b/tests/sortix/os-test-local/basic/pthread/pthread_stack_snprintf_varargs.c new file mode 100644 index 0000000000..fa4ea05162 --- /dev/null +++ b/tests/sortix/os-test-local/basic/pthread/pthread_stack_snprintf_varargs.c @@ -0,0 +1,67 @@ +/* Test pthread initial stack alignment for variadic calls. */ + +#include +#include +#include + +#include "../basic.h" + +struct test_case { + unsigned long long value; + const char* expected; +}; + +static const struct test_case test_cases[] = { + { 0x333c7d7900000000ULL, "/tmp/etilqs_333c7d7900000000" }, + { 0x17887ec000000000ULL, "/tmp/etilqs_17887ec000000000" }, + { 0x09f49ef200000000ULL, "/tmp/etilqs_9f49ef200000000" }, +}; + +static void check_format(const struct test_case* test) +{ + char buffer[80]; + memset(buffer, 0x5a, sizeof(buffer)); + + int ret = snprintf(buffer, sizeof(buffer), "%s/etilqs_%llx%c", + "/tmp", test->value, 0); + size_t expected_len = strlen(test->expected); + if ( ret != (int) expected_len + 1 ) + errx(1, "snprintf returned %d, expected %zu", ret, + expected_len + 1); + if ( strlen(buffer) != expected_len ) + errx(1, "snprintf wrote visible length %zu, expected %zu", + strlen(buffer), expected_len); + if ( strcmp(buffer, test->expected) != 0 ) + errx(1, "snprintf wrote '%s', expected '%s'", buffer, + test->expected); + if ( buffer[expected_len] != '\0' || buffer[expected_len + 1] != '\0' ) + errx(1, "snprintf did not write the %%c NUL and terminator"); +} + +static void* start(void* arg) +{ + check_format((const struct test_case*) arg); + return NULL; +} + +int main(void) +{ + for ( size_t i = 0; i < sizeof(test_cases) / sizeof(test_cases[0]); i++ ) + { + pthread_t thread; + int errnum = pthread_create(&thread, NULL, start, + (void*) &test_cases[i]); + if ( errnum ) + { + errno = errnum; + err(1, "pthread_create"); + } + errnum = pthread_join(thread, NULL); + if ( errnum ) + { + errno = errnum; + err(1, "pthread_join"); + } + } + return 0; +}