From 6280845e51eb839c14f56fa5d33f810f748aa8bd Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 03:35:53 +0000 Subject: [PATCH 01/81] spec: paths, namespace, and a content-addressed package cache Co-Authored-By: Claude Opus 5.5 --- ...28-paths-namespace-content-cache-design.md | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md new file mode 100644 index 000000000..9304878c4 --- /dev/null +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -0,0 +1,353 @@ +# Paths, namespace, and a content-addressed package cache + +Status: design, approved in conversation 2026-10-28. Not implemented. + +## Goals + +- Clicking an item in the sidebar sets `&path=` in the hash to a readable path + rooted at the account, and a tool can be opened at a path. +- Paths are real: the service worker serves them, packages are imported + through them, and the filesystem is a namespace rather than only the + `.docs` walk of the root folder. +- Package management is a folder: whatever is bound at `/packages`. +- URLs don't carry raw document ids unless a portable share link is wanted. +- The package cache can't be poisoned by a crafted link, by code running on + the origin writing to CacheStorage, or by a stale name resolution. + +## Non-goals + +- Cache isolation between two accounts in the same browser profile. +- Running package code on a separate origin. This design is meant not to + preclude it (the store and index work the same there), but doesn't do it. +- Mount points below the top level of the namespace. +- Path providers supplied by packages. +- Garbage collection of the content store. + +## Threat model + +| Attacker | In scope | What stops it | +|---|---|---| +| A crafted link makes the service worker cache content under a key your own imports later hit | yes | Only pinned URLs are stored, and the index maps a pinned URL to the hash of what is really at those heads | +| Code on the origin (a package or tool) writes to CacheStorage | yes | Every store read is hashed and compared; the index lives only in service worker memory and is filled only by the automerge worker over an authenticated channel | +| A name path resolved once keeps serving old content | yes | Names are never stored; only pinned URLs are, and they can't change meaning | +| Another account on the same origin | no | — | + +Everything on the origin shares one realm boundary, so this raises the bar +rather than closing it. What stays open is listed under +[Remaining gaps](#remaining-gaps). + +## 1. Path grammar + +### Identifiers + +- **idhash** of a document: the first 12 bytes of SHA-256 over the UTF-8 of + its `documentId`, in base58. It only distinguishes a doc from its siblings + and keeps the id out of URLs. It is not a secret and not a security + boundary. +- **acctHash**: the idhash of the account document. +- **heads**: the doc's heads in base58, sorted, joined with `.`. Sorting + makes one set of heads one URL. Base58 has no `.`, so it's unambiguous. + +### Live path + +What the address bar holds and what people read. Names resolve at the +current heads of everything along the way. + +``` +&path=/Projects/foo/index.js +``` + +### Pinned path + +What imports and share links use. Every document the walk enters is pinned: + +1. A step that follows a link **by name** is written `@` in + place of the name. +2. A step that follows a link **without consuming a name** (a union member, + a link that points at a link) adds its own `@` segment. +3. A step **inside** a document (a directory key, a mount name in the + account doc) stays a plain name, since the enclosing doc's heads already + pin it. + +``` +live: /Projects/foo/dist/index.js +pinned: @h/@h/@h/@h/dist/index.js + +live: /packages/foo/dist/index.js +pinned: @h/packages/@h/@h/dist/index.js +``` + +When resolving, a segment is a doc step if it parses as `@` +and a link reachable from the current node hashes to that idhash; otherwise +it's a name. + +The first segment of a pinned path is either an acctHash the resolver knows +locally, or `automerge:@` for a portable share link (safe +to hand out once the doc is on keyhive, since the id no longer grants +access). The `automerge:` prefix is what tells the two apart. + +A pinned path never changes meaning, so whoever builds an import URL builds +the whole chain first. Nothing in a module graph goes through a redirect: the +ES module map keys an entry by the URL requested, so a redirect is memoized +for the life of the realm (the same reason `importModule` retries with +`?retry=1`). + +## 2. Path providers and the namespace + +### Providers + +A provider matches a document and lists its children as +`{ name, link | value }`. Walking a path asks the current doc's provider for +the next child. This generalizes `FolderStrategy` in +`core/filesystem/src/resolve.ts`. + +Built-in providers, checked in order, first match wins: + +- **account**: a doc whose `@patchwork.type` is `account`; its children are + the namespace (below). +- **directory**: keys, longest prefix first, as `walkDirectoryDoc` does now. +- **folder**: `.docs` by `name`. +- **branches**: each branch by name. A path that ends at a branches doc + stands for its `default` branch. This replaces the special case in + `core/bootloader/src/resolver.ts`. + +Providers are built in only. The automerge worker runs them to build the +cache index, so a package-supplied provider would put package code inside +the resolver the index trusts. The provider interface is exported, shaped so +package providers can be added later. + +### Namespace + +The account doc gets an ordered mount table: + +```ts +namespace?: { + [mountPoint: string]: Array<{ url: AutomergeUrl }>; +}; +// e.g. +// "/": [{ url: rootFolderUrl }] +// "/packages": [{ url: myPackagesUrl }, { url: teamPackagesUrl }] +``` + +Union semantics, after plan9: + +- Lookup tries each binding of a mount in order; the first that has the name + wins. +- Listing concatenates every binding's children. +- Array position takes the place of `bind -b` / `bind -a`. +- A mount shadows an entry of the same name in `/`'s folder. +- Mount points are top-level names only, and may not contain `@`. +- Only automerge documents can be bound. + +If `namespace` is missing, `/` falls back to `rootFolderUrl`, so older +accounts and older readers keep working. + +The automerge worker needs to know acctHash → account url to resolve a path's +root. Tabs register it with the worker at boot. + +## 3. Service worker URLs and routing + +### URLs + +All new forms live under a reserved prefix, so no site file can be mistaken +for a path: + +``` +/@/ e.g. /@//Projects/foo/x.js +/@/ e.g. /@/@h/@h/.../x.js +``` + +- A **live** URL answers with a 307 to its pinned form, computed by a + resolver. Fine for `fetch()`; never use one for `import()`. +- A **pinned** URL is served from the content store (section 5). +- Existing `/automerge%3A…/` URLs keep working as they do today, keyed by URL + and unchecked. They are deprecated. + +### Hash routing + +- `path` joins `HASH_KEY_ORDER` in `core/patchwork/src/router.ts`. Each + segment is `encodeURIComponent`-encoded and the slashes stay literal. +- `doc=` says what is open; `path=` says where it was opened from. +- A hash with `path=` and no `doc=` resolves the path and opens the result. +- If `doc=` and `path=` disagree (something moved or was renamed), `doc=` + wins and `path=` is dropped with a console warning. + +### Opening at a path + +Builds on the `open-document-path` branch (48f4ab98): + +- `patchwork:open-document`'s `path` field holds the **live path** (it was a + service-worker path on that branch), and the router writes it to `&path=`. + This reverses that branch's choice to keep it out of the hash. +- `` hands it to the tool without remounting, as on that + branch. +- The sidebar (an installed tool, not in this repo) dispatches + `{ url, path }` on click, using the names it walked. + +### Helpers in `@inkandswitch/patchwork-filesystem` + +- `idhash(documentId)` +- `livePath(acctHash, names)` +- `parsePath(path)`: live or pinned, into segments +- `resolveLivePath(repo, path)` → handle +- `pinPath(repo, path)` → pinned service-worker URL, for fetching siblings or + importing + +## 4. Packages via `/packages` + +- Package management is whatever is bound at `/packages`. The folder can live + anywhere in the tree, or nowhere. +- `ModuleWatcher` gets a third source kind, `folder` (alongside automerge + module-settings docs and HTTP manifests), pointed at the `/packages` union: + - The module list is the union listing, deduplicated by document id. Two + packages with the same name both load; the name only decides which one a + path lookup finds. + - Packages are tracked by document id, so a rename doesn't reload anything. + - When a package's own heads move, the watcher builds its pinned chain with + `pinPath` and imports it. Other packages keep their existing URLs: a + pinned URL stays valid forever, so a change elsewhere in the chain + doesn't reload them. + - `PackageTracker.findLatest` keeps asking the service worker, in the new + grammar. +- The site's `system` HTTP manifest stays a separate source. +- A link under `/packages` that isn't a package (no `package.json`) is + warned about and skipped. + +### Accounts + +- `createDefaultAccount` binds `/` to the root folder and `/packages` to a new + folder. +- Migration, once at boot: if the account has `moduleSettingsUrl` and no + `/packages` binding, create a folder, add a link for each automerge entry + (named by `package.json` `name`, falling back to the doc title), and bind + it at `/packages`. The module-settings doc is left untouched. +- Builds older than this one keep reading the frozen module-settings list, so + packages installed afterwards won't show up there. +- `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` + binding. + +## 5. Content store and index + +### Store + +- A CacheStorage cache named `patchwork-content-`, separate from + the `patchwork` cache that holds site pages and assets. +- Key: `/.content/`, an internal key that is never fetched. +- Value: the bytes only. The content type lives in the index, so identical + bytes are stored once whatever their type. +- Reads and writes go through one small interface (`get(hash)`, + `put(hash, bytes)`), so entries can later be encrypted under an at-rest key. +- `clearAccountStore(acctHash)` deletes the cache. There is no sign-out flow + today; a site calls this from its own. + +### What gets stored + +- Content from unprotected documents (`isUnprotectedDocId`). +- Content from keyhive-protected documents only when reached through + `/packages`. Package code needs to load fast at boot; user documents don't. +- Everything else is served inline: checked, never stored. + +### Index + +A `Map` in service worker memory: pinned URL → `{ sha256, contentType }`. + +- Lost when the service worker is stopped, which is fine. +- On a miss, the service worker asks the automerge worker for the **manifest + of the last document in the URL at its heads**: hash and type of every file + that resolves pinned inside it. One round trip per package rather than per + file. +- The automerge worker memoizes manifests by `doc@heads` in an LRU; they never + change. + +### Request flow for a pinned URL + +1. **Index hit:** read the store, hash the bytes, compare. + - Match: serve. + - Mismatch: delete the entry, warn loudly, go to step 3. +2. **Index miss:** fetch the manifest, then step 1. +3. **Store miss:** ask a resolver for the bytes and hash them in the service + worker. + - Matches the index entry: serve, and store it if + [What gets stored](#what-gets-stored) allows. + - Doesn't match: fail as a network error, store nothing. + - No index entry, because no automerge worker is available: serve inline, + store nothing. + +Hashing is `crypto.subtle` SHA-256 on every hit, since a store entry can +change between two reads. `sites/bench` gets a case measuring it. + +### Who is trusted for what + +- **Resolvers never write the cache.** Today `resolver.ts` calls `cache.put` + itself. Instead a resolver returns bytes, and the service worker decides. +- **Only the automerge worker fills the index.** Tab resolvers run in the + same realm as packages; their answers are served inline and never stored. +- **The index channel is authenticated.** Today's `HANDOFF_CHANNEL` is a + BroadcastChannel: any context on the origin can listen, see request ids, + and reply first with a forged response. That is a hole in the current code + as well. Index answers must arrive from a `Client` whose `url` is the + automerge worker's hashed script URL, which the service worker knows at + build time. + +## Remaining gaps + +These replace the deleted `package-load-gate.md` note. + +- Offline, `servePassthrough` falls back to the URL-keyed `patchwork` cache, + which any code on the origin can write. Site assets, the automerge worker's + script included, come from there. +- Packages can write the automerge worker's IndexedDB/OPFS storage. Whether + loading a document at given heads verifies change hashes needs checking. +- A live → pinned redirect answered by a tab can point at the wrong version. + It can't poison the store, but it decides which version a tab loads. +- Legacy `/automerge%3A…` URLs stay keyed by URL and unchecked. +- Protected package code is stored in plaintext. Today keyhive's secrets sit + in the same storage as its ciphertext, so this exposes nothing new; + revisit when keyhive protects its keys at rest. +- Nothing cleans up the content store. Deduplication slows growth, and the + existing quota warning still fires. + +## Spikes before planning + +- Can the service worker `postMessage` to a SharedWorker client, and can that + worker reply with the service worker able to check its `Client.url`, in + Chromium, Firefox and WebKit? Fallback: the worker posts answers to the + service worker as `fetch()` requests, whose `clientId` the service worker + can look up. +- Does automerge verify change hashes when loading a document from storage? + This decides how big the storage-tampering gap is. + +## Testing + +- Unit: + - Path grammar: live and pinned round trips, heads sorting, segments that + look like doc steps but aren't. + - Providers: account namespace (union order, shadowing, fallback to + `rootFolderUrl`), directory, folder, branches. + - `pinPath` against a fixture folder that is renamed between versions. + - Router: `path=` parsing and serializing; `path=` alone; `doc=`/`path=` + disagreement. + - `ModuleWatcher` `folder` source: add, remove, rename without reload, only + the changed package reloading, union deduplication. + - Migration from a module-settings doc. + - Store and index with a fake `Cache` and an injected hash: forged store + entry rejected, tab answer never stored, matching answer stored, + protected non-package content never stored. + - Manifest builder against folder and directory fixtures. +- E2E, run in CI: + - Open a document with `&path=` alone. + - A page `cache.put`s forged bytes under a real content key, reloads, and + gets the real module. + +## Implementation phases + +Each phase can ship on its own and gets its own plan: + +1. **Grammar, providers, namespace** in `patchwork-filesystem`, plus the + account `namespace` field and its fallback. No behaviour change yet. +2. **Service worker:** `/@/` URLs, the content store, the index, the + authenticated channel (after its spike), resolvers no longer writing the + cache. +3. **Routing:** `&path=`, opening at a path, rebased on `open-document-path`. +4. **Packages:** the `folder` source, `/packages`, migration, deprecating + `moduleSettingsUrl`. From 60829f3b29d7fa0a9941113b7edea63942bf2033 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 04:56:50 +0000 Subject: [PATCH 02/81] plan: paths phase 1 (grammar, providers, namespace) --- ...ths-phase-1-grammar-providers-namespace.md | 513 ++++++++++++++++++ 1 file changed, 513 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md b/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md new file mode 100644 index 000000000..d115a14e5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md @@ -0,0 +1,513 @@ +# Paths phase 1: grammar, providers, namespace — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give `@inkandswitch/patchwork-filesystem` the path grammar, the built-in path providers, the account namespace, and the helpers that walk live and pinned paths. Nothing calls them yet except `resolvePath`, which moves onto the providers. + +**Architecture:** Three new modules in `core/filesystem/src`: `paths.ts` (pure string grammar, no repo), `providers.ts` (sync, one doc or node at a time: "which child does this name pick?"), and `walk.ts` (async, follows links through a `Repo`, emits the pinned form as it goes). `resolve.ts` keeps its public `resolvePath` but dispatches through `providers.ts` instead of `FolderStrategy`. + +**Tech Stack:** TypeScript, `@automerge/automerge-repo` (catalog pin 2.6.0-subduction.48), vitest in node, `bs58@^6.0.0`, `crypto.subtle`. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. This plan covers **Implementation phases → 1**, plus the section 3 filesystem helpers (`idhash`, `livePath`, `parsePath`, `resolveLivePath`, `pinPath`), because they are pure filesystem code and phases 2–4 all consume them. Phases 2–4 get their own plans. Phase 2 waits on the two spikes. + +## Global Constraints + +- idhash = first 12 bytes of SHA-256 over the UTF-8 of the `documentId`, base58 (bitcoin alphabet). Not a secret. +- heads segment = the doc's heads in base58, **sorted**, joined with `.`. +- Pinned first segment: `@` or `automerge:@`. The `automerge:` prefix tells them apart. +- Doc step syntax `@`. A segment is a doc step only if it parses that way **and** a link reachable from the current node has that idhash; otherwise it is a name. +- Provider order, first match wins: account, directory, folder, branches. Built-in only; the interface is exported. +- Namespace: `namespace?: { [mountPoint: string]: Array<{ url: AutomergeUrl }> }` on the account doc. Mount points are `/` or a top-level `/` with no `/` or `@` in ``. A mount shadows a same-named entry in `/`'s folder. Lookup: first binding with the name wins. Listing: concatenation. +- No `namespace` → `/` is `[{ url: rootFolderUrl }]`. +- Path segments are `encodeURIComponent`-encoded and slashes stay literal (same encoding the router will use in phase 3). +- Service worker URLs live under `/@/`. `pinPath` returns `/@/`; the service worker starts serving it in phase 2. +- No behaviour change at runtime: `createDefaultAccount`, the bootloader resolver, `ModuleWatcher` and the router are untouched in this phase. +- One changeset for the whole phase: `@inkandswitch/patchwork-filesystem: patch`, `@inkandswitch/patchwork-plugins: patch`. + +## Decisions this plan makes that the spec leaves open + +1. **The tail after the last container stays as names.** A link followed *by name* to a doc that has no provider (a file doc, a markdown doc) is written as its name(s), not `@`. This is what the spec's example `@h/dist/index.js` shows, and the only form under which relative imports (`./util.js`) inside a package resolve. The cost: a leaf file doc reached by a bare link can still change under a pinned path. Phase 2's manifest, memoized by `container@heads`, is what freezes leaves. Doc steps that consume no name (union members, a branches doc's default) are always written as doc steps. +2. **A branches doc's lookup falls through to `default`.** If the next name isn't a branch name, the branches provider follows `default` without consuming the name. Packages published as a branches doc then resolve as `/packages/foo/dist/index.js`, the way the bootloader resolves them today. A branch name wins over a same-named file. +3. **The `/` fallback also applies when `namespace` exists but has no `/` key.** +4. **Child gets a third variant, `union`**, for a mount's bindings. The spec's `{ name, link | value }` can't express an ordered union. +5. **`resolveLivePath` and `pinPath` take an options bag** `{ accounts }` mapping acctHash → account url. In phase 2 the automerge worker fills this map from what tabs register. +6. **`FolderStrategy` is removed.** Its signature referenced the private `Walk` type, so nothing outside the package could implement it. `PathProvider` replaces it. + +## Review Focus + +- **Names with reserved characters:** a folder entry called `a/b`, `x@y`, `50%`, `my notes`, or `日本` should round-trip through `livePath` → `parsePath` and resolve to that entry. Pinned in Task 1 and Task 3. +- **A name that looks like a doc step:** a folder entry literally named `@` should resolve as a name when no reachable link has that idhash. Pinned in Task 3. +- **Malformed or partial namespaces:** keys `packages` (no slash), `/a/b`, `/x@y`, a binding without a valid url, an account with neither `namespace` nor `rootFolderUrl`. Bad entries should be skipped with a `console.warn`. Nothing should throw. Pinned in Task 2. +- **Pinned heads the repo doesn't have, or malformed heads:** `resolveLivePath` should return `undefined`, not throw. Pinned in Task 3. +- **Cycles through steps that consume no name:** a branches doc whose `default` is itself, or a `/` binding that is the account doc itself. The walk should stop and return `undefined`. Pinned in Task 3. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `core/filesystem/src/paths.ts` (create) | `idhash`, heads segment, live/pinned path parse and format. No `Repo`. | +| `core/filesystem/src/providers.ts` (create) | `Child`, `PathProvider`, the four providers, `providerFor`, `accountBindings`, `asLink` (moved from `resolve.ts`). | +| `core/filesystem/src/walk.ts` (create) | `walkPath`, `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren`. | +| `core/filesystem/src/resolve.ts` (modify) | `resolvePath` dispatches through `providerFor`; `FolderStrategy` deleted. | +| `core/filesystem/src/types.ts` (modify) | `NamespaceBinding`, `Namespace`, `AccountNamespaceDoc`. | +| `core/filesystem/src/index.ts` (modify) | export the three new modules. | +| `core/filesystem/package.json` (modify) | add `"bs58": "^6.0.0"` to `dependencies`. | +| `core/plugins/src/account.ts` (modify) | `AccountDoc.namespace?: Namespace`. | +| `core/filesystem/test/paths.test.ts`, `providers.test.ts`, `walk.test.ts` (create) | unit tests, in-memory `new Repo()` as in `test/resolve.test.ts`. | +| `.changeset/paths-grammar-providers.md` (create) | the changeset. | + +--- + +### Task 1: Path grammar + +**Files:** +- Create: `core/filesystem/src/paths.ts` +- Modify: `core/filesystem/package.json` (add `bs58`), `core/filesystem/src/index.ts` +- Test: `core/filesystem/test/paths.test.ts` + +**Interfaces:** +- Produces: + ```ts + export type DocStep = { kind: "doc"; idhash: string; heads: UrlHeads; raw: string }; + export type NameStep = { kind: "name"; name: string }; + export type Segment = DocStep | NameStep; + export type PathRoot = + | { kind: "account"; acctHash: string; heads?: UrlHeads } + | { kind: "share"; documentId: DocumentId; heads: UrlHeads }; + export type ParsedPath = { root: PathRoot; segments: Segment[] }; + export class PathSyntaxError extends Error {} + + export function idhash(documentId: string): Promise; + export function formatHeads(heads: readonly string[]): string; // sorted copy, "."-joined + export function parseHeads(segment: string): UrlHeads | undefined; // undefined unless every part is base58 and non-empty + export function docStep(idhash: string, heads: readonly string[]): string; // `${idhash}@${formatHeads(heads)}` + export function livePath(acctHash: string, names: string[]): string; + export function parsePath(path: string): ParsedPath; // throws PathSyntaxError + ``` +- `raw` on `DocStep` is the undecoded segment, so the walker can fall back to treating it as a name (`decodeURIComponent(raw)`). + +- [ ] **Step 1: Write the failing tests** + +```ts +describe("idhash", () => { + it("is base58 of the first 12 bytes of sha-256 over the document id", async () => { + expect(await idhash("2j9knpCseyhnK8izDmLpGP5WMdZQ")).toBe("vjRneszhMkDwr5tb"); + }); +}); + +describe("heads", () => { + it("sorts so one set of heads is one string", () => { + expect(formatHeads(["c", "a", "b"])).toBe("a.b.c"); + expect(formatHeads(["b", "a"])).toBe(formatHeads(["a", "b"])); + }); + it("parses only base58 parts", () => { + expect(parseHeads("a.b")).toEqual(["a", "b"]); + expect(parseHeads("a.0")).toBeUndefined(); // 0 isn't base58 + expect(parseHeads("a..b")).toBeUndefined(); + expect(parseHeads("")).toBeUndefined(); + }); +}); + +describe("parsePath", () => { + it("round-trips a live path, encoding each name", () => { + const names = ["Projects", "a/b", "x@y", "50%", "my notes", "日本"]; + const path = livePath("acct", names); + expect(path.split("/")).toHaveLength(1 + names.length); // a slash inside a name is encoded + expect(parsePath(path)).toEqual({ + root: { kind: "account", acctHash: "acct" }, + segments: names.map((name) => ({ kind: "name", name })), + }); + }); + it("reads a pinned path's doc steps and plain names", () => { + const p = parsePath("acct@h2.h1/packages/mine@h3/foo@h4/dist/index.js"); + expect(p.root).toEqual({ kind: "account", acctHash: "acct", heads: ["h1", "h2"] }); + expect(p.segments.map((s) => s.kind)).toEqual(["name", "doc", "doc", "name", "name"]); + expect(p.segments[1]).toEqual({ kind: "doc", idhash: "mine", heads: ["h3"], raw: "mine@h3" }); + }); + it("reads a share root", () => { + expect(parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h1/x").root).toEqual({ + kind: "share", documentId: "2j9knpCseyhnK8izDmLpGP5WMdZQ", heads: ["h1"], + }); + // the router and service worker hand over encoded segments + expect(parsePath("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h1").root.kind).toBe("share"); + }); + it("treats an encoded @ as part of a name", () => { + expect(parsePath("acct/a%40b").segments).toEqual([{ kind: "name", name: "a@b" }]); + }); + it("ignores a leading slash and empty segments", () => { + expect(parsePath("/acct//a/").segments).toEqual([{ kind: "name", name: "a" }]); + }); + it("rejects a share root without heads, an empty path, and a bad root", () => { + expect(() => parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")).toThrow(PathSyntaxError); + expect(() => parsePath("")).toThrow(PathSyntaxError); + expect(() => parsePath("acct@0")).toThrow(PathSyntaxError); + expect(() => parsePath("automerge:not-an-id@h1")).toThrow(PathSyntaxError); + }); +}); +``` + +(`h1`…`h4`, `acct`, `mine`, `foo` are all valid base58. Keep test fixtures base58.) + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts` +Expected: FAIL, cannot resolve `../src/paths.js`. + +- [ ] **Step 3: Add `"bs58": "^6.0.0"` to `dependencies` and run `pnpm install`, then implement `paths.ts`** + +- Base58: `/^[1-9A-HJ-NP-Za-km-z]+$/`. +- `idhash`: `crypto.subtle.digest("SHA-256", new TextEncoder().encode(id))`, take `.slice(0, 12)`, `bs58.encode`. +- Root segment: if `decodeURIComponent(raw)` starts with `automerge:`, it's a share root. Split on the **last** `@`, check the id with `isValidDocumentId`, and require heads. Otherwise `` or `@`; anything else throws. +- Other segments: a raw segment containing a literal `@` whose two halves are base58 and `parseHeads` is a `DocStep`. Everything else becomes `{ kind: "name", name: decodeURIComponent(raw) }`. +- Export it from `index.ts`. + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/package.json pnpm-lock.yaml core/filesystem/src/paths.ts core/filesystem/src/index.ts core/filesystem/test/paths.test.ts +git commit -m "filesystem: path grammar (idhash, heads, live and pinned paths)" +``` + +--- + +### Task 2: Providers and the account namespace + +**Files:** +- Create: `core/filesystem/src/providers.ts` +- Modify: `core/filesystem/src/types.ts`, `core/filesystem/src/index.ts`, `core/plugins/src/account.ts`, `core/filesystem/src/resolve.ts` (only to import `asLink` from `providers.ts` instead of defining it) +- Test: `core/filesystem/test/providers.test.ts` + +**Interfaces:** +- Consumes: `getType`, `getDefaultBranch`, `BranchesDoc`, `FolderDoc` (existing). +- Produces: + ```ts + // types.ts + export type NamespaceBinding = { url: AutomergeUrl }; + export type Namespace = { [mountPoint: string]: NamespaceBinding[] }; + export type AccountNamespaceDoc = HasPatchworkMetadata<"account"> & { + namespace?: Namespace; + rootFolderUrl?: AutomergeUrl; + }; + + // providers.ts + export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] }; + /** `consumed` is how many leading names the step used: 0 for a step that consumes none (the `/` mount, a branches doc's default). */ + export type Lookup = { child: Child; consumed: number }; + export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; // names.length >= 1 + list(node: unknown): Child[]; + } + export const accountProvider: PathProvider; // kind "account" + export const directoryProvider: PathProvider; // kind "directory" + export const folderProvider: PathProvider; // kind "folder" + export const branchesProvider: PathProvider; // kind "branches" + export const PROVIDERS: readonly PathProvider[]; // [account, directory, folder, branches] + export function providerFor(doc: unknown): PathProvider | undefined; + /** Mount point ("/" or "/name") → bindings, with "/" falling back to rootFolderUrl. Invalid keys and bindings are skipped with a warning. */ + export function accountBindings(doc: AccountNamespaceDoc): Map; + export function asLink(value: unknown): AutomergeUrl | undefined; // moved verbatim from resolve.ts + ``` +- `AccountDoc` in plugins gains `namespace?: Namespace` (imported as a type from `@inkandswitch/patchwork-filesystem`). + +Provider behaviour: +- **account**: `lookup` returns the union of `/` with `consumed: 1` if that mount exists (and isn't `/`). Otherwise it returns `{ child: { name: "", union: bindings("/") }, consumed: 0 }`, or `undefined` if `/` has no bindings. `list` returns a union child per mount: `name` is the mount without its slash, and `/` is `""`. +- **directory**: on the node it's given (the doc or an object nested in it), the longest prefix of `names` joined with `/` that is a key. A link value gives `{ link }`, anything else `{ value }`. `consumed` = prefix length. `list` returns every key except `@patchwork`. +- **folder**: first `.docs` entry whose `String(name) === names[0]` with a valid link, `consumed: 1`. `list` returns each entry with a valid link. +- **branches**: `names[0]` names a branch → that link, `consumed: 1`. Otherwise `default`, `consumed: 0`; `undefined` if there is no default (`getDefaultBranch` throws, so catch it). `list` returns each branch. + +- [ ] **Step 1: Write the failing tests** + +All tests use plain objects. Providers are synchronous and never touch a repo. Use real automerge urls from `generateAutomergeUrl()`. + +```ts +describe("accountProvider", () => { + it("looks a mount up by name and consumes it", ...) + // namespace { "/": [{url: R}], "/packages": [{url: P1}, {url: P2}] } + // lookup(doc, ["packages", "foo"]) → { child: { name: "packages", union: [P1, P2] }, consumed: 1 } + it("sends any other name to / without consuming it", ...) + // lookup(doc, ["Projects"]) → { child: { name: "", union: [R] }, consumed: 0 } + it("falls back to rootFolderUrl with no namespace, or no / key", ...) + // { rootFolderUrl: R } and { rootFolderUrl: R, namespace: { "/packages": [...] } } both → "/" union [R] + it("finds nothing with neither namespace nor rootFolderUrl", ...) + // lookup(doc, ["x"]) → undefined; list(doc) → [] + it("skips malformed mounts and bindings with a warning", ...) + // keys "packages", "/a/b", "/x@y" and a binding { url: "nope" } are absent from accountBindings; + // console.warn spied, called; no throw + it("lists every mount as a union child", ...) + // list → [{ name: "", union: [R] }, { name: "packages", union: [P1, P2] }] (order: "/" first, then object order) +}); + +describe("directoryProvider", () => { + it("matches the longest prefix", ...) + // { "dist": {..}, "dist/index.js": L } lookup ["dist","index.js"] → { child: { name: "dist/index.js", link: L }, consumed: 2 } + it("returns a nested object as a value child", ...) + it("reads ImmutableString links", ...) + it("does not list @patchwork", ...) +}); + +describe("folderProvider", () => { + it("first entry with the name wins", ...) + // docs [{name:"a", url: A1}, {name:"a", url: A2}] → link A1 + it("misses an unknown name", ...) +}); + +describe("branchesProvider", () => { + it("follows a named branch", ...) // consumed 1 + it("falls through to default for any other name", ...) // consumed 0, link = default + it("finds nothing with no default branch", ...) // undefined, no throw +}); + +describe("providerFor", () => { + it("checks account, directory, folder, branches in order", ...) + // an object with @patchwork.type "account" and a docs array → accountProvider + // a directory-typed doc with docs: [] → directoryProvider + // a file doc → undefined +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/providers.test.ts` +Expected: FAIL, cannot resolve `../src/providers.js`. + +- [ ] **Step 3: Implement `providers.ts`, the namespace types, and `AccountDoc.namespace`** + +Mount key regex: `/^\/[^/@]*$/`. `accountBindings` builds the map from `namespace`, then sets `/` from `rootFolderUrl` if `/` is absent. `account` matches on `getType(doc) === "account"`. + +- [ ] **Step 4: Run them to see them pass, along with the existing resolve tests** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/providers.test.ts test/resolve.test.ts test/resolve-pinning.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/providers.ts core/filesystem/src/types.ts core/filesystem/src/index.ts core/filesystem/src/resolve.ts core/plugins/src/account.ts core/filesystem/test/providers.test.ts +git commit -m "filesystem: path providers and the account namespace" +``` + +--- + +### Task 3: Walking live and pinned paths + +**Files:** +- Create: `core/filesystem/src/walk.ts` +- Modify: `core/filesystem/src/index.ts` +- Test: `core/filesystem/test/walk.test.ts` + +**Interfaces:** +- Consumes: Task 1's `parsePath`, `idhash`, `docStep`, `formatHeads`, `ParsedPath`. Task 2's `providerFor`, `PathProvider`, `Child`, `accountBindings`. `documentBaseOrigin` from `urls.ts`. +- Produces: + ```ts + export type WalkOptions = { + /** acctHash → account doc url. The roots a live or pinned account path may start at. */ + accounts: ReadonlyMap; + }; + export type WalkEnd = { + /** The last document the walk entered, as a view at the heads it was read at. */ + handle: DocHandle; + /** Where the walk stopped inside it: handle.doc(), or a value nested in it. */ + node: unknown; + /** The pinned path, without a leading slash. */ + pinned: string; + }; + export function walkPath(repo: Repo, path: string, options: WalkOptions): Promise; + /** The document a live or pinned path names. undefined if it doesn't resolve or ends at a value inside a doc. */ + export function resolveLivePath(repo: Repo, path: string, options: WalkOptions): Promise | undefined>; + export function pinnedPath(repo: Repo, path: string, options: WalkOptions): Promise; + /** `${documentBaseOrigin()}/@/${pinnedPath}` */ + export function pinPath(repo: Repo, path: string, options: WalkOptions): Promise; + /** The children of a doc, with unions expanded: an account's `/` listing concatenated and shadowed by its mounts; any other doc's provider list. */ + export function listChildren(repo: Repo, handle: DocHandle): Promise; + /** Every binding's children, concatenated in binding order. Phase 4 lists `/packages` with this. */ + export function listUnion(repo: Repo, urls: AutomergeUrl[]): Promise; + ``` +- `resolveLivePath` returns the live handle (`repo.find(url)`) when the input was a live path, and the view when it was pinned. To decide, check whether the parsed root has heads. + +Walk algorithm. The signatures don't determine this: + +``` +root: + account → url = accounts.get(acctHash) ?? fail; handle = find(url); heads = root.heads ?? handle.heads() + view = handle.view(heads); emit docStep(acctHash, heads) + share → handle = find(documentId).view(heads); emit `automerge:${documentId}@${formatHeads(heads)}` +state: { handle, node = handle.doc(), provider = providerFor(node) } + +loop while segments remain: + if next is DocStep: + for each link reachable from node (provider.list(node): link children and every union member, + plus the branches default): + if await idhash(documentIdOf(link)) == step.idhash → enter(link, step.heads), emit step.raw, consume 1, continue loop + otherwise fall through and treat it as NameStep(decodeURIComponent(step.raw)) + names = the run of leading name segments + { child, consumed } = provider.lookup(node, names) ?? fail + value child → node = value; emit names[0..consumed] encoded; stay with the same provider + link child → doc = find(link); if providerFor(doc): enter at current heads, emit docStep + else (leaf): emit names[0..consumed] encoded; if segments remain after, fail; end here + union child → emit names[0..consumed] encoded (none for "/"); + rest = names after consumed + member = rest.length ? first member whose providerFor(doc).lookup(doc, rest) is defined + : first member + enter(member, current heads), emit docStep +at the end: if provider is branches and nothing remains → enter default, emit docStep +``` + +- "enter" = `find(url)`, a `view(heads)`, and `providerFor(doc)`. If `view` throws (heads the repo doesn't have), fail. +- "fail" returns `undefined`. The only thing that throws is `parsePath` on syntax. +- Cycle guard: keep a `Set` of `documentId|segmentsRemaining` for steps that consume nothing (union member, branches default). Revisiting one fails. +- `idhash` results are memoized per walk in a `Map`. + +- [ ] **Step 1: Write the failing tests** + +Fixture: build one in-memory account with `new Repo()`: +`account { @patchwork: account, namespace: { "/": [root], "/packages": [mine, team] } }`, where +- root is a folder `Projects`, and `Projects` is a folder containing `foo` (a folder holding `notes` → a file doc), +- mine is a folder with `foo` → a pushwork-shaped directory doc (`package.json`, `dist/index.js` file links), +- team is a folder with `foo` (a different package) and `bar`. + +Set `accounts = new Map([[await idhash(account.documentId), account.url]])`. + +```ts +it("resolves a live path through / and returns the live handle", ...) + // resolveLivePath(repo, livePath(a, ["Projects","foo","notes"]), {accounts}) → handle.url === notes.url (no heads) +it("pins every container the walk enters, and keeps the tail after the last one as names", async () => { + const pinned = await pinnedPath(repo, livePath(a, ["packages", "foo", "dist", "index.js"]), { accounts }); + expect(pinned).toBe( + `${docStep(a, account.heads())}/packages/${docStep(await idhash(mine.documentId), mine.heads())}` + + `/${docStep(await idhash(minePkg.documentId), minePkg.heads())}/dist/index.js` + ); +}); +it("writes the / binding as a doc step with no name", ...) + // pinnedPath(a/Projects/foo) === `${acct}/${root}/${projects}/${foo}` (doc steps only) +it("union: the first binding with the name wins", ...) + // packages/foo → mine's foo; packages/bar → team's bar +it("a mount shadows a same-named entry in /'s folder", ...) + // root folder also has an entry "packages"; a/packages/foo still goes through the mount +it("a pinned path keeps resolving to the same version after a rename", async () => { + // pin a/Projects/foo/notes, then rename foo → bar and change notes' content + // resolveLivePath(pinned) → the foo doc's view at its old heads (the same documentId as foo) + // resolveLivePath(live a/Projects/foo/notes) → undefined; a/Projects/bar/notes → resolves +}); +it("a pinned path that walks into a branches doc follows its default", ...) + // packages entry "branchy" → a branches doc whose default is a directory doc; packages/branchy/dist/index.js resolves, + // and the pinned form has a doc step for the branches doc and another for its default +it("resolves a name that looks like a doc step but matches no link", ...) + // a folder entry named `${await idhash(someOtherId)}@h1`; the literal segment resolves to that entry +it("resolves a name with reserved characters", ...) + // entry "a/b c@%" under Projects; resolveLivePath(livePath(a, ["Projects","a/b c@%"])) finds it +it("returns undefined for an unknown account, missing names, and heads the repo doesn't have", ...) +it("stops on a branches doc whose default is itself", ...) // resolves undefined, doesn't hang +it("stops when / is bound to the account itself", ...) // a/x → undefined +it("pinPath prefixes the origin and /@/", ...) // stub globalThis.location for node +it("lists an account's children: mounts shadow / entries, unions concatenate", ...) + // listChildren(repo, account) → names include "Projects" and "packages", each once + // listUnion(repo, [mine.url, team.url]) → names ["foo", "foo", "bar"]: concatenated, not deduplicated +``` + +`listChildren` on an account keeps each mount as one `{ name, union }` entry. It replaces the `/` union with its members' children, minus any name a mount shadows. + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts` +Expected: FAIL, cannot resolve `../src/walk.js`. + +- [ ] **Step 3: Implement `walk.ts` per the algorithm above, and export it from `index.ts`** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/walk.ts core/filesystem/src/index.ts core/filesystem/test/walk.test.ts +git commit -m "filesystem: resolve and pin live paths through the namespace" +``` + +--- + +### Task 4: `resolvePath` on providers + +**Files:** +- Modify: `core/filesystem/src/resolve.ts` +- Test: `core/filesystem/test/resolve.test.ts` (add one case) + +**Interfaces:** +- Consumes: `providerFor`, `Lookup`, `asLink` from Task 2. +- Produces: unchanged `resolvePath(repo, rootHandle, parts): Promise` and `Resolved`. `FolderStrategy` is deleted. + +`resolvePath` keeps its contract: it serves bytes, follows bare links, and reports `pinned` honestly. It does not go through `walkPath`, which pins and doesn't materialize. Replace `STRATEGIES` with `providerFor(doc)`, and `walkDirectoryDoc`'s prefix loop with `provider.lookup(node, parts)`. A value child keeps walking in the same provider, a link child keeps `walk.pinned &&= hasHeads(link)` and the visited-set cycle check, and a union child (only an account doc yields one) follows its first member. `materialize` is unchanged. + +- [ ] **Step 1: Write the failing test** + +```ts +it("follows a branches doc's default branch", async () => { + const repo = new Repo(); + const pkg = publish(repo, (url) => url); + const branches = repo.create({ "@patchwork": { type: "branches" }, branches: { default: pkg.url } }); + const index = await resolvePath(repo, branches, ["dist", "index.js"]); + expect(decode(index!.content)).toBe("export default 1"); +}); +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/resolve.test.ts` +Expected: FAIL. `index` is undefined, because no strategy matches a branches doc. + +- [ ] **Step 3: Rewrite the dispatch in `resolve.ts` as described** + +- [ ] **Step 4: Run the whole filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS, including `resolve-pinning.test.ts` unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/resolve.ts core/filesystem/test/resolve.test.ts +git commit -m "filesystem: resolvePath dispatches through path providers" +``` + +--- + +### Task 5: Typecheck downstream and add the changeset + +**Files:** +- Create: `.changeset/paths-grammar-providers.md` + +- [ ] **Step 1: Build the two touched packages and typecheck their dependents** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins build && pnpm --filter @inkandswitch/patchwork-bootloader --filter @inkandswitch/patchwork exec tsc --noEmit` +Expected: no errors. (Don't run e2e or site builds locally; CI does.) + +- [ ] **Step 2: Write the changeset** + +```markdown +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +--- + +Add paths: `idhash`, `livePath`, `parsePath`, `resolveLivePath`, `pinnedPath`, `pinPath` and `listChildren` walk an account's namespace by name and write the pinned `@` form that imports and share links will use. Path providers (`accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `PathProvider`) replace `FolderStrategy`, which is removed; `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. +``` + +- [ ] **Step 3: Commit** + +```bash +git add .changeset/paths-grammar-providers.md +git commit -m "changeset: paths grammar and providers" +``` From 91c220db10f9475d36cba120f6190c9cf7247c40 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 04:58:05 +0000 Subject: [PATCH 03/81] filesystem: path grammar (idhash, heads, live and pinned paths) Co-Authored-By: Claude Opus 5.5 --- core/filesystem/package.json | 1 + core/filesystem/src/index.ts | 1 + core/filesystem/src/paths.ts | 128 +++++++++++++++++++++++++++++ core/filesystem/test/paths.test.ts | 101 +++++++++++++++++++++++ pnpm-lock.yaml | 3 + 5 files changed, 234 insertions(+) create mode 100644 core/filesystem/src/paths.ts create mode 100644 core/filesystem/test/paths.test.ts diff --git a/core/filesystem/package.json b/core/filesystem/package.json index 7f4b71120..d65666eec 100644 --- a/core/filesystem/package.json +++ b/core/filesystem/package.json @@ -23,6 +23,7 @@ "@automerge/automerge-repo": "*" }, "dependencies": { + "bs58": "^6.0.0", "debug": "^4.4.3", "resolve.exports": "^2.0.3" }, diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index c2e1278e8..c3f55c685 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -3,6 +3,7 @@ export * from "./metadata.js"; export * from "./module-watcher.js"; export * from "./package-tracker.js"; export * from "./packages.js"; +export * from "./paths.js"; export * from "./resolve.js"; export * from "./urls.js"; export type * from "./types.js"; diff --git a/core/filesystem/src/paths.ts b/core/filesystem/src/paths.ts new file mode 100644 index 000000000..d54395cf8 --- /dev/null +++ b/core/filesystem/src/paths.ts @@ -0,0 +1,128 @@ +import { + type DocumentId, + type UrlHeads, + isValidDocumentId, +} from "@automerge/automerge-repo/slim"; +import bs58 from "bs58"; + +/** + * A step that enters a document at given heads: `@`. `raw` is the segment as + * written, so a walk that finds no link with this idhash can read it as a name instead. + */ +export type DocStep = { + kind: "doc"; + idhash: string; + heads: UrlHeads; + raw: string; +}; +export type NameStep = { kind: "name"; name: string }; +export type Segment = DocStep | NameStep; + +/** + * Where a path starts: an account the resolver knows by its idhash, or (for a portable share + * link) a document named by its id. A share root is always pinned. + */ +export type PathRoot = + | { kind: "account"; acctHash: string; heads?: UrlHeads } + | { kind: "share"; documentId: DocumentId; heads: UrlHeads }; + +export type ParsedPath = { root: PathRoot; segments: Segment[] }; + +export class PathSyntaxError extends Error { + name = "PathSyntaxError"; +} + +const BASE58 = /^[1-9A-HJ-NP-Za-km-z]+$/; +const SHARE_PREFIX = "automerge:"; + +/** + * The first 12 bytes of SHA-256 over the UTF-8 of a document id, in base58. It tells a document + * from its siblings and keeps the id out of URLs. It is not a secret. + */ +export async function idhash(documentId: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(documentId) + ); + return bs58.encode(new Uint8Array(digest).slice(0, 12)); +} + +/** Heads sorted and joined with `.`, so one set of heads is one string. */ +export function formatHeads(heads: readonly string[]): string { + return [...heads].sort().join("."); +} + +export function parseHeads(segment: string): UrlHeads | undefined { + const parts = segment.split("."); + if (!parts.every((part) => BASE58.test(part))) return undefined; + return parts.sort() as UrlHeads; +} + +export function docStep(idhash: string, heads: readonly string[]): string { + return `${idhash}@${formatHeads(heads)}`; +} + +export function livePath(acctHash: string, names: string[]): string { + return [acctHash, ...names.map(encodeURIComponent)].join("/"); +} + +function decode(raw: string): string { + try { + return decodeURIComponent(raw); + } catch { + throw new PathSyntaxError(`malformed escape in path segment ${raw}`); + } +} + +/** `@`, split, or undefined if it isn't that shape. */ +function splitDocStep( + raw: string +): { id: string; heads: UrlHeads } | undefined { + const at = raw.indexOf("@"); + if (at < 0) return undefined; + const id = raw.slice(0, at); + const heads = parseHeads(raw.slice(at + 1)); + if (!BASE58.test(id) || !heads) return undefined; + return { id, heads }; +} + +function parseRoot(raw: string): PathRoot { + const decoded = decode(raw); + if (decoded.startsWith(SHARE_PREFIX)) { + const rest = decoded.slice(SHARE_PREFIX.length); + const at = rest.lastIndexOf("@"); + const documentId = rest.slice(0, at); + const heads = at < 0 ? undefined : parseHeads(rest.slice(at + 1)); + if (!heads || !isValidDocumentId(documentId)) { + throw new PathSyntaxError( + `a share link must be automerge:@, got ${decoded}` + ); + } + return { kind: "share", documentId, heads }; + } + if (BASE58.test(raw)) return { kind: "account", acctHash: raw }; + const step = splitDocStep(raw); + if (!step) { + throw new PathSyntaxError( + `a path must start with an account hash or a share link, got ${raw}` + ); + } + return { kind: "account", acctHash: step.id, heads: step.heads }; +} + +/** + * Read a live or pinned path. Segments are percent-encoded and split on literal slashes; a + * segment with a literal `@` that parses as `@` is a doc step, everything else a + * name. Whether a doc step really is one is the walk's call, since only it can see the links. + */ +export function parsePath(path: string): ParsedPath { + const [first, ...rest] = path.split("/").filter(Boolean); + if (!first) throw new PathSyntaxError("empty path"); + const segments = rest.map((raw): Segment => { + const step = splitDocStep(raw); + return step + ? { kind: "doc", idhash: step.id, heads: step.heads, raw } + : { kind: "name", name: decode(raw) }; + }); + return { root: parseRoot(first), segments }; +} diff --git a/core/filesystem/test/paths.test.ts b/core/filesystem/test/paths.test.ts new file mode 100644 index 000000000..c4ac611ec --- /dev/null +++ b/core/filesystem/test/paths.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect } from "vitest"; +import { + formatHeads, + idhash, + livePath, + parseHeads, + parsePath, + PathSyntaxError, +} from "../src/paths.js"; + +describe("idhash", () => { + it("is base58 of the first 12 bytes of sha-256 over the document id", async () => { + expect(await idhash("2j9knpCseyhnK8izDmLpGP5WMdZQ")).toBe( + "vjRneszhMkDwr5tb" + ); + }); +}); + +describe("heads", () => { + it("sorts so one set of heads is one string", () => { + expect(formatHeads(["c", "a", "b"])).toBe("a.b.c"); + expect(formatHeads(["b", "a"])).toBe(formatHeads(["a", "b"])); + }); + + it("parses only base58 parts", () => { + expect(parseHeads("a.b")).toEqual(["a", "b"]); + expect(parseHeads("a.0")).toBeUndefined(); + expect(parseHeads("a..b")).toBeUndefined(); + expect(parseHeads("")).toBeUndefined(); + }); +}); + +describe("parsePath", () => { + it("round-trips a live path, encoding each name", () => { + const names = ["Projects", "a/b", "x@y", "50%", "my notes", "日本"]; + const path = livePath("acct", names); + // a slash inside a name is encoded, so it doesn't split the name + expect(path.split("/")).toHaveLength(1 + names.length); + expect(parsePath(path)).toEqual({ + root: { kind: "account", acctHash: "acct" }, + segments: names.map((name) => ({ kind: "name", name })), + }); + }); + + it("reads a pinned path's doc steps and plain names", () => { + const p = parsePath("acct@h2.h1/packages/mine@h3/foo@h4/dist/index.js"); + expect(p.root).toEqual({ + kind: "account", + acctHash: "acct", + heads: ["h1", "h2"], + }); + expect(p.segments.map((s) => s.kind)).toEqual([ + "name", + "doc", + "doc", + "name", + "name", + ]); + expect(p.segments[1]).toEqual({ + kind: "doc", + idhash: "mine", + heads: ["h3"], + raw: "mine@h3", + }); + }); + + it("reads a share root", () => { + expect( + parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h1/x").root + ).toEqual({ + kind: "share", + documentId: "2j9knpCseyhnK8izDmLpGP5WMdZQ", + heads: ["h1"], + }); + // the router and the service worker hand over encoded segments + expect( + parsePath("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h1").root.kind + ).toBe("share"); + }); + + it("treats an encoded @ as part of a name", () => { + expect(parsePath("acct/a%40b").segments).toEqual([ + { kind: "name", name: "a@b" }, + ]); + }); + + it("ignores a leading slash and empty segments", () => { + expect(parsePath("/acct//a/").segments).toEqual([ + { kind: "name", name: "a" }, + ]); + }); + + it("rejects a share root without heads, an empty path, and a bad root", () => { + expect(() => parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")).toThrow( + PathSyntaxError + ); + expect(() => parsePath("")).toThrow(PathSyntaxError); + expect(() => parsePath("acct@0")).toThrow(PathSyntaxError); + expect(() => parsePath("automerge:not-an-id@h1")).toThrow(PathSyntaxError); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f1593ba90..271648470 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -217,6 +217,9 @@ importers: '@automerge/automerge-repo': specifier: 2.6.0-subduction.48 version: 2.6.0-subduction.48(patch_hash=3560381ba62410bed0716e7ef08af4200bcf8c0af4aa0794c1d5d05d23e97d5e) + bs58: + specifier: ^6.0.0 + version: 6.0.0 debug: specifier: ^4.4.3 version: 4.4.3 From 851fbd9360e8d424bb8ae350300db696d0326a91 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 04:59:31 +0000 Subject: [PATCH 04/81] filesystem: path providers and the account namespace Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/index.ts | 1 + core/filesystem/src/providers.ts | 210 +++++++++++++++++++++++++ core/filesystem/src/resolve.ts | 13 +- core/filesystem/src/types.ts | 15 ++ core/filesystem/test/providers.test.ts | 175 +++++++++++++++++++++ core/plugins/src/account.ts | 9 +- 6 files changed, 410 insertions(+), 13 deletions(-) create mode 100644 core/filesystem/src/providers.ts create mode 100644 core/filesystem/test/providers.test.ts diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index c3f55c685..9859f75b2 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -4,6 +4,7 @@ export * from "./module-watcher.js"; export * from "./package-tracker.js"; export * from "./packages.js"; export * from "./paths.js"; +export * from "./providers.js"; export * from "./resolve.js"; export * from "./urls.js"; export type * from "./types.js"; diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts new file mode 100644 index 000000000..2025c1578 --- /dev/null +++ b/core/filesystem/src/providers.ts @@ -0,0 +1,210 @@ +import { + type AutomergeUrl, + isImmutableString, + isValidAutomergeUrl, +} from "@automerge/automerge-repo/slim"; +import { getType } from "./metadata.js"; +import { getDefaultBranch } from "./package-tracker.js"; +import type { + AccountNamespaceDoc, + BranchesDoc, + FolderDoc, +} from "./types.js"; + +/** + * What a provider finds under a name: a link to another document, a value inside the same + * document (a nested directory, a file's bytes), or an ordered union of documents (a mount). + */ +export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] }; + +/** + * A child and how many of the leading names it used. A step that consumes no name (the `/` + * mount, a branches doc's default) has `consumed: 0`. + */ +export type Lookup = { child: Child; consumed: number }; + +/** + * How to walk one kind of document. `lookup` and `list` take the document or a value nested in + * it, and never leave it: following a link is the walker's job. `lookup` is called with at least + * one name. + */ +export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; + list(node: unknown): Child[]; +} + +/** + * `value` as a link, if it is one. A link written as an `ImmutableString` + * (one op, where a plain string is an op per character) reads back as an + * object, not a string; it's the same url. + */ +export function asLink(value: unknown): AutomergeUrl | undefined { + const url = isImmutableString(value) ? String(value) : value; + return typeof url === "string" && isValidAutomergeUrl(url) ? url : undefined; +} + +const isObject = (node: unknown): node is Record => + !!node && + typeof node === "object" && + !Array.isArray(node) && + !(node instanceof Uint8Array) && + !isImmutableString(node); + +function childOf(name: string, value: unknown): Child { + const link = asLink(value); + return link ? { name, link } : { name, value }; +} + +const MOUNT_POINT = /^\/[^/@]*$/; + +/** + * Mount point → its bindings, in order. `/` falls back to `rootFolderUrl` when the namespace + * doesn't bind it, so older accounts keep working. Malformed mount points and bindings are + * skipped with a warning. + */ +export function accountBindings( + doc: AccountNamespaceDoc +): Map { + const bindings = new Map(); + for (const [mount, list] of Object.entries(doc.namespace ?? {})) { + if (!MOUNT_POINT.test(mount) || !Array.isArray(list)) { + console.warn(`namespace: skipping malformed mount ${JSON.stringify(mount)}`); + continue; + } + const urls: AutomergeUrl[] = []; + for (const binding of list) { + const url = asLink(binding?.url); + if (url) urls.push(url); + else console.warn(`namespace: skipping a binding at ${mount} without a valid url`); + } + bindings.set(mount, urls); + } + const root = asLink(doc.rootFolderUrl); + if (!bindings.has("/") && root) bindings.set("/", [root]); + return bindings; +} + +export const accountProvider: PathProvider = { + kind: "account", + matches(doc) { + return getType(doc as AccountNamespaceDoc) === "account"; + }, + lookup(node, names) { + const bindings = accountBindings(node as AccountNamespaceDoc); + const mount = bindings.get(`/${names[0]}`); + if (names[0] && mount) { + return { child: { name: names[0], union: mount }, consumed: 1 }; + } + const root = bindings.get("/"); + if (!root?.length) return undefined; + return { child: { name: "", union: root }, consumed: 0 }; + }, + list(node) { + const children: Child[] = []; + for (const [mount, union] of accountBindings(node as AccountNamespaceDoc)) { + const child = { name: mount.slice(1), union }; + if (mount === "/") children.unshift(child); + else children.push(child); + } + return children; + }, +}; + +export const directoryProvider: PathProvider = { + kind: "directory", + matches(doc) { + return getType(doc as AccountNamespaceDoc) === "directory"; + }, + // Longest-prefix match: try "main/dist/index.js", then "main/dist", then "main" + lookup(node, names) { + if (!isObject(node)) return undefined; + for (let i = names.length; i >= 1; i--) { + const key = names.slice(0, i).join("/"); + if (Object.hasOwn(node, key)) return { child: childOf(key, node[key]), consumed: i }; + } + return undefined; + }, + list(node) { + if (!isObject(node)) return []; + return Object.entries(node) + .filter(([key]) => key !== "@patchwork") + .map(([key, value]) => childOf(key, value)); + }, +}; + +export const folderProvider: PathProvider = { + kind: "folder", + matches(doc) { + return ( + !!doc && + typeof doc === "object" && + "docs" in doc && + Array.isArray((doc as { docs: unknown }).docs) + ); + }, + lookup(node, names) { + const entry = (node as FolderDoc).docs.find( + (doc) => String(doc.name) === names[0] + ); + const link = asLink(entry?.url); + return link ? { child: { name: names[0], link }, consumed: 1 } : undefined; + }, + list(node) { + return (node as FolderDoc).docs.flatMap((doc) => { + const link = asLink(doc.url); + return link ? [{ name: String(doc.name), link }] : []; + }); + }, +}; + +/** A branches doc's default branch, or undefined if it has none or isn't one. */ +export function defaultBranch(doc: unknown): AutomergeUrl | undefined { + try { + return asLink(getDefaultBranch(doc)); + } catch { + return undefined; + } +} + +/** + * A branches doc stands for its default branch: a name that isn't a branch is looked up in + * `default`, so a package published as a branches doc resolves like the package itself. A branch + * name wins over a same-named file. + */ +export const branchesProvider: PathProvider = { + kind: "branches", + matches(doc) { + return getType(doc as BranchesDoc) === "branches"; + }, + lookup(node, names) { + const named = asLink((node as BranchesDoc).branches?.[names[0]]); + if (named) return { child: { name: names[0], link: named }, consumed: 1 }; + const link = defaultBranch(node); + return link ? { child: { name: "default", link }, consumed: 0 } : undefined; + }, + list(node) { + return Object.entries((node as BranchesDoc).branches ?? {}).flatMap( + ([name, url]) => { + const link = asLink(url); + return link ? [{ name, link }] : []; + } + ); + }, +}; + +/** Checked in order; the first that matches a document walks it. */ +export const PROVIDERS: readonly PathProvider[] = [ + accountProvider, + directoryProvider, + folderProvider, + branchesProvider, +]; + +export function providerFor(doc: unknown): PathProvider | undefined { + return PROVIDERS.find((provider) => provider.matches(doc)); +} diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 98a8368ca..55ce77f87 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -3,11 +3,10 @@ import { type DocHandle, type Repo, isImmutableString, - isValidAutomergeUrl, - parseAutomergeUrl, } from "@automerge/automerge-repo/slim"; import type { FolderDoc } from "./types.js"; import { getType } from "./metadata.js"; +import { asLink } from "./providers.js"; export type Resolved = { content: string | Uint8Array; @@ -37,16 +36,6 @@ type Walk = { /** A URL carries heads when it has a fragment: `automerge:abc#heads`. */ const hasHeads = (url: unknown): boolean => String(url).includes("#"); -/** - * `value` as a link, if it is one. A link written as an `ImmutableString` - * (one op, where a plain string is an op per character) reads back as an - * object, not a string; it's the same url. - */ -function asLink(value: unknown): AutomergeUrl | undefined { - const url = isImmutableString(value) ? String(value) : value; - return typeof url === "string" && isValidAutomergeUrl(url) ? url : undefined; -} - export interface FolderStrategy { matches(doc: unknown): boolean; resolve( diff --git a/core/filesystem/src/types.ts b/core/filesystem/src/types.ts index 45898012b..836e70885 100644 --- a/core/filesystem/src/types.ts +++ b/core/filesystem/src/types.ts @@ -33,3 +33,18 @@ export type UnixFileEntry = { mimeType: string; name: string; }; + +/** One document bound at a mount point. Only automerge documents can be bound. */ +export type NamespaceBinding = { url: AutomergeUrl }; + +/** + * An account's mount table, after plan9: each mount point is `/` or a top-level `/`, bound + * to an ordered union of documents. Lookup tries each binding in order; listing concatenates. + */ +export type Namespace = { [mountPoint: string]: NamespaceBinding[] }; + +/** The parts of an account doc a path walk reads. */ +export type AccountNamespaceDoc = HasPatchworkMetadata<"account"> & { + namespace?: Namespace; + rootFolderUrl?: AutomergeUrl; +}; diff --git a/core/filesystem/test/providers.test.ts b/core/filesystem/test/providers.test.ts new file mode 100644 index 000000000..6a2424188 --- /dev/null +++ b/core/filesystem/test/providers.test.ts @@ -0,0 +1,175 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { ImmutableString } from "@automerge/automerge"; +import { generateAutomergeUrl } from "@automerge/automerge-repo"; +import { + accountBindings, + accountProvider, + branchesProvider, + directoryProvider, + folderProvider, + providerFor, +} from "../src/providers.js"; + +const [R, P1, P2, A, B] = Array.from({ length: 5 }, generateAutomergeUrl); +const account = (rest: object) => ({ + "@patchwork": { type: "account" as const }, + ...rest, +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("accountProvider", () => { + const doc = account({ + namespace: { + "/": [{ url: R }], + "/packages": [{ url: P1 }, { url: P2 }], + }, + }); + + it("looks a mount up by name and consumes it", () => { + expect(accountProvider.lookup(doc, ["packages", "foo"])).toEqual({ + child: { name: "packages", union: [P1, P2] }, + consumed: 1, + }); + }); + + it("sends any other name to / without consuming it", () => { + expect(accountProvider.lookup(doc, ["Projects"])).toEqual({ + child: { name: "", union: [R] }, + consumed: 0, + }); + }); + + it("falls back to rootFolderUrl with no namespace, or no / key", () => { + for (const d of [ + account({ rootFolderUrl: R }), + account({ rootFolderUrl: R, namespace: { "/packages": [{ url: P1 }] } }), + ]) { + expect(accountProvider.lookup(d, ["Projects"])?.child).toEqual({ + name: "", + union: [R], + }); + } + }); + + it("finds nothing with neither namespace nor rootFolderUrl", () => { + expect(accountProvider.lookup(account({}), ["x"])).toBeUndefined(); + expect(accountProvider.list(account({}))).toEqual([]); + }); + + it("skips malformed mounts and bindings with a warning", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const bindings = accountBindings( + account({ + namespace: { + "/": [{ url: R }, { url: "nope" }], + packages: [{ url: P1 }], + "/a/b": [{ url: P1 }], + "/x@y": [{ url: P1 }], + "/ok": "not an array", + }, + }) + ); + expect([...bindings]).toEqual([["/", [R]]]); + expect(warn).toHaveBeenCalled(); + }); + + it("lists every mount as a union child", () => { + expect(accountProvider.list(doc)).toEqual([ + { name: "", union: [R] }, + { name: "packages", union: [P1, P2] }, + ]); + }); +}); + +describe("directoryProvider", () => { + it("matches the longest prefix", () => { + const doc = { dist: { other: A }, "dist/index.js": B }; + expect(directoryProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "dist/index.js", link: B }, + consumed: 2, + }); + }); + + it("returns a nested object as a value child", () => { + const doc = { dist: { "index.js": A } }; + expect(directoryProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "dist", value: { "index.js": A } }, + consumed: 1, + }); + }); + + it("reads ImmutableString links", () => { + const doc = { "a.js": new ImmutableString(A) }; + expect(directoryProvider.lookup(doc, ["a.js"])?.child).toEqual({ + name: "a.js", + link: A, + }); + }); + + it("does not list @patchwork", () => { + const doc = { "@patchwork": { type: "directory" }, "a.js": A }; + expect(directoryProvider.list(doc)).toEqual([{ name: "a.js", link: A }]); + }); +}); + +describe("folderProvider", () => { + it("first entry with the name wins", () => { + const doc = { + docs: [ + { name: "a", type: "x", url: A }, + { name: "a", type: "x", url: B }, + ], + }; + expect(folderProvider.lookup(doc, ["a", "b"])).toEqual({ + child: { name: "a", link: A }, + consumed: 1, + }); + }); + + it("misses an unknown name", () => { + expect(folderProvider.lookup({ docs: [] }, ["a"])).toBeUndefined(); + }); +}); + +describe("branchesProvider", () => { + const doc = { + "@patchwork": { type: "branches" }, + branches: { default: A, feature: B }, + }; + + it("follows a named branch", () => { + expect(branchesProvider.lookup(doc, ["feature", "x"])).toEqual({ + child: { name: "feature", link: B }, + consumed: 1, + }); + }); + + it("falls through to default for any other name", () => { + expect(branchesProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "default", link: A }, + consumed: 0, + }); + }); + + it("finds nothing with no default branch", () => { + const none = { "@patchwork": { type: "branches" }, branches: { x: B } }; + expect(branchesProvider.lookup(none, ["y"])).toBeUndefined(); + }); +}); + +describe("providerFor", () => { + it("checks account, directory, folder, branches in order", () => { + expect(providerFor(account({ docs: [] }))).toBe(accountProvider); + expect( + providerFor({ "@patchwork": { type: "directory" }, docs: [] }) + ).toBe(directoryProvider); + expect(providerFor({ docs: [] })).toBe(folderProvider); + expect( + providerFor({ "@patchwork": { type: "branches" }, branches: {} }) + ).toBe(branchesProvider); + expect( + providerFor({ "@patchwork": { type: "file" }, content: "x" }) + ).toBeUndefined(); + }); +}); diff --git a/core/plugins/src/account.ts b/core/plugins/src/account.ts index f3b1f264d..41719592b 100644 --- a/core/plugins/src/account.ts +++ b/core/plugins/src/account.ts @@ -4,7 +4,10 @@ import { type DocHandle, type Repo, } from "@automerge/automerge-repo/slim"; -import type { HasPatchworkMetadata } from "@inkandswitch/patchwork-filesystem"; +import type { + HasPatchworkMetadata, + Namespace, +} from "@inkandswitch/patchwork-filesystem"; import type { AutomergeRepoKeyhiveBase as AutomergeRepoKeyhive } from "@automerge/automerge-repo-keyhive"; import { isKeyhiveDoc } from "./keyhive.js"; @@ -20,6 +23,10 @@ export type AccountDoc = { contextToolIds: string[]; documentToolbarToolIds: string[]; + /** + * Mount point → ordered bindings. Without one, `/` is {@link AccountDoc.rootFolderUrl}. + */ + namespace?: Namespace; rootFolderUrl?: AutomergeUrl; moduleSettingsUrl?: AutomergeUrl; contactUrl?: AutomergeUrl; From 5e83fe0b5b8dcb853183e6c2da0b87c5486da57a Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:03:07 +0000 Subject: [PATCH 05/81] filesystem: resolve and pin live paths through the namespace Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/index.ts | 1 + core/filesystem/src/walk.ts | 312 ++++++++++++++++++++++++++++++ core/filesystem/test/walk.test.ts | 274 ++++++++++++++++++++++++++ 3 files changed, 587 insertions(+) create mode 100644 core/filesystem/src/walk.ts create mode 100644 core/filesystem/test/walk.test.ts diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index 9859f75b2..9886de7fd 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -7,4 +7,5 @@ export * from "./paths.js"; export * from "./providers.js"; export * from "./resolve.js"; export * from "./urls.js"; +export * from "./walk.js"; export type * from "./types.js"; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts new file mode 100644 index 000000000..c17458133 --- /dev/null +++ b/core/filesystem/src/walk.ts @@ -0,0 +1,312 @@ +import { + type AutomergeUrl, + type DocHandle, + type Repo, + type UrlHeads, + parseAutomergeUrl, + stringifyAutomergeUrl, +} from "@automerge/automerge-repo/slim"; +import { + type DocStep, + docStep, + formatHeads, + idhash, + parsePath, +} from "./paths.js"; +import { + type Child, + type Lookup, + type PathProvider, + accountProvider, + branchesProvider, + defaultBranch, + providerFor, +} from "./providers.js"; +import { documentBaseOrigin } from "./urls.js"; + +export type WalkOptions = { + /** acctHash → account doc url. The roots a live or pinned account path may start at. */ + accounts: ReadonlyMap; +}; + +export type WalkEnd = { + /** The last document the walk entered, as a view at the heads it was read at. */ + handle: DocHandle; + /** Where the walk stopped: the document itself, or a value nested in it. */ + node: unknown; + /** False when the path ends at a value inside the document rather than at the document. */ + atDocument: boolean; + /** The pinned path, without a leading slash. */ + pinned: string; +}; + +/** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ +type At = + | { + kind: "doc"; + handle: DocHandle; + node: unknown; + provider?: PathProvider; + inside: boolean; + } + | { kind: "union"; members: AutomergeUrl[] }; + +type AtDoc = Extract; + +/** + * Walk a live or pinned path and write its pinned form as it goes. + * + * Every document that has a provider (a container) is written as `@` in place of + * the name that led to it. A step that consumes no name (a union member, a branches doc's + * default) adds its own doc step. Steps inside a document, and the names that lead from the last + * container to a document without a provider (a file), stay as names: that tail is what lets a + * module's relative imports resolve. + * + * Returns undefined when the path doesn't resolve. Throws only for a path that doesn't parse. + */ +export async function walkPath( + repo: Repo, + path: string, + options: WalkOptions +): Promise { + const { root, segments } = parsePath(path); + const hashes = new Map>(); + const hashOf = (url: AutomergeUrl) => { + const { documentId } = parseAutomergeUrl(url); + let hash = hashes.get(documentId); + if (!hash) hashes.set(documentId, (hash = idhash(documentId))); + return hash; + }; + const stepOf = async (handle: DocHandle) => + docStep(await hashOf(handle.url), handle.heads()); + + async function enter( + url: AutomergeUrl, + heads?: UrlHeads + ): Promise { + try { + const parsed = parseAutomergeUrl(url); + const live = await repo.find( + stringifyAutomergeUrl({ documentId: parsed.documentId }) + ); + const handle = live.view(heads ?? parsed.heads ?? live.heads()); + const node = handle.doc(); + if (node === undefined) return undefined; + return { kind: "doc", handle, node, provider: providerFor(node), inside: false }; + } catch { + return undefined; + } + } + + async function matchLink(links: AutomergeUrl[], step: DocStep) { + for (const link of links) { + if ((await hashOf(link)) === step.idhash) return link; + } + return undefined; + } + + // Steps that consume no name can loop (a branches doc whose default is itself, `/` bound to the + // account); revisiting a document with as many segments left is a cycle. + const seen = new Set(); + const guard = (url: AutomergeUrl, remaining: number) => { + const key = `${parseAutomergeUrl(url).documentId}|${remaining}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }; + + // The run of names starting at i. A doc step that matched no link reads as a name. + const namesFrom = (i: number) => { + const names: string[] = []; + for (let j = i; j < segments.length; j++) { + const segment = segments[j]; + if (segment.kind === "name") names.push(segment.name); + else if (j === i) names.push(decodeURIComponent(segment.raw)); + else break; + } + return names; + }; + + const out: string[] = []; + let at: At | undefined; + if (root.kind === "account") { + const url = options.accounts.get(root.acctHash); + at = url && (await enter(url, root.heads)); + if (!at) return undefined; + out.push(docStep(root.acctHash, at.handle.heads())); + } else { + at = await enter(stringifyAutomergeUrl({ documentId: root.documentId }), root.heads); + if (!at) return undefined; + out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); + } + + let i = 0; + while (true) { + const segment = segments[i] as (typeof segments)[number] | undefined; + + if (at.kind === "union") { + let member: AutomergeUrl | undefined; + let next: AtDoc | undefined; + if (segment?.kind === "doc") { + member = await matchLink(at.members, segment); + if (member) { + i++; + if (!guard(member, segments.length - i)) return undefined; + next = await enter(member, segment.heads); + } + } + if (!member) { + const names = namesFrom(i); + for (const url of at.members) { + const candidate = await enter(url); + if (!names.length || candidate?.provider?.lookup(candidate.node, names)) { + member = url; + next = candidate; + break; + } + } + if (!member || !guard(member, segments.length - i)) return undefined; + } + if (!next) return undefined; + out.push(await stepOf(next.handle)); + at = next; + continue; + } + + if (!segment) { + // A path that ends at a branches doc stands for its default branch. + if (at.provider !== branchesProvider || at.inside) break; + const link = defaultBranch(at.node); + if (!link || !guard(link, 0)) return undefined; + const next = await enter(link); + if (!next) return undefined; + out.push(await stepOf(next.handle)); + at = next; + continue; + } + + if (segment.kind === "doc") { + const links = (at.provider?.list(at.node) ?? []).flatMap((child) => + "link" in child ? [child.link] : "union" in child ? child.union : [] + ); + const link = await matchLink(links, segment); + if (link) { + const next = await enter(link, segment.heads); + if (!next) return undefined; + out.push(await stepOf(next.handle)); + at = next; + i++; + continue; + } + } + + // A document without a provider has no children. + if (!at.provider) return undefined; + const names = namesFrom(i); + const found: Lookup | undefined = at.provider.lookup(at.node, names); + if (!found) return undefined; + const { child, consumed }: Lookup = found; + const written = names.slice(0, consumed).map(encodeURIComponent); + i += consumed; + + if ("value" in child) { + out.push(...written); + at = { ...at, node: child.value, inside: true }; + } else if ("union" in child) { + out.push(...written); + at = { kind: "union", members: child.union }; + } else { + if (!consumed && !guard(child.link, segments.length - i)) return undefined; + const next = await enter(child.link); + if (!next) return undefined; + if (next.provider || !consumed) out.push(await stepOf(next.handle)); + else out.push(...written); + at = next; + } + } + + return { + handle: at.handle, + node: at.node, + atDocument: !at.inside, + pinned: out.join("/"), + }; +} + +/** + * The document a live or pinned path names: the live handle for a live path, a view at the + * pinned heads for a pinned one. Undefined if the path doesn't resolve, or ends at a value inside + * a document. + */ +export async function resolveLivePath( + repo: Repo, + path: string, + options: WalkOptions +): Promise | undefined> { + const { root } = parsePath(path); + const end = await walkPath(repo, path, options); + if (!end?.atDocument) return undefined; + if (root.heads) return end.handle; + return repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); +} + +export async function pinnedPath( + repo: Repo, + path: string, + options: WalkOptions +): Promise { + return (await walkPath(repo, path, options))?.pinned; +} + +/** The pinned service worker URL for a path, for fetching or importing. */ +export async function pinPath( + repo: Repo, + path: string, + options: WalkOptions +): Promise { + const pinned = await pinnedPath(repo, path, options); + return pinned && `${documentBaseOrigin()}/@/${pinned}`; +} + +/** Every binding's children, concatenated in binding order. */ +export async function listUnion( + repo: Repo, + urls: AutomergeUrl[] +): Promise { + const lists = await Promise.all( + urls.map(async (url) => { + try { + const doc = (await repo.find(url)).doc(); + return providerFor(doc)?.list(doc) ?? []; + } catch { + return []; + } + }) + ); + return lists.flat(); +} + +/** + * A document's children. For an account, `/`'s union is expanded and each mount shadows a + * same-named entry in it; mounts stay as union children. + */ +export async function listChildren( + repo: Repo, + handle: DocHandle +): Promise { + const doc = handle.doc(); + const provider = providerFor(doc); + if (!provider) return []; + const children = provider.list(doc); + if (provider !== accountProvider) return children; + + const mounts = children.filter((child) => child.name !== ""); + const root = children.find((child) => child.name === ""); + const shadowed = new Set(mounts.map((child) => child.name)); + const rootChildren = + root && "union" in root ? await listUnion(repo, root.union) : []; + return [ + ...rootChildren.filter((child) => !shadowed.has(child.name)), + ...mounts, + ]; +} diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts new file mode 100644 index 000000000..29092ecc4 --- /dev/null +++ b/core/filesystem/test/walk.test.ts @@ -0,0 +1,274 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { docStep, idhash, livePath } from "../src/paths.js"; +import { + listChildren, + listUnion, + pinnedPath, + pinPath, + resolveLivePath, +} from "../src/walk.js"; + +const step = async (handle: DocHandle) => + docStep(await idhash(handle.documentId), handle.heads()); + +const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ + name, + type: "x", + url, + })), + }); + +const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + +// A package the way pushwork writes one: a directory doc of path → file doc url. +const pkg = (repo: Repo, main: string) => + repo.create({ + "@patchwork": { type: "directory" }, + "package.json": file(repo, "{}").url, + "dist/index.js": file(repo, main).url, + }); + +async function fixture() { + const repo = new Repo(); + const notes = file(repo, "hi"); + const foo = folder(repo, { notes: notes.url }); + const projects = folder(repo, { foo: foo.url }); + const decoy = folder(repo, { foo: file(repo, "decoy").url }); + const loop = repo.create<{ "@patchwork": { type: string }; branches: Record }>({ + "@patchwork": { type: "branches" }, + branches: {}, + }); + loop.change((d) => (d.branches.default = loop.url)); + const root = folder(repo, { + Projects: projects.url, + packages: decoy.url, + loop: loop.url, + }); + const minePkg = pkg(repo, "mine"); + const mine = folder(repo, { foo: minePkg.url }); + const branchyPkg = pkg(repo, "branchy"); + const branchy = repo.create({ + "@patchwork": { type: "branches" }, + branches: { default: branchyPkg.url }, + }); + const team = folder(repo, { + foo: pkg(repo, "team foo").url, + bar: pkg(repo, "team bar").url, + branchy: branchy.url, + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { + "/": [{ url: root.url }], + "/packages": [{ url: mine.url }, { url: team.url }], + }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + return { + repo, a, accounts, account, root, projects, foo, notes, mine, minePkg, + team, branchy, branchyPkg, + }; +} + +const text = (handle: DocHandle | undefined) => handle?.doc()?.content; + +afterEach(() => vi.unstubAllGlobals()); + +describe("resolveLivePath", () => { + it("resolves a live path through / and returns the live handle", async () => { + const { repo, a, accounts, notes } = await fixture(); + const handle = await resolveLivePath( + repo, + livePath(a, ["Projects", "foo", "notes"]), + { accounts } + ); + expect(handle?.url).toBe(notes.url); + }); + + it("union: the first binding with the name wins", async () => { + const { repo, a, accounts, minePkg, team } = await fixture(); + const foo = await resolveLivePath(repo, livePath(a, ["packages", "foo"]), { accounts }); + expect(foo?.url).toBe(minePkg.url); + const bar = await resolveLivePath(repo, livePath(a, ["packages", "bar"]), { accounts }); + expect(bar?.url).toBe(team.doc().docs[1].url); + }); + + it("a mount shadows a same-named entry in /'s folder", async () => { + const { repo, a, accounts } = await fixture(); + const index = await resolveLivePath( + repo, + livePath(a, ["packages", "foo", "dist", "index.js"]), + { accounts } + ); + expect(text(index)).toBe("mine"); + }); + + it("a pinned path keeps resolving to the same version after a rename", async () => { + const { repo, a, accounts, projects, foo } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const before = foo.heads(); + + projects.change((d: any) => (d.docs[0].name = "bar")); + foo.change((d: any) => (d.title = "renamed")); + + const old = await resolveLivePath(repo, pinned!, { accounts }); + expect(old?.documentId).toBe(foo.documentId); + expect(old?.heads()).toEqual(before); + expect((old?.doc() as any).title).toBe("folder"); + + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts }) + ).toBeUndefined(); + const bar = await resolveLivePath(repo, livePath(a, ["Projects", "bar"]), { accounts }); + expect(bar?.url).toBe(foo.url); + }); + + it("resolves a name that looks like a doc step but matches no link", async () => { + const { repo, a, accounts, projects } = await fixture(); + const lookalike = `${await idhash("someOtherDocumentId")}@h1`; + const target = file(repo, "lookalike"); + projects.change((d: any) => + d.docs.push({ name: lookalike, type: "x", url: target.url }) + ); + const handle = await resolveLivePath(repo, `${a}/Projects/${lookalike}`, { accounts }); + expect(handle?.url).toBe(target.url); + }); + + it("resolves a name with reserved characters", async () => { + const { repo, a, accounts, projects } = await fixture(); + const name = "a/b c@%日本"; + const target = file(repo, "reserved"); + projects.change((d: any) => d.docs.push({ name, type: "x", url: target.url })); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", name]), { accounts }); + expect(handle?.url).toBe(target.url); + }); + + it("returns undefined for an unknown account, missing names, and heads the repo doesn't have", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await resolveLivePath(repo, livePath("unknown", ["Projects"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "nope"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes", "deeper"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, `${a}@3vQB7B6MrGQZaxCuFg4oh/Projects`, { accounts }) + ).toBeUndefined(); + }); + + it("returns undefined for a path that ends inside a document", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.nested = { x: "y" })); + expect( + await resolveLivePath(repo, livePath(a, ["packages", "foo", "nested"]), { accounts }) + ).toBeUndefined(); + }); + + it("stops on a branches doc whose default is itself", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await resolveLivePath(repo, livePath(a, ["loop", "x"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["loop"]), { accounts }) + ).toBeUndefined(); + }); + + it("stops when / is bound to the account itself", async () => { + const repo = new Repo(); + const account = repo.create({ "@patchwork": { type: "account" } }); + account.change((d) => (d.namespace = { "/": [{ url: account.url }] })); + const a = await idhash(account.documentId); + expect( + await resolveLivePath(repo, livePath(a, ["x"]), { + accounts: new Map([[a, account.url]]), + }) + ).toBeUndefined(); + }); +}); + +describe("pinnedPath", () => { + it("pins every container the walk enters, and keeps the tail after the last one as names", async () => { + const { repo, a, accounts, account, mine, minePkg } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "foo", "dist", "index.js"]), + { accounts } + ); + expect(pinned).toBe( + `${docStep(a, account.heads())}/packages/${await step(mine)}/${await step(minePkg)}/dist/index.js` + ); + const index = await resolveLivePath(repo, pinned!, { accounts }); + expect(text(index)).toBe("mine"); + }); + + it("writes the / binding as a doc step with no name", async () => { + const { repo, a, accounts, account, root, projects, foo } = await fixture(); + expect( + await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }) + ).toBe( + [docStep(a, account.heads()), await step(root), await step(projects), await step(foo)].join("/") + ); + }); + + it("walks into a branches doc and follows its default", async () => { + const { repo, a, accounts, account, team, branchy, branchyPkg } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "branchy", "dist", "index.js"]), + { accounts } + ); + expect(pinned).toBe( + `${docStep(a, account.heads())}/packages/${await step(team)}/${await step(branchy)}/${await step(branchyPkg)}/dist/index.js` + ); + expect(text(await resolveLivePath(repo, pinned!, { accounts }))).toBe("branchy"); + // a path that ends at a branches doc stands for its default branch + const end = await resolveLivePath(repo, livePath(a, ["packages", "branchy"]), { accounts }); + expect(end?.url).toBe(branchyPkg.url); + }); + + it("re-pins a pinned path to the same string", async () => { + const { repo, a, accounts } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "bar", "dist", "index.js"]), + { accounts } + ); + expect(await pinnedPath(repo, pinned!, { accounts })).toBe(pinned); + }); +}); + +describe("pinPath", () => { + it("prefixes the origin and /@/", async () => { + vi.stubGlobal("location", { origin: "https://patchwork.test" }); + const { repo, a, accounts } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); + expect(await pinPath(repo, livePath(a, ["Projects"]), { accounts })).toBe( + `https://patchwork.test/@/${pinned}` + ); + }); +}); + +describe("listing", () => { + it("lists an account's children: mounts shadow / entries", async () => { + const { repo, account } = await fixture(); + const names = (await listChildren(repo, account)).map((c) => c.name); + expect(names.filter((n) => n === "packages")).toHaveLength(1); + expect(names).toContain("Projects"); + }); + + it("concatenates a union's bindings without deduplicating", async () => { + const { repo, mine, team } = await fixture(); + const names = (await listUnion(repo, [mine.url, team.url])).map((c) => c.name); + expect(names).toEqual(["foo", "foo", "bar", "branchy"]); + }); +}); From 41be2fd947da2dc4a654916b784c9c349ebb92e3 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:05:06 +0000 Subject: [PATCH 06/81] filesystem: resolvePath dispatches through path providers Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/resolve.ts | 115 +++++++-------------------- core/filesystem/test/resolve.test.ts | 11 +++ 2 files changed, 40 insertions(+), 86 deletions(-) diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 55ce77f87..d347b3114 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -1,12 +1,9 @@ import { - type AutomergeUrl, type DocHandle, type Repo, isImmutableString, } from "@automerge/automerge-repo/slim"; -import type { FolderDoc } from "./types.js"; -import { getType } from "./metadata.js"; -import { asLink } from "./providers.js"; +import { type PathProvider, asLink, providerFor } from "./providers.js"; export type Resolved = { content: string | Uint8Array; @@ -36,85 +33,37 @@ type Walk = { /** A URL carries heads when it has a fragment: `automerge:abc#heads`. */ const hasHeads = (url: unknown): boolean => String(url).includes("#"); -export interface FolderStrategy { - matches(doc: unknown): boolean; - resolve( - repo: Repo, - handle: DocHandle, - parts: string[], - walk: Walk - ): Promise; -} - -const folderStrategy: FolderStrategy = { - matches(doc) { - return ( - !!doc && - typeof doc === "object" && - "docs" in doc && - Array.isArray((doc as { docs: unknown }).docs) - ); - }, - async resolve(repo, handle, parts, walk) { - const folder = handle.doc() as FolderDoc | undefined; - if (!folder?.docs) return undefined; - - const part = parts[0]; - const docLink = folder.docs.find((doc) => String(doc.name) === part); - const url = asLink(docLink?.url); - if (!url) return undefined; - - // Following a bare link means everything below this point is current, not fixed. - walk.pinned &&= hasHeads(url); - const next = await repo.find(url); - return resolvePathInternal(repo, next, parts.slice(1), walk); - }, -}; - -const directoryStrategy: FolderStrategy = { - matches(doc) { - return getType(doc as Parameters[0]) === "directory"; - }, - async resolve(repo, handle, parts, walk) { - return walkDirectoryDoc(repo, handle.doc(), parts, walk); - }, -}; - -async function walkDirectoryDoc( +/** + * Resolve `parts` against `node`, a document or a value inside one, using the provider of the + * document it belongs to. Following a link hands the rest to that document's own provider. + */ +async function resolveIn( repo: Repo, + provider: PathProvider, node: unknown, parts: string[], walk: Walk ): Promise { - const link = asLink(node); - if (link) { - // Following a url consumes no parts, so revisiting the same url with the - // same remaining parts is a cycle. - const key = `${link}|${parts.join("/")}`; - if (walk.visited.has(key)) return undefined; - walk.visited.add(key); - walk.pinned &&= hasHeads(link); - const next = await repo.find(link); - return resolvePathInternal(repo, next, parts, walk); - } - - if (parts.length === 0) { - return materialize(repo, node, undefined, walk); - } - - if (!node || typeof node !== "object" || node instanceof Uint8Array) { - return undefined; - } - - // Longest-prefix match: try "main/dist/index.js", then "main/dist", then "main" - const obj = node as Record; - for (let i = parts.length; i >= 1; i--) { - const key = parts.slice(0, i).join("/"); - if (key in obj) { - return walkDirectoryDoc(repo, obj[key], parts.slice(i), walk); - } - } - return undefined; + if (parts.length === 0) return materialize(repo, node, undefined, walk); + + const found = provider.lookup(node, parts); + if (!found) return undefined; + const { child, consumed } = found; + const rest = parts.slice(consumed); + if ("value" in child) return resolveIn(repo, provider, child.value, rest, walk); + + // A union (an account's mount) is read through its first binding. + const link = "link" in child ? child.link : child.union[0]; + if (!link) return undefined; + // A step that consumes no parts can revisit the same url with the same remaining parts: a + // cycle. + const key = `${link}|${rest.join("/")}`; + if (walk.visited.has(key)) return undefined; + walk.visited.add(key); + // Following a bare link means everything below this point is current, not fixed. + walk.pinned &&= hasHeads(link); + const next = await repo.find(link); + return resolvePathInternal(repo, next, rest, walk); } // rule of thumb: bytes pass through, anything else gets JSON.stringify'd unless @@ -196,8 +145,6 @@ async function materialize( return undefined; } -const STRATEGIES: FolderStrategy[] = [directoryStrategy, folderStrategy]; - async function resolvePathInternal( repo: Repo, handle: DocHandle, @@ -209,12 +156,8 @@ async function resolvePathInternal( } const doc = handle.doc(); - for (const strategy of STRATEGIES) { - if (strategy.matches(doc)) { - return strategy.resolve(repo, handle, parts, walk); - } - } - return undefined; + const provider = providerFor(doc); + return provider ? resolveIn(repo, provider, doc, parts, walk) : undefined; } export async function resolvePath( diff --git a/core/filesystem/test/resolve.test.ts b/core/filesystem/test/resolve.test.ts index eb05a3d53..ab6020c50 100644 --- a/core/filesystem/test/resolve.test.ts +++ b/core/filesystem/test/resolve.test.ts @@ -45,6 +45,17 @@ describe("resolvePath", () => { }); } + it("follows a branches doc's default branch", async () => { + const repo = new Repo(); + const pkg = publish(repo, (url) => url); + const branches = repo.create({ + "@patchwork": { type: "branches" }, + branches: { default: pkg.url }, + }); + const index = await resolvePath(repo, branches, ["dist", "index.js"]); + expect(index && decode(index.content)).toBe("export default 1"); + }); + it("follows ImmutableString links in a folder doc", async () => { const repo = new Repo(); const file = repo.create({ From 5a4eff30c71bc6d4763dc46e597c40b0b01130cf Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:05:37 +0000 Subject: [PATCH 07/81] changeset: paths grammar and providers Co-Authored-By: Claude Opus 5.5 --- .changeset/paths-grammar-providers.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/paths-grammar-providers.md diff --git a/.changeset/paths-grammar-providers.md b/.changeset/paths-grammar-providers.md new file mode 100644 index 000000000..53596470a --- /dev/null +++ b/.changeset/paths-grammar-providers.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +--- + +Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@/…`). `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is removed: it named a private type, so nothing outside the package could implement it. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. From a8372d8e6a5ed23367273a067db066869c004487 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:16:50 +0000 Subject: [PATCH 08/81] filesystem: review fixes for paths Literal doc-shaped segments inside a multi-segment directory key resolve; a pinned root followed by names is reported non-canonical and resolves live; a null folder entry is skipped; / falls back to rootFolderUrl when every binding is malformed; FolderStrategy stays as a deprecated alias. Co-Authored-By: Claude Opus 5.5 --- .changeset/paths-grammar-providers.md | 2 +- core/filesystem/src/providers.ts | 8 ++--- core/filesystem/src/resolve.ts | 3 ++ core/filesystem/src/walk.ts | 48 ++++++++++++++++---------- core/filesystem/test/providers.test.ts | 8 +++++ core/filesystem/test/walk.test.ts | 32 +++++++++++++++++ 6 files changed, 78 insertions(+), 23 deletions(-) diff --git a/.changeset/paths-grammar-providers.md b/.changeset/paths-grammar-providers.md index 53596470a..327ee9ba2 100644 --- a/.changeset/paths-grammar-providers.md +++ b/.changeset/paths-grammar-providers.md @@ -3,4 +3,4 @@ "@inkandswitch/patchwork-plugins": patch --- -Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@/…`). `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is removed: it named a private type, so nothing outside the package could implement it. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. +Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@/…`). `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is now a deprecated alias of `PathProvider`. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts index 2025c1578..3b8c5f75f 100644 --- a/core/filesystem/src/providers.ts +++ b/core/filesystem/src/providers.ts @@ -64,7 +64,7 @@ const MOUNT_POINT = /^\/[^/@]*$/; /** * Mount point → its bindings, in order. `/` falls back to `rootFolderUrl` when the namespace - * doesn't bind it, so older accounts keep working. Malformed mount points and bindings are + * binds nothing valid there, so older accounts keep working. Malformed mount points and bindings are * skipped with a warning. */ export function accountBindings( @@ -85,7 +85,7 @@ export function accountBindings( bindings.set(mount, urls); } const root = asLink(doc.rootFolderUrl); - if (!bindings.has("/") && root) bindings.set("/", [root]); + if (!bindings.get("/")?.length && root) bindings.set("/", [root]); return bindings; } @@ -149,14 +149,14 @@ export const folderProvider: PathProvider = { }, lookup(node, names) { const entry = (node as FolderDoc).docs.find( - (doc) => String(doc.name) === names[0] + (doc) => doc && String(doc.name) === names[0] ); const link = asLink(entry?.url); return link ? { child: { name: names[0], link }, consumed: 1 } : undefined; }, list(node) { return (node as FolderDoc).docs.flatMap((doc) => { - const link = asLink(doc.url); + const link = asLink(doc?.url); return link ? [{ name: String(doc.name), link }] : []; }); }, diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index d347b3114..1d80cf290 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -33,6 +33,9 @@ type Walk = { /** A URL carries heads when it has a fragment: `automerge:abc#heads`. */ const hasHeads = (url: unknown): boolean => String(url).includes("#"); +/** @deprecated Use {@link PathProvider}, which replaces it. */ +export type FolderStrategy = PathProvider; + /** * Resolve `parts` against `node`, a document or a value inside one, using the provider of the * document it belongs to. Following a link hands the rest to that document's own provider. diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index c17458133..b52188189 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -38,6 +38,12 @@ export type WalkEnd = { atDocument: boolean; /** The pinned path, without a leading slash. */ pinned: string; + /** + * True when every document the pinned path names was entered at heads the input gave: the + * input was already pinned and means what `pinned` means. A pinned root followed by names is + * not, since the names resolve at whatever the documents below it hold now. + */ + canonical: boolean; }; /** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ @@ -115,19 +121,19 @@ export async function walkPath( return true; }; - // The run of names starting at i. A doc step that matched no link reads as a name. - const namesFrom = (i: number) => { - const names: string[] = []; - for (let j = i; j < segments.length; j++) { - const segment = segments[j]; - if (segment.kind === "name") names.push(segment.name); - else if (j === i) names.push(decodeURIComponent(segment.raw)); - else break; - } - return names; - }; + // The names from i on, for a lookup that may consume several (a directory key like + // "vendor/react@18.3.1/index.js"). A doc step reads as a name here: one is only taken as a step + // at the front, and only when it matches a link. + const namesFrom = (i: number) => + segments + .slice(i) + .map((segment) => + segment.kind === "name" ? segment.name : decodeURIComponent(segment.raw) + ); const out: string[] = []; + // Set once a document the pinned path names was entered at its current heads. + let live = !root.heads; let at: At | undefined; if (root.kind === "account") { const url = options.accounts.get(root.acctHash); @@ -166,6 +172,7 @@ export async function walkPath( } } if (!member || !guard(member, segments.length - i)) return undefined; + live = true; } if (!next) return undefined; out.push(await stepOf(next.handle)); @@ -181,6 +188,7 @@ export async function walkPath( const next = await enter(link); if (!next) return undefined; out.push(await stepOf(next.handle)); + live = true; at = next; continue; } @@ -219,8 +227,10 @@ export async function walkPath( if (!consumed && !guard(child.link, segments.length - i)) return undefined; const next = await enter(child.link); if (!next) return undefined; - if (next.provider || !consumed) out.push(await stepOf(next.handle)); - else out.push(...written); + if (next.provider || !consumed) { + out.push(await stepOf(next.handle)); + live = true; + } else out.push(...written); at = next; } } @@ -230,23 +240,25 @@ export async function walkPath( node: at.node, atDocument: !at.inside, pinned: out.join("/"), + canonical: !live, }; } /** - * The document a live or pinned path names: the live handle for a live path, a view at the - * pinned heads for a pinned one. Undefined if the path doesn't resolve, or ends at a value inside - * a document. + * The document a live or pinned path names. For a canonical pinned path (see + * {@link WalkEnd.canonical}) that is a view at the heads it was read at; a file reached by name + * after the last pinned document is read at its current heads, or the heads its link carries. + * Anything else gets the live handle. Undefined if the path doesn't resolve, or ends at a value + * inside a document. */ export async function resolveLivePath( repo: Repo, path: string, options: WalkOptions ): Promise | undefined> { - const { root } = parsePath(path); const end = await walkPath(repo, path, options); if (!end?.atDocument) return undefined; - if (root.heads) return end.handle; + if (end.canonical) return end.handle; return repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); } diff --git a/core/filesystem/test/providers.test.ts b/core/filesystem/test/providers.test.ts index 6a2424188..99825cf54 100644 --- a/core/filesystem/test/providers.test.ts +++ b/core/filesystem/test/providers.test.ts @@ -74,6 +74,14 @@ describe("accountProvider", () => { expect(warn).toHaveBeenCalled(); }); + it("falls back to rootFolderUrl when every / binding is malformed", () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const bindings = accountBindings( + account({ rootFolderUrl: R, namespace: { "/": [{ url: "nope" }] } }) + ); + expect(bindings.get("/")).toEqual([R]); + }); + it("lists every mount as a union child", () => { expect(accountProvider.list(doc)).toEqual([ { name: "", union: [R] }, diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts index 29092ecc4..cc975d5e0 100644 --- a/core/filesystem/test/walk.test.ts +++ b/core/filesystem/test/walk.test.ts @@ -7,6 +7,7 @@ import { pinnedPath, pinPath, resolveLivePath, + walkPath, } from "../src/walk.js"; const step = async (handle: DocHandle) => @@ -165,6 +166,37 @@ describe("resolveLivePath", () => { ).toBeUndefined(); }); + it("reaches a directory key whose later segments look like doc steps", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const vendored = file(repo, "react"); + minePkg.change((d: any) => (d["vendor/react@18.3.1/index.js"] = vendored.url)); + const handle = await resolveLivePath( + repo, + `${a}/packages/foo/vendor/react@18.3.1/index.js`, + { accounts } + ); + expect(handle?.url).toBe(vendored.url); + }); + + it("skips a null folder entry", async () => { + const { repo, a, accounts, projects, foo } = await fixture(); + projects.change((d: any) => d.docs.unshift(null)); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + expect(handle?.url).toBe(foo.url); + }); + + it("treats a pinned root followed by live steps as live, not pinned", async () => { + const { repo, a, accounts, account, foo } = await fixture(); + const mixed = `${docStep(a, account.heads())}/Projects/foo`; + const end = await walkPath(repo, mixed, { accounts }); + expect(end?.canonical).toBe(false); + // a live path's answer: the live handle, not a view whose heads look pinned + expect((await resolveLivePath(repo, mixed, { accounts }))?.url).toBe(foo.url); + + const canonical = await walkPath(repo, end!.pinned, { accounts }); + expect(canonical?.canonical).toBe(true); + }); + it("returns undefined for a path that ends inside a document", async () => { const { repo, a, accounts, minePkg } = await fixture(); minePkg.change((d: any) => (d.nested = { x: "y" })); From 5ebd6085a9eac940f42bc589e1218f3b2cf8878f Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:37:07 +0000 Subject: [PATCH 09/81] =?UTF-8?q?spike:=20service=20worker=20=E2=86=94=20s?= =?UTF-8?q?hared=20worker=20messaging=20across=20browsers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records, per browser, whether the service worker can reach a module SharedWorker client, and what it learns about the sender on each reply path (controller postMessage, a transferred port, fetch clientId, and a worker-initiated port handoff), alongside a dedicated worker loaded from the same URL. Runs in CI; the results land in the job summary. Also records in the spec that automerge verifies change hashes on load. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/spike-sw-sharedworker.yml | 53 +++++++ ...28-paths-namespace-content-cache-design.md | 12 +- .../e2e/spikes/sw-sharedworker/.gitignore | 1 + .../sw-sharedworker/playwright.config.mjs | 23 +++ .../spikes/sw-sharedworker/public/index.html | 64 ++++++++ .../spikes/sw-sharedworker/public/shared.js | 64 ++++++++ .../e2e/spikes/sw-sharedworker/public/sw.js | 145 ++++++++++++++++++ packages/e2e/spikes/sw-sharedworker/serve.mjs | 31 ++++ .../e2e/spikes/sw-sharedworker/spike.spec.mjs | 25 +++ 9 files changed, 414 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/spike-sw-sharedworker.yml create mode 100644 packages/e2e/spikes/sw-sharedworker/.gitignore create mode 100644 packages/e2e/spikes/sw-sharedworker/playwright.config.mjs create mode 100644 packages/e2e/spikes/sw-sharedworker/public/index.html create mode 100644 packages/e2e/spikes/sw-sharedworker/public/shared.js create mode 100644 packages/e2e/spikes/sw-sharedworker/public/sw.js create mode 100644 packages/e2e/spikes/sw-sharedworker/serve.mjs create mode 100644 packages/e2e/spikes/sw-sharedworker/spike.spec.mjs diff --git a/.github/workflows/spike-sw-sharedworker.yml b/.github/workflows/spike-sw-sharedworker.yml new file mode 100644 index 000000000..d29dc6807 --- /dev/null +++ b/.github/workflows/spike-sw-sharedworker.yml @@ -0,0 +1,53 @@ +name: spike sw-sharedworker + +# Records how each browser lets a service worker and a module SharedWorker +# message each other, and what it tells the service worker about the sender. +# See packages/e2e/spikes/sw-sharedworker. The JSON lands in the job summary. + +permissions: + contents: read + +on: + workflow_dispatch: + push: + paths: + - packages/e2e/spikes/sw-sharedworker/** + - .github/workflows/spike-sw-sharedworker.yml + +jobs: + spike: + name: spike (${{ matrix.browser }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + run_install: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + + - name: install the e2e package + run: pnpm install --frozen-lockfile --filter @inkandswitch/patchwork-e2e + + - name: install ${{ matrix.browser }} + working-directory: packages/e2e + run: pnpm exec playwright install --with-deps ${{ matrix.browser }} + + - name: run the spike + working-directory: packages/e2e + run: pnpm exec playwright test -c spikes/sw-sharedworker/playwright.config.mjs --project=${{ matrix.browser }} + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: spike-sw-sharedworker-${{ matrix.browser }} + path: packages/e2e/spikes/sw-sharedworker/results/ + retention-days: 14 diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 9304878c4..45abfbbc2 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -296,8 +296,10 @@ These replace the deleted `package-load-gate.md` note. - Offline, `servePassthrough` falls back to the URL-keyed `patchwork` cache, which any code on the origin can write. Site assets, the automerge worker's script included, come from there. -- Packages can write the automerge worker's IndexedDB/OPFS storage. Whether - loading a document at given heads verifies change hashes needs checking. +- Packages can write the automerge worker's IndexedDB/OPFS storage. + Automerge verifies change hashes when it loads a document, so tampered + changes don't load as the heads they claim. That check is planned to become + optional; when it does, this gap reopens unless patchwork keeps it on. - A live → pinned redirect answered by a tab can point at the wrong version. It can't poison the store, but it decides which version a tab loads. - Legacy `/automerge%3A…` URLs stay keyed by URL and unchecked. @@ -313,9 +315,11 @@ These replace the deleted `package-load-gate.md` note. worker reply with the service worker able to check its `Client.url`, in Chromium, Firefox and WebKit? Fallback: the worker posts answers to the service worker as `fetch()` requests, whose `clientId` the service worker - can look up. + can look up. Spike: `packages/e2e/spikes/sw-sharedworker`, run in CI by + `.github/workflows/spike-sw-sharedworker.yml`. - Does automerge verify change hashes when loading a document from storage? - This decides how big the storage-tampering gap is. + **Answered: yes**, though it's planned to become optional (see + [Remaining gaps](#remaining-gaps)). ## Testing diff --git a/packages/e2e/spikes/sw-sharedworker/.gitignore b/packages/e2e/spikes/sw-sharedworker/.gitignore new file mode 100644 index 000000000..fbca22537 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs b/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs new file mode 100644 index 000000000..070c5efa8 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs @@ -0,0 +1,23 @@ +import { defineConfig, devices } from "@playwright/test"; + +const PORT = 5199; + +export default defineConfig({ + testDir: ".", + testMatch: "*.spec.mjs", + timeout: 60_000, + retries: 0, + reporter: "list", + use: { baseURL: `http://localhost:${PORT}` }, + webServer: { + command: `node serve.mjs`, + env: { PORT: String(PORT) }, + url: `http://localhost:${PORT}/`, + reuseExistingServer: false, + }, + projects: [ + { name: "chromium", use: devices["Desktop Chrome"] }, + { name: "firefox", use: devices["Desktop Firefox"] }, + { name: "webkit", use: devices["Desktop Safari"] }, + ], +}); diff --git a/packages/e2e/spikes/sw-sharedworker/public/index.html b/packages/e2e/spikes/sw-sharedworker/public/index.html new file mode 100644 index 000000000..e8722eb8e --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/index.html @@ -0,0 +1,64 @@ + + +spike: service worker ↔ shared worker +
running…
+ diff --git a/packages/e2e/spikes/sw-sharedworker/public/shared.js b/packages/e2e/spikes/sw-sharedworker/public/shared.js new file mode 100644 index 000000000..9363d16a3 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/shared.js @@ -0,0 +1,64 @@ +// Loaded twice by the page: as the module SharedWorker the spike is about, and +// as a dedicated Worker from the same URL, the impostor any page on the origin +// can start. The service worker should be able to tell them apart. + +const isShared = + typeof SharedWorkerGlobalScope !== "undefined" && + self instanceof SharedWorkerGlobalScope; +const kind = isShared ? "shared" : "dedicated"; +const sw = self.navigator?.serviceWorker; + +const env = { + kind, + hasNavigatorServiceWorker: !!sw, + hasController: !!sw?.controller, + pingsReceived: [], +}; + +function onPing(event, via) { + const data = event.data; + if (data?.type !== "ping") return; + env.pingsReceived.push(via); + sw?.controller?.postMessage({ type: "pong", nonce: data.nonce }); + event.ports?.[0]?.postMessage({ type: "pong", nonce: data.nonce }); + fetch(`/__spike/pong?nonce=${data.nonce}`).catch(() => {}); +} + +if (sw) { + sw.addEventListener("message", (event) => onPing(event, "navigator.serviceWorker")); + sw.startMessages?.(); +} +// Not expected to fire for a service worker's postMessage; recorded in case. +self.addEventListener("message", (event) => onPing(event, "self")); + +// Worker-initiated: hand the service worker a port to call us on. +const { port1, port2 } = new MessageChannel(); +port1.onmessage = (event) => { + if (event.data?.type === "ping-on-hello-port") { + port1.postMessage({ type: "pong-on-hello-port", nonce: event.data.nonce }); + } +}; +try { + sw?.controller?.postMessage({ type: "hello", kind }, [port2]); + env.helloPosted = !!sw?.controller; +} catch (error) { + env.helloPosted = `threw: ${error}`; +} + +const ready = (async () => { + env.probe = await fetch("/__spike/probe") + .then((response) => response.text()) + .catch((error) => `error: ${error}`); + env.fetchesReachServiceWorker = env.probe === "sw"; + await fetch(`/__spike/hello?kind=${kind}`).catch(() => {}); + return env; +})(); + +if (isShared) { + self.addEventListener("connect", (event) => { + const port = event.ports[0]; + ready.then((env) => port.postMessage({ type: "env", env })); + }); +} else { + ready.then((env) => self.postMessage({ type: "env", env })); +} diff --git a/packages/e2e/spikes/sw-sharedworker/public/sw.js b/packages/e2e/spikes/sw-sharedworker/public/sw.js new file mode 100644 index 000000000..7d645a3e7 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/sw.js @@ -0,0 +1,145 @@ +// Spike: can the service worker talk to a SharedWorker client and tell, from +// what the platform reports, that an answer came from that worker? +// +// Three reply paths are measured for a service-worker-initiated ping: +// controller: the worker posts to navigator.serviceWorker.controller, and +// we read event.source (a Client: url, type) +// port: the worker answers on a MessagePort we sent with the ping +// fetch: the worker fetch()es /__spike/pong, and we look its clientId up +// and one worker-initiated path: +// hello: the worker posts a MessagePort to the controller at startup; +// we check event.source, keep the port, and ping over it. + +const WORKER_PATH = "/shared.js?v=spike"; + +self.addEventListener("install", () => self.skipWaiting()); +self.addEventListener("activate", (event) => + event.waitUntil(self.clients.claim()) +); + +const describe = (source) => { + if (!source) return null; + if (typeof Client !== "undefined" && source instanceof Client) { + return { kind: "Client", url: source.url, type: source.type, id: source.id }; + } + if (typeof MessagePort !== "undefined" && source instanceof MessagePort) { + return { kind: "MessagePort" }; + } + return { kind: Object.prototype.toString.call(source) }; +}; + +const replies = new Map(); // nonce -> { via -> observation } +const record = (nonce, via, observation) => { + if (!replies.has(nonce)) replies.set(nonce, {}); + replies.get(nonce)[via] = { at: Date.now(), ...observation }; +}; +const collect = (nonce, ms) => + new Promise((resolve) => setTimeout(() => resolve(replies.get(nonce) ?? {}), ms)); + +const hellos = []; // { source, port } +const helloFetches = []; +const forged = []; + +self.addEventListener("message", (event) => { + const data = event.data ?? {}; + switch (data.type) { + case "run": + event.waitUntil(run(event.ports[0])); + break; + case "hello": + hellos.push({ + kind: data.kind, + source: describe(event.source), + hasPort: !!event.ports[0], + port: event.ports[0], + }); + break; + case "pong": + record(data.nonce, "controller", { source: describe(event.source) }); + break; + case "forged-pong": + forged.push({ source: describe(event.source) }); + break; + } +}); + +self.addEventListener("fetch", (event) => { + const url = new URL(event.request.url); + if (!url.pathname.startsWith("/__spike/")) return; + event.respondWith( + (async () => { + const client = event.clientId + ? await self.clients.get(event.clientId) + : undefined; + const observation = { + clientId: event.clientId, + client: describe(client), + }; + if (url.pathname === "/__spike/pong") { + record(url.searchParams.get("nonce"), "fetch", observation); + } else if (url.pathname === "/__spike/hello") { + helloFetches.push({ kind: url.searchParams.get("kind"), ...observation }); + } + // The worker compares this with what the server says, to tell whether + // its fetches reach us at all. + return new Response("sw", { headers: { "cache-control": "no-store" } }); + })() + ); +}); + +async function pingOverHelloPort(hello) { + const nonce = crypto.randomUUID(); + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 3000); + hello.port.onmessage = (event) => { + if (event.data?.nonce !== nonce) return; + clearTimeout(timer); + resolve(true); + }; + hello.port.postMessage({ type: "ping-on-hello-port", nonce }); + }); +} + +async function run(reply) { + const results = { workerHref: new URL(WORKER_PATH, self.location).href }; + + const all = await self.clients.matchAll({ type: "all", includeUncontrolled: true }); + results.allClients = all.map(describe); + results.controlledSharedWorkers = ( + await self.clients.matchAll({ type: "sharedworker" }) + ).map(describe); + results.allSharedWorkers = ( + await self.clients.matchAll({ type: "sharedworker", includeUncontrolled: true }) + ).map(describe); + + const target = all.find((client) => client.type === "sharedworker"); + results.sharedWorkerFound = !!target; + results.sharedWorkerUrlMatches = target?.url === results.workerHref; + + if (target) { + const nonce = crypto.randomUUID(); + const { port1, port2 } = new MessageChannel(); + port1.onmessage = (event) => { + if (event.data?.nonce === nonce) { + record(nonce, "port", { source: describe(event.source) }); + } + }; + try { + target.postMessage({ type: "ping", nonce }, [port2]); + results.swToWorkerPostMessage = "sent"; + } catch (error) { + results.swToWorkerPostMessage = `threw: ${error}`; + } + results.swToWorkerReplies = await collect(nonce, 3000); + } + + results.hellos = hellos.map(({ port, ...rest }) => rest); + results.helloFetches = helloFetches; + const sharedHello = hellos.find((hello) => hello.source?.type === "sharedworker"); + results.helloPortRoundTrip = sharedHello?.port + ? await pingOverHelloPort(sharedHello) + : null; + results.forgedFromWindow = forged; + + reply.postMessage(results); +} diff --git a/packages/e2e/spikes/sw-sharedworker/serve.mjs b/packages/e2e/spikes/sw-sharedworker/serve.mjs new file mode 100644 index 000000000..9d5e8b685 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/serve.mjs @@ -0,0 +1,31 @@ +// A static server for the spike. /__spike/* answers "server", so a worker +// can tell whether its fetches went through the service worker. +import { createServer } from "node:http"; +import { readFile } from "node:fs/promises"; +import { extname, join, normalize } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("./public/", import.meta.url)); +const port = Number(process.env.PORT ?? 5199); +const types = { ".html": "text/html", ".js": "text/javascript" }; + +createServer(async (request, response) => { + const { pathname } = new URL(request.url, "http://localhost"); + const headers = { "cache-control": "no-store" }; + if (pathname.startsWith("/__spike/")) { + response.writeHead(200, headers).end("server"); + return; + } + const file = normalize(join(root, pathname === "/" ? "index.html" : pathname)); + if (!file.startsWith(root)) { + response.writeHead(403).end(); + return; + } + try { + const body = await readFile(file); + headers["content-type"] = types[extname(file)] ?? "application/octet-stream"; + response.writeHead(200, headers).end(body); + } catch { + response.writeHead(404).end(); + } +}).listen(port, () => console.log(`spike on http://localhost:${port}`)); diff --git a/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs b/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs new file mode 100644 index 000000000..c782a173b --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs @@ -0,0 +1,25 @@ +import { test, expect } from "@playwright/test"; +import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; + +// Records what each browser does; it only fails if the page never reports. +test("service worker ↔ shared worker messaging", async ({ page, browserName }) => { + page.on("console", (message) => console.log(`[${browserName}] ${message.text()}`)); + await page.goto("/"); + const handle = await page.waitForFunction(() => window.__results, null, { + timeout: 45_000, + }); + const results = await handle.jsonValue(); + const json = JSON.stringify(results, null, 2); + console.log(json); + + const dir = new URL("./results/", import.meta.url); + mkdirSync(dir, { recursive: true }); + writeFileSync(new URL(`${browserName}.json`, dir), json); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `### ${browserName}\n\n\`\`\`json\n${json}\n\`\`\`\n` + ); + } + expect(results.error).toBeUndefined(); +}); From e66af9492e6dd8f52d1b0b19070eceaac3a0c5fe Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:55:20 +0000 Subject: [PATCH 10/81] =?UTF-8?q?spec:=20record=20the=20service=20worker?= =?UTF-8?q?=20=E2=86=94=20shared=20worker=20spike=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only a fetch() from the worker, checked by clientId → Client type and url, works in Chromium, Firefox and WebKit. Index answers go that way; requests can stay broadcast. Co-Authored-By: Claude Opus 5.5 --- ...28-paths-namespace-content-cache-design.md | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 45abfbbc2..0ac4e5eaa 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -285,9 +285,18 @@ change between two reads. `sites/bench` gets a case measuring it. - **The index channel is authenticated.** Today's `HANDOFF_CHANNEL` is a BroadcastChannel: any context on the origin can listen, see request ids, and reply first with a forged response. That is a hole in the current code - as well. Index answers must arrive from a `Client` whose `url` is the - automerge worker's hashed script URL, which the service worker knows at - build time. + as well. Index answers must arrive from a `Client` whose `type` is + `sharedworker` and whose `url` is the automerge worker's script URL, which + the service worker knows at build time. + - The worker sends answers as `fetch()` requests to a reserved path. The + service worker looks up `event.clientId` with `clients.get` and checks + the client. This is the only reply path that works in all three engines + (see [Spikes](#spikes-before-planning)). + - Requests can stay on a BroadcastChannel. A listener learns which pinned + URLs are being loaded, which it could see anyway, but it can't answer. + - Checking `type` is not optional. Any page can start a dedicated `Worker` + from the same script URL, and it shows up with the same `url` and + `type: "worker"`. ## Remaining gaps @@ -316,7 +325,28 @@ These replace the deleted `package-load-gate.md` note. Chromium, Firefox and WebKit? Fallback: the worker posts answers to the service worker as `fetch()` requests, whose `clientId` the service worker can look up. Spike: `packages/e2e/spikes/sw-sharedworker`, run in CI by - `.github/workflows/spike-sw-sharedworker.yml`. + `.github/workflows/spike-sw-sharedworker.yml` (run 40640485949: Chrome 147, + Firefox 148, Safari 26.4). + + **Answered: only the fetch path works everywhere.** + + | | Chromium | Firefox | WebKit | + |---|---|---|---| + | Worker is a controlled `sharedworker` client, `url` includes the query | yes | yes | yes | + | `navigator.serviceWorker` in the worker | no | yes | yes | + | `navigator.serviceWorker.controller` in the worker | no | null | yes | + | Service worker `client.postMessage` reaches the worker | no | yes | yes | + | Worker replies with `controller.postMessage`, `event.source` checkable | no | no | yes | + | Worker replies on a port sent with the ping (`source` is null) | no | yes | yes | + | Worker `fetch()`, `clients.get(clientId)` gives `sharedworker` and its url | yes | yes | yes | + | Dedicated `Worker` from the same URL: same `url`, `type: "worker"` | yes | yes | yes | + + Chromium doesn't expose `ServiceWorkerContainer` in workers, so the service + worker can't message the automerge worker at all there. In Firefox the + worker's first `fetch()`, made while its module was still evaluating, went + to the network instead of the service worker. Its later fetches were + intercepted. So answers should be retried, or the worker should hold them + until it has seen one of its own fetches answered by the service worker. - Does automerge verify change hashes when loading a document from storage? **Answered: yes**, though it's planned to become optional (see [Remaining gaps](#remaining-gaps)). From 3d43fa44cf7acd5dcf37d9cf26dfcbe915c7bd47 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 05:59:05 +0000 Subject: [PATCH 11/81] spec: a port relayed through a tab doesn't authenticate the worker Co-Authored-By: Claude Opus 5.5 --- .../2026-10-28-paths-namespace-content-cache-design.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 0ac4e5eaa..a000889a3 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -294,6 +294,12 @@ change between two reads. `sites/bench` gets a case measuring it. (see [Spikes](#spikes-before-planning)). - Requests can stay on a BroadcastChannel. A listener learns which pinned URLs are being loaded, which it could see anyway, but it can't answer. + - A `MessagePort` relayed through a tab doesn't authenticate anything. + Messages on a port have `source: null`, and the tab runs package code, + which can keep the port and answer itself, or sit in the middle and edit + answers. A relayed port is only safe if every answer on it is signed + with a key the worker got from the response to a checked `fetch()`. Not + worth that machinery unless one fetch per manifest proves too slow. - Checking `type` is not optional. Any page can start a dedicated `Worker` from the same script URL, and it shows up with the same `url` and `type: "worker"`. From 43d0fecbcc74e3e8e1d31b4a8bd9434b88f3dbb8 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:06:57 +0000 Subject: [PATCH 12/81] plan: paths phase 2 (service worker, content store, index) Co-Authored-By: Claude Opus 5.5 --- ...2026-10-28-paths-phase-2-service-worker.md | 621 ++++++++++++++++++ 1 file changed, 621 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md b/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md new file mode 100644 index 000000000..43d850ad8 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md @@ -0,0 +1,621 @@ +# Paths phase 2: service worker, content store, index — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** The service worker serves `/@/` URLs: live ones redirect (307) to their pinned form, and pinned ones come from a content-addressed store. Every read from that store is checked against an index that only the automerge worker can fill. + +**Architecture:** +- `patchwork-filesystem` learns to read one file at a path (`readPath`) and to list every file under a pinned document (`listFiles`). +- The bootloader gets four small modules the service worker wires together, unit-tested under vitest: + - `content-hash.ts`: the one hash function. + - `content-store.ts`: the store. + - `pinned.ts`: key normalization, the index, and the request flow. + - `index-channel.ts`: checks who sent an answer and posts answers. +- The automerge worker builds manifests (`manifest.ts`) and answers them with an authenticated `fetch()`. +- Resolvers, in tabs and in the worker, stop writing the cache. They return bytes and the service worker decides what to keep. + +**Tech Stack:** TypeScript; `@automerge/automerge-repo` 2.6.0-subduction.48; the service worker and Cache Storage APIs; `crypto.subtle`; `isUnprotectedDocId` from `@automerge/automerge-repo-keyhive`; vitest in node; Playwright e2e and bench in CI only. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. This plan covers section 3 *URLs*, section 5 in full, and **Implementation phases → 2**. Read the spike results under *Spikes before planning*: they decide the channel. Phase 1's plan is `docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md`, and its walker is in `core/filesystem/src/walk.ts`. + +## Global Constraints + +- Store: a CacheStorage cache named `patchwork-content-`. Key `/.content/`, on this origin, never fetched. Value: the bytes only. +- Store reads and writes go through `get(hash)` / `put(hash, bytes)` (plus `delete(hash)`), so they can be encrypted later. +- Index: a `Map` in service worker memory from pinned URL to `{ sha256, contentType }` (plus `store`; see Decisions). Only the automerge worker fills it. +- A store hit is hashed with `crypto.subtle` SHA-256 **on every read**. A mismatch deletes the entry, warns loudly, and goes back to a resolver. +- Bytes a resolver returns are hashed in the service worker: + - matching the index entry: serve, and store if the entry allows; + - not matching: a network error, nothing stored; + - no index entry (no automerge worker): serve inline, store nothing. +- What gets stored: content from unprotected documents (`isUnprotectedDocId`), and keyhive-protected content only when it's reached through `/packages`. +- Index answers count only from a `Client` with `type === "sharedworker"` and the automerge worker's script URL. The worker sends them as `fetch()` requests to a reserved path, and the service worker looks up `event.clientId`. Requests may ride the BroadcastChannel. +- A live `/@/` URL answers 307 to its pinned form. A pinned URL never redirects. +- Legacy `/automerge%3A…/` URLs keep working, keyed by URL and unchecked. +- `clearAccountStore(acctHash)` deletes the account's store. +- Stale workers outlive deploys, so `HANDOFF_CHANNEL` messages and handoff replies only change additively (`core/bootloader/src/types.ts`, `HandoffRequest`). +- No local e2e, `vite preview` or shell builds. Unit tests and typechecks only; CI runs the rest. + +## Decisions this plan makes that the spec leaves open + +1. **The worker's script URL is fixed, not hashed.** The vite plugin serves `/automerge-protocol-handler-worker.js` under that name in every build (`core/patchwork/src/vite/service-worker-plugin.ts`, `workers`). The service worker compares the client's origin and pathname with a constant `AUTOMERGE_WORKER_PATH`, and ignores the query. A site that overrides `workerPath` in `setupServiceWorker` gets its answers rejected: everything is then served inline and nothing is stored. The spec's word "hashed" is corrected in Task 9. +2. **Answers go to `POST /@/.index`.** `.` isn't base58, so it can't be an acctHash. The service worker replies `204` with `x-patchwork-index: accepted`. A reply without that header means the fetch skipped the service worker, as the spike saw in Firefox. The worker then retries 5 times, 200ms apart. +3. **Manifest requests ride `HANDOFF_CHANNEL`** as a new message type, `manifest`. Old workers ignore it, since they only read `type: "request"`. +4. **The manifest covers the last document the URL enters at its heads.** Its entries are keyed by the tail (the names after the last doc step, each `encodeURIComponent`'d, joined with `/`). The document itself is the empty tail `""`. Containers below it aren't listed: they have their own doc steps. The worker returns the **canonical** prefix, and the service worker indexes `prefix + (tail ? "/" + tail : "")`. If the prefix the service worker guessed isn't canonical (a name that looks like a doc step), the request misses the index and is served inline, unstored. That's safe. +5. **The store flag is the worker's call**, made per entry: + - `store = rootIsAccount && (unprotected(contentDocId) || prefixSegments[1] === "packages")`. + - A share root (`automerge:@h`) never stores. +6. **The tab tells its account to the resolvers.** `registerAccount(url)` in the bootloader adds the account to the tab's own resolver and posts `{ type: "register-account", url }` to the automerge worker. It posts again whenever the worker respawns. The worker computes the acctHash itself, so a tab can't map a hash to the wrong document. This settles phase 1's carry-over "the accounts map isn't checked". +7. **Waiting for documents moves into the walk.** `WalkOptions` gains `find?: (url, heads?) => Promise`. The bootloader passes one that waits for heads (pinned) or for peers to catch up (live), within the resolver's 30s signal. This settles phase 1's carry-over "no signal / waitForHeads". +8. **Legacy replies are inline.** For `automerge:` URLs the resolver now always answers `type: "response"`. The service worker caches it by URL itself when it's immutable (status 200/203/204, no `no-store`). The service worker still accepts `type: "cached"` from older workers. +9. **The manifest wait is bounded at 10s** (`MANIFEST_TIMEOUT_MS`). After that the request is served as if there were no worker. +10. **Cold start cost, accepted and measured.** The index lives in memory, so after a browser restart every package needs a manifest, and building one loads and hashes every file document in the package. Today the URL-keyed cache serves those without loading any document. Task 8's bench measures it. If it's too slow, persisting manifests is the follow-up, and it reopens a gap. That trade-off isn't this plan's to make. + +## Review Focus + +- **Request URL spellings:** browsers send `@`, `:` and non-ASCII in path segments in different encodings (`a@b.js` vs `a%40b.js`, `日本` vs `%E6%97%A5%E6%9C%AC`). Every spelling of one file must hit one index key. Pinned in Task 3 (`normalizePinned`). +- **The service worker stops mid-session:** the index is empty again and the worker's LRU answers. Content is identical, nothing is re-stored that was already there, and no request hangs. Pinned in Task 3 (fresh index, warm store). +- **No worker, or a worker that is slow to boot:** the manifest times out, the request is served inline within the timeout, and nothing is stored. Pinned in Task 3. +- **Many files of one package requested at once:** exactly one manifest request. Pinned in Task 3. +- **Non-ASCII string content:** the worker hashes the same UTF-8 bytes the service worker receives. Pinned in Task 4. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `core/filesystem/src/walk.ts` (modify) | `WalkOptions.find`; module-level idhash memo; `readPath`; `listFiles` | +| `core/filesystem/src/providers.ts` (modify) | `accountBindings` memoized per doc object, warning once per bad mount | +| `core/filesystem/src/resolve.ts` (modify) | export `materialize` for `walk.ts` | +| `core/bootloader/vitest.config.ts`, `core/bootloader/test/setup.ts` (create), `package.json` (modify) | test infra (same shape as `core/filesystem`) | +| `core/bootloader/src/content-hash.ts` (create) | `contentHash(bytes)`, `toBytes(content)` | +| `core/bootloader/src/content-store.ts` (create) | `ContentStore`, `cacheStore`, `contentCacheName`, `clearAccountStore` | +| `core/bootloader/src/pinned.ts` (create) | `normalizePinned`, `indexManifest`, `servePinned` | +| `core/bootloader/src/index-channel.ts` (create) | `AUTOMERGE_WORKER_PATH` check, `acceptIndexAnswer`, `postIndexAnswer` | +| `core/bootloader/src/manifest.ts` (create) | `buildManifest`: the worker's side | +| `core/bootloader/src/types.ts` (modify) | `Manifest`, `IndexEntry`, `ManifestRequestMessage`, `IndexAnswer`, `HandoffRequest.path`, `RegisterAccountMessage` | +| `core/bootloader/src/resolver.ts` (modify) | answers `path` requests (307 or bytes), legacy inline, no `cache.put`, `find` with waits | +| `core/bootloader/src/tab-resolver.ts`, `setup.ts` (modify) | `registerAccount`, account map, re-register on respawn | +| `core/bootloader/src/automerge-protocol-handler.ts` (modify) | accounts, manifest requests, LRU, answers via `postIndexAnswer` | +| `core/bootloader/src/service-worker.ts` (modify) | `/@/` routing, `POST /@/.index`, legacy caching moved in, content caches survive `clearOtherCaches` | +| `core/patchwork/src/index.ts` (modify) | call `registerAccount(accountDocHandle.url)` | +| `sites/bench/tests/content-hash.spec.ts` (create) | hashing cost on a hit | +| `packages/e2e/tests/content-store.spec.ts` (create) | forged store entry and forged answer | + +--- + +### Task 1: Walk carry-overs and `WalkOptions.find` + +**Files:** Modify `core/filesystem/src/walk.ts`, `core/filesystem/src/providers.ts`. Test `core/filesystem/test/walk.test.ts`, `core/filesystem/test/providers.test.ts`. + +**Interfaces:** +- Produces: `WalkOptions.find?: (url: AutomergeUrl, heads?: UrlHeads) => Promise>`. It returns a handle that `enter` then `.view()`s at `heads ?? link heads ?? current`. The default is `repo.find` of the bare documentId url, as `enter` does now. `idhash` results are memoized in a module-level `Map>`, cleared when it passes 10_000 entries. + +- [ ] **Step 1: Write the failing tests** + +```ts +it("walks through the find it's given", async () => { + const { repo, a, accounts } = await fixture(); + const seen: string[] = []; + const find = (url: AutomergeUrl) => (seen.push(url), repo.find(url)); + await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts, find }); + expect(seen.length).toBeGreaterThanOrEqual(4); // account, /, Projects, foo +}); +it("fails, not throws, when find rejects", ...) + // find = () => Promise.reject(new Error("offline")) → resolveLivePath → undefined +``` + +```ts +it("warns about a malformed mount once, however often it's read", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doc = account({ namespace: { "/x@y": [{ url: R }], "/": [{ url: R }] } }); + for (let i = 0; i < 5; i++) accountProvider.lookup(doc, ["a"]); + expect(warn).toHaveBeenCalledTimes(1); +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts test/providers.test.ts` +Expected: FAIL. `find` is never called (seen is empty), and warn is called 5 times. + +- [ ] **Step 3: Implement** + +- `accountBindings`: memoize in a `WeakMap>` keyed by the doc object. +- Warn once per malformed mount key, using a module-level `Set`. +- `enter` calls `(options.find ?? defaultFind)(bareUrl, heads)` inside its existing `try`. + +- [ ] **Step 4: Run them to see them pass, plus the whole filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS. + +- [ ] **Step 5: Commit** `filesystem: walk takes a find, and remembers idhashes and warnings` + +--- + +### Task 2: `readPath` and `listFiles` + +**Files:** Modify `core/filesystem/src/walk.ts`, and `core/filesystem/src/resolve.ts` (export `materialize`). Test `core/filesystem/test/files.test.ts`. + +**Interfaces:** +- Consumes: `walkPath`, `WalkEnd`, `providerFor`, `Child` (phase 1); `materialize(repo, node, typeHint?, walk?)` from `resolve.ts`. +- Produces: + ```ts + export type FileContent = { content: string | Uint8Array; type: string }; + /** What a path names, read: its pinned form, whether the input was canonical, and the bytes. */ + export function readPath(repo: Repo, path: string, options: WalkOptions): + Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined>; + export type ListedFile = FileContent & { tail: string; documentId: DocumentId }; + /** + * Every file under the last document a canonical pinned path enters, keyed by tail: "" for the + * document itself, encoded names joined with "/" below it. Containers aren't listed; they have + * their own doc steps. Undefined unless `path` is canonical and ends at a document. + */ + export function listFiles(repo: Repo, path: string, options: WalkOptions): + Promise<{ prefix: string; files: AsyncIterable } | undefined>; + ``` +- `documentId` is the document holding the bytes: the leaf doc for a link, the enclosing doc for a value. + +`listFiles` enumeration (the walk doesn't determine this): + +``` +emit("", end.handle.doc(), end.handle.documentId) +visit(node, names, docId) for each child of provider.list(node): + value child, object (not bytes, not ImmutableString) → visit(value, [...names, ...name.split("/")], docId) + value child, otherwise → emit(join(names + name parts), value, docId) + link child → doc = find(link) at link heads or current; providerFor(doc) ? skip : emit(tail, doc, doc's id) + union child → skip +tail = parts.map(encodeURIComponent).join("/") +If two entries produce the same tail, the one whose directory key had more "/"-separated parts wins, matching the walk's longest-prefix lookup. +``` + +- [ ] **Step 1: Write the failing tests** + +```ts +it("reads a file and its pinned form", ...) + // readPath(repo, livePath(a, ["packages","foo","dist","index.js"])) → + // content "mine", type "text/plain", canonical false, pinned === pinnedPath(...) +it("lists a directory package's files", ...) + // tails sorted → ["", "dist/index.js", "package.json"]; each documentId is the file doc's +it("lists a folder's leaves and skips its subfolders", ...) + // Projects/foo has notes (file) and sub (folder) → tails ["", "notes"] +it("lists nested values inside a directory doc", ...) + // d.dist = { "a.js": "x" } with mimeType hint absent → tail "dist/a.js" +it("encodes names in tails", ...) // key "my file@1.js" → tail "my%20file%401.js" +it("agrees with readPath for every file it lists", async () => { + // for the directory package and the folder: for each listed file f, + // readPath(prefix + (f.tail ? "/" + f.tail : "")) has the same content and type +}); +it("lists nothing for a path that isn't canonical", ...) // live path → undefined +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/files.test.ts` +Expected: FAIL, no export `readPath`. + +- [ ] **Step 3: Implement both in `walk.ts`, exported from `index.ts` through the existing `export *`** + +- [ ] **Step 4: Run the filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS. + +- [ ] **Step 5: Commit** `filesystem: readPath and listFiles` + +--- + +### Task 3: Store, index, and the request flow (bootloader, pure) + +**Files:** +- Create `core/bootloader/vitest.config.ts` and `core/bootloader/test/setup.ts`, copied from `core/filesystem`. +- Add `"test": "vitest run"` to `core/bootloader/package.json`. +- Create `src/content-hash.ts`, `src/content-store.ts` and `src/pinned.ts`; modify `src/types.ts`. +- Test `core/bootloader/test/pinned.test.ts` and `test/content-store.test.ts`. + +**Interfaces:** +- Produces: + ```ts + // types.ts + export type IndexEntry = { sha256: string; contentType: string; store: boolean }; + export type Manifest = { prefix: string; entries: Record }; + + // content-hash.ts + export function toBytes(content: string | Uint8Array): Uint8Array; // strings → UTF-8 + export function contentHash(bytes: Uint8Array): Promise; // SHA-256, base64url, no padding + + // content-store.ts + export interface ContentStore { + get(hash: string): Promise; + put(hash: string, bytes: Uint8Array): Promise; + delete(hash: string): Promise; + } + export const contentCacheName = (acctHash: string) => `patchwork-content-${acctHash}`; + export const CONTENT_CACHE_PREFIX = "patchwork-content-"; + export function cacheStore(cache: Cache, origin: string): ContentStore; // key `${origin}/.content/${hash}` + export function clearAccountStore(acctHash: string): Promise; // caches.delete + + // pinned.ts + /** The `/@/` path, normalized so every spelling of one file is one key. */ + export function normalizePinned(path: string): + { key: string; prefix: string; tail: string; acctHash?: string; pinned: boolean } | undefined; + export function indexManifest(index: Map, manifest: Manifest): void; + export class PinnedAbort extends Error {} + export type PinnedDeps = { + index: Map; + store: ContentStore | undefined; // undefined: nothing is read or written (share roots) + hash(bytes: Uint8Array): Promise; + manifest(key: string): Promise; // deduped by the caller, see below + resolve(key: string): Promise<{ body: Uint8Array; contentType: string } | undefined>; + warn(message: string): void; + keep(work: Promise): void; // fetchEvent.waitUntil + }; + export function servePinned(key: string, deps: PinnedDeps): Promise; // throws PinnedAbort + /** Wrap a manifest fetcher so concurrent calls for one prefix share a request. */ + export function sharedManifests(fetch: (key: string) => Promise, prefixOf: (key: string) => string): + (key: string) => Promise; + ``` + +`normalizePinned` rules: +- Split on `/`, dropping empty segments. +- The root is pinned if it has a literal `@` or decodes to `automerge:`. A share root is written `automerge:` with the colon decoded. +- `prefix` runs up to and including the last segment that matches `/^[1-9A-HJ-NP-Za-km-z]+@[1-9A-HJ-NP-Za-km-z.]+$/`. +- In the prefix, plain names are re-encoded (`encodeURIComponent(decodeURIComponent(s))`) and doc steps are kept verbatim. +- Every tail segment is decoded and re-encoded the same way. +- `key = prefix + (tail ? "/" + tail : "")`. +- Undefined on a malformed escape. + +`servePinned` follows spec section 5's *Request flow* exactly. The response headers are `content-type: entry.contentType` and `cache-control: public, max-age=31536000, immutable`. Without an index entry it's `no-store`. + +- [ ] **Step 1: Write the failing tests** (fake store: a `Map`, real `contentHash`) + +```ts +describe("normalizePinned", () => { + it("gives one key for every spelling of a file", () => { + const keys = ["acct@h/x@h/a@b.js", "acct@h/x@h/a%40b.js", "/acct@h//x@h/a%40b.js/"] + .map((p) => normalizePinned(p)!.key); + expect(new Set(keys).size).toBe(1); + expect(normalizePinned("acct@h/x@h/日本")!.key).toBe(normalizePinned("acct@h/x@h/%E6%97%A5%E6%9C%AC")!.key); + }); + it("splits at the last doc step", () => { + expect(normalizePinned("acct@h/packages/m@h/f@h/dist/index.js")).toMatchObject({ + prefix: "acct@h/packages/m@h/f@h", tail: "dist/index.js", acctHash: "acct", pinned: true, + }); + }); + it("says a live path is live", () => expect(normalizePinned("acct/Projects")!.pinned).toBe(false)); + it("reads a share root", ...) // "automerge%3A@h/x" → prefix starts "automerge:", acctHash undefined +}); + +describe("servePinned", () => { + it("serves a store hit whose bytes match, without asking a resolver", ...) + it("deletes a forged store entry, warns, and serves the resolver's real bytes", ...) + // store.put(realHash, forged); index has realHash → body === real; warn called; store now holds real + it("asks for one manifest on an index miss, then serves", ...) + it("stores resolver bytes that match an entry that allows it", ...) + it("never stores when the entry says not to", ...) + it("fails as a network error when resolver bytes don't match, storing nothing", ...) + // rejects with PinnedAbort; store empty + it("serves inline and stores nothing with no manifest", ...) + // manifest → undefined; resolve → bytes; response cache-control no-store; store empty; index empty + it("rebuilds the index after a restart and re-uses the store", ...) + // second deps with empty index, same store: one manifest, zero resolves, same bytes + it("shares one manifest among concurrent requests for a package", ...) + // sharedManifests: 5 parallel servePinned for 5 files of one prefix → fetch called once +}); + +describe("cacheStore", () => { + it("round-trips bytes under /.content/", ...) // fake Cache with put/match/delete over a Map keyed by url +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, missing modules. + +- [ ] **Step 3: Implement the three modules and the types** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: content store, index and the pinned request flow` + +--- + +### Task 4: Manifests and the authenticated answer channel + +**Files:** Create `core/bootloader/src/manifest.ts` and `core/bootloader/src/index-channel.ts`; modify `src/types.ts`. Test `test/manifest.test.ts` and `test/index-channel.test.ts`. + +**Interfaces:** +- Consumes: `listFiles`, `WalkOptions` (Task 2); `contentHash`, `toBytes`, `Manifest` (Task 3). +- Produces: + ```ts + // types.ts + export const AUTOMERGE_WORKER_PATH = "/automerge-protocol-handler-worker.js"; // setup.ts's default uses it + export const INDEX_ANSWER_PATH = "/@/.index"; + export interface ManifestRequestMessage { type: "manifest"; id: string; path: string } + export type IndexAnswer = { id: string; manifest?: Manifest; error?: string }; + export interface RegisterAccountMessage { type: "register-account"; url: AutomergeUrl } + + // manifest.ts + export function buildManifest(repo: Repo, path: string, options: WalkOptions & { + isProtected?: (documentId: DocumentId) => boolean; // default: !isUnprotectedDocId(documentIdToBinary(id)) + }): Promise; + + // index-channel.ts + /** True only for the automerge worker: a sharedworker on this origin at AUTOMERGE_WORKER_PATH, any query. */ + export function isAutomergeWorker(client: Pick | undefined, origin: string): boolean; + /** The service worker's side of POST /@/.index: 403 unless the client is the worker, else parse, hand the answer to onAnswer, and reply 204 with x-patchwork-index: accepted. */ + export function acceptIndexAnswer(request: Request, client: Pick | undefined, + origin: string, onAnswer: (answer: IndexAnswer) => void): Promise; + /** The worker's side: POST the answer, retrying (5 × 200ms) until a reply carries x-patchwork-index: accepted. */ + export function postIndexAnswer(answer: IndexAnswer, fetcher?: typeof fetch): Promise; + ``` +- `buildManifest`: + - Takes `listFiles(path)` and hashes `toBytes(content)` for each file. + - Sets `store` per Decision 5. `prefixSegments[1]` is read from `parsePath(prefix).segments[0]`, which must be the name `packages`. + - Returns `{ prefix, entries }`. + +- [ ] **Step 1: Write the failing tests** + +```ts +describe("buildManifest", () => { + it("hashes each file as the service worker will", ...) + // a file doc with content "héllo ✓" → entries["…"].sha256 === await contentHash(new TextEncoder().encode("héllo ✓")) + it("stores unprotected content and, through /packages, protected content", ...) + // isProtected: () => true → entries under a/Projects/... store false; under a/packages/... store true + it("never stores under a share root", ...) + it("returns nothing for a live path", ...) +}); + +describe("isAutomergeWorker", () => { + const o = "https://p.test"; + it.each([ + [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js` }, true], + [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js?x=1` }, true], + [{ type: "worker", url: `${o}/automerge-protocol-handler-worker.js` }, false], // same url, dedicated + [{ type: "window", url: `${o}/` }, false], + [{ type: "sharedworker", url: `${o}/@/acct@h/evil.js` }, false], + [{ type: "sharedworker", url: `https://evil.test/automerge-protocol-handler-worker.js` }, false], + [undefined, false], + ])("%j → %s", (client, ok) => expect(isAutomergeWorker(client as any, o)).toBe(ok)); +}); + +describe("acceptIndexAnswer", () => { + it("refuses a window with 403 and hands nothing on", ...) + it("accepts the worker with 204 and the header", ...) +}); + +describe("postIndexAnswer", () => { + it("retries a reply that skipped the service worker", async () => { + const replies = [new Response("", { status: 404 }), new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } })]; + const fetcher = vi.fn(async () => replies.shift()!); + expect(await postIndexAnswer({ id: "1" }, fetcher)).toBe(true); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + it("gives up after 5 tries", ...) +}); +``` + +Bootloader tests that use a `Repo` need the wasm setup from Task 3's `test/setup.ts`. + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, missing modules. + +- [ ] **Step 3: Implement** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: manifests, and the worker's authenticated answers` + +--- + +### Task 5: Resolvers answer paths and stop writing the cache + +**Files:** Modify `core/bootloader/src/resolver.ts`, `src/types.ts` (`HandoffRequest.path?: string`) and `src/tab-resolver.ts`. Test `test/resolver.test.ts`. + +**Interfaces:** +- Consumes: `readPath`, `pinnedPath`, `walkPath`, `WalkOptions` (filesystem); `toBytes`. +- Produces: + - `HandoffRequest.path?: string`: the `/@/`-stripped path. For these requests the service worker sets `handoffURL` to `patchwork-path:` + path, so an old resolver parses it, sees a protocol that isn't `automerge:`, and declines. + - `createResolver(getRepo, log?, accounts?: () => ReadonlyMap)`: the new third parameter defaults to an empty map. + - `registerAccount(url: AutomergeUrl): Promise` in `tab-resolver.ts` returns the acctHash. It adds the account to the map the tab's resolver reads. Task 6 makes it also post to the worker. +- `answer()`: + - A `path` request that isn't pinned (`parsePath(path).root.heads` is missing), or isn't canonical: + - 307 to `/@/` + `pinnedPath(...)`, with `cache-control: no-store`; + - 404 if it doesn't resolve. + - A canonical pinned `path`: inline `{ status: 200, headers: { "content-type": type }, body: toBytes(content) }`; 404 if it doesn't resolve. + - An `automerge:` URL: as now, but always inline. The `cache-control` header is kept so the service worker can decide. + - No `caches` calls remain in `resolver.ts`. +- The resolver's `find` for the walk: + - With heads: `waitForHeads` (abort → `AbortHandoff`). + - Without heads: `caughtUpWithPeers`. + - Both use `AbortSignal.timeout(RESOLVE_TIMEOUT_MS)`. + +- [ ] **Step 1: Write the failing tests** (node, in-memory `Repo`, no `caches` global: any call would throw) + +```ts +it("redirects a live path to its pinned form", ...) + // answer({ request: { path: livePath(a, ["Projects","foo","notes"]), handoffURL: "patchwork-path:…", … } }) + // → type "response", status 307, headers.location === "/@/" + await pinnedPath(...), cache-control no-store +it("answers a pinned path with its bytes inline", ...) + // status 200, content-type text/plain, body bytes === UTF-8 of "hi" +it("redirects a pinned-looking path that isn't canonical", ...) +it("answers an unknown path with 404", ...) +it("answers legacy automerge URLs inline, without touching caches", ...) + // pinned automerge:#heads/dist/index.js → type "response", cache-control includes immutable +it("registerAccount makes the tab's resolver answer for that account", ...) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL. Path requests return undefined, and the legacy test throws `caches is not defined`. + +- [ ] **Step 3: Implement** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: resolvers answer /@/ paths and leave caching to the service worker` + +--- + +### Task 6: The automerge worker builds and posts manifests; tabs register accounts + +**Files:** +- Modify `core/bootloader/src/automerge-protocol-handler.ts`, `src/setup.ts`, `src/tab-resolver.ts`. +- Modify `core/patchwork/src/index.ts`. +- Test `core/bootloader/test/worker-manifests.test.ts`. + +**Interfaces:** +- Consumes: `buildManifest`, `postIndexAnswer` (Task 4); `ManifestRequestMessage`, `RegisterAccountMessage`. +- Produces: + - `createManifestHandler(getRepo, accounts, post = postIndexAnswer)` in `manifest.ts`. It returns `(message: ManifestRequestMessage) => Promise`, and keeps an LRU of 64 manifests keyed by canonical prefix. Requests already in flight for the same `path` share one build. A failed build posts `{ id, error }`. + - The worker wires it to `HANDOFF_CHANNEL` messages of `type: "manifest"`. It handles `register-account` port messages through `startWorkerControl`'s `onMessage`, adding `[await idhash(documentId), url]` to its accounts map. It counts manifest work in `handoffStarted`/`handoffFinished`, so a recycle waits for it. + - `setup.ts`: `registerAccount` posts `{ type: "register-account", url }` to `automergeProtocolHandlerWorker`. It remembers the url and posts it again from the handle's `onSpawn`. Re-export `registerAccount` and `clearAccountStore` from `setup.ts`, the package's `.` entry. + - `core/patchwork/src/index.ts`: after `resolveAccountHandle`, `await registerAccount(accountDocHandle.url)`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it("builds a manifest once per prefix and posts it", ...) + // two messages for files of one package → post called twice with the same manifest, listFiles walked once (spy on find) +it("posts an error for a path it can't list", ...) +it("evicts the oldest manifest after 64", ...) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, no `createManifestHandler`. + +- [ ] **Step 3: Implement and wire, then typecheck the dependents** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins --filter @inkandswitch/patchwork-bootloader build && pnpm --filter @inkandswitch/patchwork exec tsc --noEmit` +Expected: no errors. + +- [ ] **Step 4: Run the tests** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: the automerge worker answers manifests; tabs register their account` + +--- + +### Task 7: The service worker serves `/@/` + +**Files:** Modify `core/bootloader/src/service-worker.ts`. Test `core/bootloader/test/pinned.test.ts` (one case for the request builder) and a typecheck. + +**Interfaces:** +- Consumes: everything from Tasks 3–5. +- Wiring, in order, in the `fetch` listener: + 1. `POST` to `INDEX_ANSWER_PATH`: `respondWith(acceptIndexAnswer(request, await clients.get(event.clientId), origin, settle))`. `settle` resolves the pending manifest request by `id`, and `indexManifest`s the answer. + 2. `GET` whose pathname starts with `/@/` and isn't `/@/.index`: + - `normalizePinned(path)`; on undefined, 400. + - Live → `handoff()` with `path`. The resolver's 307 is served as is (never cached). + - Pinned → `servePinned(key, deps)`, with: + - `store`: `cacheStore(await caches.open(contentCacheName(acctHash)), origin)`, or undefined for a share root; + - `manifest`: `sharedManifests(requestManifest, prefixOf)`. `requestManifest` posts `{ type: "manifest", id, path: key }` on `HANDOFF_CHANNEL`, calls `startWorkerIfMissing`, and resolves on `settle` or after `MANIFEST_TIMEOUT_MS = 10_000` with undefined; + - `resolve`: `handoff()` with `path: key`, read inline; + - `keep`: `event.waitUntil`. + - `PinnedAbort` → rethrow (a network error). + 3. Everything else, as today. +- `serveHandoff`, for legacy URLs: an inline `response` that is immutable (status 200/203/204, no `no-store` in `cache-control`) is `cacheInBackground` by its request URL. The `cached` branch stays for old workers. +- `clearOtherCaches` keeps every cache whose name starts with `CONTENT_CACHE_PREFIX`. +- Move the handoff-message construction into `pinned.ts` as `handoffRequest(request: Request, path?: string): HandoffRequestMessage` so it's testable. + +- [ ] **Step 1: Write the failing test** + +```ts +it("builds a path handoff that old resolvers decline", () => { + const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "acct/Projects"); + expect(m.request.path).toBe("acct/Projects"); + expect(new URL(m.request.handoffURL).protocol).not.toBe("automerge:"); +}); +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, no export `handoffRequest`. + +- [ ] **Step 3: Wire the service worker as above** + +- [ ] **Step 4: Run the bootloader tests and typecheck** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test && pnpm --filter @inkandswitch/patchwork-bootloader exec tsc --noEmit` +Expected: PASS, no type errors. + +- [ ] **Step 5: Commit** `service worker: serve /@/ paths from the checked content store` + +--- + +### Task 8: E2E and bench (CI) + +**Files:** Create `packages/e2e/tests/content-store.spec.ts` and `sites/bench/tests/content-hash.spec.ts`. + +**Interfaces:** The page exposes `window.patchwork.repo` and `window.patchwork.account`, as the existing e2e helpers use them. The acctHash is base58 of the first 12 bytes of SHA-256 over the account's `documentId`; the test computes it in the page with `crypto.subtle` and a short base58 loop. + +- [ ] **Step 1: Write the e2e spec** (two tests) + +```ts +test("a forged store entry is never served", async ({ page }) => { + // boot; create a file doc "forge.js" (content "export default 42", mimeType text/javascript) + // in the account's root folder; pinned = (await fetch(`/@/${acct}/forge.js`, { redirect: "manual" }) …) + // follow the 307 to get the pinned URL; fetch it once (stores it) + // in the page: cache.put(`/.content/${sha256(real)}`, new Response("export default 666")) + // into caches.open(`patchwork-content-${acct}`) + // reload; expect((await import(pinned)).default).toBe(42) +}); +test("an index answer from a page is refused", async ({ page }) => { + // fetch("/@/.index", { method: "POST", body: JSON.stringify({ id: "x", manifest: {...} }) }) → status 403 +}); +``` + +- [ ] **Step 2: Write the bench** `content-hash.spec.ts`: in `patchwork` mode, time `crypto.subtle.digest("SHA-256", …)` over 10 KB, 100 KB and 1 MB, 200 runs each. `record({ metric: "sha256 ", mode: "patchwork", value: median, unit: "ms" })`. + +- [ ] **Step 3: Typecheck both** + +Run: `pnpm --filter @inkandswitch/patchwork-e2e exec tsc --noEmit && pnpm --filter patchwork-bench exec tsc --noEmit` +Expected: no errors. Don't run them locally. CI's e2e matrix runs the spec on the PR. + +- [ ] **Step 4: Commit** `e2e: forged store entries and forged answers; bench: hashing a hit` + +--- + +### Task 9: Spec corrections and the changeset + +**Files:** Modify `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. Create `.changeset/paths-content-store.md`. + +- [ ] **Step 1: Correct the spec** + +- Section 5, *The index channel is authenticated*: "the automerge worker's script URL is fixed (`/automerge-protocol-handler-worker.js`), not hashed. A site that sets `workerPath` gets inline, unstored serving." +- *Remaining gaps*: add the cold-start cost from Decision 10, and say that Task 8's bench measures it. + +- [ ] **Step 2: Write the changeset** + +```markdown +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form; a pinned one is served from a content-addressed store (`patchwork-content-`) whose every read is hashed and checked against an index that only the automerge worker fills, over `fetch()` requests the service worker checks are really from it. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker. New: `registerAccount` and `clearAccountStore` from `@inkandswitch/patchwork-bootloader`; `readPath`, `listFiles` and `WalkOptions.find` from `@inkandswitch/patchwork-filesystem`. The e2e suite gains a forged-store-entry test. +``` + +- [ ] **Step 3: Commit** `spec and changeset: content store` From ee28c9dde4bd4a53df3a15d3ae08109868754993 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:19:09 +0000 Subject: [PATCH 13/81] filesystem: walk takes a find, and remembers idhashes and warnings Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/providers.ts | 22 ++++++++++++++-- core/filesystem/src/walk.ts | 36 +++++++++++++++++++------- core/filesystem/test/providers.test.ts | 9 +++++++ core/filesystem/test/walk.test.ts | 21 +++++++++++++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts index 3b8c5f75f..0716aa4fe 100644 --- a/core/filesystem/src/providers.ts +++ b/core/filesystem/src/providers.ts @@ -62,6 +62,17 @@ function childOf(name: string, value: unknown): Child { const MOUNT_POINT = /^\/[^/@]*$/; +// The same malformed mount is read on every lookup, so say so once. +const warned = new Set(); +const warnOnce = (key: string, message: string) => { + if (warned.has(key)) return; + warned.add(key); + console.warn(message); +}; + +// Bindings are read on every lookup and list, from a doc that doesn't change under them. +const bindingsByDoc = new WeakMap>(); + /** * Mount point → its bindings, in order. `/` falls back to `rootFolderUrl` when the namespace * binds nothing valid there, so older accounts keep working. Malformed mount points and bindings are @@ -70,22 +81,29 @@ const MOUNT_POINT = /^\/[^/@]*$/; export function accountBindings( doc: AccountNamespaceDoc ): Map { + const memo = bindingsByDoc.get(doc); + if (memo) return memo; const bindings = new Map(); for (const [mount, list] of Object.entries(doc.namespace ?? {})) { if (!MOUNT_POINT.test(mount) || !Array.isArray(list)) { - console.warn(`namespace: skipping malformed mount ${JSON.stringify(mount)}`); + warnOnce(mount, `namespace: skipping malformed mount ${JSON.stringify(mount)}`); continue; } const urls: AutomergeUrl[] = []; for (const binding of list) { const url = asLink(binding?.url); if (url) urls.push(url); - else console.warn(`namespace: skipping a binding at ${mount} without a valid url`); + else + warnOnce( + `binding ${mount}`, + `namespace: skipping a binding at ${mount} without a valid url` + ); } bindings.set(mount, urls); } const root = asLink(doc.rootFolderUrl); if (!bindings.get("/")?.length && root) bindings.set("/", [root]); + bindingsByDoc.set(doc, bindings); return bindings; } diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index b52188189..6450eccc3 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -27,8 +27,26 @@ import { documentBaseOrigin } from "./urls.js"; export type WalkOptions = { /** acctHash → account doc url. The roots a live or pinned account path may start at. */ accounts: ReadonlyMap; + /** + * Find a document, given its bare url and the heads the walk will read it at (none: current). + * The walk views what it returns at those heads. A resolver passes one that waits for the + * heads to arrive, or for peers to catch up; the default is `repo.find`. A rejection fails + * the walk. + */ + find?: (url: AutomergeUrl, heads?: UrlHeads) => Promise>; }; +// idhashes never change, and every walk hashes the same few documents again. +const idhashes = new Map>(); +function memoIdhash(documentId: string): Promise { + let hash = idhashes.get(documentId); + if (!hash) { + if (idhashes.size >= 10_000) idhashes.clear(); + idhashes.set(documentId, (hash = idhash(documentId))); + } + return hash; +} + export type WalkEnd = { /** The last document the walk entered, as a view at the heads it was read at. */ handle: DocHandle; @@ -76,13 +94,9 @@ export async function walkPath( options: WalkOptions ): Promise { const { root, segments } = parsePath(path); - const hashes = new Map>(); - const hashOf = (url: AutomergeUrl) => { - const { documentId } = parseAutomergeUrl(url); - let hash = hashes.get(documentId); - if (!hash) hashes.set(documentId, (hash = idhash(documentId))); - return hash; - }; + const hashOf = (url: AutomergeUrl) => + memoIdhash(parseAutomergeUrl(url).documentId); + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); const stepOf = async (handle: DocHandle) => docStep(await hashOf(handle.url), handle.heads()); @@ -92,10 +106,12 @@ export async function walkPath( ): Promise { try { const parsed = parseAutomergeUrl(url); - const live = await repo.find( - stringifyAutomergeUrl({ documentId: parsed.documentId }) + const at = heads ?? parsed.heads; + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + at ); - const handle = live.view(heads ?? parsed.heads ?? live.heads()); + const handle = live.view(at ?? live.heads()); const node = handle.doc(); if (node === undefined) return undefined; return { kind: "doc", handle, node, provider: providerFor(node), inside: false }; diff --git a/core/filesystem/test/providers.test.ts b/core/filesystem/test/providers.test.ts index 99825cf54..1d12a3e2f 100644 --- a/core/filesystem/test/providers.test.ts +++ b/core/filesystem/test/providers.test.ts @@ -82,6 +82,15 @@ describe("accountProvider", () => { expect(bindings.get("/")).toEqual([R]); }); + it("warns about a malformed mount once, however often it's read", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doc = account({ + namespace: { "/once@only": [{ url: R }], "/": [{ url: R }] }, + }); + for (let i = 0; i < 5; i++) accountProvider.lookup(doc, ["a"]); + expect(warn).toHaveBeenCalledTimes(1); + }); + it("lists every mount as a union child", () => { expect(accountProvider.list(doc)).toEqual([ { name: "", union: [R] }, diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts index cc975d5e0..04e91ed92 100644 --- a/core/filesystem/test/walk.test.ts +++ b/core/filesystem/test/walk.test.ts @@ -197,6 +197,27 @@ describe("resolveLivePath", () => { expect(canonical?.canonical).toBe(true); }); + it("walks through the find it's given", async () => { + const { repo, a, accounts } = await fixture(); + const seen: string[] = []; + const find = (url: AutomergeUrl) => (seen.push(url), repo.find(url)); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { + accounts, + find, + }); + expect(handle).toBeDefined(); + // the account, /, Projects, foo + expect(seen.length).toBeGreaterThanOrEqual(4); + }); + + it("fails, not throws, when find rejects", async () => { + const { repo, a, accounts } = await fixture(); + const find = () => Promise.reject(new Error("offline")); + expect( + await resolveLivePath(repo, livePath(a, ["Projects"]), { accounts, find }) + ).toBeUndefined(); + }); + it("returns undefined for a path that ends inside a document", async () => { const { repo, a, accounts, minePkg } = await fixture(); minePkg.change((d: any) => (d.nested = { x: "y" })); From 8eac5e79fcac04458d54fbb0677a50e2e79c904a Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:22:28 +0000 Subject: [PATCH 14/81] filesystem: readPath and listFiles Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/resolve.ts | 12 +++ core/filesystem/src/walk.ts | 109 ++++++++++++++++++++ core/filesystem/test/files.test.ts | 117 ++++++++++++++++++++++ core/filesystem/test/namespace-fixture.ts | 72 +++++++++++++ core/filesystem/test/walk.test.ts | 72 +------------ 5 files changed, 312 insertions(+), 70 deletions(-) create mode 100644 core/filesystem/test/files.test.ts create mode 100644 core/filesystem/test/namespace-fixture.ts diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 1d80cf290..0306176b0 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -175,3 +175,15 @@ export async function resolvePath( pinned: hasHeads(rootHandle.url), }); } + +/** + * What `node` serves as, the way {@link resolvePath} serves the end of a path: a file doc's + * content with its mime type, bytes as they are, anything else as JSON. + */ +export async function readNode( + repo: Repo, + node: unknown +): Promise<{ content: string | Uint8Array; type: string } | undefined> { + const read = await materialize(repo, node); + return read && { content: read.content, type: read.type }; +} diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 6450eccc3..847e32b90 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -1,8 +1,10 @@ import { type AutomergeUrl, type DocHandle, + type DocumentId, type Repo, type UrlHeads, + isImmutableString, parseAutomergeUrl, stringifyAutomergeUrl, } from "@automerge/automerge-repo/slim"; @@ -16,12 +18,14 @@ import { import { type Child, type Lookup, + directoryProvider, type PathProvider, accountProvider, branchesProvider, defaultBranch, providerFor, } from "./providers.js"; +import { readNode } from "./resolve.js"; import { documentBaseOrigin } from "./urls.js"; export type WalkOptions = { @@ -338,3 +342,108 @@ export async function listChildren( ...mounts, ]; } + +export type FileContent = { content: string | Uint8Array; type: string }; + +/** What a path names, read: its pinned form, whether the input was canonical, and the bytes. */ +export async function readPath( + repo: Repo, + path: string, + options: WalkOptions +): Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined> { + const end = await walkPath(repo, path, options); + if (!end) return undefined; + const read = await readNode(repo, end.node); + return read && { ...read, pinned: end.pinned, canonical: end.canonical }; +} + +/** A file under a pinned document, and the document that holds its bytes. */ +export type ListedFile = FileContent & { tail: string; documentId: DocumentId }; + +const isNested = (value: unknown): value is Record => + !!value && + typeof value === "object" && + !Array.isArray(value) && + !(value instanceof Uint8Array) && + !isImmutableString(value) && + !("content" in value); + +// Which of two spellings of one tail the walk finds: at the first level where their directory +// keys differ in length, the longer key, since lookup tries the longest prefix first. +function walkPrefers(a: number[], b: number[]): boolean { + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (a[i] !== b[i]) return a[i] > b[i]; + } + return false; +} + +/** + * Every file under the last document a canonical pinned path enters, keyed by tail: "" for the + * document itself, encoded names joined with "/" below it. Containers aren't listed; they have + * their own doc steps. Undefined unless `path` is canonical and ends at a document. + * + * A file reached by a link without heads is read as it is now, so the list is a snapshot. + */ +export async function listFiles( + repo: Repo, + path: string, + options: WalkOptions +): Promise<{ prefix: string; files: AsyncIterable } | undefined> { + const end = await walkPath(repo, path, options); + if (!end?.canonical || !end.atDocument) return undefined; + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const provider = providerFor(end.node); + const documentId = end.handle.documentId; + + async function* files(): AsyncIterable { + const found = new Map(); + const add = async ( + parts: string[], + keys: number[], + node: unknown, + holder: DocumentId + ) => { + const tail = parts.map(encodeURIComponent).join("/"); + const existing = found.get(tail); + if (existing && !walkPrefers(keys, existing.keys)) return; + const read = await readNode(repo, node); + if (read) found.set(tail, { keys, file: { ...read, tail, documentId: holder } }); + }; + + const visit = async (node: unknown, parts: string[], keys: number[]) => { + for (const child of provider?.list(node) ?? []) { + // Only a directory's keys span several path segments; a folder entry's name is one. + const names = + provider === directoryProvider ? child.name.split("/") : [child.name]; + const childParts = [...parts, ...names]; + const childKeys = [...keys, names.length]; + if ("value" in child) { + await add(childParts, childKeys, child.value, documentId); + // Nested values are addressable by path only inside a directory doc. + if (provider === directoryProvider && isNested(child.value)) { + await visit(child.value, childParts, childKeys); + } + } else if ("link" in child) { + try { + const parsed = parseAutomergeUrl(child.link); + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + parsed.heads + ); + const doc = live.view(parsed.heads ?? live.heads()).doc(); + if (doc === undefined || providerFor(doc)) continue; + await add(childParts, childKeys, doc, parsed.documentId); + } catch { + // A file that can't be found isn't listed; reading it fails the same way. + } + } + } + }; + + await add([], [], end!.node, documentId); + await visit(end!.node, [], []); + for (const { file } of found.values()) yield file; + } + + return { prefix: end.pinned, files: files() }; +} diff --git a/core/filesystem/test/files.test.ts b/core/filesystem/test/files.test.ts new file mode 100644 index 000000000..be618968d --- /dev/null +++ b/core/filesystem/test/files.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect } from "vitest"; +import { parseAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; +import { livePath } from "../src/paths.js"; +import { listFiles, pinnedPath, readPath, type ListedFile } from "../src/walk.js"; +import { file, fixture, folder } from "./namespace-fixture.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +async function collect(files: AsyncIterable) { + const out: ListedFile[] = []; + for await (const f of files) out.push(f); + return out.sort((a, b) => a.tail.localeCompare(b.tail)); +} + +const idOf = (url: AutomergeUrl) => parseAutomergeUrl(url).documentId; + +describe("readPath", () => { + it("reads a file and its pinned form", async () => { + const { repo, a, accounts } = await fixture(); + const path = livePath(a, ["packages", "foo", "dist", "index.js"]); + const read = await readPath(repo, path, { accounts }); + expect(read && decode(read.content)).toBe("mine"); + expect(read?.type).toBe("text/plain"); + expect(read?.canonical).toBe(false); + expect(read?.pinned).toBe(await pinnedPath(repo, path, { accounts })); + }); + + it("reads nothing for a path that doesn't resolve", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await readPath(repo, livePath(a, ["packages", "nope"]), { accounts }) + ).toBeUndefined(); + }); +}); + +describe("listFiles", () => { + it("lists a directory package's files", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const listed = await listFiles(repo, prefix!, { accounts }); + expect(listed?.prefix).toBe(prefix); + const files = await collect(listed!.files); + expect(files.map((f) => f.tail)).toEqual(["", "dist/index.js", "package.json"]); + const index = files.find((f) => f.tail === "dist/index.js")!; + expect(index.documentId).toBe(idOf((minePkg.doc() as any)["dist/index.js"])); + expect(files[0].documentId).toBe(minePkg.documentId); + }); + + it("lists a folder's leaves and skips its subfolders", async () => { + const { repo, a, accounts, foo } = await fixture(); + const sub = folder(repo, { inner: file(repo, "inner").url }); + foo.change((d: any) => d.docs.push({ name: "sub", type: "folder", url: sub.url })); + const prefix = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toEqual(["", "notes"]); + }); + + it("lists nested values inside a directory doc", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.dist = { "a.js": "x" })); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toContain("dist/a.js"); + expect(files.find((f) => f.tail === "dist/a.js")?.documentId).toBe(minePkg.documentId); + }); + + it("encodes names in tails", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d["my file@1.js"] = file(repo, "spaced").url)); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toContain("my%20file%401.js"); + }); + + it("lists a flat key over a nested one at the same path, as the walk finds it", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => { + d["x/y"] = file(repo, "flat").url; + d.x = { y: "nested" }; + }); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + const xy = files.filter((f) => f.tail === "x/y"); + expect(xy).toHaveLength(1); + expect(decode(xy[0].content)).toBe("flat"); + }); + + it("agrees with readPath for every file it lists", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => { + d.dist = { "a.js": "x" }; + d["x/y"] = file(repo, "flat").url; + d.x = { y: "nested" }; + }); + for (const names of [["packages", "foo"], ["Projects", "foo"]]) { + const prefix = await pinnedPath(repo, livePath(a, names), { accounts }); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.length).toBeGreaterThan(1); + for (const f of files) { + const read = await readPath(repo, f.tail ? `${prefix}/${f.tail}` : prefix!, { accounts }); + expect([f.tail, read && decode(read.content), read?.type]).toEqual([ + f.tail, + decode(f.content), + f.type, + ]); + } + } + }); + + it("lists nothing for a path that isn't canonical", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await listFiles(repo, livePath(a, ["packages", "foo"]), { accounts }) + ).toBeUndefined(); + }); +}); diff --git a/core/filesystem/test/namespace-fixture.ts b/core/filesystem/test/namespace-fixture.ts new file mode 100644 index 000000000..869612c7f --- /dev/null +++ b/core/filesystem/test/namespace-fixture.ts @@ -0,0 +1,72 @@ +// A small account to walk: / is a folder of Projects, /packages a union of two folders. +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { docStep, idhash } from "../src/paths.js"; + +export const step = async (handle: DocHandle) => + docStep(await idhash(handle.documentId), handle.heads()); + +export const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ + name, + type: "x", + url, + })), + }); + +export const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + +// A package the way pushwork writes one: a directory doc of path → file doc url. +export const pkg = (repo: Repo, main: string) => + repo.create({ + "@patchwork": { type: "directory" }, + "package.json": file(repo, "{}").url, + "dist/index.js": file(repo, main).url, + }); + +export async function fixture() { + const repo = new Repo(); + const notes = file(repo, "hi"); + const foo = folder(repo, { notes: notes.url }); + const projects = folder(repo, { foo: foo.url }); + const decoy = folder(repo, { foo: file(repo, "decoy").url }); + const loop = repo.create<{ "@patchwork": { type: string }; branches: Record }>({ + "@patchwork": { type: "branches" }, + branches: {}, + }); + loop.change((d) => (d.branches.default = loop.url)); + const root = folder(repo, { + Projects: projects.url, + packages: decoy.url, + loop: loop.url, + }); + const minePkg = pkg(repo, "mine"); + const mine = folder(repo, { foo: minePkg.url }); + const branchyPkg = pkg(repo, "branchy"); + const branchy = repo.create({ + "@patchwork": { type: "branches" }, + branches: { default: branchyPkg.url }, + }); + const team = folder(repo, { + foo: pkg(repo, "team foo").url, + bar: pkg(repo, "team bar").url, + branchy: branchy.url, + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { + "/": [{ url: root.url }], + "/packages": [{ url: mine.url }, { url: team.url }], + }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + return { + repo, a, accounts, account, root, projects, foo, notes, mine, minePkg, + team, branchy, branchyPkg, + }; +} + +export const text = (handle: DocHandle | undefined) => handle?.doc()?.content; diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts index 04e91ed92..5c12f3745 100644 --- a/core/filesystem/test/walk.test.ts +++ b/core/filesystem/test/walk.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, afterEach } from "vitest"; -import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; import { docStep, idhash, livePath } from "../src/paths.js"; +import { file, fixture, step, text } from "./namespace-fixture.js"; import { listChildren, listUnion, @@ -10,75 +11,6 @@ import { walkPath, } from "../src/walk.js"; -const step = async (handle: DocHandle) => - docStep(await idhash(handle.documentId), handle.heads()); - -const folder = (repo: Repo, entries: Record) => - repo.create({ - title: "folder", - docs: Object.entries(entries).map(([name, url]) => ({ - name, - type: "x", - url, - })), - }); - -const file = (repo: Repo, content: string) => - repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); - -// A package the way pushwork writes one: a directory doc of path → file doc url. -const pkg = (repo: Repo, main: string) => - repo.create({ - "@patchwork": { type: "directory" }, - "package.json": file(repo, "{}").url, - "dist/index.js": file(repo, main).url, - }); - -async function fixture() { - const repo = new Repo(); - const notes = file(repo, "hi"); - const foo = folder(repo, { notes: notes.url }); - const projects = folder(repo, { foo: foo.url }); - const decoy = folder(repo, { foo: file(repo, "decoy").url }); - const loop = repo.create<{ "@patchwork": { type: string }; branches: Record }>({ - "@patchwork": { type: "branches" }, - branches: {}, - }); - loop.change((d) => (d.branches.default = loop.url)); - const root = folder(repo, { - Projects: projects.url, - packages: decoy.url, - loop: loop.url, - }); - const minePkg = pkg(repo, "mine"); - const mine = folder(repo, { foo: minePkg.url }); - const branchyPkg = pkg(repo, "branchy"); - const branchy = repo.create({ - "@patchwork": { type: "branches" }, - branches: { default: branchyPkg.url }, - }); - const team = folder(repo, { - foo: pkg(repo, "team foo").url, - bar: pkg(repo, "team bar").url, - branchy: branchy.url, - }); - const account = repo.create({ - "@patchwork": { type: "account" }, - namespace: { - "/": [{ url: root.url }], - "/packages": [{ url: mine.url }, { url: team.url }], - }, - }); - const a = await idhash(account.documentId); - const accounts = new Map([[a, account.url]]); - return { - repo, a, accounts, account, root, projects, foo, notes, mine, minePkg, - team, branchy, branchyPkg, - }; -} - -const text = (handle: DocHandle | undefined) => handle?.doc()?.content; - afterEach(() => vi.unstubAllGlobals()); describe("resolveLivePath", () => { From 50fad4ccf08a23b9bd78cb90ab525a298e46d8a8 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:26:11 +0000 Subject: [PATCH 15/81] bootloader: content store, index and the pinned request flow Co-Authored-By: Claude Opus 5.5 --- core/bootloader/package.json | 3 +- core/bootloader/src/content-hash.ts | 16 ++ core/bootloader/src/content-store.ts | 41 ++++ core/bootloader/src/pinned.ts | 195 ++++++++++++++++++ core/bootloader/src/types.ts | 13 ++ core/bootloader/test/content-store.test.ts | 46 +++++ core/bootloader/test/pinned.test.ts | 222 +++++++++++++++++++++ core/bootloader/test/setup.ts | 10 + core/bootloader/vitest.config.ts | 16 ++ 9 files changed, 561 insertions(+), 1 deletion(-) create mode 100644 core/bootloader/src/content-hash.ts create mode 100644 core/bootloader/src/content-store.ts create mode 100644 core/bootloader/src/pinned.ts create mode 100644 core/bootloader/test/content-store.test.ts create mode 100644 core/bootloader/test/pinned.test.ts create mode 100644 core/bootloader/test/setup.ts create mode 100644 core/bootloader/vitest.config.ts diff --git a/core/bootloader/package.json b/core/bootloader/package.json index 207a095d6..2faa571df 100644 --- a/core/bootloader/package.json +++ b/core/bootloader/package.json @@ -113,6 +113,7 @@ }, "scripts": { "build": "tsc && cp src/global.css dist/global.css", - "dev": "tsc -w --preserveWatchOutput" + "dev": "tsc -w --preserveWatchOutput", + "test": "vitest run" } } diff --git a/core/bootloader/src/content-hash.ts b/core/bootloader/src/content-hash.ts new file mode 100644 index 000000000..a5784565b --- /dev/null +++ b/core/bootloader/src/content-hash.ts @@ -0,0 +1,16 @@ +// The one way content is hashed, by the automerge worker building a manifest and by the +// service worker checking what it serves: SHA-256 over the bytes, base64url without padding. + +/** Bytes as served: strings as UTF-8, which is what a Response body of a string sends. */ +export function toBytes(content: string | Uint8Array): Uint8Array { + return typeof content === "string" ? new TextEncoder().encode(content) : content; +} + +export async function contentHash(bytes: Uint8Array): Promise { + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", bytes as BufferSource) + ); + let binary = ""; + for (const byte of digest) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} diff --git a/core/bootloader/src/content-store.ts b/core/bootloader/src/content-store.ts new file mode 100644 index 000000000..bd3135495 --- /dev/null +++ b/core/bootloader/src/content-store.ts @@ -0,0 +1,41 @@ +// Bytes by their hash, in CacheStorage. Nothing here is trusted: any code on the origin can +// write these caches, so every read is checked against the service worker's index before it's +// served (see pinned.ts). Reads and writes go through this one interface so the bytes can be +// encrypted at rest later. + +export interface ContentStore { + get(hash: string): Promise; + put(hash: string, bytes: Uint8Array): Promise; + delete(hash: string): Promise; +} + +export const CONTENT_CACHE_PREFIX = "patchwork-content-"; + +/** One store per account, apart from the `patchwork` cache of site pages and assets. */ +export const contentCacheName = (acctHash: string) => + `${CONTENT_CACHE_PREFIX}${acctHash}`; + +/** A store in `cache`, keyed by `/.content/`: an internal key, never fetched. */ +export function cacheStore(cache: Cache, origin: string): ContentStore { + const key = (hash: string) => `${origin}/.content/${hash}`; + return { + async get(hash) { + const response = await cache.match(key(hash)); + return response ? new Uint8Array(await response.arrayBuffer()) : undefined; + }, + async put(hash, bytes) { + await cache.put(key(hash), new Response(bytes as BlobPart)); + }, + async delete(hash) { + await cache.delete(key(hash)); + }, + }; +} + +/** + * Forget everything stored for an account. There's no sign-out flow; a site calls this from its + * own. The service worker's index may still name these hashes, which then read as misses. + */ +export function clearAccountStore(acctHash: string): Promise { + return caches.delete(contentCacheName(acctHash)); +} diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts new file mode 100644 index 000000000..c0a14ce9f --- /dev/null +++ b/core/bootloader/src/pinned.ts @@ -0,0 +1,195 @@ +// Serving a pinned `/@/` URL: find what its bytes must hash to, read them from the content +// store or a resolver, check them, and decide whether to keep them. Only the automerge worker's +// manifests say what a URL's bytes must hash to (see index-channel.ts); a store entry or a +// resolver's answer that doesn't match is never served. +import type { ContentStore } from "./content-store.js"; +import type { IndexEntry, Manifest } from "./types.js"; + +const BASE58 = "[1-9A-HJ-NP-Za-km-z]+"; +const DOC_STEP = new RegExp(`^${BASE58}@${BASE58}(\\.${BASE58})*$`); +const ACCOUNT_ROOT = new RegExp(`^(${BASE58})(@${BASE58}(\\.${BASE58})*)?$`); +const SHARE_PREFIX = "automerge:"; + +const reencode = (segment: string) => encodeURIComponent(decodeURIComponent(segment)); + +export type NormalizedPath = { + /** + * The keys this path may be indexed under, most likely first. A segment shaped like a doc step + * (`react@18.3.1`) may be a name, and only the worker's manifest can say, so there's a key for + * each place the doc steps could end; every name after that point is re-encoded. + */ + keys: string[]; + /** The first key up to and including its last doc step: the document its manifest covers. */ + prefix: string; + acctHash?: string; + /** False for a live path, whose names resolve at current heads. */ + pinned: boolean; + /** A share link, `automerge:@`: nothing under it is stored. */ + share: boolean; +}; + +/** + * The path after `/@/`, normalized so every spelling of one file has one key among its + * candidates: segments are percent-decoded and re-encoded, and empty ones dropped. Undefined for + * a malformed escape or a root that isn't an account hash or a share link. + */ +export function normalizePinned(path: string): NormalizedPath | undefined { + const [root, ...rest] = path.split("/").filter(Boolean); + if (!root) return undefined; + try { + const decodedRoot = decodeURIComponent(root); + let head: string; + let acctHash: string | undefined; + let pinned: boolean; + const share = decodedRoot.startsWith(SHARE_PREFIX); + if (share) { + head = decodedRoot; + pinned = true; + } else { + const match = ACCOUNT_ROOT.exec(root); + if (!match) return undefined; + head = root; + acctHash = match[1]; + pinned = !!match[2]; + } + + // Doc-step-shaped segments after the root, latest first. + const splits = rest + .map((segment, i) => (DOC_STEP.test(segment) ? i : -1)) + .filter((i) => i >= 0) + .reverse(); + const keyAt = (split: number) => + [ + head, + ...rest.map((segment, i) => + i <= split && DOC_STEP.test(segment) ? segment : reencode(segment) + ), + ].join("/"); + const keys = splits.length ? splits.map(keyAt) : [keyAt(-1)]; + // Encoded segments hold no "/", so the first key splits back into its segments. + const prefix = keys[0] + .split("/") + .slice(0, (splits[0] ?? -1) + 2) + .join("/"); + return { keys, prefix, acctHash, pinned, share }; + } catch { + return undefined; + } +} + +/** Add a manifest's entries to the index, under `prefix/tail`. */ +export function indexManifest( + index: Map, + manifest: Manifest +): void { + for (const [tail, entry] of Object.entries(manifest.entries)) { + index.set(tail ? `${manifest.prefix}/${tail}` : manifest.prefix, entry); + } +} + +/** Thrown to fail the request as a network error: nothing that can be memoized. */ +export class PinnedAbort extends Error {} + +export type PinnedDeps = { + index: Map; + /** Undefined: nothing is read or written (a share link). */ + store: ContentStore | undefined; + hash(bytes: Uint8Array): Promise; + /** Ask the automerge worker for the manifest covering `key`; undefined if it can't answer. */ + manifest(key: string): Promise; + /** Ask a tab or the automerge worker for `key`'s bytes; undefined if it doesn't resolve. */ + resolve(key: string): Promise<{ body: Uint8Array; contentType: string } | undefined>; + warn(message: string): void; + /** Keep the service worker alive for work that outlives the response. */ + keep(work: Promise): void; +}; + +const lookup = (index: Map, keys: string[]) => { + for (const key of keys) { + const entry = index.get(key); + if (entry) return entry; + } + return undefined; +}; + +const respond = (bytes: Uint8Array, contentType: string, immutable: boolean) => + new Response(bytes as BlobPart, { + headers: { + "content-type": contentType, + "cache-control": immutable ? "public, max-age=31536000, immutable" : "no-store", + }, + }); + +/** + * Serve a pinned URL, given its candidate keys (see {@link normalizePinned}): + * + * 1. Find its index entry, asking for a manifest on a miss. + * 2. With an entry, read the store and hash what's there: a match is served; a mismatch is + * deleted, warned about, and treated as a miss. + * 3. Ask a resolver and hash its bytes. With an entry, a match is served (and stored, if the + * entry allows) and a mismatch fails as a network error. Without one, because no worker + * answered, the bytes are served as they are and kept nowhere. + */ +export async function servePinned( + keys: string[], + deps: PinnedDeps +): Promise { + let entry = lookup(deps.index, keys); + if (!entry) { + const manifest = await deps.manifest(keys[0]); + if (manifest) indexManifest(deps.index, manifest); + entry = lookup(deps.index, keys); + } + + if (entry && deps.store) { + const stored = await deps.store.get(entry.sha256); + if (stored) { + if ((await deps.hash(stored)) === entry.sha256) { + return respond(stored, entry.contentType, true); + } + deps.warn( + `content store entry ${entry.sha256} for ${keys[0]} doesn't hash to its key; deleting it` + ); + await deps.store.delete(entry.sha256); + } + } + + const answer = await deps.resolve(keys[0]); + if (!answer) { + return new Response(`nothing at ${keys[0]}`, { + status: 404, + headers: { "content-type": "text/plain", "cache-control": "no-store" }, + }); + } + if (!entry) return respond(answer.body, answer.contentType, false); + + if ((await deps.hash(answer.body)) !== entry.sha256) { + throw new PinnedAbort( + `${keys[0]} resolved to bytes that don't match the automerge worker's manifest` + ); + } + if (entry.store && deps.store) { + deps.keep(deps.store.put(entry.sha256, answer.body)); + } + return respond(answer.body, entry.contentType, true); +} + +/** + * Wrap a manifest fetcher so concurrent calls for one prefix share a request. A settled request + * is forgotten: its entries are in the index by then. + */ +export function sharedManifests( + fetch: (key: string) => Promise, + prefixOf: (key: string) => string +): (key: string) => Promise { + const inFlight = new Map>(); + return (key) => { + const prefix = prefixOf(key); + let request = inFlight.get(prefix); + if (!request) { + request = fetch(key).finally(() => inFlight.delete(prefix)); + inFlight.set(prefix, request); + } + return request; + }; +} diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 0ddc6d4ec..f4ce24b6c 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -193,3 +193,16 @@ export type SetupServiceWorkerResult = { shared?: SharedWorker; kill?: () => void; }; + +/** + * What the service worker knows about one pinned URL: the hash its bytes must have, the type to + * serve them as, and whether they may be kept in the content store. + */ +export type IndexEntry = { sha256: string; contentType: string; store: boolean }; + +/** + * Automerge worker → service worker: every file under the last document a pinned URL enters, at + * its heads. `prefix` is the canonical pinned path of that document; each entry's key is the tail + * below it ("" for the document itself). + */ +export type Manifest = { prefix: string; entries: Record }; diff --git a/core/bootloader/test/content-store.test.ts b/core/bootloader/test/content-store.test.ts new file mode 100644 index 000000000..39be24cd9 --- /dev/null +++ b/core/bootloader/test/content-store.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from "vitest"; +import { cacheStore } from "../src/content-store.js"; +import { contentHash, toBytes } from "../src/content-hash.js"; + +/** Enough of Cache for the store: put, match and delete by url. */ +export class FakeCache { + entries = new Map(); + async put(request: RequestInfo | URL, response: Response) { + this.entries.set(new Request(request).url, response.clone()); + } + async match(request: RequestInfo | URL) { + return this.entries.get(new Request(request).url)?.clone(); + } + async delete(request: RequestInfo | URL) { + return this.entries.delete(new Request(request).url); + } +} + +describe("contentHash", () => { + it("is base64url sha-256 without padding", async () => { + // sha256("") = e3b0c442…b855 + expect(await contentHash(new Uint8Array())).toBe( + "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" + ); + }); + + it("hashes strings as UTF-8", async () => { + expect(await contentHash(toBytes("héllo ✓"))).toBe( + await contentHash(new TextEncoder().encode("héllo ✓")) + ); + }); +}); + +describe("cacheStore", () => { + it("round-trips bytes under /.content/", async () => { + const cache = new FakeCache(); + const store = cacheStore(cache as unknown as Cache, "https://p.test"); + const bytes = toBytes("export default 1"); + const hash = await contentHash(bytes); + await store.put(hash, bytes); + expect([...cache.entries.keys()]).toEqual([`https://p.test/.content/${hash}`]); + expect(await store.get(hash)).toEqual(bytes); + await store.delete(hash); + expect(await store.get(hash)).toBeUndefined(); + }); +}); diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts new file mode 100644 index 000000000..38fe784bd --- /dev/null +++ b/core/bootloader/test/pinned.test.ts @@ -0,0 +1,222 @@ +import { describe, it, expect, vi } from "vitest"; +import { contentHash, toBytes } from "../src/content-hash.js"; +import type { ContentStore } from "../src/content-store.js"; +import { + PinnedAbort, + normalizePinned, + servePinned, + sharedManifests, + type PinnedDeps, +} from "../src/pinned.js"; +import type { IndexEntry, Manifest } from "../src/types.js"; + +const keysOf = (path: string) => normalizePinned(path)!.keys; + +describe("normalizePinned", () => { + it("gives every spelling of a file one key among its candidates", () => { + const canonical = keysOf("acct@h/x@h/a%40b.js")[0]; + expect(canonical).toBe("acct@h/x@h/a%40b.js"); + for (const spelling of [ + "acct@h/x@h/a@b.js", + "/acct@h//x@h/a%40b.js/", + "acct@h/x@h/a%40b.js", + ]) { + expect(keysOf(spelling)).toContain(canonical); + } + expect(keysOf("acct@h/x@h/日本")).toEqual( + keysOf("acct@h/x@h/%E6%97%A5%E6%9C%AC") + ); + }); + + it("tries the latest doc step first, then each earlier one", () => { + expect(keysOf("acct@h/packages/m@h/f@h/vendor/react@18.3.1/index.js")).toEqual([ + "acct@h/packages/m@h/f@h/vendor/react@18.3.1/index.js", + "acct@h/packages/m@h/f@h/vendor/react%4018.3.1/index.js", + "acct@h/packages/m@h/f%40h/vendor/react%4018.3.1/index.js", + ]); + }); + + it("splits the first key at its last doc step", () => { + expect(normalizePinned("acct@h/packages/m@h/f@h/dist/index.js")).toMatchObject({ + prefix: "acct@h/packages/m@h/f@h", + acctHash: "acct", + pinned: true, + share: false, + }); + }); + + it("says a live path is live", () => { + expect(normalizePinned("acct/Projects")).toMatchObject({ + pinned: false, + acctHash: "acct", + }); + }); + + it("reads a share root", () => { + const n = normalizePinned("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h/x"); + expect(n).toMatchObject({ pinned: true, share: true, acctHash: undefined }); + expect(n!.keys[0]).toBe("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h/x"); + }); + + it("refuses a malformed escape or a bad root", () => { + expect(normalizePinned("acct@h/%zz")).toBeUndefined(); + expect(normalizePinned("")).toBeUndefined(); + expect(normalizePinned("not-base58!/x")).toBeUndefined(); + }); +}); + +class MemoryStore implements ContentStore { + bytes = new Map(); + async get(hash: string) { + return this.bytes.get(hash); + } + async put(hash: string, bytes: Uint8Array) { + this.bytes.set(hash, bytes); + } + async delete(hash: string) { + this.bytes.delete(hash); + } +} + +const KEY = "acct@h/pkg@h/index.js"; +const REAL = toBytes("export default 42"); + +async function setup(options: { store?: boolean; manifest?: boolean } = {}) { + const sha256 = await contentHash(REAL); + const entry: IndexEntry = { + sha256, + contentType: "text/javascript", + store: options.store ?? true, + }; + const manifest: Manifest = { + prefix: "acct@h/pkg@h", + entries: { "index.js": entry }, + }; + const kept: Promise[] = []; + const deps = { + index: new Map(), + store: new MemoryStore(), + hash: contentHash, + manifest: vi.fn(async () => (options.manifest === false ? undefined : manifest)), + resolve: vi.fn(async () => ({ body: REAL, contentType: "text/javascript" })), + warn: vi.fn(), + keep: (work: Promise) => void kept.push(work), + } satisfies PinnedDeps; + const settle = () => Promise.all(kept); + return { deps, sha256, entry, manifest, settle }; +} + +const text = (response: Response) => response.text(); + +describe("servePinned", () => { + it("serves a store hit whose bytes match, without asking a resolver", async () => { + const { deps, sha256 } = await setup(); + await deps.store.put(sha256, REAL); + const response = await servePinned([KEY], deps); + expect(await text(response)).toBe("export default 42"); + expect(response.headers.get("content-type")).toBe("text/javascript"); + expect(response.headers.get("cache-control")).toContain("immutable"); + expect(deps.resolve).not.toHaveBeenCalled(); + }); + + it("deletes a forged store entry, warns, and serves the resolver's real bytes", async () => { + const { deps, sha256, settle } = await setup(); + await deps.store.put(sha256, toBytes("export default 666")); + const response = await servePinned([KEY], deps); + await settle(); + expect(await text(response)).toBe("export default 42"); + expect(deps.warn).toHaveBeenCalled(); + expect(deps.store.bytes.get(sha256)).toEqual(REAL); + }); + + it("asks for one manifest on an index miss, then serves", async () => { + const { deps } = await setup(); + await servePinned([KEY], deps); + await servePinned([KEY], deps); + expect(deps.manifest).toHaveBeenCalledTimes(1); + }); + + it("finds the entry under a later candidate key", async () => { + const { deps } = await setup(); + const response = await servePinned(["acct@h/pkg@h/index@x.js", KEY], deps); + expect(response.headers.get("cache-control")).toContain("immutable"); + }); + + it("stores resolver bytes that match an entry that allows it", async () => { + const { deps, sha256, settle } = await setup(); + await servePinned([KEY], deps); + await settle(); + expect(deps.store.bytes.get(sha256)).toEqual(REAL); + }); + + it("never stores when the entry says not to", async () => { + const { deps, settle } = await setup({ store: false }); + expect(await text(await servePinned([KEY], deps))).toBe("export default 42"); + await settle(); + expect(deps.store.bytes.size).toBe(0); + }); + + it("fails as a network error when resolver bytes don't match, storing nothing", async () => { + const { deps, settle } = await setup(); + deps.resolve.mockResolvedValue({ + body: toBytes("export default 666"), + contentType: "text/javascript", + }); + await expect(servePinned([KEY], deps)).rejects.toBeInstanceOf(PinnedAbort); + await settle(); + expect(deps.store.bytes.size).toBe(0); + }); + + it("serves inline and stores nothing with no manifest", async () => { + const { deps, settle } = await setup({ manifest: false }); + const response = await servePinned([KEY], deps); + await settle(); + expect(await text(response)).toBe("export default 42"); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(deps.store.bytes.size).toBe(0); + expect(deps.index.size).toBe(0); + }); + + it("answers 404 when nothing resolves the path", async () => { + const { deps } = await setup({ manifest: false }); + deps.resolve.mockResolvedValue(undefined as never); + expect((await servePinned([KEY], deps)).status).toBe(404); + }); + + it("rebuilds the index after a restart and reuses the store", async () => { + const first = await setup(); + await servePinned([KEY], first.deps); + await first.settle(); + const second = await setup(); + second.deps.store = first.deps.store; + expect(await text(await servePinned([KEY], second.deps))).toBe("export default 42"); + expect(second.deps.manifest).toHaveBeenCalledTimes(1); + expect(second.deps.resolve).not.toHaveBeenCalled(); + }); + + it("reads and writes nothing without a store", async () => { + const { deps } = await setup(); + const response = await servePinned([KEY], { ...deps, store: undefined }); + expect(await text(response)).toBe("export default 42"); + }); +}); + +describe("sharedManifests", () => { + it("shares one manifest request among concurrent requests for a package", async () => { + const { deps, manifest } = await setup(); + let release!: () => void; + const gate = new Promise((resolve) => (release = resolve)); + const fetcher = vi.fn(async () => (await gate, manifest)); + const shared = sharedManifests(fetcher, (key) => normalizePinned(key)!.prefix); + const all = ["a.js", "b.js", "c.js", "d.js", "index.js"].map((file) => + shared(`acct@h/pkg@h/${file}`) + ); + release(); + await Promise.all(all); + expect(fetcher).toHaveBeenCalledTimes(1); + // settled requests aren't remembered: the next asks again + await shared(KEY); + expect(fetcher).toHaveBeenCalledTimes(2); + void deps; + }); +}); diff --git a/core/bootloader/test/setup.ts b/core/bootloader/test/setup.ts new file mode 100644 index 000000000..c8dc9e97e --- /dev/null +++ b/core/bootloader/test/setup.ts @@ -0,0 +1,10 @@ +// Initialize both Wasm modules before any test runs. +// automerge-repo@subduction.9 always creates a SubductionSource, +// which imports from @automerge/automerge-subduction/slim — the +// Wasm must be initialized first. +// +// Importing the fat entry points auto-calls initSync / UseApi(). +// The vitest.config.ts resolve aliases ensure a single copy is used +// even when automerge-repo is linked locally. +import "@automerge/automerge"; +import "@automerge/automerge-subduction"; diff --git a/core/bootloader/vitest.config.ts b/core/bootloader/vitest.config.ts new file mode 100644 index 000000000..f4ad63c16 --- /dev/null +++ b/core/bootloader/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["test/**/*.{test,spec}.ts"], + testTimeout: 30_000, + setupFiles: ["./test/setup.ts"], + // automerge-repo's DocumentQuery rejects internally-held promises when a + // pending query is torn down after a test ends; that's upstream noise, not + // a test failure (see patchwork-filesystem's config). This vitest has no + // per-error filter, only the big switch. + dangerouslyIgnoreUnhandledErrors: true, + }, +}); From 3ee67d6520ffb640227d71b23846c4d48fff70c1 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:28:09 +0000 Subject: [PATCH 16/81] bootloader: manifests, and the worker's authenticated answers Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/index-channel.ts | 80 ++++++++++++++++++++++ core/bootloader/src/manifest.ts | 50 ++++++++++++++ core/bootloader/src/setup.ts | 12 ++-- core/bootloader/src/types.ts | 29 ++++++++ core/bootloader/test/index-channel.test.ts | 75 ++++++++++++++++++++ core/bootloader/test/manifest.test.ts | 65 ++++++++++++++++++ vitest.config.ts | 1 + 7 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 core/bootloader/src/index-channel.ts create mode 100644 core/bootloader/src/manifest.ts create mode 100644 core/bootloader/test/index-channel.test.ts create mode 100644 core/bootloader/test/manifest.test.ts diff --git a/core/bootloader/src/index-channel.ts b/core/bootloader/src/index-channel.ts new file mode 100644 index 000000000..ec6111f5f --- /dev/null +++ b/core/bootloader/src/index-channel.ts @@ -0,0 +1,80 @@ +// The automerge worker's answers to the service worker, and how the service worker tells they +// really are its. A BroadcastChannel can't: anything on the origin can listen and answer first. +// Messages on a port carry no sender at all. A fetch() carries its client: the service worker +// looks `clientId` up and checks the client is a SharedWorker running the worker's script. +// That's the only reply path every engine supports; Chromium has no navigator.serviceWorker in +// workers (see the spec's spike results). +import { + AUTOMERGE_WORKER_PATH, + INDEX_ANSWER_PATH, + type IndexAnswer, +} from "./types.js"; + +const ACCEPTED_HEADER = "x-patchwork-index"; + +/** + * True only for the automerge worker: a SharedWorker on this origin running its script, whatever + * the query. Any page can start a dedicated Worker from the same URL, which reports the same url + * with type "worker", so the type matters as much as the url. + */ +export function isAutomergeWorker( + client: Pick | undefined, + origin: string +): boolean { + if (client?.type !== "sharedworker") return false; + try { + const url = new URL(client.url); + return url.origin === origin && url.pathname === AUTOMERGE_WORKER_PATH; + } catch { + return false; + } +} + +const isAnswer = (value: unknown): value is IndexAnswer => + !!value && + typeof value === "object" && + typeof (value as IndexAnswer).id === "string"; + +/** The service worker's side of a POST to INDEX_ANSWER_PATH. */ +export async function acceptIndexAnswer( + request: Request, + client: Pick | undefined, + origin: string, + onAnswer: (answer: IndexAnswer) => void +): Promise { + if (!isAutomergeWorker(client, origin)) { + console.warn( + `refusing an index answer from ${client?.type ?? "an unknown client"} at ${client?.url ?? "?"}` + ); + return new Response("only the automerge worker answers here", { status: 403 }); + } + const body: unknown = await request.json().catch(() => undefined); + if (!isAnswer(body)) return new Response("not an index answer", { status: 400 }); + onAnswer(body); + return new Response(null, { status: 204, headers: { [ACCEPTED_HEADER]: "accepted" } }); +} + +const TRIES = 5; +const RETRY_MS = 200; + +/** + * The automerge worker's side: POST an answer. A reply without the accepted header never reached + * the service worker (Firefox sends a worker's first fetch straight to the network), so try + * again a few times. False if it never got through. + */ +export async function postIndexAnswer( + answer: IndexAnswer, + fetcher: typeof fetch = fetch +): Promise { + const body = JSON.stringify(answer); + for (let i = 0; i < TRIES; i++) { + if (i) await new Promise((resolve) => setTimeout(resolve, RETRY_MS)); + const response = await fetcher(INDEX_ANSWER_PATH, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + }).catch(() => undefined); + if (response?.headers.get(ACCEPTED_HEADER) === "accepted") return true; + } + return false; +} diff --git a/core/bootloader/src/manifest.ts b/core/bootloader/src/manifest.ts new file mode 100644 index 000000000..d3d122c38 --- /dev/null +++ b/core/bootloader/src/manifest.ts @@ -0,0 +1,50 @@ +// The automerge worker's side of the index: what every file under a pinned document hashes to, +// and whether the service worker may keep it. +import { + stringifyAutomergeUrl, + type DocumentId, + type Repo, +} from "@automerge/automerge-repo/slim"; +import { + listFiles, + parsePath, + type WalkOptions, +} from "@inkandswitch/patchwork-filesystem"; +import { isKeyhiveDoc } from "@inkandswitch/patchwork-plugins"; +import { contentHash, toBytes } from "./content-hash.js"; +import type { IndexEntry, Manifest } from "./types.js"; + +const isProtectedDoc = (documentId: DocumentId) => + isKeyhiveDoc(stringifyAutomergeUrl({ documentId })); + +/** + * The manifest covering canonical pinned `path`, or undefined if it isn't one. + * + * What may be stored: content from unprotected documents, and keyhive-protected content only + * when it's reached through `/packages`, since package code needs to load fast at boot and user + * documents don't. Nothing under a share link. + */ +export async function buildManifest( + repo: Repo, + path: string, + options: WalkOptions & { isProtected?: (documentId: DocumentId) => boolean } +): Promise { + const listed = await listFiles(repo, path, options); + if (!listed) return undefined; + const { root, segments } = parsePath(listed.prefix); + const first = segments[0]; + const throughPackages = first?.kind === "name" && first.name === "packages"; + const isProtected = options.isProtected ?? isProtectedDoc; + + const entries: Record = {}; + for await (const file of listed.files) { + entries[file.tail] = { + sha256: await contentHash(toBytes(file.content)), + contentType: file.type, + store: + root.kind === "account" && + (throughPackages || !isProtected(file.documentId)), + }; + } + return { prefix: listed.prefix, entries }; +} diff --git a/core/bootloader/src/setup.ts b/core/bootloader/src/setup.ts index 0b1eb1335..b7b363b17 100644 --- a/core/bootloader/src/setup.ts +++ b/core/bootloader/src/setup.ts @@ -1,7 +1,8 @@ -import type { - NewBuildInfo, - SetupServiceWorkerOptions, - SetupServiceWorkerResult, +import { + AUTOMERGE_WORKER_PATH, + type NewBuildInfo, + type SetupServiceWorkerOptions, + type SetupServiceWorkerResult, } from "./types.js"; import { buildId, isNewerBuild } from "./build-id.js"; import { promptForNewBuild } from "./new-build-prompt.js"; @@ -107,8 +108,7 @@ function serviceWorkerBuild(sw: ServiceWorker): Promise { // port closes, or when the Web Lock it holds for its lifetime is released (see // AUTOMERGE_WORKER_LOCK). -let automergeProtocolHandlerWorkerPath = - "/automerge-protocol-handler-worker.js"; +let automergeProtocolHandlerWorkerPath = AUTOMERGE_WORKER_PATH; const automergeProtocolHandlerWorker = sharedWorkerHandle( "patchwork-automerge-protocol-handler", diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index f4ce24b6c..9cbfb31a7 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -206,3 +206,32 @@ export type IndexEntry = { sha256: string; contentType: string; store: boolean } * below it ("" for the document itself). */ export type Manifest = { prefix: string; entries: Record }; + +/** + * Where the automerge worker's script is served. The vite plugin keeps this name from one build + * to the next (it's started by URL), and the service worker trusts index answers only from a + * SharedWorker running it. + */ +export const AUTOMERGE_WORKER_PATH = "/automerge-protocol-handler-worker.js"; + +/** + * Where the automerge worker POSTs its answers to the service worker. Under the reserved `/@/` + * prefix, and `.` isn't base58, so no account path can collide with it. + */ +export const INDEX_ANSWER_PATH = "/@/.index"; + +/** Service worker → automerge worker, on HANDOFF_CHANNEL: build the manifest covering `path`. */ +export interface ManifestRequestMessage { + type: "manifest"; + id: string; + path: string; +} + +/** Automerge worker → service worker, as the body of a POST to INDEX_ANSWER_PATH. */ +export type IndexAnswer = { id: string; manifest?: Manifest; error?: string }; + +/** Tab → automerge worker: resolve paths under this account. */ +export interface RegisterAccountMessage { + type: "register-account"; + url: string; +} diff --git a/core/bootloader/test/index-channel.test.ts b/core/bootloader/test/index-channel.test.ts new file mode 100644 index 000000000..5774fb54b --- /dev/null +++ b/core/bootloader/test/index-channel.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect, vi } from "vitest"; +import { + acceptIndexAnswer, + isAutomergeWorker, + postIndexAnswer, +} from "../src/index-channel.js"; + +const o = "https://p.test"; +const worker = { type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js` } as const; + +describe("isAutomergeWorker", () => { + it.each([ + [worker, true], + [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js?x=1` }, true], + // what any page can start from the same url: a dedicated worker + [{ type: "worker", url: `${o}/automerge-protocol-handler-worker.js` }, false], + [{ type: "window", url: `${o}/` }, false], + [{ type: "sharedworker", url: `${o}/@/acct@h/evil.js` }, false], + [{ type: "sharedworker", url: `https://evil.test/automerge-protocol-handler-worker.js` }, false], + [undefined, false], + ])("%j → %s", (client, ok) => { + expect(isAutomergeWorker(client as any, o)).toBe(ok); + }); +}); + +const answerRequest = (body: unknown) => + new Request(`${o}/@/.index`, { method: "POST", body: JSON.stringify(body) }); + +describe("acceptIndexAnswer", () => { + it("refuses a window with 403 and hands nothing on", async () => { + const onAnswer = vi.fn(); + const response = await acceptIndexAnswer( + answerRequest({ id: "1" }), + { type: "window", url: `${o}/` }, + o, + onAnswer + ); + expect(response.status).toBe(403); + expect(onAnswer).not.toHaveBeenCalled(); + }); + + it("accepts the worker with 204 and the header", async () => { + const onAnswer = vi.fn(); + const answer = { id: "1", manifest: { prefix: "a@h", entries: {} } }; + const response = await acceptIndexAnswer(answerRequest(answer), worker, o, onAnswer); + expect(response.status).toBe(204); + expect(response.headers.get("x-patchwork-index")).toBe("accepted"); + expect(onAnswer).toHaveBeenCalledWith(answer); + }); + + it("refuses a body that isn't an answer", async () => { + const onAnswer = vi.fn(); + const response = await acceptIndexAnswer(answerRequest("nope"), worker, o, onAnswer); + expect(response.status).toBe(400); + expect(onAnswer).not.toHaveBeenCalled(); + }); +}); + +describe("postIndexAnswer", () => { + it("retries a reply that skipped the service worker", async () => { + const replies = [ + new Response("", { status: 404 }), + new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } }), + ]; + const fetcher = vi.fn(async () => replies.shift()!); + expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(true); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + + it("gives up after 5 tries", async () => { + const fetcher = vi.fn(async () => new Response("", { status: 404 })); + expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(false); + expect(fetcher).toHaveBeenCalledTimes(5); + }); +}); diff --git a/core/bootloader/test/manifest.test.ts b/core/bootloader/test/manifest.test.ts new file mode 100644 index 000000000..b7c62e908 --- /dev/null +++ b/core/bootloader/test/manifest.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; +import { formatHeads, idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; +import { buildManifest } from "../src/manifest.js"; +import { contentHash } from "../src/content-hash.js"; + +const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; +const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + +async function fixture() { + const repo = new Repo(); + const projects = folder(repo, { note: file(repo, "héllo ✓") }); + const root = folder(repo, { Projects: projects.url }); + const pkg = repo.create({ + "@patchwork": { type: "directory" }, + "index.js": file(repo, "export default 1"), + }); + const packages = folder(repo, { pkg: pkg.url }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }], "/packages": [{ url: packages.url }] }, + }); + const a = await idhash(account.documentId); + return { repo, a, accounts: new Map([[a, account.url]]), projects }; +} + +describe("buildManifest", () => { + it("hashes each file as the service worker will", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); + const manifest = await buildManifest(repo, prefix!, { accounts }); + expect(manifest?.prefix).toBe(prefix); + expect(manifest?.entries.note).toEqual({ + sha256: await contentHash(new TextEncoder().encode("héllo ✓")), + contentType: "text/plain", + store: true, + }); + }); + + it("stores protected content only through /packages", async () => { + const { repo, a, accounts } = await fixture(); + const isProtected = () => true; + const projects = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); + const pkg = await pinnedPath(repo, livePath(a, ["packages", "pkg"]), { accounts }); + expect((await buildManifest(repo, projects!, { accounts, isProtected }))?.entries.note.store).toBe(false); + expect((await buildManifest(repo, pkg!, { accounts, isProtected }))?.entries["index.js"].store).toBe(true); + }); + + it("never stores under a share root", async () => { + const { repo, accounts, projects } = await fixture(); + const share = `automerge:${projects.documentId}@${formatHeads(projects.heads())}`; + const manifest = await buildManifest(repo, share, { accounts }); + expect(manifest?.entries.note.store).toBe(false); + }); + + it("returns nothing for a live path", async () => { + const { repo, a, accounts } = await fixture(); + expect(await buildManifest(repo, livePath(a, ["Projects"]), { accounts })).toBeUndefined(); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index baa96d0fa..d7c73226d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { workspace: [ + "core/bootloader", "core/filesystem", "core/patchwork", "packages/edge-handles", From 9aaf485646e5f95809ceddb291d328eeec5704eb Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:31:36 +0000 Subject: [PATCH 17/81] bootloader: resolvers answer /@/ paths and leave caching to the service worker Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/resolver.ts | 194 ++++++++++++++++++-------- core/bootloader/src/tab-resolver.ts | 28 +++- core/bootloader/src/types.ts | 11 +- core/bootloader/test/resolver.test.ts | 157 +++++++++++++++++++++ 4 files changed, 325 insertions(+), 65 deletions(-) create mode 100644 core/bootloader/test/resolver.test.ts diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 72e0f0395..a7c47481b 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -1,21 +1,30 @@ -// Turning a handoff request into a reply, given a Repo: find the automerge -// URL's document, resolve the path in it, and either write the response into -// the service worker's cache or hand it back inline. A tab answers the service -// worker with its own Repo this way (see tab-resolver.ts), and the automerge -// worker answers the broadcasts nobody else could (automerge-protocol-handler). +// Turning a handoff request into a reply, given a Repo: walk a `/@/` path, or +// find a legacy automerge URL's document and resolve the path in it, and hand +// the response back inline. What to keep is the service worker's call, never a +// resolver's. A tab answers the service worker with its own Repo this way (see +// tab-resolver.ts), and the automerge worker answers the broadcasts nobody else +// could (automerge-protocol-handler). import { hasHeads } from "@automerge/automerge/slim"; import { + decodeHeads, isValidAutomergeUrl, parseAutomergeUrl, stringifyAutomergeUrl, + type AutomergeUrl, type DocHandle, type Repo, type StorageId, + type UrlHeads, } from "@automerge/automerge-repo/slim"; import { getDefaultBranch, + parsePath, + readNode, resolvePath, + walkPath, + type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; +import { toBytes } from "./content-hash.js"; import type { HandoffReplyMessage, @@ -24,8 +33,6 @@ import type { const RESOLVE_TIMEOUT_MS = 30_000; -const CACHEABLE_STATUSES = [200, 203, 204]; - const CATCH_UP_MS = 3_000; function waitForHeads( @@ -178,19 +185,96 @@ function impatience(limit: number) { ); } +/** + * The walk's `find`: wait for pinned heads to arrive, or for a live document to catch up with + * the sync server, within `signal`. Heads that never arrive are remembered in `missing`, so a + * walk that fails because of them answers with a network error rather than a 404. + */ +function waitingFind( + repo: Repo, + signal: AbortSignal, + missing: Set +): NonNullable { + return async (url: AutomergeUrl, heads?: UrlHeads) => { + const handle = await repo.find(url, { signal }); + if (heads) { + if (!(await waitForHeads(handle, decodeHeads(heads), signal))) { + missing.add(url); + throw new AbortHandoff(`heads ${heads.join(",")} of ${url} not found`); + } + } else { + await caughtUpWithPeers(repo, handle, signal); + } + return handle; + }; +} + +/** + * A `/@/` path, answered inline: a 307 to the pinned form for a live path (or a pinned-looking + * one that isn't canonical), the bytes for a canonical pinned one, 404 for one that doesn't + * resolve. Pinned heads that never arrive throw {@link AbortHandoff}. + */ +async function resolveWalkPath( + repo: Repo, + path: string, + accounts: ReadonlyMap, + signal: AbortSignal +): Promise { + try { + parsePath(path); + } catch (error) { + return new Response(String(error), { status: 400 }); + } + const missing = new Set(); + const end = await walkPath(repo, path, { + accounts, + find: waitingFind(repo, signal, missing), + }); + if (!end) { + if (missing.size) { + throw new AbortHandoff(`heads not found for ${[...missing].join(", ")}`); + } + return new Response(`nothing at ${path}`, { + status: 404, + headers: { "cache-control": "no-store" }, + }); + } + if (!end.canonical) { + return new Response(null, { + status: 307, + headers: { location: `/@/${end.pinned}`, "cache-control": "no-store" }, + }); + } + const read = await readNode(repo, end.node); + if (!read) return new Response(`can't read ${path}`, { status: 404 }); + return new Response(toBytes(read.content) as BlobPart, { + status: 200, + headers: { "content-type": read.type }, + }); +} + export type Resolver = { /** - * Resolve `message`'s request and say how the service worker should answer - * it, having written the response into its cache if it's one to keep. - * Undefined if the special URL isn't an `automerge:` one. + * Resolve `message`'s request and hand the response back inline, for the + * service worker to serve and, if it chooses, keep. Undefined if it's neither + * a `/@/` path nor an `automerge:` URL. `signal` cuts the 30s allowance short. */ - answer(message: HandoffRequestMessage): Promise; + answer( + message: HandoffRequestMessage, + signal?: AbortSignal + ): Promise; }; export function createResolver( getRepo: () => Promise, - log: (...args: unknown[]) => void = () => {} + log: (...args: unknown[]) => void = () => {}, + accounts: () => ReadonlyMap = () => new Map() ): Resolver { + const within = (signal?: AbortSignal) => + signal + ? AbortSignal.any([signal, AbortSignal.timeout(RESOLVE_TIMEOUT_MS)]) + : AbortSignal.timeout(RESOLVE_TIMEOUT_MS); + const resolveWithin = async (handoff: URL): Promise => { const repo = await getRepo(); const signal = AbortSignal.timeout(RESOLVE_TIMEOUT_MS); @@ -237,21 +321,44 @@ export function createResolver( response: { status, body, headers: { "content-type": "text/plain" } }, }) satisfies HandoffReplyMessage; - return { - async answer({ id, cachename, request }) { - let handoff: URL; - try { - handoff = new URL(request.handoffURL); - } catch { - console.error("couldn't parse handoff url", request); - return inline(id, 400, `couldn't parse a special url out of ${request.url}`); - } - if (handoff.protocol !== "automerge:") return undefined; + const reply = async (id: string, response: Response): Promise => ({ + id, + type: "response", + response: { + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: response.body ? new Uint8Array(await response.arrayBuffer()) : undefined, + }, + }); + return { + async answer({ id, request }, signal) { let response: Response; try { - log(`resolving handoff ${id} for ${handoff}`); - response = await resolveHandoff(handoff); + if (request.path !== undefined) { + const repo = await getRepo(); + const limit = within(signal); + log(`resolving handoff ${id} for /@/${request.path}`); + response = await Promise.race([ + resolveWalkPath(repo, request.path, accounts(), limit), + new Promise((_, reject) => + limit.addEventListener("abort", () => + reject(new AbortHandoff(`resolving /@/${request.path} timed out`)) + ) + ), + ]); + } else { + let handoff: URL; + try { + handoff = new URL(request.handoffURL); + } catch { + console.error("couldn't parse handoff url", request); + return inline(id, 400, `couldn't parse a special url out of ${request.url}`); + } + if (handoff.protocol !== "automerge:") return undefined; + log(`resolving handoff ${id} for ${handoff}`); + response = await resolveHandoff(handoff); + } } catch (error) { if (error instanceof AbortHandoff) { return { id, type: "abort", reason: error.message }; @@ -267,41 +374,10 @@ export function createResolver( } try { - const immutable = - CACHEABLE_STATUSES.includes(response.status) && - !/(^|,)\s*no-store(\s*,|$)/.test( - response.headers.get("cache-control") ?? "" - ); - if (!immutable) { - // Errors, redirects and the like go back inline for the service - // worker to serve directly. Mutable responses aren't cached because - // the same URL can resolve to different content later. - log(`responding inline to ${request.url} with ${response.status}`); - return { - id, - type: "response", - response: { - status: response.status, - headers: Object.fromEntries(response.headers.entries()), - body: response.body - ? new Uint8Array(await response.arrayBuffer()) - : undefined, - }, - }; - } - - // Reconstruct the request the service worker is holding so the entry - // matches its cache.match. `destination` isn't constructible but - // doesn't participate in cache matching. - const cacheKey = new Request(request.url, { - method: request.method, - headers: request.headers, - referrer: request.referrer, - }); - const cache = await caches.open(cachename); - await cache.put(cacheKey, response); - log(`cached ${cacheKey.url} in ${cachename}`); - return { id, type: "cached" }; + // Everything goes back inline. The service worker keeps what's + // immutable (see its serveHandoff), or checks it against the index. + log(`responding inline to ${request.url} with ${response.status}`); + return await reply(id, response); } catch (error) { console.error(`failed to reply for ${request.url}`, error); return inline(id, 558, String(error)); diff --git a/core/bootloader/src/tab-resolver.ts b/core/bootloader/src/tab-resolver.ts index c0f6ec0af..18a58652a 100644 --- a/core/bootloader/src/tab-resolver.ts +++ b/core/bootloader/src/tab-resolver.ts @@ -2,7 +2,12 @@ // service worker asks the tab that made a request first, then any other tab, // and only broadcasts to the automerge worker when none of them can; a tab // that hasn't been handed its Repo yet says so straight away. -import type { Repo } from "@automerge/automerge-repo/slim"; +import { + parseAutomergeUrl, + type AutomergeUrl, + type Repo, +} from "@automerge/automerge-repo/slim"; +import { idhash } from "@inkandswitch/patchwork-filesystem"; import { createResolver, type Resolver } from "./resolver.js"; import type { @@ -14,13 +19,26 @@ import type { let resolver: Resolver | undefined; +/** acctHash → account url: the accounts whose `/@/` paths this tab resolves. */ +const accounts = new Map(); + /** - * Resolve the service worker's `automerge:` requests with `repo` from now on, - * rather than leaving them to the automerge worker. What they load stays in - * `repo`. + * Resolve the service worker's `automerge:` requests and `/@/` paths with + * `repo` from now on, rather than leaving them to the automerge worker. What + * they load stays in `repo`. */ export function resolveHandoffsWith(repo: Repo): void { - resolver = createResolver(() => Promise.resolve(repo)); + resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts); +} + +/** + * Resolve `/@/` paths rooted at this account in this tab. Returns its acctHash, + * computed here from the url, so nothing can name an account by the wrong hash. + */ +export async function registerAccount(url: AutomergeUrl): Promise { + const acctHash = await idhash(parseAutomergeUrl(url).documentId); + accounts.set(acctHash, url); + return acctHash; } /** Answer a ResolveRequestMessage; false if `event` isn't one. */ diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 9cbfb31a7..1dc62ada9 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -40,8 +40,17 @@ export interface HandoffRequest { * `https://…/automerge%3Aabc/…` form) — the cache key. */ url: string; - /** the decoded special URL, e.g. `automerge:abc/some/path` */ + /** + * the decoded special URL, e.g. `automerge:abc/some/path`. For a `/@/` path + * it's `patchwork-path:` and the path, which older resolvers parse and then + * decline, since it isn't `automerge:`. + */ handoffURL: string; + /** + * For a `/@/` URL, the path after `/@/`: live (answered with a redirect to + * its pinned form) or pinned (answered with its bytes, inline). + */ + path?: string; /** * @deprecated A briefly-deployed shape put the special URL in `url` and * the cache key here. Only read, never sent. diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts new file mode 100644 index 000000000..c955691be --- /dev/null +++ b/core/bootloader/test/resolver.test.ts @@ -0,0 +1,157 @@ +import { describe, it, expect } from "vitest"; +import { + Repo, + encodeHeads, + stringifyAutomergeUrl, + type AutomergeUrl, +} from "@automerge/automerge-repo"; +import { idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; +import { createResolver } from "../src/resolver.js"; +import { registerAccount, resolveHandoffsWith, handleResolveRequest } from "../src/tab-resolver.js"; +import type { HandoffRequestMessage, HandoffResponseMessage } from "../src/types.js"; + +// No `caches` in node: a resolver that still wrote the cache would throw. + +const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; +const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + +async function fixture() { + const repo = new Repo(); + const foo = folder(repo, { notes: file(repo, "hi") }); + const root = folder(repo, { Projects: folder(repo, { foo: foo.url }).url }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + const resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts); + return { repo, a, accounts, account, resolver }; +} + +const pathRequest = (path: string): HandoffRequestMessage => ({ + id: "1", + type: "request", + cachename: "patchwork", + request: { + url: `https://p.test/@/${path}`, + handoffURL: `patchwork-path:${path}`, + path, + headers: {}, + method: "GET", + destination: "script", + referrer: "", + }, +}); + +const inline = (reply: unknown) => (reply as HandoffResponseMessage).response; +const body = (reply: unknown) => { + const b = inline(reply).body; + return typeof b === "string" ? b : new TextDecoder().decode(b); +}; + +describe("resolver: /@/ paths", () => { + it("redirects a live path to its pinned form", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const live = livePath(a, ["Projects", "foo", "notes"]); + const reply = await resolver.answer(pathRequest(live)); + expect(reply?.type).toBe("response"); + expect(inline(reply).status).toBe(307); + expect(inline(reply).headers?.location).toBe( + `/@/${await pinnedPath(repo, live, { accounts })}` + ); + expect(inline(reply).headers?.["cache-control"]).toBe("no-store"); + }); + + it("answers a pinned path with its bytes inline", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo", "notes"]), { accounts }); + const reply = await resolver.answer(pathRequest(pinned!)); + expect(inline(reply).status).toBe(200); + expect(inline(reply).headers?.["content-type"]).toBe("text/plain"); + expect(body(reply)).toBe("hi"); + }); + + it("redirects a pinned-looking path that isn't canonical", async () => { + const { repo, a, account, resolver } = await fixture(); + const heads = [...account.heads()].sort().join("."); + const reply = await resolver.answer(pathRequest(`${a}@${heads}/Projects`)); + expect(inline(reply).status).toBe(307); + void repo; + }); + + it("answers an unknown path with 404", async () => { + const { a, resolver } = await fixture(); + const reply = await resolver.answer(pathRequest(livePath(a, ["nope"]))); + expect(inline(reply).status).toBe(404); + }); + + it("aborts a pinned path whose heads never arrive", async () => { + const { a, resolver } = await fixture(); + const [unknown] = encodeHeads(["00".repeat(32)]); + const reply = await resolver.answer( + pathRequest(`${a}@${unknown}/Projects`), + AbortSignal.timeout(200) + ); + expect(reply?.type).toBe("abort"); + }); + + it("answers 404 for heads that can't be heads", async () => { + const { a, resolver } = await fixture(); + const reply = await resolver.answer(pathRequest(`${a}@3vQB7B6MrGQZaxCuFg4oh/Projects`)); + expect(inline(reply).status).toBe(404); + }); +}); + +describe("resolver: legacy automerge URLs", () => { + it("answers inline, leaving caching to the service worker", async () => { + const { repo, resolver } = await fixture(); + const pkg = repo.create({ + "@patchwork": { type: "directory" }, + "index.js": { content: "export default 1", mimeType: "text/javascript" }, + }); + const url = stringifyAutomergeUrl({ documentId: pkg.documentId, heads: pkg.heads() }); + const reply = await resolver.answer({ + ...pathRequest(""), + request: { + url: `https://p.test/${encodeURIComponent(url)}/index.js`, + handoffURL: `${url}/index.js`, + headers: {}, + method: "GET", + destination: "script", + referrer: "", + }, + }); + expect(reply?.type).toBe("response"); + expect(inline(reply).status).toBe(200); + expect(inline(reply).headers?.["cache-control"]).toContain("immutable"); + expect(body(reply)).toBe("export default 1"); + }); +}); + +describe("registerAccount", () => { + it("makes the tab's resolver answer for that account", async () => { + const { repo, a, account } = await fixture(); + resolveHandoffsWith(repo); + expect(await registerAccount(account.url)).toBe(a); + + const { port1, port2 } = new MessageChannel(); + const replies: unknown[] = []; + port1.onmessage = (event) => replies.push(event.data); + const handled = handleResolveRequest( + new MessageEvent("message", { + data: { ...pathRequest(livePath(a, ["Projects"])), type: "resolve" }, + ports: [port2], + }) + ); + expect(handled).toBe(true); + await expect.poll(() => replies.length, { timeout: 5000 }).toBeGreaterThanOrEqual(2); + port1.close(); + expect(inline(replies.at(-1)).status).toBe(307); + }); +}); From 9c17a49af34d7ed1302eabd8e3acc289dfa16b93 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:39:53 +0000 Subject: [PATCH 18/81] bootloader: the automerge worker answers manifests; tabs register their account The walk now records the last container it entered, and listFiles lists under it, so a manifest request for a file covers its whole package. Co-Authored-By: Claude Opus 5.5 --- .../src/automerge-protocol-handler.ts | 53 +++++++++++- core/bootloader/src/manifest.ts | 73 +++++++++++++++- core/bootloader/src/resolver.ts | 2 +- core/bootloader/src/setup.ts | 33 ++++++- core/bootloader/test/worker-manifests.test.ts | 85 +++++++++++++++++++ core/filesystem/src/walk.ts | 35 ++++++-- core/filesystem/test/files.test.ts | 15 ++++ core/patchwork/src/index.ts | 3 + 8 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 core/bootloader/test/worker-manifests.test.ts diff --git a/core/bootloader/src/automerge-protocol-handler.ts b/core/bootloader/src/automerge-protocol-handler.ts index 9f0a457ec..d1bc939a2 100644 --- a/core/bootloader/src/automerge-protocol-handler.ts +++ b/core/bootloader/src/automerge-protocol-handler.ts @@ -17,9 +17,12 @@ import initSubduction from "@automerge/automerge-subduction/slim"; import { Repo, + parseAutomergeUrl, + type AutomergeUrl, type DocumentId, type PeerId, } from "@automerge/automerge-repo/slim"; +import { idhash } from "@inkandswitch/patchwork-filesystem"; import { IndexedDBStorageAdapter } from "@automerge/automerge-repo-storage-indexeddb"; import type { @@ -31,12 +34,15 @@ import { loadOrCreateSigner } from "./signer.js"; import { keyhiveStorageName, storagePrefix } from "./storage.js"; import { startWorkerControl } from "./worker-control.js"; import { AUTOMERGE_WASM, SUBDUCTION_WASM } from "./emitted.js"; -import { createResolver } from "./resolver.js"; +import { createManifestHandler } from "./manifest.js"; +import { createResolver, waitingFind } from "./resolver.js"; import { AUTOMERGE_WORKER_LOCK, HANDOFF_CHANNEL, type HandoffOnlineMessage, type HandoffRequestMessage, + type ManifestRequestMessage, + type RegisterAccountMessage, } from "./types.js"; declare const __SYNC_SERVER__: { @@ -50,11 +56,30 @@ const syncServer = ? __SYNC_SERVER__ : { url: "wss://subduction.sync.inkandswitch.com" }; +// acctHash → account url: the accounts whose `/@/` paths this worker resolves. +// Tabs tell us theirs; the hash is computed here, so no tab can name an +// account by the wrong one. +const accounts = new Map(); + +async function registerAccount({ url }: RegisterAccountMessage) { + try { + const { documentId } = parseAutomergeUrl(url as AutomergeUrl); + accounts.set(await idhash(documentId), url as AutomergeUrl); + } catch (error) { + console.warn(`ignoring a register-account for ${url}`, error); + } +} + const control = startWorkerControl("automerge-protocol-handler-worker", { onRestart: drainForRestart, // Held only so the tabs and the service worker can tell whether this // instance is still around; nothing here waits for it. lock: AUTOMERGE_WORKER_LOCK, + onMessage(data) { + if (data?.type === "register-account") { + void registerAccount(data as RegisterAccountMessage); + } + }, }); const log = control.log; @@ -169,7 +194,19 @@ async function buildKeyhiveRepo( // them can (one still booting, one from an older build, a site that brings // its own Repo). -const resolver = createResolver(getRepo, log); +const resolver = createResolver(getRepo, log, () => accounts); + +// The service worker stops waiting for a manifest after 10s; walking to it +// gets no longer than that. +const MANIFEST_WALK_TIMEOUT_MS = 10_000; + +// Manifests are what the service worker checks every pinned `/@/` URL +// against. Only this worker answers them: tabs run package code. +const answerManifest = createManifestHandler(getRepo, () => accounts, { + find: (repo) => + waitingFind(repo, AbortSignal.timeout(MANIFEST_WALK_TIMEOUT_MS), new Set()), + log, +}); const handoffChannel = new BroadcastChannel(HANDOFF_CHANNEL); const EVICT_IDLE_MS = 5_000; @@ -312,9 +349,17 @@ async function drainForRestart(): Promise { } handoffChannel.addEventListener("message", (event) => { - if (event.data?.type !== "request") return; + const type = event.data?.type; + if (type !== "request" && type !== "manifest") return; if (draining) { - log(`ignoring handoff ${event.data.id} while restarting`); + log(`ignoring ${type} ${event.data.id} while restarting`); + return; + } + if (type === "manifest") { + handoffStarted(); + void answerManifest(event.data as ManifestRequestMessage).finally( + handoffFinished + ); return; } void handleHandoffRequest(event.data as HandoffRequestMessage); diff --git a/core/bootloader/src/manifest.ts b/core/bootloader/src/manifest.ts index d3d122c38..63814bcdb 100644 --- a/core/bootloader/src/manifest.ts +++ b/core/bootloader/src/manifest.ts @@ -2,17 +2,25 @@ // and whether the service worker may keep it. import { stringifyAutomergeUrl, + type AutomergeUrl, type DocumentId, type Repo, } from "@automerge/automerge-repo/slim"; import { listFiles, parsePath, + walkPath, type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; import { isKeyhiveDoc } from "@inkandswitch/patchwork-plugins"; import { contentHash, toBytes } from "./content-hash.js"; -import type { IndexEntry, Manifest } from "./types.js"; +import { postIndexAnswer } from "./index-channel.js"; +import type { + IndexAnswer, + IndexEntry, + Manifest, + ManifestRequestMessage, +} from "./types.js"; const isProtectedDoc = (documentId: DocumentId) => isKeyhiveDoc(stringifyAutomergeUrl({ documentId })); @@ -48,3 +56,66 @@ export async function buildManifest( } return { prefix: listed.prefix, entries }; } + +const REMEMBERED_MANIFESTS = 64; + +/** + * The automerge worker's answer to the service worker's manifest requests. A request names a + * file; its manifest covers the container the file is in, so every file of a package shares one. + * Manifests are remembered by that container's pinned path, which never changes meaning, and + * concurrent requests for one share a build. + */ +export function createManifestHandler( + getRepo: () => Promise, + accounts: () => ReadonlyMap, + options: { + post?: (answer: IndexAnswer) => Promise; + /** The walk's find, per request: one that waits for heads, in the worker. */ + find?: (repo: Repo) => WalkOptions["find"]; + log?: (...args: unknown[]) => void; + } = {} +): (message: ManifestRequestMessage) => Promise { + const post = options.post ?? ((answer) => postIndexAnswer(answer)); + const remembered = new Map(); + const building = new Map>(); + + const manifestFor = async (path: string): Promise => { + const repo = await getRepo(); + const walk: WalkOptions = { accounts: accounts(), find: options.find?.(repo) }; + const end = await walkPath(repo, path, walk); + const prefix = end?.canonical ? end.container?.pinned : undefined; + if (!prefix) throw new Error(`${path} isn't a pinned path this worker can walk`); + + const known = remembered.get(prefix); + if (known) { + remembered.delete(prefix); + remembered.set(prefix, known); + return known; + } + let build = building.get(prefix); + if (!build) { + build = buildManifest(repo, prefix, walk).finally(() => building.delete(prefix)); + building.set(prefix, build); + } + const manifest = await build; + if (!manifest) throw new Error(`couldn't list the files under ${prefix}`); + remembered.set(prefix, manifest); + if (remembered.size > REMEMBERED_MANIFESTS) { + remembered.delete(remembered.keys().next().value!); + } + return manifest; + }; + + return async ({ id, path }) => { + let answer: IndexAnswer; + try { + answer = { id, manifest: await manifestFor(path) }; + } catch (error) { + answer = { id, error: error instanceof Error ? error.message : String(error) }; + } + options.log?.(`manifest ${id} for ${path}: ${answer.error ?? "built"}`); + if (!(await post(answer))) { + console.warn(`couldn't get manifest ${id} to the service worker`); + } + }; +} diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index a7c47481b..b5be34e76 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -190,7 +190,7 @@ function impatience(limit: number) { * the sync server, within `signal`. Heads that never arrive are remembered in `missing`, so a * walk that fails because of them answers with a network error rather than a 404. */ -function waitingFind( +export function waitingFind( repo: Repo, signal: AbortSignal, missing: Set diff --git a/core/bootloader/src/setup.ts b/core/bootloader/src/setup.ts index b7b363b17..ac2febdd7 100644 --- a/core/bootloader/src/setup.ts +++ b/core/bootloader/src/setup.ts @@ -1,5 +1,6 @@ import { AUTOMERGE_WORKER_PATH, + type RegisterAccountMessage, type NewBuildInfo, type SetupServiceWorkerOptions, type SetupServiceWorkerResult, @@ -7,7 +8,11 @@ import { import { buildId, isNewerBuild } from "./build-id.js"; import { promptForNewBuild } from "./new-build-prompt.js"; import debug from "debug"; -import { handleResolveRequest } from "./tab-resolver.js"; +import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { + handleResolveRequest, + registerAccount as resolveAccountInTab, +} from "./tab-resolver.js"; import { forwardWorkerConsole, lifecycleLog, @@ -16,6 +21,7 @@ import { export { lifecycleLog }; export { resolveHandoffsWith } from "./tab-resolver.js"; +export { clearAccountStore } from "./content-store.js"; export { connectClassicSync } from "./classic-sync.js"; const serviceWorkerDebugging = debug.enabled("patchwork:serviceworker"); @@ -110,6 +116,22 @@ function serviceWorkerBuild(sw: ServiceWorker): Promise { let automergeProtocolHandlerWorkerPath = AUTOMERGE_WORKER_PATH; +const registeredAccounts = new Set(); + +/** + * Let `/@//…` paths under this account resolve: in this tab, and in + * the automerge worker, which is told again whenever it's replaced. Returns the + * account's acctHash. + */ +export async function registerAccount(url: AutomergeUrl): Promise { + registeredAccounts.add(url); + automergeProtocolHandlerWorker.post({ + type: "register-account", + url, + } satisfies RegisterAccountMessage); + return resolveAccountInTab(url); +} + const automergeProtocolHandlerWorker = sharedWorkerHandle( "patchwork-automerge-protocol-handler", () => automergeProtocolHandlerWorkerPath, @@ -118,6 +140,15 @@ const automergeProtocolHandlerWorker = sharedWorkerHandle( onMessage(event) { forwardWorkerConsole("automerge-protocol-handler-worker", event.data); }, + // A new instance knows no accounts until it's told again. + onSpawn(worker) { + for (const url of registeredAccounts) { + worker.port.postMessage({ + type: "register-account", + url, + } satisfies RegisterAccountMessage); + } + }, onRestarting(message) { // The worker restarts because a newer tab connected. If that build is // newer than ours too, we're the stale one. diff --git a/core/bootloader/test/worker-manifests.test.ts b/core/bootloader/test/worker-manifests.test.ts new file mode 100644 index 000000000..cee511ea6 --- /dev/null +++ b/core/bootloader/test/worker-manifests.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, vi } from "vitest"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; +import { idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; +import { createManifestHandler } from "../src/manifest.js"; +import type { IndexAnswer } from "../src/types.js"; + +const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; + +async function fixture(packages = 1) { + const repo = new Repo(); + const pkgs = Array.from({ length: packages }, (_, i) => + repo.create({ + "@patchwork": { type: "directory" }, + "a.js": file(repo, `a${i}`), + "b.js": file(repo, `b${i}`), + }) + ); + const folder = repo.create({ + title: "packages", + docs: pkgs.map((pkg, i) => ({ name: `p${i}`, type: "x", url: pkg.url })), + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: folder.url as AutomergeUrl }] }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + const prefixes = await Promise.all( + pkgs.map((_, i) => pinnedPath(repo, livePath(a, ["packages", `p${i}`]), { accounts })) + ); + return { repo, accounts, prefixes: prefixes as string[] }; +} + +const setup = async (packages?: number) => { + const f = await fixture(packages); + const posted: IndexAnswer[] = []; + const post = vi.fn(async (answer: IndexAnswer) => (posted.push(answer), true)); + const finds: string[] = []; + const find = (repo: Repo) => (url: AutomergeUrl) => (finds.push(url), repo.find(url)); + const handle = createManifestHandler(() => Promise.resolve(f.repo), () => f.accounts, { + post, + find, + }); + return { ...f, handle, posted, finds }; +}; + +describe("createManifestHandler", () => { + it("builds a package's manifest once and answers every file's request with it", async () => { + const { handle, prefixes, posted, finds } = await setup(); + await Promise.all([ + handle({ type: "manifest", id: "1", path: `${prefixes[0]}/a.js` }), + handle({ type: "manifest", id: "2", path: `${prefixes[0]}/b.js` }), + ]); + await handle({ type: "manifest", id: "3", path: `${prefixes[0]}/a.js` }); + expect(posted.map((p) => p.id).sort()).toEqual(["1", "2", "3"]); + expect(posted[0].manifest).toEqual(posted[1].manifest); + expect(Object.keys(posted[0].manifest!.entries).sort()).toEqual(["", "a.js", "b.js"]); + // a request for a package already built walks to it and lists nothing again + const fileFinds = finds.length; + await handle({ type: "manifest", id: "4", path: `${prefixes[0]}/b.js` }); + expect(finds.length - fileFinds).toBeLessThanOrEqual(4); // a walk, no relisting + }); + + it("posts an error for a path it can't list", async () => { + const { handle, posted } = await setup(); + await handle({ type: "manifest", id: "x", path: "unknown@h/a.js" }); + expect(posted[0]).toMatchObject({ id: "x", error: expect.any(String) }); + expect(posted[0].manifest).toBeUndefined(); + }); + + it("evicts the oldest manifest after 64", async () => { + const { handle, prefixes, posted, finds } = await setup(65); + for (const [i, prefix] of prefixes.entries()) { + await handle({ type: "manifest", id: `${i}`, path: prefix }); + } + const before = finds.length; + await handle({ type: "manifest", id: "again", path: prefixes[64] }); + const cached = finds.length - before; + await handle({ type: "manifest", id: "evicted", path: prefixes[0] }); + const rebuilt = finds.length - before - cached; + expect(rebuilt).toBeGreaterThan(cached); + expect(posted.at(-1)?.manifest?.prefix).toBe(prefixes[0]); + }); +}); diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 847e32b90..020290f67 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -66,6 +66,11 @@ export type WalkEnd = { * not, since the names resolve at whatever the documents below it hold now. */ canonical: boolean; + /** + * The last document the walk entered that has a provider (a folder, a directory, …), and its + * pinned path: what a manifest of the files around the end of the path covers. + */ + container?: { handle: DocHandle; pinned: string }; }; /** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ @@ -166,8 +171,16 @@ export async function walkPath( out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); } + let container: WalkEnd["container"]; + const noteContainer = () => { + if (at?.kind === "doc" && at.provider && !at.inside) { + container = { handle: at.handle, pinned: out.join("/") }; + } + }; + let i = 0; while (true) { + noteContainer(); const segment = segments[i] as (typeof segments)[number] | undefined; if (at.kind === "union") { @@ -255,12 +268,14 @@ export async function walkPath( } } + noteContainer(); return { handle: at.handle, node: at.node, atDocument: !at.inside, pinned: out.join("/"), canonical: !live, + container, }; } @@ -378,9 +393,10 @@ function walkPrefers(a: number[], b: number[]): boolean { } /** - * Every file under the last document a canonical pinned path enters, keyed by tail: "" for the - * document itself, encoded names joined with "/" below it. Containers aren't listed; they have - * their own doc steps. Undefined unless `path` is canonical and ends at a document. + * Every file under the last container a canonical pinned path enters (see + * {@link WalkEnd.container}), keyed by tail: "" for the container itself, encoded names joined + * with "/" below it. Containers below it aren't listed; they have their own doc steps. `path` + * may name the container or anything in it. Undefined unless `path` is canonical. * * A file reached by a link without heads is read as it is now, so the list is a snapshot. */ @@ -390,10 +406,11 @@ export async function listFiles( options: WalkOptions ): Promise<{ prefix: string; files: AsyncIterable } | undefined> { const end = await walkPath(repo, path, options); - if (!end?.canonical || !end.atDocument) return undefined; + if (!end?.canonical || !end.container) return undefined; const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); - const provider = providerFor(end.node); - const documentId = end.handle.documentId; + const root = end.container.handle.doc(); + const provider = providerFor(root); + const documentId = end.container.handle.documentId; async function* files(): AsyncIterable { const found = new Map(); @@ -440,10 +457,10 @@ export async function listFiles( } }; - await add([], [], end!.node, documentId); - await visit(end!.node, [], []); + await add([], [], root, documentId); + await visit(root, [], []); for (const { file } of found.values()) yield file; } - return { prefix: end.pinned, files: files() }; + return { prefix: end.container.pinned, files: files() }; } diff --git a/core/filesystem/test/files.test.ts b/core/filesystem/test/files.test.ts index be618968d..81d5efbc1 100644 --- a/core/filesystem/test/files.test.ts +++ b/core/filesystem/test/files.test.ts @@ -108,6 +108,21 @@ describe("listFiles", () => { } }); + it("lists the package a file path is in", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const listed = await listFiles(repo, `${prefix}/dist/index.js`, { accounts }); + expect(listed?.prefix).toBe(prefix); + expect((await collect(listed!.files)).map((f) => f.tail)).toContain("dist/index.js"); + }); + + it("lists the package for a path that ends inside it", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.dist = { "a.js": "x" })); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + expect((await listFiles(repo, `${prefix}/dist`, { accounts }))?.prefix).toBe(prefix); + }); + it("lists nothing for a path that isn't canonical", async () => { const { repo, a, accounts } = await fixture(); expect( diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index b3b784b47..617477829 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -49,6 +49,7 @@ import { import * as plugins from "@inkandswitch/patchwork-plugins"; import setupServiceWorker, { connectClassicSync, + registerAccount, resolveHandoffsWith, } from "@inkandswitch/patchwork-bootloader"; import debug from "debug"; @@ -202,6 +203,8 @@ async function doSetup(options: PatchworkOptions): Promise { hive, createAccount: options.createAccount ?? createDefaultAccount, })) as DocHandle; + // So /@//… paths under it resolve, here and in the automerge worker. + await registerAccount(accountDocHandle.url); wireModuleSettings(accountDocHandle, moduleWatcher); From f5cdb27a5912a0a4d3e75604221210048fd94109 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:46:58 +0000 Subject: [PATCH 19/81] service worker: serve /@/ paths from the checked content store Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/pinned.ts | 49 +++++- core/bootloader/src/service-worker.ts | 242 +++++++++++++++++++++++--- core/bootloader/test/pinned.test.ts | 33 ++++ 3 files changed, 297 insertions(+), 27 deletions(-) diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index c0a14ce9f..091af19a7 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -3,7 +3,11 @@ // manifests say what a URL's bytes must hash to (see index-channel.ts); a store entry or a // resolver's answer that doesn't match is never served. import type { ContentStore } from "./content-store.js"; -import type { IndexEntry, Manifest } from "./types.js"; +import type { + HandoffRequestMessage, + IndexEntry, + Manifest, +} from "./types.js"; const BASE58 = "[1-9A-HJ-NP-Za-km-z]+"; const DOC_STEP = new RegExp(`^${BASE58}@${BASE58}(\\.${BASE58})*$`); @@ -97,8 +101,13 @@ export type PinnedDeps = { hash(bytes: Uint8Array): Promise; /** Ask the automerge worker for the manifest covering `key`; undefined if it can't answer. */ manifest(key: string): Promise; - /** Ask a tab or the automerge worker for `key`'s bytes; undefined if it doesn't resolve. */ - resolve(key: string): Promise<{ body: Uint8Array; contentType: string } | undefined>; + /** + * Ask a tab or the automerge worker for `key`'s bytes: undefined if it doesn't resolve, a + * Response for any answer that isn't bytes (a redirect for a path that isn't canonical). + */ + resolve( + key: string + ): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; warn(message: string): void; /** Keep the service worker alive for work that outlives the response. */ keep(work: Promise): void; @@ -161,6 +170,14 @@ export async function servePinned( headers: { "content-type": "text/plain", "cache-control": "no-store" }, }); } + if (answer instanceof Response) { + if (entry) { + throw new PinnedAbort( + `${keys[0]} is in the automerge worker's manifest, but a resolver answered ${answer.status}` + ); + } + return answer; + } if (!entry) return respond(answer.body, answer.contentType, false); if ((await deps.hash(answer.body)) !== entry.sha256) { @@ -193,3 +210,29 @@ export function sharedManifests( return request; }; } + +/** + * The handoff the service worker sends tabs and the automerge worker: `to` is a legacy special + * URL (`automerge:…`), or a `/@/` path. A path's `handoffURL` is `patchwork-path:` and the path, + * which resolvers from before paths parse and then decline. + */ +export function handoffRequest( + request: Request, + cachename: string, + to: URL | string +): HandoffRequestMessage { + return { + id: crypto.randomUUID(), + type: "request", + cachename, + request: { + url: request.url, + handoffURL: typeof to === "string" ? `patchwork-path:${to}` : to.href, + ...(typeof to === "string" ? { path: to } : {}), + headers: Object.fromEntries(request.headers.entries()), + method: request.method, + destination: request.destination, + referrer: request.referrer, + }, + }; +} diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 4a4878598..ebefc2e98 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -3,12 +3,32 @@ import { AUTOMERGE_WORKER_LOCK, HANDOFF_CHANNEL, + INDEX_ANSWER_PATH, type AutomergeWorkerMissingMessage, type HandoffReplyMessage, type HandoffRequestMessage, + type HandoffResponse, + type IndexAnswer, + type IndexEntry, + type Manifest, + type ManifestRequestMessage, type ResolveAcceptedMessage, type ResolveRequestMessage, } from "./types.js"; +import { contentHash, toBytes } from "./content-hash.js"; +import { + CONTENT_CACHE_PREFIX, + cacheStore, + contentCacheName, +} from "./content-store.js"; +import { acceptIndexAnswer } from "./index-channel.js"; +import { + PinnedAbort, + handoffRequest, + normalizePinned, + servePinned, + sharedManifests, +} from "./pinned.js"; // Referenced so this script's bytes change with every build. The browser only // installs a replacement service worker when they do, and the controllerchange // that follows is how open tabs learn a deploy happened (see setup.ts). @@ -85,7 +105,11 @@ async function clearOtherCaches() { const names = await caches.keys(); await Promise.all( names - .filter((name) => name !== cachename) + // Content stores are per account and checked on every read; they + // outlive cache renames and deploys. + .filter( + (name) => name !== cachename && !name.startsWith(CONTENT_CACHE_PREFIX) + ) .map((name) => caches.delete(name)) ); } @@ -175,15 +199,87 @@ handoffChannel.addEventListener("message", (event) => { if (data?.type !== "online") return; // The automerge worker (re)started — re-broadcast anything still in flight so // requests that raced its boot aren't stranded. - const stranded = [...pendingHandoffs.values()]; + const stranded = [ + ...[...pendingHandoffs.values()].map(({ message }) => message), + ...[...pendingManifests.values()].map(({ message }) => message), + ]; if (stranded.length === 0) return; lifecycle( "info", - `automerge worker (re)started; re-broadcasting ${stranded.length} in-flight asset handoff(s)` + `automerge worker (re)started; re-broadcasting ${stranded.length} in-flight request(s)` ); - for (const { message } of stranded) handoffChannel.postMessage(message); + for (const message of stranded) handoffChannel.postMessage(message); }); +// ── The content index ────────────────────────────────────────────────── +// Pinned `/@/` URL → what its bytes must hash to. Held only here, in memory, +// and filled only from the automerge worker's manifests, whose answers arrive +// as fetches this worker checks the sender of (see index-channel.ts). Lost +// whenever this worker stops, and rebuilt from the automerge worker on demand. + +const contentIndex = new Map(); + +// The automerge worker answers within its own 10s; past that, serve unchecked. +const MANIFEST_TIMEOUT_MS = 10_000; + +type PendingManifest = { + message: ManifestRequestMessage; + settle: (manifest: Manifest | undefined) => void; +}; + +const pendingManifests = new Map(); + +function requestManifest(path: string): Promise { + const message: ManifestRequestMessage = { + type: "manifest", + id: crypto.randomUUID(), + path, + }; + return new Promise((resolve) => { + const timer = setTimeout(() => { + lifecycle( + "warn", + `no manifest for /@/${path} after ${MANIFEST_TIMEOUT_MS}ms; serving it unchecked` + ); + settle(undefined); + }, MANIFEST_TIMEOUT_MS); + const settle = (manifest: Manifest | undefined) => { + clearTimeout(timer); + pendingManifests.delete(message.id); + resolve(manifest); + }; + pendingManifests.set(message.id, { message, settle }); + handoffChannel.postMessage(message); + void startWorkerIfMissing(message.id); + }); +} + +const manifests = sharedManifests( + requestManifest, + (key) => normalizePinned(key)?.prefix ?? key +); + +function settleIndexAnswer(answer: IndexAnswer) { + const pending = pendingManifests.get(answer.id); + if (!pending) return log(`no pending manifest for id ${answer.id}`); + if (answer.error) { + log(`no manifest for /@/${pending.message.path}: ${answer.error}`); + } + pending.settle(answer.manifest); +} + +async function answerIndex(fetchEvent: FetchEvent): Promise { + const client = fetchEvent.clientId + ? await self.clients.get(fetchEvent.clientId) + : undefined; + return acceptIndexAnswer( + fetchEvent.request, + client, + self.location.origin, + settleIndexAnswer + ); +} + const MISSING_WORKER_NUDGE_MS = 5_000; // A worker that's been spawned only asks for the lock once its imports have // evaluated, so on a cold start it looks missing for a while. @@ -208,7 +304,7 @@ async function startWorkerIfMissing(id: string) { if (!navigator.locks) return; if (await automergeWorkerExists()) return; await new Promise((resolve) => setTimeout(resolve, MISSING_WORKER_GRACE_MS)); - if (!pendingHandoffs.has(id)) return; + if (!pendingHandoffs.has(id) && !pendingManifests.has(id)) return; if (await automergeWorkerExists()) return; // Check and claim the throttle with no await in between: every handoff that // raced the same cold start comes through here at once, and would all pass a @@ -324,22 +420,9 @@ function askTab( */ async function handoff( fetchEvent: FetchEvent, - handoffURL: URL + to: URL | string ): Promise { - const request = fetchEvent.request; - const message: HandoffRequestMessage = { - id: crypto.randomUUID(), - type: "request", - cachename, - request: { - url: request.url, - handoffURL: handoffURL.href, - headers: Object.fromEntries(request.headers.entries()), - method: request.method, - destination: request.destination, - referrer: request.referrer, - }, - }; + const message = handoffRequest(fetchEvent.request, cachename, to); const deadline = Date.now() + HANDOFF_TIMEOUT_MS; for (const tab of await resolvingTabs(fetchEvent.clientId)) { const answer = await askTab(tab, message, deadline); @@ -479,10 +562,13 @@ async function serveHandoff( if (reply.type === "response") { log(`serving handed-off response for ${handoffURL}`, reply); - return new Response(reply.response.body ?? null, { - status: reply.response.status ?? 200, - headers: reply.response.headers, - }); + const response = inlineResponse(reply.response); + // Resolvers hand everything back inline; what's immutable is kept here, + // by URL, as legacy URLs always were. + if (isImmutable(response)) { + cacheInBackground(fetchEvent, cache, fetchEvent.request, response.clone()); + } + return response; } const stored = await cache.match(fetchEvent.request); @@ -496,6 +582,96 @@ async function serveHandoff( return stored; } +const inlineResponse = (response: HandoffResponse) => + new Response(response.body ?? null, { + status: response.status ?? 200, + headers: response.headers, + }); + +const IMMUTABLE_STATUSES = [200, 203, 204]; + +function isImmutable(response: Response): boolean { + return ( + IMMUTABLE_STATUSES.includes(response.status) && + !/(^|,)\s*no-store(\s*,|$)/.test(response.headers.get("cache-control") ?? "") + ); +} + +/** + * A `/@/` path: a live one is handed off and its redirect served as is; a + * pinned one is served from the content store, checked against the index. + */ +async function servePath(fetchEvent: FetchEvent, path: string): Promise { + const normalized = normalizePinned(path); + if (!normalized) { + return new Response(`not a path: /@/${path}`, { + status: 400, + headers: { "content-type": "text/plain" }, + }); + } + + const resolve = async (key: string) => { + const reply = await handoff(fetchEvent, key); + if (reply.type === "abort") throw new HandoffAborted(reply.reason); + if (reply.type !== "response") { + return new Response(`unexpected ${reply.type} reply for /@/${key}`, { + status: 502, + }); + } + const { status = 200, body, headers } = reply.response; + if (status !== 200) return inlineResponse(reply.response); + return { + body: toBytes(body ?? new Uint8Array()), + contentType: headers?.["content-type"] ?? "application/octet-stream", + }; + }; + + try { + if (!normalized.pinned) { + const answer = await resolve(normalized.keys[0]); + return answer instanceof Response + ? answer + : new Response(answer.body as BlobPart, { + headers: { "content-type": answer.contentType, "cache-control": "no-store" }, + }); + } + const store = + normalized.acctHash && !normalized.share + ? cacheStore( + await caches.open(contentCacheName(normalized.acctHash)), + self.location.origin + ) + : undefined; + return await servePinned(normalized.keys, { + index: contentIndex, + store, + hash: contentHash, + manifest: manifests, + resolve, + warn: (message) => lifecycle("warn", message), + keep: (work) => + fetchEvent.waitUntil( + work.catch((error) => { + // Always loud, as for the page cache: a QuotaExceededError here + // is the first sign the origin is under storage pressure. + console.warn(`error keeping /@/${path}`, error); + }) + ), + }); + } catch (error) { + // Deliberate: fail the request as a network error, with no response. + if (error instanceof PinnedAbort || error instanceof HandoffAborted) { + lifecycle("warn", `failing /@/${path}: ${(error as Error).message}`); + throw error; + } + console.error(`service worker error serving /@/${path}`, error); + return new Response( + error instanceof Error ? `${error.message}\n\n${error.stack}` : String(error), + { status: 556, headers: { "content-type": "text/plain" } } + ); + } +} + async function servePassthrough( fetchEvent: FetchEvent, cache: Cache, @@ -570,6 +746,24 @@ async function respond( self.addEventListener("fetch", (fetchEvent: FetchEvent) => { const request = fetchEvent.request; log("fetch event", request.url); + const url = new URL(request.url); + const ours = url.origin === self.location.origin; + + if (ours && url.pathname === INDEX_ANSWER_PATH) { + fetchEvent.respondWith( + request.method === "POST" + ? answerIndex(fetchEvent) + : new Response(null, { status: 405 }) + ); + return; + } + if (request.method !== "GET") return; + + if (ours && url.pathname.startsWith("/@/")) { + fetchEvent.respondWith(servePath(fetchEvent, url.pathname.slice("/@/".length))); + return; + } + fetchEvent.respondWith(respond(fetchEvent, specialURLFor(request))); }); diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index 38fe784bd..f2cca58ea 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -3,6 +3,7 @@ import { contentHash, toBytes } from "../src/content-hash.js"; import type { ContentStore } from "../src/content-store.js"; import { PinnedAbort, + handoffRequest, normalizePinned, servePinned, sharedManifests, @@ -183,6 +184,22 @@ describe("servePinned", () => { expect((await servePinned([KEY], deps)).status).toBe(404); }); + it("passes a resolver's other answer through when there's no entry", async () => { + const { deps } = await setup({ manifest: false }); + deps.resolve.mockResolvedValue( + new Response(null, { status: 307, headers: { location: "/@/acct@h/x@h" } }) as never + ); + const response = await servePinned([KEY], deps); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toBe("/@/acct@h/x@h"); + }); + + it("fails a resolver's other answer as a network error when there's an entry", async () => { + const { deps } = await setup(); + deps.resolve.mockResolvedValue(new Response("nope", { status: 404 }) as never); + await expect(servePinned([KEY], deps)).rejects.toBeInstanceOf(PinnedAbort); + }); + it("rebuilds the index after a restart and reuses the store", async () => { const first = await setup(); await servePinned([KEY], first.deps); @@ -220,3 +237,19 @@ describe("sharedManifests", () => { void deps; }); }); + +describe("handoffRequest", () => { + it("builds a path handoff that old resolvers decline", () => { + const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "patchwork", "acct/Projects"); + expect(m.request.path).toBe("acct/Projects"); + expect(new URL(m.request.handoffURL).protocol).not.toBe("automerge:"); + expect(m.request.url).toBe("https://p.test/@/acct/Projects"); + }); + + it("builds a legacy handoff with no path", () => { + const special = new URL("automerge:abc/x.js"); + const m = handoffRequest(new Request("https://p.test/automerge%3Aabc/x.js"), "patchwork", special); + expect(m.request.handoffURL).toBe(special.href); + expect(m.request.path).toBeUndefined(); + }); +}); From e9006ccbeee6917aadee9486dea4f647b92f0487 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:48:42 +0000 Subject: [PATCH 20/81] e2e: forged store entries and forged answers; bench: hashing a hit Co-Authored-By: Claude Opus 5.5 --- packages/e2e/tests/content-store.spec.ts | 117 +++++++++++++++++++++++ sites/bench/README.md | 3 + sites/bench/tests/content-hash.spec.ts | 38 ++++++++ 3 files changed, 158 insertions(+) create mode 100644 packages/e2e/tests/content-store.spec.ts create mode 100644 sites/bench/tests/content-hash.spec.ts diff --git a/packages/e2e/tests/content-store.spec.ts b/packages/e2e/tests/content-store.spec.ts new file mode 100644 index 000000000..8bdde13db --- /dev/null +++ b/packages/e2e/tests/content-store.spec.ts @@ -0,0 +1,117 @@ +import { expect, test } from "@playwright/test"; +import { waitForRepoReady, waitForServiceWorkerActive } from "./helpers.js"; + +/** + * The service worker's content store for pinned `/@/` URLs. Anything on the + * origin can write CacheStorage, so the store is untrusted: every read is + * hashed and checked against an index only the automerge worker can fill. + * These tests play the page that tries to get around that. + */ + +declare global { + interface Window { + patchwork?: { + account: { + documentId: string; + doc(): { rootFolderUrl?: string }; + }; + }; + } +} + +test("a forged store entry is never served", async ({ page, browserName }) => { + test.skip( + browserName !== "chromium", + "needs the full-UI boot for an account, which is only reliable on chromium (see install-tool.spec.ts)" + ); + test.setTimeout(300_000); + + await page.goto("/", { timeout: 120_000 }); + await page.waitForFunction(() => !!window.patchwork?.account, null, { + timeout: 180_000, + }); + + const { pinned, acctHash, sha256 } = await page.evaluate(async () => { + const encoder = new TextEncoder(); + const ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + const base58 = (bytes: Uint8Array) => { + let n = 0n; + for (const byte of bytes) n = n * 256n + BigInt(byte); + let out = ""; + while (n > 0n) { + out = ALPHABET[Number(n % 58n)] + out; + n /= 58n; + } + for (const byte of bytes) { + if (byte !== 0) break; + out = "1" + out; + } + return out; + }; + const sha = async (text: string) => + new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(text))); + + const account = window.patchwork!.account; + const acctHash = base58((await sha(account.documentId)).slice(0, 12)); + + const file = window.repo.create>(); + file.change((d) => { + d.content = "export default 42"; + d.mimeType = "text/javascript"; + }); + const root = await window.repo.find<{ docs: unknown[] }>( + account.doc().rootFolderUrl! + ); + root.change((d) => { + d.docs.push({ name: "forge-test.js", type: "file", url: file.url }); + }); + + // The live URL redirects to the pinned one; fetch follows it. + let pinned = ""; + for (let i = 0; i < 30 && !pinned; i++) { + const response = await fetch(`/@/${acctHash}/forge-test.js`); + if (response.ok && response.redirected) pinned = response.url; + else await new Promise((resolve) => setTimeout(resolve, 1000)); + } + const digest = await sha("export default 42"); + const sha256 = btoa(String.fromCharCode(...digest)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); + return { pinned, acctHash, sha256 }; + }); + expect(pinned).toContain("/@/"); + + await page.evaluate( + async ({ acctHash, sha256 }) => { + const cache = await caches.open(`patchwork-content-${acctHash}`); + await cache.put(`/.content/${sha256}`, new Response("export default 666")); + }, + { acctHash, sha256 } + ); + + await page.reload({ timeout: 120_000 }); + await waitForRepoReady(page, 180_000); + const value = await page.evaluate( + async (url) => (await import(/* @vite-ignore */ url)).default, + pinned + ); + expect(value).toBe(42); +}); + +test("an index answer from a page is refused", async ({ page }) => { + await page.goto("/"); + await waitForServiceWorkerActive(page); + const status = await page.evaluate(async () => { + const response = await fetch("/@/.index", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "forged", + manifest: { prefix: "x@y", entries: {} }, + }), + }); + return response.status; + }); + expect(status).toBe(403); +}); diff --git a/sites/bench/README.md b/sites/bench/README.md index 730bbb565..4d73ba9bd 100644 --- a/sites/bench/README.md +++ b/sites/bench/README.md @@ -129,6 +129,9 @@ server holds our heads" means that peer and not a sibling. from a fresh tab, three tabs flushing concurrently, sync latency with the server; with the longest main-thread stall (a 5ms timer's lateness) and long task time recorded around the heavy parts. +- `content-hash.spec` — what the service worker pays on every hit of the `/@/` + content store, which it hashes before serving since anything on the origin + can write CacheStorage: SHA-256 of 10KB, 100KB and 1MB, median of 200. Cross-tab timings use epoch milliseconds, since `performance.now()` counts from each page's own navigation start. `find()` in the helpers retries on diff --git a/sites/bench/tests/content-hash.spec.ts b/sites/bench/tests/content-hash.spec.ts new file mode 100644 index 000000000..b0d23740b --- /dev/null +++ b/sites/bench/tests/content-hash.spec.ts @@ -0,0 +1,38 @@ +// What the service worker pays on every hit of the content store: it hashes +// what it read before serving it, since anything on the origin can write +// CacheStorage. SHA-256 over typical module sizes, in the page, which runs the +// same crypto.subtle the service worker does. +import { test } from "@playwright/test"; +import { median, openTab, record } from "./bench.js"; + +const SIZES = [ + ["10KB", 10 * 1024], + ["100KB", 100 * 1024], + ["1MB", 1024 * 1024], +] as const; +const RUNS = 200; + +test("patchwork: sha-256 of a content store hit", async ({ context }) => { + const page = await openTab(context, "patchwork", { server: "none" }); + for (const [label, bytes] of SIZES) { + const times = await page.evaluate( + async ({ bytes, runs }) => { + const data = new Uint8Array(bytes).map((_, i) => i % 251); + const times: number[] = []; + for (let i = 0; i < runs; i++) { + const start = performance.now(); + await crypto.subtle.digest("SHA-256", data); + times.push(performance.now() - start); + } + return times; + }, + { bytes, runs: RUNS } + ); + record({ + metric: `sha-256 of a ${label} hit`, + mode: "patchwork", + value: median(times), + unit: "ms", + }); + } +}); From 8edb11c51c6d2807406bf71e14054c240ee0f40b Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 06:48:59 +0000 Subject: [PATCH 21/81] spec and changeset: content store Co-Authored-By: Claude Opus 5.5 --- .changeset/paths-content-store.md | 8 ++++++++ ...-10-28-paths-namespace-content-cache-design.md | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 .changeset/paths-content-store.md diff --git a/.changeset/paths-content-store.md b/.changeset/paths-content-store.md new file mode 100644 index 000000000..e59ff16d1 --- /dev/null +++ b/.changeset/paths-content-store.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one is served from a content-addressed store (`patchwork-content-`) whose every read is hashed and checked against an index that only the automerge worker fills, over `fetch()` requests the service worker checks are really from it. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `readPath`, `listFiles`, `readNode`, `WalkOptions.find` and `WalkEnd.container`. The e2e suite gains tests that a forged store entry is never served and a page can't answer for the automerge worker. diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index a000889a3..80c2479a2 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -287,7 +287,11 @@ change between two reads. `sites/bench` gets a case measuring it. and reply first with a forged response. That is a hole in the current code as well. Index answers must arrive from a `Client` whose `type` is `sharedworker` and whose `url` is the automerge worker's script URL, which - the service worker knows at build time. + the service worker knows at build time. That URL is fixed, not hashed: + `/automerge-protocol-handler-worker.js`, the name the vite plugin keeps from + one build to the next. A site that sets its own `workerPath` has its + answers refused, so everything under `/@/` is served inline and nothing is + stored. - The worker sends answers as `fetch()` requests to a reserved path. The service worker looks up `event.clientId` with `clients.get` and checks the client. This is the only reply path that works in all three engines @@ -323,6 +327,15 @@ These replace the deleted `package-load-gate.md` note. revisit when keyhive protects its keys at rest. - Nothing cleans up the content store. Deduplication slows growth, and the existing quota warning still fires. +- The index lives only in service worker memory, so after a browser restart + every package needs a fresh manifest, and building one loads and hashes + every file document in the package. The URL-keyed cache this replaces + served package code at boot without loading any document. The automerge + worker remembers 64 manifests, which covers service worker restarts within + a session but not a cold start. Persisting manifests would fix it, but + anything stored on the origin can be written by package code. `sites/bench`'s + `content-hash.spec` measures the per-hit hashing; the cold-start cost isn't + measured yet. ## Spikes before planning From a2299b3465e57420624ee0a2aba8aef1e7a55671 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 07:07:01 +0000 Subject: [PATCH 22/81] content store: review fixes - A manifest answer settles only the request for the same path, so a page can't get the worker to answer a forged copy and switch the check off. - The worker probes /@/.index before posting, so a fetch that skips the service worker never carries file names to the origin server. - A tab declines paths under accounts it doesn't know yet, instead of answering 404 while it boots. - Listing a package reads file docs without waiting for peers, several at once, on a budget of its own, and a partial manifest isn't kept. - After a manifest times out, requests wait 250ms instead of 10s until the worker answers again. - Immutable legacy responses are cached under the request only, so a navigation no longer replaces the offline app shell. Co-Authored-By: Claude Opus 5.5 --- .../src/automerge-protocol-handler.ts | 9 ++ core/bootloader/src/index-channel.ts | 43 +++++- core/bootloader/src/manifest.ts | 47 +++++-- core/bootloader/src/pinned.ts | 65 +++++++++ core/bootloader/src/resolver.ts | 27 +++- core/bootloader/src/service-worker.ts | 77 +++++------ core/bootloader/src/tab-resolver.ts | 4 +- core/bootloader/src/types.ts | 12 +- core/bootloader/test/index-channel.test.ts | 42 ++++-- core/bootloader/test/manifest.test.ts | 10 +- core/bootloader/test/pinned.test.ts | 52 +++++++- core/bootloader/test/resolver.test.ts | 17 +++ core/bootloader/test/worker-manifests.test.ts | 25 +++- core/filesystem/src/walk.ts | 123 ++++++++++-------- core/filesystem/test/files.test.ts | 11 ++ 15 files changed, 431 insertions(+), 133 deletions(-) diff --git a/core/bootloader/src/automerge-protocol-handler.ts b/core/bootloader/src/automerge-protocol-handler.ts index d1bc939a2..39169679b 100644 --- a/core/bootloader/src/automerge-protocol-handler.ts +++ b/core/bootloader/src/automerge-protocol-handler.ts @@ -199,12 +199,21 @@ const resolver = createResolver(getRepo, log, () => accounts); // The service worker stops waiting for a manifest after 10s; walking to it // gets no longer than that. const MANIFEST_WALK_TIMEOUT_MS = 10_000; +const MANIFEST_LIST_TIMEOUT_MS = 30_000; // Manifests are what the service worker checks every pinned `/@/` URL // against. Only this worker answers them: tabs run package code. const answerManifest = createManifestHandler(getRepo, () => accounts, { find: (repo) => waitingFind(repo, AbortSignal.timeout(MANIFEST_WALK_TIMEOUT_MS), new Set()), + // Listing a package reads every file doc in it. Each is read as this node + // has it, without waiting for peers, and the listing gets its own budget: a + // manifest finished after the service worker stopped waiting is still kept + // for the next request. + findFile: (repo) => + waitingFind(repo, AbortSignal.timeout(MANIFEST_LIST_TIMEOUT_MS), new Set(), { + catchUp: false, + }), log, }); const handoffChannel = new BroadcastChannel(HANDOFF_CHANNEL); diff --git a/core/bootloader/src/index-channel.ts b/core/bootloader/src/index-channel.ts index ec6111f5f..bff21d843 100644 --- a/core/bootloader/src/index-channel.ts +++ b/core/bootloader/src/index-channel.ts @@ -54,21 +54,56 @@ export async function acceptIndexAnswer( return new Response(null, { status: 204, headers: { [ACCEPTED_HEADER]: "accepted" } }); } +/** + * The service worker's answer to a GET of INDEX_ANSWER_PATH: nothing to get, but the header says + * the request reached it rather than the network. + */ +export function indexChannelProbe(): Response { + return new Response(null, { + status: 405, + headers: { [ACCEPTED_HEADER]: "present", allow: "POST" }, + }); +} + const TRIES = 5; const RETRY_MS = 200; +const pause = () => new Promise((resolve) => setTimeout(resolve, RETRY_MS)); + +// Fetchers whose requests have been seen to reach the service worker. +const reachesServiceWorker = new WeakSet(); + +/** + * Whether this worker's fetches reach the service worker. Firefox sent a worker's first fetch + * straight to the network in the spike, and an answer that does that hands the origin server + * the account's file names; so probe with a GET that carries nothing first. + */ +async function probe(fetcher: typeof fetch): Promise { + if (reachesServiceWorker.has(fetcher)) return true; + for (let i = 0; i < TRIES; i++) { + if (i) await pause(); + const response = await fetcher(INDEX_ANSWER_PATH).catch(() => undefined); + if (response?.headers.get(ACCEPTED_HEADER) === "present") { + reachesServiceWorker.add(fetcher); + return true; + } + } + return false; +} + /** - * The automerge worker's side: POST an answer. A reply without the accepted header never reached - * the service worker (Firefox sends a worker's first fetch straight to the network), so try - * again a few times. False if it never got through. + * The automerge worker's side: POST an answer, once a probe says this worker's fetches reach the + * service worker. A reply without the accepted header didn't get there, so try again a few + * times. False if it never got through. */ export async function postIndexAnswer( answer: IndexAnswer, fetcher: typeof fetch = fetch ): Promise { + if (!(await probe(fetcher))) return false; const body = JSON.stringify(answer); for (let i = 0; i < TRIES; i++) { - if (i) await new Promise((resolve) => setTimeout(resolve, RETRY_MS)); + if (i) await pause(); const response = await fetcher(INDEX_ANSWER_PATH, { method: "POST", headers: { "content-type": "application/json" }, diff --git a/core/bootloader/src/manifest.ts b/core/bootloader/src/manifest.ts index 63814bcdb..b8813fa70 100644 --- a/core/bootloader/src/manifest.ts +++ b/core/bootloader/src/manifest.ts @@ -35,8 +35,11 @@ const isProtectedDoc = (documentId: DocumentId) => export async function buildManifest( repo: Repo, path: string, - options: WalkOptions & { isProtected?: (documentId: DocumentId) => boolean } -): Promise { + options: WalkOptions & { + findFile?: WalkOptions["find"]; + isProtected?: (documentId: DocumentId) => boolean; + } +): Promise<{ manifest: Manifest; missing: string[] } | undefined> { const listed = await listFiles(repo, path, options); if (!listed) return undefined; const { root, segments } = parsePath(listed.prefix); @@ -45,7 +48,7 @@ export async function buildManifest( const isProtected = options.isProtected ?? isProtectedDoc; const entries: Record = {}; - for await (const file of listed.files) { + for (const file of listed.files) { entries[file.tail] = { sha256: await contentHash(toBytes(file.content)), contentType: file.type, @@ -54,7 +57,7 @@ export async function buildManifest( (throughPackages || !isProtected(file.documentId)), }; } - return { prefix: listed.prefix, entries }; + return { manifest: { prefix: listed.prefix, entries }, missing: listed.missing }; } const REMEMBERED_MANIFESTS = 64; @@ -72,12 +75,17 @@ export function createManifestHandler( post?: (answer: IndexAnswer) => Promise; /** The walk's find, per request: one that waits for heads, in the worker. */ find?: (repo: Repo) => WalkOptions["find"]; + /** The listing's find for file docs, per build, with a budget of its own. */ + findFile?: (repo: Repo) => WalkOptions["find"]; log?: (...args: unknown[]) => void; } = {} ): (message: ManifestRequestMessage) => Promise { const post = options.post ?? ((answer) => postIndexAnswer(answer)); const remembered = new Map(); - const building = new Map>(); + const building = new Map< + string, + Promise<{ manifest: Manifest; missing: string[] } | undefined> + >(); const manifestFor = async (path: string): Promise => { const repo = await getRepo(); @@ -94,24 +102,39 @@ export function createManifestHandler( } let build = building.get(prefix); if (!build) { - build = buildManifest(repo, prefix, walk).finally(() => building.delete(prefix)); + build = buildManifest(repo, prefix, { + ...walk, + findFile: options.findFile?.(repo), + }).finally(() => building.delete(prefix)); building.set(prefix, build); } - const manifest = await build; - if (!manifest) throw new Error(`couldn't list the files under ${prefix}`); - remembered.set(prefix, manifest); + const built = await build; + if (!built) throw new Error(`couldn't list the files under ${prefix}`); + // A partial manifest still checks what it lists, but isn't kept: the next + // request tries the missing files again. + if (built.missing.length) { + options.log?.( + `manifest for ${prefix} is missing ${built.missing.length} file(s); not keeping it` + ); + return built.manifest; + } + remembered.set(prefix, built.manifest); if (remembered.size > REMEMBERED_MANIFESTS) { remembered.delete(remembered.keys().next().value!); } - return manifest; + return built.manifest; }; return async ({ id, path }) => { let answer: IndexAnswer; try { - answer = { id, manifest: await manifestFor(path) }; + answer = { id, path, manifest: await manifestFor(path) }; } catch (error) { - answer = { id, error: error instanceof Error ? error.message : String(error) }; + answer = { + id, + path, + error: error instanceof Error ? error.message : String(error), + }; } options.log?.(`manifest ${id} for ${path}: ${answer.error ?? "built"}`); if (!(await post(answer))) { diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index 091af19a7..f110416da 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -5,8 +5,10 @@ import type { ContentStore } from "./content-store.js"; import type { HandoffRequestMessage, + IndexAnswer, IndexEntry, Manifest, + ManifestRequestMessage, } from "./types.js"; const BASE58 = "[1-9A-HJ-NP-Za-km-z]+"; @@ -236,3 +238,66 @@ export function handoffRequest( }, }; } + +/** + * The service worker's manifest requests to the automerge worker: sent with `send`, settled by + * an answer for the same id *and* path (requests are broadcast, so a page can get the worker to + * answer a copy with another path), or given up on after `timeoutMs`. Once one times out, the + * worker is taken to be away and later requests wait only `quickTimeoutMs`, for `quietMs` or + * until it answers again, so a missing worker costs each request a moment rather than the + * whole timeout. + */ +export function manifestRequests(options: { + send(message: ManifestRequestMessage): void; + timeoutMs: number; + quickTimeoutMs: number; + quietMs: number; + onTimeout(path: string): void; +}) { + const pending = new Map< + string, + { message: ManifestRequestMessage; finish(manifest: Manifest | undefined): void } + >(); + let quietUntil = 0; + + return { + request(path: string): Promise { + const message: ManifestRequestMessage = { + type: "manifest", + id: crypto.randomUUID(), + path, + }; + const quiet = Date.now() < quietUntil; + return new Promise((resolve) => { + const timer = setTimeout( + () => { + if (!quiet) { + quietUntil = Date.now() + options.quietMs; + options.onTimeout(path); + } + finish(undefined); + }, + quiet ? options.quickTimeoutMs : options.timeoutMs + ); + const finish = (manifest: Manifest | undefined) => { + clearTimeout(timer); + pending.delete(message.id); + resolve(manifest); + }; + pending.set(message.id, { message, finish }); + options.send(message); + }); + }, + /** False if `answer` settles nothing: an unknown id, or another path. */ + settle(answer: IndexAnswer): boolean { + const request = pending.get(answer.id); + if (!request || answer.path !== request.message.path) return false; + quietUntil = 0; + request.finish(answer.manifest); + return true; + }, + has: (id: string) => pending.has(id), + /** What's still waiting, to send again to a worker that restarted. */ + pending: () => [...pending.values()].map(({ message }) => message), + }; +} diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index b5be34e76..e0df78729 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -193,7 +193,8 @@ function impatience(limit: number) { export function waitingFind( repo: Repo, signal: AbortSignal, - missing: Set + missing: Set, + { catchUp = true }: { catchUp?: boolean } = {} ): NonNullable { return async (url: AutomergeUrl, heads?: UrlHeads) => { const handle = await repo.find(url, { signal }); @@ -202,7 +203,7 @@ export function waitingFind( missing.add(url); throw new AbortHandoff(`heads ${heads.join(",")} of ${url} not found`); } - } else { + } else if (catchUp) { await caughtUpWithPeers(repo, handle, signal); } return handle; @@ -268,8 +269,25 @@ export type Resolver = { export function createResolver( getRepo: () => Promise, log: (...args: unknown[]) => void = () => {}, - accounts: () => ReadonlyMap = () => new Map() + accounts: () => ReadonlyMap = () => new Map(), + options: { + /** + * Decline (answer undefined) a path under an account this resolver + * doesn't know, so the service worker asks someone else. A tab learns its + * account only once it has loaded; the automerge worker, asked last, + * answers 404 instead. + */ + declineUnknownAccounts?: boolean; + } = {} ): Resolver { + const knowsAccountOf = (path: string) => { + try { + const { root } = parsePath(path); + return root.kind !== "account" || accounts().has(root.acctHash); + } catch { + return true; // answered with a 400 below + } + }; const within = (signal?: AbortSignal) => signal ? AbortSignal.any([signal, AbortSignal.timeout(RESOLVE_TIMEOUT_MS)]) @@ -336,6 +354,9 @@ export function createResolver( let response: Response; try { if (request.path !== undefined) { + if (options.declineUnknownAccounts && !knowsAccountOf(request.path)) { + return undefined; + } const repo = await getRepo(); const limit = within(signal); log(`resolving handoff ${id} for /@/${request.path}`); diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index ebefc2e98..6e0920bc3 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -21,10 +21,11 @@ import { cacheStore, contentCacheName, } from "./content-store.js"; -import { acceptIndexAnswer } from "./index-channel.js"; +import { acceptIndexAnswer, indexChannelProbe } from "./index-channel.js"; import { PinnedAbort, handoffRequest, + manifestRequests, normalizePinned, servePinned, sharedManifests, @@ -201,7 +202,7 @@ handoffChannel.addEventListener("message", (event) => { // requests that raced its boot aren't stranded. const stranded = [ ...[...pendingHandoffs.values()].map(({ message }) => message), - ...[...pendingManifests.values()].map(({ message }) => message), + ...manifestRequestsInFlight.pending(), ]; if (stranded.length === 0) return; lifecycle( @@ -221,51 +222,36 @@ const contentIndex = new Map(); // The automerge worker answers within its own 10s; past that, serve unchecked. const MANIFEST_TIMEOUT_MS = 10_000; +// After one goes unanswered, how long later requests wait, and for how long. +const MANIFEST_QUICK_TIMEOUT_MS = 250; +const MANIFEST_QUIET_MS = 60_000; -type PendingManifest = { - message: ManifestRequestMessage; - settle: (manifest: Manifest | undefined) => void; -}; - -const pendingManifests = new Map(); - -function requestManifest(path: string): Promise { - const message: ManifestRequestMessage = { - type: "manifest", - id: crypto.randomUUID(), - path, - }; - return new Promise((resolve) => { - const timer = setTimeout(() => { - lifecycle( - "warn", - `no manifest for /@/${path} after ${MANIFEST_TIMEOUT_MS}ms; serving it unchecked` - ); - settle(undefined); - }, MANIFEST_TIMEOUT_MS); - const settle = (manifest: Manifest | undefined) => { - clearTimeout(timer); - pendingManifests.delete(message.id); - resolve(manifest); - }; - pendingManifests.set(message.id, { message, settle }); +const manifestRequestsInFlight = manifestRequests({ + send(message) { handoffChannel.postMessage(message); void startWorkerIfMissing(message.id); - }); -} + }, + timeoutMs: MANIFEST_TIMEOUT_MS, + quickTimeoutMs: MANIFEST_QUICK_TIMEOUT_MS, + quietMs: MANIFEST_QUIET_MS, + onTimeout(path) { + lifecycle( + "warn", + `no manifest for /@/${path} after ${MANIFEST_TIMEOUT_MS}ms; serving /@/ unchecked, and waiting only ${MANIFEST_QUICK_TIMEOUT_MS}ms for manifests, for ${MANIFEST_QUIET_MS / 1000}s or until the automerge worker answers` + ); + }, +}); const manifests = sharedManifests( - requestManifest, + (path) => manifestRequestsInFlight.request(path), (key) => normalizePinned(key)?.prefix ?? key ); function settleIndexAnswer(answer: IndexAnswer) { - const pending = pendingManifests.get(answer.id); - if (!pending) return log(`no pending manifest for id ${answer.id}`); - if (answer.error) { - log(`no manifest for /@/${pending.message.path}: ${answer.error}`); + if (answer.error) log(`no manifest for /@/${answer.path}: ${answer.error}`); + if (!manifestRequestsInFlight.settle(answer)) { + log(`no pending manifest ${answer.id} for /@/${answer.path}`); } - pending.settle(answer.manifest); } async function answerIndex(fetchEvent: FetchEvent): Promise { @@ -304,7 +290,7 @@ async function startWorkerIfMissing(id: string) { if (!navigator.locks) return; if (await automergeWorkerExists()) return; await new Promise((resolve) => setTimeout(resolve, MISSING_WORKER_GRACE_MS)); - if (!pendingHandoffs.has(id) && !pendingManifests.has(id)) return; + if (!pendingHandoffs.has(id) && !manifestRequestsInFlight.has(id)) return; if (await automergeWorkerExists()) return; // Check and claim the throttle with no await in between: every handoff that // raced the same cold start comes through here at once, and would all pass a @@ -564,9 +550,16 @@ async function serveHandoff( log(`serving handed-off response for ${handoffURL}`, reply); const response = inlineResponse(reply.response); // Resolvers hand everything back inline; what's immutable is kept here, - // by URL, as legacy URLs always were. + // under the request, as resolvers used to keep it. Not cachePage: a + // navigation to a document in a doc must not replace the app shell's + // offline copy at / and /index.html. if (isImmutable(response)) { - cacheInBackground(fetchEvent, cache, fetchEvent.request, response.clone()); + const copy = response.clone(); + fetchEvent.waitUntil( + cache.put(fetchEvent.request, copy).catch((error) => { + console.warn(`error caching ${handoffURL} in ${cachename}`, error); + }) + ); } return response; } @@ -751,9 +744,7 @@ self.addEventListener("fetch", (fetchEvent: FetchEvent) => { if (ours && url.pathname === INDEX_ANSWER_PATH) { fetchEvent.respondWith( - request.method === "POST" - ? answerIndex(fetchEvent) - : new Response(null, { status: 405 }) + request.method === "POST" ? answerIndex(fetchEvent) : indexChannelProbe() ); return; } diff --git a/core/bootloader/src/tab-resolver.ts b/core/bootloader/src/tab-resolver.ts index 18a58652a..3f3510206 100644 --- a/core/bootloader/src/tab-resolver.ts +++ b/core/bootloader/src/tab-resolver.ts @@ -28,7 +28,9 @@ const accounts = new Map(); * they load stays in `repo`. */ export function resolveHandoffsWith(repo: Repo): void { - resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts); + resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + declineUnknownAccounts: true, + }); } /** diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 1dc62ada9..5cd8047c8 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -237,7 +237,17 @@ export interface ManifestRequestMessage { } /** Automerge worker → service worker, as the body of a POST to INDEX_ANSWER_PATH. */ -export type IndexAnswer = { id: string; manifest?: Manifest; error?: string }; +export type IndexAnswer = { + id: string; + /** + * The path the request named. Requests are broadcast, so a page can send + * the worker a copy with the same id and another path; the service worker + * takes an answer only for the path it asked about. + */ + path?: string; + manifest?: Manifest; + error?: string; +}; /** Tab → automerge worker: resolve paths under this account. */ export interface RegisterAccountMessage { diff --git a/core/bootloader/test/index-channel.test.ts b/core/bootloader/test/index-channel.test.ts index 5774fb54b..49ca52c44 100644 --- a/core/bootloader/test/index-channel.test.ts +++ b/core/bootloader/test/index-channel.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi } from "vitest"; import { acceptIndexAnswer, + indexChannelProbe, isAutomergeWorker, postIndexAnswer, } from "../src/index-channel.js"; @@ -56,20 +57,43 @@ describe("acceptIndexAnswer", () => { }); }); +describe("indexChannelProbe", () => { + it("says the service worker is there, and takes nothing", () => { + const response = indexChannelProbe(); + expect(response.status).toBe(405); + expect(response.headers.get("x-patchwork-index")).toBe("present"); + }); +}); + +const present = () => + new Response(null, { status: 405, headers: { "x-patchwork-index": "present" } }); +const accepted = () => + new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } }); + describe("postIndexAnswer", () => { - it("retries a reply that skipped the service worker", async () => { - const replies = [ - new Response("", { status: 404 }), - new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } }), - ]; - const fetcher = vi.fn(async () => replies.shift()!); + it("probes before posting, so an answer never reaches the network", async () => { + const replies = [new Response("", { status: 404 }), present(), accepted()]; + const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => replies.shift()!); expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(true); - expect(fetcher).toHaveBeenCalledTimes(2); + expect(fetcher.mock.calls.map(([, init]) => init?.method ?? "GET")).toEqual([ + "GET", + "GET", + "POST", + ]); }); - it("gives up after 5 tries", async () => { - const fetcher = vi.fn(async () => new Response("", { status: 404 })); + it("never posts when the probe never reaches the service worker", async () => { + const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => new Response("", { status: 404 })); expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(false); + expect(fetcher.mock.calls.every(([, init]) => (init?.method ?? "GET") === "GET")).toBe(true); expect(fetcher).toHaveBeenCalledTimes(5); }); + + it("probes once per fetcher, then retries a post that didn't get through", async () => { + const replies = [present(), new Response("", { status: 404 }), accepted(), accepted()]; + const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => replies.shift()!); + expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(true); + expect(await postIndexAnswer({ id: "2" }, fetcher as typeof fetch)).toBe(true); + expect(fetcher).toHaveBeenCalledTimes(4); + }); }); diff --git a/core/bootloader/test/manifest.test.ts b/core/bootloader/test/manifest.test.ts index b7c62e908..fff311f4e 100644 --- a/core/bootloader/test/manifest.test.ts +++ b/core/bootloader/test/manifest.test.ts @@ -33,7 +33,7 @@ describe("buildManifest", () => { it("hashes each file as the service worker will", async () => { const { repo, a, accounts } = await fixture(); const prefix = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); - const manifest = await buildManifest(repo, prefix!, { accounts }); + const manifest = (await buildManifest(repo, prefix!, { accounts }))?.manifest; expect(manifest?.prefix).toBe(prefix); expect(manifest?.entries.note).toEqual({ sha256: await contentHash(new TextEncoder().encode("héllo ✓")), @@ -47,15 +47,15 @@ describe("buildManifest", () => { const isProtected = () => true; const projects = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); const pkg = await pinnedPath(repo, livePath(a, ["packages", "pkg"]), { accounts }); - expect((await buildManifest(repo, projects!, { accounts, isProtected }))?.entries.note.store).toBe(false); - expect((await buildManifest(repo, pkg!, { accounts, isProtected }))?.entries["index.js"].store).toBe(true); + expect((await buildManifest(repo, projects!, { accounts, isProtected }))?.manifest.entries.note.store).toBe(false); + expect((await buildManifest(repo, pkg!, { accounts, isProtected }))?.manifest.entries["index.js"].store).toBe(true); }); it("never stores under a share root", async () => { const { repo, accounts, projects } = await fixture(); const share = `automerge:${projects.documentId}@${formatHeads(projects.heads())}`; - const manifest = await buildManifest(repo, share, { accounts }); - expect(manifest?.entries.note.store).toBe(false); + const built = await buildManifest(repo, share, { accounts }); + expect(built?.manifest.entries.note.store).toBe(false); }); it("returns nothing for a live path", async () => { diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index f2cca58ea..e30640d4e 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -4,12 +4,17 @@ import type { ContentStore } from "../src/content-store.js"; import { PinnedAbort, handoffRequest, + manifestRequests, normalizePinned, servePinned, sharedManifests, type PinnedDeps, } from "../src/pinned.js"; -import type { IndexEntry, Manifest } from "../src/types.js"; +import type { + IndexEntry, + Manifest, + ManifestRequestMessage, +} from "../src/types.js"; const keysOf = (path: string) => normalizePinned(path)!.keys; @@ -253,3 +258,48 @@ describe("handoffRequest", () => { expect(m.request.path).toBeUndefined(); }); }); + +describe("manifestRequests", () => { + const manifest: Manifest = { prefix: "acct@h/p@h", entries: {} }; + const make = (timeouts = { timeoutMs: 1000, quickTimeoutMs: 20 }) => { + const sent: ManifestRequestMessage[] = []; + const requests = manifestRequests({ + send: (message) => sent.push(message), + ...timeouts, + quietMs: 60_000, + onTimeout: () => {}, + }); + return { sent, requests }; + }; + + it("ignores an answer for a different path and keeps waiting", async () => { + const { sent, requests } = make(); + const pending = requests.request("acct@h/p@h/a.js"); + requests.settle({ id: sent[0].id, path: "garbage", error: "unparseable" }); + expect(requests.has(sent[0].id)).toBe(true); + requests.settle({ id: sent[0].id, path: sent[0].path, manifest }); + expect(await pending).toEqual(manifest); + }); + + it("gives up quickly after a timeout, until the worker answers again", async () => { + const { sent, requests } = make({ timeoutMs: 150, quickTimeoutMs: 10 }); + const elapsed = async (path: string) => { + const start = performance.now(); + await requests.request(path); + return performance.now() - start; + }; + expect(await elapsed("acct@h/p@h/a.js")).toBeGreaterThanOrEqual(140); + expect(await elapsed("acct@h/p@h/b.js")).toBeLessThan(100); + const answered = requests.request("acct@h/p@h/c.js"); + const last = sent.at(-1)!; + requests.settle({ id: last.id, path: last.path, manifest }); + await answered; + expect(await elapsed("acct@h/p@h/d.js")).toBeGreaterThanOrEqual(140); + }); + + it("lists what's pending, for re-sending to a restarted worker", () => { + const { sent, requests } = make(); + void requests.request("acct@h/p@h/a.js"); + expect(requests.pending()).toEqual(sent); + }); +}); diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index c955691be..520fe736f 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -108,6 +108,23 @@ describe("resolver: /@/ paths", () => { }); }); +describe("resolver: accounts it doesn't know", () => { + it("a tab declines, so another tab or the worker is asked", async () => { + const { repo, a } = await fixture(); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => new Map(), { + declineUnknownAccounts: true, + }); + expect(await tab.answer(pathRequest(livePath(a, ["Projects"])))).toBeUndefined(); + }); + + it("the worker, asked last, answers 404", async () => { + const { repo, a } = await fixture(); + const worker = createResolver(() => Promise.resolve(repo), undefined, () => new Map()); + const reply = await worker.answer(pathRequest(livePath(a, ["Projects"]))); + expect(inline(reply).status).toBe(404); + }); +}); + describe("resolver: legacy automerge URLs", () => { it("answers inline, leaving caching to the service worker", async () => { const { repo, resolver } = await fixture(); diff --git a/core/bootloader/test/worker-manifests.test.ts b/core/bootloader/test/worker-manifests.test.ts index cee511ea6..6dc6101b3 100644 --- a/core/bootloader/test/worker-manifests.test.ts +++ b/core/bootloader/test/worker-manifests.test.ts @@ -29,15 +29,18 @@ async function fixture(packages = 1) { const prefixes = await Promise.all( pkgs.map((_, i) => pinnedPath(repo, livePath(a, ["packages", `p${i}`]), { accounts })) ); - return { repo, accounts, prefixes: prefixes as string[] }; + return { repo, accounts, pkgs, prefixes: prefixes as string[] }; } -const setup = async (packages?: number) => { +const setup = async (packages?: number, unreadable?: (url: string) => boolean) => { const f = await fixture(packages); const posted: IndexAnswer[] = []; const post = vi.fn(async (answer: IndexAnswer) => (posted.push(answer), true)); const finds: string[] = []; - const find = (repo: Repo) => (url: AutomergeUrl) => (finds.push(url), repo.find(url)); + const find = (repo: Repo) => (url: AutomergeUrl) => { + finds.push(url); + return unreadable?.(url) ? Promise.reject(new Error("unavailable")) : repo.find(url); + }; const handle = createManifestHandler(() => Promise.resolve(f.repo), () => f.accounts, { post, find, @@ -62,6 +65,22 @@ describe("createManifestHandler", () => { expect(finds.length - fileFinds).toBeLessThanOrEqual(4); // a walk, no relisting }); + it("echoes the path it answers for", async () => { + const { handle, prefixes, posted } = await setup(); + await handle({ type: "manifest", id: "1", path: `${prefixes[0]}/a.js` }); + expect(posted[0].path).toBe(`${prefixes[0]}/a.js`); + }); + + it("doesn't remember a manifest that's missing files", async () => { + let bUrl = ""; + const { handle, prefixes, posted, finds, pkgs } = await setup(1, (url) => url === bUrl); + bUrl = String((pkgs[0].doc() as Record)["b.js"]); + await handle({ type: "manifest", id: "1", path: prefixes[0] }); + expect(Object.keys(posted[0].manifest!.entries)).not.toContain("b.js"); + await handle({ type: "manifest", id: "2", path: prefixes[0] }); + expect(finds.filter((url) => url === bUrl)).toHaveLength(2); + }); + it("posts an error for a path it can't list", async () => { const { handle, posted } = await setup(); await handle({ type: "manifest", id: "x", path: "unknown@h/a.js" }); diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 020290f67..dc8f6a0cf 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -392,75 +392,96 @@ function walkPrefers(a: number[], b: number[]): boolean { return false; } +// How many file docs a listing reads at once. +const LIST_CONCURRENCY = 8; + /** * Every file under the last container a canonical pinned path enters (see * {@link WalkEnd.container}), keyed by tail: "" for the container itself, encoded names joined * with "/" below it. Containers below it aren't listed; they have their own doc steps. `path` * may name the container or anything in it. Undefined unless `path` is canonical. * - * A file reached by a link without heads is read as it is now, so the list is a snapshot. + * File docs are read with `findFile` (default: `find`), several at once. One that can't be read + * is left out and its tail named in `missing`, so a caller can tell a partial list from a whole + * one. A file reached by a link without heads is read as it is now, so the list is a snapshot. */ export async function listFiles( repo: Repo, path: string, - options: WalkOptions -): Promise<{ prefix: string; files: AsyncIterable } | undefined> { + options: WalkOptions & { findFile?: WalkOptions["find"] } +): Promise<{ prefix: string; files: ListedFile[]; missing: string[] } | undefined> { const end = await walkPath(repo, path, options); if (!end?.canonical || !end.container) return undefined; - const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const find = + options.findFile ?? options.find ?? ((url: AutomergeUrl) => repo.find(url)); const root = end.container.handle.doc(); const provider = providerFor(root); const documentId = end.container.handle.documentId; - async function* files(): AsyncIterable { - const found = new Map(); - const add = async ( - parts: string[], - keys: number[], - node: unknown, - holder: DocumentId - ) => { - const tail = parts.map(encodeURIComponent).join("/"); - const existing = found.get(tail); - if (existing && !walkPrefers(keys, existing.keys)) return; - const read = await readNode(repo, node); - if (read) found.set(tail, { keys, file: { ...read, tail, documentId: holder } }); - }; - - const visit = async (node: unknown, parts: string[], keys: number[]) => { - for (const child of provider?.list(node) ?? []) { - // Only a directory's keys span several path segments; a folder entry's name is one. - const names = - provider === directoryProvider ? child.name.split("/") : [child.name]; - const childParts = [...parts, ...names]; - const childKeys = [...keys, names.length]; - if ("value" in child) { - await add(childParts, childKeys, child.value, documentId); - // Nested values are addressable by path only inside a directory doc. - if (provider === directoryProvider && isNested(child.value)) { - await visit(child.value, childParts, childKeys); - } - } else if ("link" in child) { - try { - const parsed = parseAutomergeUrl(child.link); - const live = await find( - stringifyAutomergeUrl({ documentId: parsed.documentId }), - parsed.heads - ); - const doc = live.view(parsed.heads ?? live.heads()).doc(); - if (doc === undefined || providerFor(doc)) continue; - await add(childParts, childKeys, doc, parsed.documentId); - } catch { - // A file that can't be found isn't listed; reading it fails the same way. - } + const found = new Map(); + const missing: string[] = []; + const tailOf = (parts: string[]) => parts.map(encodeURIComponent).join("/"); + const add = async ( + parts: string[], + keys: number[], + node: unknown, + holder: DocumentId + ) => { + const tail = tailOf(parts); + const existing = found.get(tail); + if (existing && !walkPrefers(keys, existing.keys)) return; + const read = await readNode(repo, node); + if (read) found.set(tail, { keys, file: { ...read, tail, documentId: holder } }); + }; + + // Values are read in place; links are gathered and read below. + const links: { parts: string[]; keys: number[]; link: AutomergeUrl }[] = []; + const visit = async (node: unknown, parts: string[], keys: number[]) => { + for (const child of provider?.list(node) ?? []) { + // Only a directory's keys span several path segments; a folder entry's name is one. + const names = + provider === directoryProvider ? child.name.split("/") : [child.name]; + const childParts = [...parts, ...names]; + const childKeys = [...keys, names.length]; + if ("value" in child) { + await add(childParts, childKeys, child.value, documentId); + // Nested values are addressable by path only inside a directory doc. + if (provider === directoryProvider && isNested(child.value)) { + await visit(child.value, childParts, childKeys); } + } else if ("link" in child) { + links.push({ parts: childParts, keys: childKeys, link: child.link }); } - }; + } + }; + await add([], [], root, documentId); + await visit(root, [], []); - await add([], [], root, documentId); - await visit(root, [], []); - for (const { file } of found.values()) yield file; - } + let next = 0; + const readLinks = async () => { + while (next < links.length) { + const { parts, keys, link } = links[next++]; + try { + const parsed = parseAutomergeUrl(link); + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + parsed.heads + ); + const doc = live.view(parsed.heads ?? live.heads()).doc(); + if (doc === undefined || providerFor(doc)) continue; + await add(parts, keys, doc, parsed.documentId); + } catch { + missing.push(tailOf(parts)); + } + } + }; + await Promise.all( + Array.from({ length: Math.min(LIST_CONCURRENCY, links.length) }, readLinks) + ); - return { prefix: end.container.pinned, files: files() }; + return { + prefix: end.container.pinned, + files: [...found.values()].map(({ file }) => file), + missing, + }; } diff --git a/core/filesystem/test/files.test.ts b/core/filesystem/test/files.test.ts index 81d5efbc1..53bf3b81e 100644 --- a/core/filesystem/test/files.test.ts +++ b/core/filesystem/test/files.test.ts @@ -123,6 +123,17 @@ describe("listFiles", () => { expect((await listFiles(repo, `${prefix}/dist`, { accounts }))?.prefix).toBe(prefix); }); + it("says which files it couldn't read", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const bad = (minePkg.doc() as any)["package.json"] as string; + const find = (url: AutomergeUrl) => + url === bad ? Promise.reject(new Error("unavailable")) : repo.find(url); + const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const listed = await listFiles(repo, prefix!, { accounts, find }); + expect(listed?.missing).toEqual(["package.json"]); + expect(listed?.files.map((f) => f.tail)).not.toContain("package.json"); + }); + it("lists nothing for a path that isn't canonical", async () => { const { repo, a, accounts } = await fixture(); expect( From 92f7c7bf053925078651112d0fe6c7bb4b1e6f57 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 12:33:59 +0000 Subject: [PATCH 23/81] bootloader: test the service worker itself A harness runs the real service worker module in node against a fake global scope (self, clients, caches) with node's own BroadcastChannel and locks, and dispatches fetch and lifecycle events at it. Tabs are fake window clients running the real tab resolver. Covers the legacy navigation fix (an immutable response is kept under its own URL, not over the app shell's / and /index.html), the index channel refusing a page and answering the worker's probe, and activate keeping content stores. The first two fail against the service worker before a2299b34. Co-Authored-By: Claude Opus 5.5 --- .../bootloader/test/service-worker-harness.ts | 166 ++++++++++++++++++ core/bootloader/test/service-worker.test.ts | 88 ++++++++++ 2 files changed, 254 insertions(+) create mode 100644 core/bootloader/test/service-worker-harness.ts create mode 100644 core/bootloader/test/service-worker.test.ts diff --git a/core/bootloader/test/service-worker-harness.ts b/core/bootloader/test/service-worker-harness.ts new file mode 100644 index 000000000..6429637a8 --- /dev/null +++ b/core/bootloader/test/service-worker-harness.ts @@ -0,0 +1,166 @@ +// Runs the real service worker module in node against a fake global scope: +// `self` (listeners, clients, location), `caches` as in-memory maps, and node's +// own BroadcastChannel and navigator.locks. Tabs and the automerge worker are +// fake clients; give them real resolvers to test the whole round trip. +import { vi } from "vitest"; + +export const ORIGIN = "https://p.test"; + +/** Enough of Cache: put, match and delete, keyed by url. */ +export class FakeCache { + entries = new Map(); + async put(request: RequestInfo | URL, response: Response) { + this.entries.set(new Request(request).url, response.clone()); + } + async match(request: RequestInfo | URL) { + return this.entries.get(new Request(request).url)?.clone(); + } + async delete(request: RequestInfo | URL) { + return this.entries.delete(new Request(request).url); + } + async keys() { + return [...this.entries.keys()].map((url) => new Request(url)); + } +} + +export class FakeCaches { + named = new Map(); + async open(name: string) { + let cache = this.named.get(name); + if (!cache) this.named.set(name, (cache = new FakeCache())); + return cache as unknown as Cache; + } + async delete(name: string) { + return this.named.delete(name); + } + async keys() { + return [...this.named.keys()]; + } + async match(request: RequestInfo | URL) { + for (const cache of this.named.values()) { + const hit = await cache.match(request); + if (hit) return hit; + } + return undefined; + } +} + +export type FakeClient = { + id: string; + type: "window" | "worker" | "sharedworker"; + url: string; + focused?: boolean; + visibilityState?: "visible" | "hidden"; + postMessage(message: unknown, transfer?: Transferable[]): void; +}; + +/** A fetch event the test can await: what `respondWith` was given, and every `waitUntil`. */ +export type FakeFetchEvent = { + request: Request; + clientId: string; + respondWith(response: Response | Promise): void; + waitUntil(work: Promise): void; + /** The response, or undefined if the worker let the request through. */ + response(): Promise | undefined; + /** Every waitUntil, settled. */ + settled(): Promise; +}; + +export type ServiceWorkerHarness = { + caches: FakeCaches; + clients: Map; + /** Dispatch a fetch event to the worker. */ + fetch(request: Request, clientId?: string): FakeFetchEvent; + /** Dispatch a lifecycle event (install, activate) and wait for its waitUntil. */ + lifecycle(type: "install" | "activate"): Promise; + close(): void; +}; + +/** + * A request the platform builds but scripts can't: a navigation. `mode` and + * `destination` are read-only getters on Request.prototype, so shadow them. + */ +export function navigation(url: string): Request { + const request = new Request(url); + Object.defineProperty(request, "mode", { value: "navigate" }); + Object.defineProperty(request, "destination", { value: "document" }); + return request; +} + +/** Install the fake scope and import a fresh copy of the service worker module. */ +export async function startServiceWorker(): Promise { + const listeners = new Map void>>(); + const clients = new Map(); + const caches = new FakeCaches(); + + // Every channel the module opens, so a test's service worker doesn't hear + // the next test's traffic. + const channels: BroadcastChannel[] = []; + const RealBroadcastChannel = globalThis.BroadcastChannel; + class TrackedBroadcastChannel extends RealBroadcastChannel { + constructor(name: string) { + super(name); + channels.push(this); + } + } + + const scope = { + location: new URL(`${ORIGIN}/service-worker.js`), + registration: { scope: `${ORIGIN}/` }, + addEventListener(type: string, listener: (event: any) => void) { + listeners.set(type, [...(listeners.get(type) ?? []), listener]); + }, + skipWaiting: async () => {}, + clients: { + async get(id: string) { + return clients.get(id); + }, + async matchAll(options: { type?: string } = {}) { + return [...clients.values()].filter( + (client) => !options.type || options.type === "all" || client.type === options.type + ); + }, + async claim() {}, + }, + }; + + vi.stubGlobal("self", scope); + vi.stubGlobal("caches", caches); + vi.stubGlobal("BroadcastChannel", TrackedBroadcastChannel); + vi.resetModules(); + await import("../src/service-worker.js"); + // The module's own channels are open; the test's are the real class again. + vi.stubGlobal("BroadcastChannel", RealBroadcastChannel); + + const dispatch = (type: string, event: unknown) => { + for (const listener of listeners.get(type) ?? []) listener(event); + }; + + return { + caches, + clients, + fetch(request, clientId = "") { + let responded: Promise | undefined; + const waits: Promise[] = []; + const event: FakeFetchEvent = { + request, + clientId, + respondWith: (response) => void (responded = Promise.resolve(response)), + waitUntil: (work) => void waits.push(work), + response: () => responded, + settled: () => Promise.allSettled(waits), + }; + dispatch("fetch", event); + return event; + }, + async lifecycle(type) { + const waits: Promise[] = []; + dispatch(type, { waitUntil: (work: Promise) => waits.push(work) }); + await Promise.all(waits); + }, + close() { + for (const channel of channels) channel.close(); + vi.unstubAllGlobals(); + }, + }; +} diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts new file mode 100644 index 000000000..0ca80d8a5 --- /dev/null +++ b/core/bootloader/test/service-worker.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, it, expect } from "vitest"; +import { Repo, stringifyAutomergeUrl } from "@automerge/automerge-repo"; +import { handleResolveRequest, resolveHandoffsWith } from "../src/tab-resolver.js"; +import { + ORIGIN, + navigation, + startServiceWorker, + type ServiceWorkerHarness, +} from "./service-worker-harness.js"; + +let sw: ServiceWorkerHarness | undefined; +afterEach(() => { + sw?.close(); + sw = undefined; +}); + +/** A tab that answers the service worker with `repo`, as tab-resolver.ts does in a page. */ +function addTab(harness: ServiceWorkerHarness, repo: Repo, id = "tab") { + resolveHandoffsWith(repo); + harness.clients.set(id, { + id, + type: "window", + url: `${ORIGIN}/`, + focused: true, + visibilityState: "visible", + postMessage(message, transfer) { + handleResolveRequest( + new MessageEvent("message", { + data: message, + ports: (transfer ?? []) as MessagePort[], + }) + ); + }, + }); +} + +describe("service worker: legacy automerge URLs", () => { + it("keeps an immutable navigation under its own URL, not over the app shell", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const doc = repo.create({ + "@patchwork": { type: "directory" }, + "index.html": { content: "

a doc

", mimeType: "text/html" }, + }); + const url = stringifyAutomergeUrl({ documentId: doc.documentId, heads: doc.heads() }); + const request = navigation(`${ORIGIN}/${encodeURIComponent(url)}/index.html`); + + const event = sw.fetch(request, "tab"); + const response = await event.response()!; + expect(await response.text()).toBe("

a doc

"); + await event.settled(); + + const kept = [...(sw.caches.named.get("patchwork")?.entries.keys() ?? [])]; + expect(kept).toEqual([request.url]); + }); +}); + +describe("service worker: the index channel", () => { + it("refuses an answer posted by a page", async () => { + sw = await startServiceWorker(); + sw.clients.set("page", { id: "page", type: "window", url: `${ORIGIN}/`, postMessage() {} }); + const event = sw.fetch( + new Request(`${ORIGIN}/@/.index`, { + method: "POST", + body: JSON.stringify({ id: "x", path: "a@h", manifest: { prefix: "a@h", entries: {} } }), + }), + "page" + ); + expect((await event.response()!).status).toBe(403); + }); + + it("answers a probe so the worker knows its fetches arrive", async () => { + sw = await startServiceWorker(); + const response = await sw.fetch(new Request(`${ORIGIN}/@/.index`)).response()!; + expect(response.headers.get("x-patchwork-index")).toBe("present"); + }); +}); + +describe("service worker: lifecycle", () => { + it("keeps content stores when it clears old caches", async () => { + sw = await startServiceWorker(); + await sw.caches.open("patchwork-content-acct"); + await sw.caches.open("some-old-cache"); + await sw.lifecycle("activate"); + expect((await sw.caches.keys()).sort()).toEqual(["patchwork", "patchwork-content-acct"]); + }); +}); From f9446695981fc727c7da5d3ad48e38e6a84d8302 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:20:27 +0000 Subject: [PATCH 24/81] spec: pinned URLs name a hash of their folder's file list A container's pinned step carries ~, the SHA-256 of a canonical list of the files below it (bytes hash, type, heads read at). The list is content like any other, so a pinned URL is checked against itself: links no longer need heads to pin their files, the in-memory index and the authenticated channel go, and a cold start needs no document. No document ids in URLs or in the store. Co-Authored-By: Claude Opus 5.5 --- ...28-paths-namespace-content-cache-design.md | 269 +++++++++++------- 1 file changed, 171 insertions(+), 98 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 80c2479a2..609f844fc 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -1,6 +1,11 @@ # Paths, namespace, and a content-addressed package cache -Status: design, approved in conversation 2026-10-28. Not implemented. +Status: design, approved in conversation 2026-10-28. Revised the same day: +pinned URLs now name a hash of their folder's file list, so a pinned URL is +checked against itself and nothing else (sections 1 and 5). Phase 1 is +implemented. Phase 2 was implemented against the earlier section 5 (an index +in service worker memory, filled by the automerge worker over an +authenticated channel) and gets reworked to this one. ## Goals @@ -10,7 +15,8 @@ Status: design, approved in conversation 2026-10-28. Not implemented. through them, and the filesystem is a namespace rather than only the `.docs` walk of the root folder. - Package management is a folder: whatever is bound at `/packages`. -- URLs don't carry raw document ids unless a portable share link is wanted. +- URLs don't carry raw document ids unless a portable share link is wanted, + and neither does anything in the content store. - The package cache can't be poisoned by a crafted link, by code running on the origin writing to CacheStorage, or by a stale name resolution. @@ -18,7 +24,7 @@ Status: design, approved in conversation 2026-10-28. Not implemented. - Cache isolation between two accounts in the same browser profile. - Running package code on a separate origin. This design is meant not to - preclude it (the store and index work the same there), but doesn't do it. + preclude it (the store and file lists work the same there), but doesn't do it. - Mount points below the top level of the namespace. - Path providers supplied by packages. - Garbage collection of the content store. @@ -27,9 +33,9 @@ Status: design, approved in conversation 2026-10-28. Not implemented. | Attacker | In scope | What stops it | |---|---|---| -| A crafted link makes the service worker cache content under a key your own imports later hit | yes | Only pinned URLs are stored, and the index maps a pinned URL to the hash of what is really at those heads | -| Code on the origin (a package or tool) writes to CacheStorage | yes | Every store read is hashed and compared; the index lives only in service worker memory and is filled only by the automerge worker over an authenticated channel | -| A name path resolved once keeps serving old content | yes | Names are never stored; only pinned URLs are, and they can't change meaning | +| A crafted link makes the service worker cache content under a key your own imports later hit | yes | Content is stored only under its own hash, after hashing it; a crafted link can only name other content | +| Code on the origin (a package or tool) writes to CacheStorage | yes | Every store read is hashed and compared with a hash that comes from the URL, directly or through a file list that is itself checked against the URL | +| A name path resolved once keeps serving old content | yes | Names are never stored; a pinned URL carries the heads of each folder and the hash of its file list, so it can't change meaning | | Another account on the same origin | no | — | Everything on the origin shares one realm boundary, so this raises the bar @@ -47,6 +53,10 @@ rather than closing it. What stays open is listed under - **acctHash**: the idhash of the account document. - **heads**: the doc's heads in base58, sorted, joined with `.`. Sorting makes one set of heads one URL. Base58 has no `.`, so it's unambiguous. +- **listHash**: the SHA-256, in base64url, of a container's **file list** + (section 5): every file the URL may reach below that container, with the + hash of its bytes and the heads it was read at. base64url has no `@`, `.` + or `~`. ### Live path @@ -68,18 +78,33 @@ What imports and share links use. Every document the walk enters is pinned: 3. A step **inside** a document (a directory key, a mount name in the account doc) stays a plain name, since the enclosing doc's heads already pin it. +4. The names after the last container (a folder, a directory, …) lead to a + **file**: a value inside the container, or a document reached by a link. + They stay names, so a module's relative imports (`./util.js`, `../x.js`) + resolve against them. A link without heads doesn't pin the file it leads + to, so the container's step carries `~`, and the list records + each file's hash and heads. Links don't need to carry heads. ``` live: /Projects/foo/dist/index.js -pinned: @h/@h/@h/@h/dist/index.js +pinned: @h/@h/@h/@h~/dist/index.js live: /packages/foo/dist/index.js -pinned: @h/packages/@h/@h/dist/index.js +pinned: @h/packages/@h/@h~/dist/index.js ``` -When resolving, a segment is a doc step if it parses as `@` -and a link reachable from the current node hashes to that idhash; otherwise -it's a name. +Which containers carry a list: + +- `pinPath` puts one on the last container. +- Pinning a package (phase 4) puts one on every container from the + package's own step down, so `../` between a folder package's subfolders + stays pinned. +- A file reached under a container step with no list is resolved live, + served `no-store`, and never stored or checked. + +When resolving, a segment is a doc step if it parses as +`@` or `@~` and a link reachable +from the current node hashes to that idhash; otherwise it's a name. The first segment of a pinned path is either an acctHash the resolver knows locally, or `automerge:@` for a portable share link (safe @@ -87,7 +112,11 @@ to hand out once the doc is on keyhive, since the id no longer grants access). The `automerge:` prefix is what tells the two apart. A pinned path never changes meaning, so whoever builds an import URL builds -the whole chain first. Nothing in a module graph goes through a redirect: the +the whole chain, and its file lists, first. + +No raw document id appears in a live or pinned path: every step is an +idhash. The only exception is a share link's root, which carries one on +purpose. Nothing in a module graph goes through a redirect: the ES module map keys an entry by the URL requested, so a redirect is memoized for the life of the realm (the same reason `importModule` retries with `?retry=1`). @@ -111,10 +140,11 @@ Built-in providers, checked in order, first match wins: stands for its `default` branch. This replaces the special case in `core/bootloader/src/resolver.ts`. -Providers are built in only. The automerge worker runs them to build the -cache index, so a package-supplied provider would put package code inside -the resolver the index trusts. The provider interface is exported, shaped so -package providers can be added later. +Providers are built in only, for now. Nothing trusts a provider's answers +any more, since every byte is checked against the URL. But a provider runs +inside every resolver, the automerge worker's included, so a package +supplying one would run package code there. The provider interface is +exported, shaped so package providers can be added later. ### Namespace @@ -143,7 +173,9 @@ If `namespace` is missing, `/` falls back to `rootFolderUrl`, so older accounts and older readers keep working. The automerge worker needs to know acctHash → account url to resolve a path's -root. Tabs register it with the worker at boot. +root. Tabs register it with the worker at boot. The worker computes the +acctHash from the url itself, so no tab can register an account under +another's hash. ## 3. Service worker URLs and routing @@ -154,12 +186,15 @@ for a path: ``` /@/ e.g. /@//Projects/foo/x.js -/@/ e.g. /@/@h/@h/.../x.js +/@/ e.g. /@/@h/@h/.../@h~/x.js ``` - A **live** URL answers with a 307 to its pinned form, computed by a - resolver. Fine for `fetch()`; never use one for `import()`. -- A **pinned** URL is served from the content store (section 5). + resolver. Fine for `fetch()`; never use one for `import()`. The pinned + form's list covers only the file asked for, so a redirect costs one file + read, not a folder's worth (section 5, *File lists*). +- A **pinned** URL is served from the content store, checked against its + own list (section 5). - Existing `/automerge%3A…/` URLs keep working as they do today, keyed by URL and unchecked. They are deprecated. @@ -191,7 +226,7 @@ Builds on the `open-document-path` branch (48f4ab98): - `parsePath(path)`: live or pinned, into segments - `resolveLivePath(repo, path)` → handle - `pinPath(repo, path)` → pinned service-worker URL, for fetching siblings or - importing + importing, and the file list it names (section 5) ## 4. Packages via `/packages` @@ -204,7 +239,10 @@ Builds on the `open-document-path` branch (48f4ab98): path lookup finds. - Packages are tracked by document id, so a rename doesn't reload anything. - When a package's own heads move, the watcher builds its pinned chain with - `pinPath` and imports it. Other packages keep their existing URLs: a + `pinPath`, with a list on every container from the package's own step + down, and imports it. A push that changes only files behind links + without heads must still touch the package doc; pushwork already bumps + `lastSyncAt` for that. Other packages keep their existing URLs: a pinned URL stays valid forever, so a change elsewhere in the chain doesn't reload them. - `PackageTracker.findLatest` keeps asking the service worker, in the new @@ -226,87 +264,111 @@ Builds on the `open-document-path` branch (48f4ab98): - `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding. -## 5. Content store and index +## 5. Content store and file lists ### Store - A CacheStorage cache named `patchwork-content-`, separate from the `patchwork` cache that holds site pages and assets. - Key: `/.content/`, an internal key that is never fetched. -- Value: the bytes only. The content type lives in the index, so identical - bytes are stored once whatever their type. +- Value: the bytes only. The content type lives in the file list, so + identical bytes are stored once whatever their type. - Reads and writes go through one small interface (`get(hash)`, `put(hash, bytes)`), so entries can later be encrypted under an at-rest key. +- Anything on the origin can write it. That's fine: every read is hashed and + compared with a hash the URL vouches for, and a mismatch is deleted. - `clearAccountStore(acctHash)` deletes the cache. There is no sign-out flow today; a site calls this from its own. -### What gets stored +### File lists -- Content from unprotected documents (`isUnprotectedDocId`). -- Content from keyhive-protected documents only when reached through - `/packages`. Package code needs to load fast at boot; user documents don't. -- Everything else is served inline: checked, never stored. +A container's file list says what each file under it is, at the moment it +was pinned. It is canonical JSON: keys sorted, no whitespace. -### Index +```json +{"files":{"dist/index.js":{"heads":"a.b","sha256":"…","store":true,"type":"text/javascript"},"package.json":{…}},"v":1} +``` + +- **Keys** are tails: the names below the container, each + `encodeURIComponent`-encoded and joined with `/`. The container itself is + `""`. +- **Entries:** + - `sha256` is the hash of the bytes the file serves as. + - `type` is its content type. + - `heads` is the heads its document was read at. It's absent for a value + inside the container, which the container's own heads pin. + - `store` says whether the bytes may be kept (see + [What gets stored](#what-gets-stored)). +- **No document ids, by design.** A resolver that needs a file's bytes walks + to it by name from the container at its pinned heads, and reads it at the + recorded heads. Automerge keeps history, so an old version stays readable + wherever the document is. +- **The listHash is the SHA-256 of these bytes.** The list is content like + any other: stored under its hash if its container's content may be stored, + and otherwise held in service worker memory. +- **A list may cover only some of a container's files.** A package's list + covers all of them, so relative imports work. The list behind a live → + pinned redirect covers only the file asked for. A tail a list doesn't + cover answers 404. +- **Whoever pins builds the list** (`pinPath`): it loads each file once and + hashes it. A tab or the automerge worker, it doesn't matter which: a list + is checked against the URL, not trusted for who made it. + - The pinner writes the list into the store itself when it may be stored. + - Otherwise it hands the list to the service worker in a message, or with + the 307 it answers. + - The service worker hashes the list before keeping it in memory. -A `Map` in service worker memory: pinned URL → `{ sha256, contentType }`. +### What gets stored -- Lost when the service worker is stopped, which is fine. -- On a miss, the service worker asks the automerge worker for the **manifest - of the last document in the URL at its heads**: hash and type of every file - that resolves pinned inside it. One round trip per package rather than per - file. -- The automerge worker memoizes manifests by `doc@heads` in an LRU; they never - change. +- Content, file lists included, from unprotected documents + (`isUnprotectedDocId`). +- Content from keyhive-protected documents only when reached through + `/packages`. Package code needs to load fast at boot; user documents don't. +- Everything else is served after checking, never stored. ### Request flow for a pinned URL -1. **Index hit:** read the store, hash the bytes, compare. - - Match: serve. - - Mismatch: delete the entry, warn loudly, go to step 3. -2. **Index miss:** fetch the manifest, then step 1. -3. **Store miss:** ask a resolver for the bytes and hash them in the service - worker. - - Matches the index entry: serve, and store it if - [What gets stored](#what-gets-stored) allows. - - Doesn't match: fail as a network error, store nothing. - - No index entry, because no automerge worker is available: serve inline, - store nothing. +1. **The list:** + - Read the last container's list by its listHash: from the store, or + from memory. + - Hash it. + - On a miss, or a mismatch (delete the store entry and warn), ask a + resolver to build that container's list at its pinned heads. Keep the + answer only if it hashes to the listHash. + - No list: fail as a network error. +2. **The entry:** look the tail up in the list; not there, 404. +3. **The bytes:** + - Read the store by the entry's `sha256` and hash the bytes. A match + serves them; a mismatch deletes the entry, warns, and goes on. + - Ask a resolver for the file at its pinned path and the entry's `heads`, + and hash its bytes. + - A match serves them, and stores them if the entry allows it. A + mismatch fails as a network error and stores nothing. + +A container step with no list leads to files resolved live: served +`no-store` from a resolver, never stored, not checked. Hashing is `crypto.subtle` SHA-256 on every hit, since a store entry can change between two reads. `sites/bench` gets a case measuring it. +Cold start needs no document: a package's list and its files are both in the +store, and both are checked against the URL the watcher pinned. + ### Who is trusted for what -- **Resolvers never write the cache.** Today `resolver.ts` calls `cache.put` - itself. Instead a resolver returns bytes, and the service worker decides. -- **Only the automerge worker fills the index.** Tab resolvers run in the - same realm as packages; their answers are served inline and never stored. -- **The index channel is authenticated.** Today's `HANDOFF_CHANNEL` is a - BroadcastChannel: any context on the origin can listen, see request ids, - and reply first with a forged response. That is a hole in the current code - as well. Index answers must arrive from a `Client` whose `type` is - `sharedworker` and whose `url` is the automerge worker's script URL, which - the service worker knows at build time. That URL is fixed, not hashed: - `/automerge-protocol-handler-worker.js`, the name the vite plugin keeps from - one build to the next. A site that sets its own `workerPath` has its - answers refused, so everything under `/@/` is served inline and nothing is - stored. - - The worker sends answers as `fetch()` requests to a reserved path. The - service worker looks up `event.clientId` with `clients.get` and checks - the client. This is the only reply path that works in all three engines - (see [Spikes](#spikes-before-planning)). - - Requests can stay on a BroadcastChannel. A listener learns which pinned - URLs are being loaded, which it could see anyway, but it can't answer. - - A `MessagePort` relayed through a tab doesn't authenticate anything. - Messages on a port have `source: null`, and the tab runs package code, - which can keep the port and answer itself, or sit in the middle and edit - answers. A relayed port is only safe if every answer on it is signed - with a key the worker got from the response to a checked `fetch()`. Not - worth that machinery unless one fetch per manifest proves too slow. - - Checking `type` is not optional. Any page can start a dedicated `Worker` - from the same script URL, and it shows up with the same `url` and - `type: "worker"`. +- **Nothing between the URL and the bytes is trusted.** Lists, store + entries, and resolver answers from a tab or the automerge worker are all + checked against hashes that start at the URL. A forged answer can fail a + request, but it can't change what's served. +- **Whoever builds a URL chooses the version it names.** For imports that's + the tab's own ModuleWatcher. For a live → pinned redirect it's whichever + resolver answered (see [Remaining gaps](#remaining-gaps)). +- **No authenticated channel is needed.** The earlier design filled an index + from the automerge worker over `fetch()` requests checked by `clientId`. + The spike behind that is kept below, in case a later design needs to know + who sent something. +- **Legacy `/automerge%3A…` URLs** stay keyed by URL and unchecked. Their + replies still ride `HANDOFF_CHANNEL`, which any context can answer first. ## Remaining gaps @@ -319,27 +381,28 @@ These replace the deleted `package-load-gate.md` note. Automerge verifies change hashes when it loads a document, so tampered changes don't load as the heads they claim. That check is planned to become optional; when it does, this gap reopens unless patchwork keeps it on. -- A live → pinned redirect answered by a tab can point at the wrong version. - It can't poison the store, but it decides which version a tab loads. +- Whoever builds a pinned URL chooses its version. A live → pinned redirect + answered by a tab can name a wrong version, and so can a package that + builds import URLs for another. It can't poison the store, but it decides + which version gets loaded. +- A pinned URL can't be served if its file list is gone (evicted, cleared, + or held only in a service worker that stopped) and no resolver can + rebuild it. A rebuild reads each file at its current heads, so it matches + only if nothing changed since, and only for a list that covers the whole + container (not a redirect's one-file list, which a fresh request to the + live URL replaces). The ModuleWatcher pins again on boot, so + imports recover; a hand-made share link doesn't. - Legacy `/automerge%3A…` URLs stay keyed by URL and unchecked. - Protected package code is stored in plaintext. Today keyhive's secrets sit in the same storage as its ciphertext, so this exposes nothing new; revisit when keyhive protects its keys at rest. - Nothing cleans up the content store. Deduplication slows growth, and the existing quota warning still fires. -- The index lives only in service worker memory, so after a browser restart - every package needs a fresh manifest, and building one loads and hashes - every file document in the package. The URL-keyed cache this replaces - served package code at boot without loading any document. The automerge - worker remembers 64 manifests, which covers service worker restarts within - a session but not a cold start. Persisting manifests would fix it, but - anything stored on the origin can be written by package code. `sites/bench`'s - `content-hash.spec` measures the per-hit hashing; the cold-start cost isn't - measured yet. ## Spikes before planning -- Can the service worker `postMessage` to a SharedWorker client, and can that +- (For the earlier, index-based section 5. Not needed now, but kept.) Can + the service worker `postMessage` to a SharedWorker client, and can that worker reply with the service worker able to check its `Client.url`, in Chromium, Firefox and WebKit? Fallback: the worker posts answers to the service worker as `fetch()` requests, whose `clientId` the service worker @@ -383,10 +446,19 @@ These replace the deleted `package-load-gate.md` note. - `ModuleWatcher` `folder` source: add, remove, rename without reload, only the changed package reloading, union deduplication. - Migration from a module-settings doc. - - Store and index with a fake `Cache` and an injected hash: forged store - entry rejected, tab answer never stored, matching answer stored, - protected non-package content never stored. - - Manifest builder against folder and directory fixtures. + - File lists: canonical bytes (key order, no whitespace), tails for folder + and directory fixtures, no document ids, and a list that agrees with + `readPath` for every file it covers. + - The request flow with a fake `Cache` and an injected hash: + - a forged list or store entry is rejected; + - a resolver's mismatching bytes fail as a network error and are never + stored; + - a file read at its recorded heads after it has changed still serves + the old bytes; + - protected non-package content is never stored; + - a tail the list doesn't cover answers 404. + - The service worker itself, in node, against a fake global scope + (`core/bootloader/test/service-worker-harness.ts`). - E2E, run in CI: - Open a document with `&path=` alone. - A page `cache.put`s forged bytes under a real content key, reloads, and @@ -398,9 +470,10 @@ Each phase can ship on its own and gets its own plan: 1. **Grammar, providers, namespace** in `patchwork-filesystem`, plus the account `namespace` field and its fallback. No behaviour change yet. -2. **Service worker:** `/@/` URLs, the content store, the index, the - authenticated channel (after its spike), resolvers no longer writing the - cache. +2. **Service worker:** `/@/` URLs, the content store, resolvers no longer + writing the cache. Done against the earlier index-based section 5; the + rework replaces the index, the manifest handler and the authenticated + channel with file lists. 3. **Routing:** `&path=`, opening at a path, rebased on `open-document-path`. 4. **Packages:** the `folder` source, `/packages`, migration, deprecating `moduleSettingsUrl`. From 9199452c92d6c798e604751873125c9ca78c4cda Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:28:07 +0000 Subject: [PATCH 25/81] plan: paths phase 2b (pinned URLs name their file list) Co-Authored-By: Claude Opus 5.5 --- .../2026-10-28-paths-phase-2b-file-lists.md | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md b/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md new file mode 100644 index 000000000..6dcecf99e --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md @@ -0,0 +1,280 @@ +# Paths phase 2b: pinned URLs name their file list — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rework phase 2 so a pinned `/@/` URL is checked against itself. The last container step carries `~`, the hash of a canonical file list. The service worker checks the list against the URL and the bytes against the list, and trusts nothing in between. + +**Architecture:** +- `patchwork-filesystem` learns three things: + - the `~listHash` grammar; + - how to build a file list (`buildFileList`); + - how to read a file at the heads a list recorded (`readPath` with `heads`). +- `pinPath` returns the URL and the list it names. +- The bootloader's request flow (`servePinned`) is rewritten around lists. +- Resolvers answer three kinds of request: + - a live path: a 307, whose body is the one-file list; + - a pinned file at given heads: its bytes; + - a container: its full list. +- The in-memory index, the worker's manifests and the authenticated `/@/.index` channel are deleted. + +**Tech Stack:** As in phase 2. TypeScript; automerge-repo 2.6.0-subduction.48; Cache Storage; `crypto.subtle`; vitest in node, including the service worker harness (`core/bootloader/test/service-worker-harness.ts`). + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, revised in f9446695. Section 1 covers the `~listHash` grammar and rule 4; section 5 covers *File lists*, *What gets stored*, *Request flow* and *Who is trusted for what*. The plan being reworked is `docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md`. + +## Global Constraints + +- The pinned container step is `@~`. `listHash` is SHA-256 of the list's bytes, in base64url without padding: the same function as content (`contentHash`). +- A list is canonical JSON: keys sorted at every level, no whitespace. + - Shape: `{"files":{"":{"heads"?:"a.b","sha256":"…","store":bool,"type":"…"}},"v":1}`. + - `heads` is absent for a value inside the container. + - The container itself is the tail `""`. +- No document ids in lists or URLs (except a share root). +- Store a list if and only if every entry in it has `store: true`. Otherwise the service worker keeps it in memory. +- What gets stored is unchanged: unprotected documents, or protected ones reached through `/packages`, never under a share root. +- `pinPath` lists every file under the last container. A live → pinned redirect lists only the file asked for. A tail a list doesn't cover answers 404. +- A container step without a list leads to files served live: `no-store`, unstored, unchecked. +- Request flow, exactly as the spec's section 5. A list or bytes that don't match fail as a network error, and the service worker never stores anything that doesn't match. +- Phases 1 and 2 aren't released, so their changesets are rewritten, not added to. +- No local e2e, `vite preview` or site builds. + +## Decisions this plan makes + +1. **Hashing moves to filesystem** (`core/filesystem/src/content-hash.ts`: `contentHash`, `toBytes`). `pinPath` needs it, and the bootloader imports it from there. The bootloader's `content-hash.ts` is deleted. +2. **Keyhive protection moves to filesystem too:** `isKeyhiveDocumentId(documentId)`. `@inkandswitch/patchwork-plugins`'s `isKeyhiveDoc` calls it, so the rule lives once. +3. **`pinPath` and `pinnedPath` return `{ url | path, list }`** instead of a string. They're unreleased phase 1 API. +4. **A live → pinned 307 carries the one-file list as its body.** The service worker hashes it, keeps it if it matches the `~listHash` in `Location` (stored or in memory, by the rule above), and serves the 307 without the body. A list `pinPath` builds for imports (phase 4) is handed over the same way. Phase 4 adds the store write, or a message to the service worker; nothing calls it before then. +5. **Rebuilding a missing list** is a handoff with `request.list = true` on the container's pinned path, without `~`. The resolver answers the full list's bytes. Only a list that hashes to the URL's `listHash` is kept. +6. **Key normalization changes.** `normalizePinned` returns candidates of `{ listHash?, prefix, tail }`, latest doc-step-shaped segment first, as before, plus the share-root fix the review deferred. The first candidate that has a `listHash` is used. +7. **The worker keeps account registration**, since it still resolves paths for bytes and lists. Manifests, `index-channel.ts`, `INDEX_ANSWER_PATH`, the Firefox probe and the `/@/.index` route are deleted. `AUTOMERGE_WORKER_PATH` stays as `setup.ts`'s default. + +## Review Focus + +- **A file changed after pinning:** the pinned URL still serves the old bytes. The resolver reads the file at the list's `heads`, and they match the list. Pinned in Task 2 (`readPath` with `heads`) and Task 5 (the harness round trip). +- **A list in the store forged or truncated:** it's deleted and rebuilt. With nothing changed the rebuild matches; with something changed, the request fails as a network error, and no forged bytes are ever served. Pinned in Tasks 3 and 5. +- **Canonical bytes:** the same files in a different insertion order give the same `listHash`. Pinned in Task 2. +- **A redirect's one-file list:** a sibling requested under it answers 404, not someone else's bytes. Pinned in Task 3. +- **No document ids:** the list bytes contain no documentId. Pinned in Task 2. + +--- + +## File structure + +| File | Change | +|---|---| +| `core/filesystem/src/paths.ts` | `DocStep.listHash?`, `docStep(idhash, heads, listHash?)`, parse `~` | +| `core/filesystem/src/content-hash.ts` (create) | `contentHash`, `toBytes`, `isKeyhiveDocumentId` | +| `core/filesystem/src/file-list.ts` (create) | `FileList`, `FileEntry`, `canonicalJson`, `buildFileList` | +| `core/filesystem/src/walk.ts` | the walk keeps `~listHash` and exposes `container.listHash` and `tail`; `readPath(…, { heads })`; `pinPath`/`pinnedPath` return the list | +| `core/plugins/src/keyhive.ts` | `isKeyhiveDoc` via `isKeyhiveDocumentId` | +| `core/bootloader/src/pinned.ts` | candidates with `listHash`; `servePinned` rewritten; manifest functions deleted | +| `core/bootloader/src/resolver.ts` | heads reads, `list` requests, a 307 with its list | +| `core/bootloader/src/types.ts` | `HandoffRequest.heads?`, `HandoffRequest.list?`; manifest/index types deleted | +| `core/bootloader/src/service-worker.ts` | wiring; index, manifests and `/@/.index` deleted | +| `core/bootloader/src/automerge-protocol-handler.ts` | manifest wiring deleted | +| delete | `core/bootloader/src/{manifest,index-channel,content-hash}.ts` and their tests | +| `packages/e2e/tests/content-store.spec.ts` | drop the forged-answer test | +| `.changeset/paths-*.md` | rewrite | + +--- + +### Task 1: Grammar and walk carry `~listHash` + +**Files:** `core/filesystem/src/paths.ts`, `src/walk.ts`; tests `test/paths.test.ts`, `test/walk.test.ts`. + +**Interfaces:** +- Produces: + - `DocStep = { kind: "doc"; idhash; heads; listHash?: string; raw }`. + - `docStep(idhash, heads, listHash?)`. + - `parsePath` reads `@~`: `/^[A-Za-z0-9_-]+$/` after the `~`, and a `~` with nothing after it is a syntax error for the root and a name otherwise. + - `WalkEnd.container` gains `listHash?: string`. The walk writes a matched doc step back with its `listHash`, so re-pinning a listed path gives the same string. + - `WalkEnd.tail: string`: the encoded names after the container, joined with `/`. + +- [ ] **Step 1: Failing tests** + +```ts +// paths.test.ts +it("reads a doc step's file list hash", () => { + expect(parsePath("acct@h/foo@h~AbC-_9/dist/x.js").segments[0]).toEqual({ + kind: "doc", idhash: "foo", heads: ["h"], listHash: "AbC-_9", raw: "foo@h~AbC-_9", + }); + expect(docStep("foo", ["b", "a"], "L")).toBe("foo@a.b~L"); +}); +// walk.test.ts +it("keeps a container's list hash and says which tail follows it", async () => { + // pinned = (await pinnedPath(repo, live(packages/foo), {accounts}))!.path, then append "~L" by hand + // walkPath(`${pinned}~L/dist/index.js`) → container.listHash "L", tail "dist/index.js", + // pinned === `${pinned}~L/dist/index.js`, canonical true +}); +``` + +- [ ] **Step 2: Run, expect FAIL** (`listHash` undefined): `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts test/walk.test.ts` +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run the filesystem suite, expect PASS:** `pnpm --filter @inkandswitch/patchwork-filesystem test` +- [ ] **Step 5: Commit** `filesystem: doc steps carry a file list hash` + +### Task 2: File lists, reading at heads, `pinPath` returns the list + +**Files:** create `core/filesystem/src/content-hash.ts` and `src/file-list.ts`; modify `src/walk.ts`, `src/index.ts` and `core/plugins/src/keyhive.ts`; test `test/file-list.test.ts`. + +**Interfaces:** +- Produces: + ```ts + // content-hash.ts + export function toBytes(content: string | Uint8Array): Uint8Array; + export function contentHash(bytes: Uint8Array): Promise; // moved from bootloader, same output + export function isKeyhiveDocumentId(documentId: DocumentId): boolean; + // file-list.ts + export type FileEntry = { sha256: string; type: string; heads?: string; store: boolean }; + export type FileList = { v: 1; files: Record }; + export function canonicalJson(value: unknown): string; + export function parseFileList(bytes: Uint8Array): FileList | undefined; + /** The list for the container `path` is in (all its files, or only `only`'s tails), its bytes, hash, and the container's pinned path without `~`. */ + export function buildFileList(repo: Repo, path: string, options: WalkOptions & { + findFile?: WalkOptions["find"]; only?: string[]; isProtected?: (id: DocumentId) => boolean; + }): Promise<{ prefix: string; list: FileList; bytes: Uint8Array; listHash: string; missing: string[] } | undefined>; + // walk.ts + export function readPath(repo, path, options: WalkOptions & { heads?: UrlHeads }): … // heads: read the final doc at these + export function pinnedPath(repo, path, options): Promise<{ path: string; list: Uint8Array } | undefined>; + export function pinPath(repo, path, options): Promise<{ url: string; list: Uint8Array } | undefined>; + ``` +- `buildFileList` uses `listFiles` (phase 2), drops `documentId` from each entry, and records `heads` (`formatHeads` of the heads the file doc was read at) for files reached by a link. `ListedFile` gains `heads?: UrlHeads`. `store` follows the spec rule, with `isProtected` defaulting to `isKeyhiveDocumentId`. +- `pinnedPath` builds the full list for the container the path ends in, or the one-file list when called with `{ only: [tail] }`, and writes `~listHash` on that container's step. + +- [ ] **Step 1: Failing tests** + +```ts +it("is canonical: the same files in another order hash the same", ...) + // two directory docs with the same keys inserted in opposite order and equal file contents/heads? — use one doc, + // build twice → identical bytes; canonicalJson({b:1,a:{d:1,c:2}}) === '{"a":{"c":2,"d":1},"b":1}' +it("records each file's hash, type and heads, and no document ids", ...) + // bytes as text contains no documentId of any fixture doc; entry for dist/index.js has heads === formatHeads(file.heads()) +it("lists only what it's asked to", ...) // only: ["dist/index.js"] → keys ["dist/index.js"] +it("stores protected content only through /packages", ...) // as phase 2's manifest test +it("reads a file at the heads its list recorded, after it changed", async () => { + // pin; change the file doc's content; readPath(pinned path, {heads: parseHeads(entry.heads)}) → old content +}); +it("pins a path with its container's list hash", ...) + // pinnedPath(live packages/foo/dist/index.js) → path ends `~${await contentHash(list)}/dist/index.js` +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/file-list.test.ts` +- [ ] **Step 3: Implement.** Update existing callers of `pinnedPath` and `pinPath` in the filesystem tests and in `core/bootloader/src/resolver.ts` (it reads `.path`). +- [ ] **Step 4: Run the filesystem suite and typecheck the dependents, expect PASS:** `pnpm --filter @inkandswitch/patchwork-filesystem test && pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins build && pnpm --filter @inkandswitch/patchwork-bootloader exec tsc --noEmit` +- [ ] **Step 5: Commit** `filesystem: file lists, reading at recorded heads, pinPath names its list` + +### Task 3: The request flow around lists (bootloader, pure) + +**Files:** `core/bootloader/src/pinned.ts`, `src/types.ts`; tests `test/pinned.test.ts`. Delete `src/manifest.ts`, `src/index-channel.ts`, `src/content-hash.ts` and their tests. Change imports to `@inkandswitch/patchwork-filesystem`. + +**Interfaces:** +- Produces: + ```ts + export type Candidate = { prefix: string; tail: string; listHash?: string }; + export function normalizePinned(path: string): + { candidates: Candidate[]; acctHash?: string; pinned: boolean; share: boolean } | undefined; + export type PinnedDeps = { + lists: Map; // in memory, by listHash + store: ContentStore | undefined; + hash(bytes: Uint8Array): Promise; + /** Ask for the full list of the container at `prefix`; its bytes, or undefined. */ + buildList(prefix: string): Promise; + /** Ask for the file at `path`, read at `heads` if given. */ + resolve(path: string, heads?: string): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; + warn(message: string): void; + keep(work: Promise): void; + }; + /** Check `bytes` against `listHash` and keep the list: stored if every entry allows it, else in memory. False if it doesn't match. */ + export function keepFileList(bytes: Uint8Array, listHash: string, deps: PinnedDeps): Promise; + export function servePinned(normalized: { candidates: Candidate[] }, deps: PinnedDeps): Promise; // throws PinnedAbort + ``` +- `handoffRequest(request, cachename, to: URL | { path: string; heads?: string; list?: boolean })`. +- Flow: the spec's *Request flow for a pinned URL*. A candidate with no `listHash` anywhere is served live through `resolve(path)`: `no-store`, unchecked. Concurrent `buildList` calls for one `listHash` share a request (`sharedRequests(fn)`, keyed by its argument). + +- [ ] **Step 1: Failing tests** (fake store, real `contentHash`, lists built with `canonicalJson`) + +```ts +it("serves a file whose list and bytes are in the store, asking no resolver", ...) +it("deletes a forged list, rebuilds it, and serves", ...) // store has garbage under listHash; buildList returns real bytes +it("fails when the rebuilt list doesn't match the URL", ...) // buildList returns a different list → PinnedAbort +it("deletes a forged file, rereads it at the list's heads, and serves", ...) // resolve called with entry.heads +it("fails on a resolver's bytes that don't match, storing nothing", ...) +it("answers 404 for a tail the list doesn't cover", ...) +it("keeps a list in memory when an entry may not be stored", ...) // keepFileList → lists has it, store empty +it("serves a path with no list live and unchecked", ...) // cache-control no-store +it("normalizes a share root's doc-step-shaped name", ...) // the deferred review minor +it("shares one rebuild among concurrent requests", ...) +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork-bootloader exec vitest run test/pinned.test.ts` +- [ ] **Step 3: Implement**, and delete the manifest and index-channel code. +- [ ] **Step 4: Typecheck, expect a clean run.** Expect errors only in `service-worker.ts`, `resolver.ts` and `automerge-protocol-handler.ts`, which Tasks 4–5 fix. Run the pinned tests, expect PASS. +- [ ] **Step 5: Commit** `bootloader: check pinned URLs against their own file list` + +### Task 4: Resolvers answer lists and reads at heads; the worker sheds manifests + +**Files:** `core/bootloader/src/resolver.ts`, `src/automerge-protocol-handler.ts`, `src/types.ts`; test `test/resolver.test.ts`. + +**Interfaces:** +- `HandoffRequest.heads?: string` and `HandoffRequest.list?: boolean`, both additive. +- `answer()` for a `path`: + - **`list`:** `buildFileList(path)`, which must be canonical. Answers `200 application/json` with the list's bytes, or 404. + - **`heads`:** `readPath(path, { heads: parseHeads(heads) })` → 200 with the bytes. + - **Live, or not canonical:** `pinnedPath(path, { only: [tail] })` → 307. `Location` is `/@/` plus the pinned path, and the body is the one-file list. + - **Canonical pinned, no heads:** the bytes, as now. +- The worker: drop `createManifestHandler` and the `manifest` channel branch. Keep `register-account`. + +- [ ] **Step 1: Failing tests** + +```ts +it("answers a live path with a 307 whose body is the list it names", ...) + // contentHash(body) === the ~listHash in location; parseFileList(body) has one entry +it("answers a container's full list", ...) +it("reads a file at given heads, after it changed", ...) +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** `pnpm --filter @inkandswitch/patchwork-bootloader exec vitest run test/resolver.test.ts` +- [ ] **Step 5: Commit** `bootloader: resolvers answer file lists and reads at heads` + +### Task 5: Service worker wiring, through the harness + +**Files:** `core/bootloader/src/service-worker.ts`; test `test/service-worker.test.ts`. + +**Interfaces:** +- `servePath`: + - Live → `handoff({ path })`. On a 307 with a body, `keepFileList(body, listHash from location)`, then serve the 307 without the body. + - Pinned → `servePinned` with: + - `buildList = handoff({ path: prefix, list: true })`; + - `resolve = handoff({ path, heads })`; + - `lists`: a module-level `Map`. +- Delete: `contentIndex`, manifest requests, `settleIndexAnswer`, `answerIndex`, the `/@/.index` route, and the manifest re-broadcast on `online`. + +- [ ] **Step 1: Failing harness tests** (a fake tab running the real resolver, a real in-memory Repo, an account registered in the tab) + +```ts +it("redirects a live path, then serves the pinned file from its list", ...) + // fetch live → 307; fetch location → bytes; content store now holds the list and the bytes +it("serves the old bytes for a pinned URL after the file changed", ...) +it("never serves a forged store entry", ...) + // put forged bytes under the file's sha256 → fetch again → real bytes +it("fails, rather than serving, when a tab answers with other bytes", ...) + // swap the tab's postMessage to answer with forged bytes → event.response() rejects +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** `pnpm --filter @inkandswitch/patchwork-bootloader test`, plus `pnpm --filter @inkandswitch/patchwork exec tsc --noEmit` after building filesystem, plugins and bootloader. +- [ ] **Step 5: Commit** `service worker: serve pinned paths checked against their file list` + +### Task 6: e2e, changesets, cleanup + +- [ ] **Step 1:** In `packages/e2e/tests/content-store.spec.ts`, delete the forged-answer test. The forged-store test stays as is, since the live → 307 → pinned flow still holds. Typecheck: `pnpm --filter @inkandswitch/patchwork-e2e exec tsc --noEmit`. +- [ ] **Step 2: Rewrite `.changeset/paths-content-store.md`** for what ships: + - `/@/` paths, file lists, the store; + - `registerAccount` and `clearAccountStore`; + - no index and no answer channel. + + Rewrite `.changeset/paths-grammar-providers.md` for `pinPath` and `pinnedPath` now returning `{ url | path, list }`. Add `@inkandswitch/patchwork-plugins` for `isKeyhiveDoc` now going through filesystem. +- [ ] **Step 3: Run the full suites and lint.** `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-bootloader test` should pass. In `pnpm lint`, ignore hits under the git-ignored `sites/bench/bench-results`. +- [ ] **Step 4: Commit** `content store: changesets and e2e for file lists` From 4cad85f8ab0b31e2a8b868ae6d1a0dbaff05d9b6 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:46:40 +0000 Subject: [PATCH 26/81] filesystem: doc steps carry a file list hash Co-Authored-By: Claude Opus 5.5 --- core/filesystem/src/paths.ts | 63 ++++++++++++++++++++++-------- core/filesystem/src/walk.ts | 37 ++++++++++++++---- core/filesystem/test/paths.test.ts | 30 ++++++++++++++ core/filesystem/test/walk.test.ts | 19 +++++++++ 4 files changed, 126 insertions(+), 23 deletions(-) diff --git a/core/filesystem/src/paths.ts b/core/filesystem/src/paths.ts index d54395cf8..7541d77bf 100644 --- a/core/filesystem/src/paths.ts +++ b/core/filesystem/src/paths.ts @@ -6,13 +6,16 @@ import { import bs58 from "bs58"; /** - * A step that enters a document at given heads: `@`. `raw` is the segment as - * written, so a walk that finds no link with this idhash can read it as a name instead. + * A step that enters a document at given heads: `@`, or + * `@~` for a container whose files the URL names by their list (see + * file-list.ts). `raw` is the segment as written, so a walk that finds no link with this idhash + * can read it as a name instead. */ export type DocStep = { kind: "doc"; idhash: string; heads: UrlHeads; + listHash?: string; raw: string; }; export type NameStep = { kind: "name"; name: string }; @@ -23,8 +26,8 @@ export type Segment = DocStep | NameStep; * link) a document named by its id. A share root is always pinned. */ export type PathRoot = - | { kind: "account"; acctHash: string; heads?: UrlHeads } - | { kind: "share"; documentId: DocumentId; heads: UrlHeads }; + | { kind: "account"; acctHash: string; heads?: UrlHeads; listHash?: string } + | { kind: "share"; documentId: DocumentId; heads: UrlHeads; listHash?: string }; export type ParsedPath = { root: PathRoot; segments: Segment[] }; @@ -33,6 +36,7 @@ export class PathSyntaxError extends Error { } const BASE58 = /^[1-9A-HJ-NP-Za-km-z]+$/; +const BASE64URL = /^[A-Za-z0-9_-]+$/; const SHARE_PREFIX = "automerge:"; /** @@ -58,8 +62,12 @@ export function parseHeads(segment: string): UrlHeads | undefined { return parts.sort() as UrlHeads; } -export function docStep(idhash: string, heads: readonly string[]): string { - return `${idhash}@${formatHeads(heads)}`; +export function docStep( + idhash: string, + heads: readonly string[], + listHash?: string +): string { + return `${idhash}@${formatHeads(heads)}${listHash ? `~${listHash}` : ""}`; } export function livePath(acctHash: string, names: string[]): string { @@ -74,16 +82,28 @@ function decode(raw: string): string { } } -/** `@`, split, or undefined if it isn't that shape. */ +/** `` or `~`, split, or undefined if it isn't that shape. */ +function splitHeads( + text: string +): { heads: UrlHeads; listHash?: string } | undefined { + const tilde = text.indexOf("~"); + const heads = parseHeads(tilde < 0 ? text : text.slice(0, tilde)); + if (!heads) return undefined; + if (tilde < 0) return { heads }; + const listHash = text.slice(tilde + 1); + return BASE64URL.test(listHash) ? { heads, listHash } : undefined; +} + +/** `@[~]`, split, or undefined if it isn't that shape. */ function splitDocStep( raw: string -): { id: string; heads: UrlHeads } | undefined { +): { id: string; heads: UrlHeads; listHash?: string } | undefined { const at = raw.indexOf("@"); if (at < 0) return undefined; const id = raw.slice(0, at); - const heads = parseHeads(raw.slice(at + 1)); - if (!BASE58.test(id) || !heads) return undefined; - return { id, heads }; + const rest = splitHeads(raw.slice(at + 1)); + if (!BASE58.test(id) || !rest) return undefined; + return { id, ...rest }; } function parseRoot(raw: string): PathRoot { @@ -92,13 +112,13 @@ function parseRoot(raw: string): PathRoot { const rest = decoded.slice(SHARE_PREFIX.length); const at = rest.lastIndexOf("@"); const documentId = rest.slice(0, at); - const heads = at < 0 ? undefined : parseHeads(rest.slice(at + 1)); - if (!heads || !isValidDocumentId(documentId)) { + const pinned = at < 0 ? undefined : splitHeads(rest.slice(at + 1)); + if (!pinned || !isValidDocumentId(documentId)) { throw new PathSyntaxError( `a share link must be automerge:@, got ${decoded}` ); } - return { kind: "share", documentId, heads }; + return { kind: "share", documentId, ...pinned }; } if (BASE58.test(raw)) return { kind: "account", acctHash: raw }; const step = splitDocStep(raw); @@ -107,7 +127,12 @@ function parseRoot(raw: string): PathRoot { `a path must start with an account hash or a share link, got ${raw}` ); } - return { kind: "account", acctHash: step.id, heads: step.heads }; + return { + kind: "account", + acctHash: step.id, + heads: step.heads, + ...(step.listHash ? { listHash: step.listHash } : {}), + }; } /** @@ -121,7 +146,13 @@ export function parsePath(path: string): ParsedPath { const segments = rest.map((raw): Segment => { const step = splitDocStep(raw); return step - ? { kind: "doc", idhash: step.id, heads: step.heads, raw } + ? { + kind: "doc", + idhash: step.id, + heads: step.heads, + ...(step.listHash ? { listHash: step.listHash } : {}), + raw, + } : { kind: "name", name: decode(raw) }; }); return { root: parseRoot(first), segments }; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index dc8f6a0cf..8f2f7f99d 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -67,10 +67,13 @@ export type WalkEnd = { */ canonical: boolean; /** - * The last document the walk entered that has a provider (a folder, a directory, …), and its - * pinned path: what a manifest of the files around the end of the path covers. + * The last document the walk entered that has a provider (a folder, a directory, …): its + * pinned path, without its own `~listHash`, and that list hash if the input gave one. Its file + * list covers the files around the end of the path. */ - container?: { handle: DocHandle; pinned: string }; + container?: { handle: DocHandle; pinned: string; listHash?: string }; + /** The encoded names after the container, joined with "/": "" at the container itself. */ + tail: string; }; /** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ @@ -157,6 +160,13 @@ export async function walkPath( ); const out: string[] = []; + // Index in `out` → the file list hash the input gave that doc step. + const lists = new Map(); + const render = (from: number, to: number) => + out + .slice(from, to) + .map((step, j) => (lists.has(from + j) ? `${step}~${lists.get(from + j)}` : step)) + .join("/"); // Set once a document the pinned path names was entered at its current heads. let live = !root.heads; let at: At | undefined; @@ -165,16 +175,18 @@ export async function walkPath( at = url && (await enter(url, root.heads)); if (!at) return undefined; out.push(docStep(root.acctHash, at.handle.heads())); + if (root.listHash) lists.set(0, root.listHash); } else { at = await enter(stringifyAutomergeUrl({ documentId: root.documentId }), root.heads); if (!at) return undefined; out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); + if (root.listHash) lists.set(0, root.listHash); } - let container: WalkEnd["container"]; + let container: { handle: DocHandle; index: number } | undefined; const noteContainer = () => { if (at?.kind === "doc" && at.provider && !at.inside) { - container = { handle: at.handle, pinned: out.join("/") }; + container = { handle: at.handle, index: out.length - 1 }; } }; @@ -186,9 +198,11 @@ export async function walkPath( if (at.kind === "union") { let member: AutomergeUrl | undefined; let next: AtDoc | undefined; + let listHash: string | undefined; if (segment?.kind === "doc") { member = await matchLink(at.members, segment); if (member) { + listHash = segment.listHash; i++; if (!guard(member, segments.length - i)) return undefined; next = await enter(member, segment.heads); @@ -209,6 +223,7 @@ export async function walkPath( } if (!next) return undefined; out.push(await stepOf(next.handle)); + if (listHash) lists.set(out.length - 1, listHash); at = next; continue; } @@ -235,6 +250,7 @@ export async function walkPath( const next = await enter(link, segment.heads); if (!next) return undefined; out.push(await stepOf(next.handle)); + if (segment.listHash) lists.set(out.length - 1, segment.listHash); at = next; i++; continue; @@ -273,9 +289,16 @@ export async function walkPath( handle: at.handle, node: at.node, atDocument: !at.inside, - pinned: out.join("/"), + pinned: render(0, out.length), canonical: !live, - container, + container: container && { + handle: container.handle, + pinned: [render(0, container.index), out[container.index]] + .filter(Boolean) + .join("/"), + listHash: lists.get(container.index), + }, + tail: container ? out.slice(container.index + 1).join("/") : "", }; } diff --git a/core/filesystem/test/paths.test.ts b/core/filesystem/test/paths.test.ts index c4ac611ec..484e4db93 100644 --- a/core/filesystem/test/paths.test.ts +++ b/core/filesystem/test/paths.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; import { + docStep, formatHeads, idhash, livePath, @@ -99,3 +100,32 @@ describe("parsePath", () => { expect(() => parsePath("automerge:not-an-id@h1")).toThrow(PathSyntaxError); }); }); + +describe("file list hashes", () => { + it("reads a doc step's file list hash", () => { + expect(parsePath("acct@h/foo@h~AbC-_9/dist/x.js").segments[0]).toEqual({ + kind: "doc", + idhash: "foo", + heads: ["h"], + listHash: "AbC-_9", + raw: "foo@h~AbC-_9", + }); + expect(docStep("foo", ["b", "a"], "L")).toBe("foo@a.b~L"); + expect(docStep("foo", ["a"])).toBe("foo@a"); + }); + + it("reads a root's file list hash", () => { + expect(parsePath("acct@h~L").root).toEqual({ + kind: "account", + acctHash: "acct", + heads: ["h"], + listHash: "L", + }); + }); + + it("takes a malformed list hash as a name, and refuses it at the root", () => { + expect(parsePath("acct@h/foo@h~/x").segments[0]).toEqual({ kind: "name", name: "foo@h~" }); + expect(parsePath("acct@h/foo@h~a.b/x").segments[0].kind).toBe("name"); + expect(() => parsePath("acct@h~")).toThrow(PathSyntaxError); + }); +}); diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts index 5c12f3745..886bf27ac 100644 --- a/core/filesystem/test/walk.test.ts +++ b/core/filesystem/test/walk.test.ts @@ -232,6 +232,25 @@ describe("pinnedPath", () => { }); }); +describe("file list hashes in a walk", () => { + it("keeps a container's list hash and says which tail follows it", async () => { + const { repo, a, accounts } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const end = await walkPath(repo, `${pinned}~L/dist/index.js`, { accounts }); + expect(end?.container?.listHash).toBe("L"); + expect(end?.container?.pinned).toBe(pinned); + expect(end?.tail).toBe("dist/index.js"); + expect(end?.pinned).toBe(`${pinned}~L/dist/index.js`); + expect(end?.canonical).toBe(true); + }); + + it("says the tail is empty at the container itself", async () => { + const { repo, a, accounts } = await fixture(); + const end = await walkPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + expect(end?.tail).toBe(""); + }); +}); + describe("pinPath", () => { it("prefixes the origin and /@/", async () => { vi.stubGlobal("location", { origin: "https://patchwork.test" }); From ddfa1d6fe72b93632556022580cc5344d5a99cb9 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:51:09 +0000 Subject: [PATCH 27/81] filesystem: file lists, reading at recorded heads, pinPath names its list Hashing and the keyhive check move into filesystem; plugins' isKeyhiveDoc goes through it. Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/manifest.ts | 2 +- core/bootloader/src/resolver.ts | 2 +- core/bootloader/src/service-worker.ts | 2 +- core/bootloader/test/content-store.test.ts | 2 +- core/bootloader/test/manifest.test.ts | 2 +- core/bootloader/test/pinned.test.ts | 2 +- .../src/content-hash.ts | 5 +- core/filesystem/src/file-list.ts | 106 ++++++++++++++++++ core/filesystem/src/index.ts | 3 + core/filesystem/src/keyhive.ts | 15 +++ core/filesystem/src/walk.ts | 86 +++++++++++--- core/filesystem/test/file-list.test.ts | 84 ++++++++++++++ core/filesystem/test/files.test.ts | 24 ++-- core/filesystem/test/namespace-fixture.ts | 8 ++ core/filesystem/test/walk.test.ts | 64 +++++------ core/plugins/src/keyhive.ts | 7 +- 16 files changed, 336 insertions(+), 78 deletions(-) rename core/{bootloader => filesystem}/src/content-hash.ts (70%) create mode 100644 core/filesystem/src/file-list.ts create mode 100644 core/filesystem/src/keyhive.ts create mode 100644 core/filesystem/test/file-list.test.ts diff --git a/core/bootloader/src/manifest.ts b/core/bootloader/src/manifest.ts index b8813fa70..3b55d8cdc 100644 --- a/core/bootloader/src/manifest.ts +++ b/core/bootloader/src/manifest.ts @@ -13,7 +13,7 @@ import { type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; import { isKeyhiveDoc } from "@inkandswitch/patchwork-plugins"; -import { contentHash, toBytes } from "./content-hash.js"; +import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; import { postIndexAnswer } from "./index-channel.js"; import type { IndexAnswer, diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index e0df78729..2a358a09a 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -24,7 +24,7 @@ import { walkPath, type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; -import { toBytes } from "./content-hash.js"; +import { toBytes } from "@inkandswitch/patchwork-filesystem"; import type { HandoffReplyMessage, diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 6e0920bc3..88e6e123e 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -15,7 +15,7 @@ import { type ResolveAcceptedMessage, type ResolveRequestMessage, } from "./types.js"; -import { contentHash, toBytes } from "./content-hash.js"; +import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; import { CONTENT_CACHE_PREFIX, cacheStore, diff --git a/core/bootloader/test/content-store.test.ts b/core/bootloader/test/content-store.test.ts index 39be24cd9..2c626244c 100644 --- a/core/bootloader/test/content-store.test.ts +++ b/core/bootloader/test/content-store.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { cacheStore } from "../src/content-store.js"; -import { contentHash, toBytes } from "../src/content-hash.js"; +import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; /** Enough of Cache for the store: put, match and delete by url. */ export class FakeCache { diff --git a/core/bootloader/test/manifest.test.ts b/core/bootloader/test/manifest.test.ts index fff311f4e..c1b70e66b 100644 --- a/core/bootloader/test/manifest.test.ts +++ b/core/bootloader/test/manifest.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from "vitest"; import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; import { formatHeads, idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; import { buildManifest } from "../src/manifest.js"; -import { contentHash } from "../src/content-hash.js"; +import { contentHash } from "@inkandswitch/patchwork-filesystem"; const file = (repo: Repo, content: string) => repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index e30640d4e..baaabc794 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from "vitest"; -import { contentHash, toBytes } from "../src/content-hash.js"; +import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; import type { ContentStore } from "../src/content-store.js"; import { PinnedAbort, diff --git a/core/bootloader/src/content-hash.ts b/core/filesystem/src/content-hash.ts similarity index 70% rename from core/bootloader/src/content-hash.ts rename to core/filesystem/src/content-hash.ts index a5784565b..01b33fcda 100644 --- a/core/bootloader/src/content-hash.ts +++ b/core/filesystem/src/content-hash.ts @@ -1,5 +1,6 @@ -// The one way content is hashed, by the automerge worker building a manifest and by the -// service worker checking what it serves: SHA-256 over the bytes, base64url without padding. +// The one way content is hashed, by whoever builds a file list and by the service worker +// checking what it serves: SHA-256 over the bytes, base64url without padding. A file list's own +// hash, the `~listHash` in a pinned path, is this too. /** Bytes as served: strings as UTF-8, which is what a Response body of a string sends. */ export function toBytes(content: string | Uint8Array): Uint8Array { diff --git a/core/filesystem/src/file-list.ts b/core/filesystem/src/file-list.ts new file mode 100644 index 000000000..5f061e2be --- /dev/null +++ b/core/filesystem/src/file-list.ts @@ -0,0 +1,106 @@ +// A container's file list: what each file under it is, at the moment it was pinned. A pinned +// path names one by its hash (`@~`), so the service worker checks the +// list against the URL and each file's bytes against the list, and trusts nothing in between. +// No document ids: a resolver finds a file by walking to it by name from the container at its +// pinned heads, and reads it at the heads the list recorded. +import type { DocumentId, Repo } from "@automerge/automerge-repo/slim"; +import { contentHash, toBytes } from "./content-hash.js"; +import { isKeyhiveDocumentId } from "./keyhive.js"; +import { formatHeads, parsePath } from "./paths.js"; +import { listFiles, type WalkOptions } from "./walk.js"; + +export type FileEntry = { + /** SHA-256 of the bytes the file serves as. */ + sha256: string; + type: string; + /** The heads the file's document was read at; absent for a value inside the container. */ + heads?: string; + /** Whether the bytes may be kept in the content store. */ + store: boolean; +}; + +export type FileList = { v: 1; files: Record }; + +/** JSON with every object's keys sorted and no whitespace, so equal values are equal bytes. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + const entries = Object.entries(value) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; + } + return JSON.stringify(value); +} + +const isEntry = (value: unknown): value is FileEntry => { + const entry = value as FileEntry; + return ( + !!entry && + typeof entry.sha256 === "string" && + typeof entry.type === "string" && + typeof entry.store === "boolean" && + (entry.heads === undefined || typeof entry.heads === "string") + ); +}; + +/** A list from its bytes, or undefined if they aren't one. Check the hash before trusting it. */ +export function parseFileList(bytes: Uint8Array): FileList | undefined { + try { + const value = JSON.parse(new TextDecoder().decode(bytes)); + if (value?.v !== 1 || !value.files || typeof value.files !== "object") return undefined; + if (!Object.values(value.files).every(isEntry)) return undefined; + return value as FileList; + } catch { + return undefined; + } +} + +/** + * The file list of the container `path` is in: every file under it, or only the tails in + * `only`. Returns its bytes and hash, and the container's pinned path without `~listHash`. + * Undefined unless `path` is canonical. `missing` names files that couldn't be read, which a + * whole list then leaves out. + * + * What may be stored: content from unprotected documents, and keyhive-protected content only + * when it's reached through `/packages`, since package code needs to load fast at boot and user + * documents don't. Nothing under a share link. + */ +export async function buildFileList( + repo: Repo, + path: string, + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: string[]; + isProtected?: (documentId: DocumentId) => boolean; + } +): Promise< + | { prefix: string; list: FileList; bytes: Uint8Array; listHash: string; missing: string[] } + | undefined +> { + const listed = await listFiles(repo, path, options); + if (!listed) return undefined; + const { root, segments } = parsePath(listed.prefix); + const first = segments[0]; + const throughPackages = first?.kind === "name" && first.name === "packages"; + const isProtected = options.isProtected ?? isKeyhiveDocumentId; + + const files: Record = {}; + for (const file of listed.files) { + files[file.tail] = { + sha256: await contentHash(toBytes(file.content)), + type: file.type, + ...(file.heads ? { heads: formatHeads(file.heads) } : {}), + store: root.kind === "account" && (throughPackages || !isProtected(file.documentId)), + }; + } + const list: FileList = { v: 1, files }; + const bytes = new TextEncoder().encode(canonicalJson(list)); + return { + prefix: listed.prefix, + list, + bytes, + listHash: await contentHash(bytes), + missing: listed.missing, + }; +} diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index 9886de7fd..5c148d184 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -1,4 +1,7 @@ +export * from "./content-hash.js"; +export * from "./file-list.js"; export * from "./find-handle.js"; +export * from "./keyhive.js"; export * from "./metadata.js"; export * from "./module-watcher.js"; export * from "./package-tracker.js"; diff --git a/core/filesystem/src/keyhive.ts b/core/filesystem/src/keyhive.ts new file mode 100644 index 000000000..d54518f23 --- /dev/null +++ b/core/filesystem/src/keyhive.ts @@ -0,0 +1,15 @@ +import { + documentIdToBinary, + type DocumentId, +} from "@automerge/automerge-repo/slim"; + +/** + * Whether a document is keyhive-protected: its binary id carries a non-zero + * second half, where a pre-keyhive document's is empty. + */ +export function isKeyhiveDocumentId(documentId: DocumentId): boolean { + const bytes = documentIdToBinary(documentId); + return ( + !!bytes && bytes.length >= 32 && bytes.subarray(16, 32).some((byte) => byte !== 0) + ); +} diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 8f2f7f99d..e98e95fe0 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -25,6 +25,7 @@ import { defaultBranch, providerFor, } from "./providers.js"; +import { buildFileList } from "./file-list.js"; import { readNode } from "./resolve.js"; import { documentBaseOrigin } from "./urls.js"; @@ -320,22 +321,41 @@ export async function resolveLivePath( return repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); } +/** + * The pinned form of a path, with a file list on the last container it enters: every file + * under that container, or with `only: "end"`, only the one the path ends at (what a live → + * pinned redirect hands out). Returns the path and the list's bytes, which whoever pins hands to + * the service worker. No list if the path enters no container. + */ export async function pinnedPath( repo: Repo, path: string, - options: WalkOptions -): Promise { - return (await walkPath(repo, path, options))?.pinned; + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: "end"; + isProtected?: (documentId: DocumentId) => boolean; + } +): Promise<{ path: string; list?: Uint8Array } | undefined> { + const end = await walkPath(repo, path, options); + if (!end) return undefined; + if (!end.container) return { path: end.pinned }; + const built = await buildFileList(repo, end.container.pinned, { + ...options, + only: options.only === "end" ? [end.tail] : undefined, + }); + if (!built) return { path: end.pinned }; + const listed = `${end.container.pinned}~${built.listHash}`; + return { path: end.tail ? `${listed}/${end.tail}` : listed, list: built.bytes }; } -/** The pinned service worker URL for a path, for fetching or importing. */ +/** The pinned service worker URL for a path, for fetching or importing, and its file list. */ export async function pinPath( repo: Repo, path: string, - options: WalkOptions -): Promise { + options: Parameters[2] +): Promise<{ url: string; list?: Uint8Array } | undefined> { const pinned = await pinnedPath(repo, path, options); - return pinned && `${documentBaseOrigin()}/@/${pinned}`; + return pinned && { url: `${documentBaseOrigin()}/@/${pinned.path}`, list: pinned.list }; } /** Every binding's children, concatenated in binding order. */ @@ -383,20 +403,41 @@ export async function listChildren( export type FileContent = { content: string | Uint8Array; type: string }; -/** What a path names, read: its pinned form, whether the input was canonical, and the bytes. */ +/** + * What a path names, read: its pinned form, whether the input was canonical, and the bytes. + * With `heads`, the document the path ends at is read at those heads instead: how a file is + * read at the heads its file list recorded, after it has changed. + */ export async function readPath( repo: Repo, path: string, - options: WalkOptions + options: WalkOptions & { heads?: UrlHeads } ): Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined> { const end = await walkPath(repo, path, options); if (!end) return undefined; - const read = await readNode(repo, end.node); + let node = end.node; + if (options.heads && end.atDocument) { + try { + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const bare = stringifyAutomergeUrl({ documentId: end.handle.documentId }); + node = (await find(bare, options.heads)).view(options.heads).doc(); + } catch { + return undefined; + } + } + const read = await readNode(repo, node); return read && { ...read, pinned: end.pinned, canonical: end.canonical }; } -/** A file under a pinned document, and the document that holds its bytes. */ -export type ListedFile = FileContent & { tail: string; documentId: DocumentId }; +/** + * A file under a pinned document, the document that holds its bytes, and the heads that + * document was read at if it was reached by a link. + */ +export type ListedFile = FileContent & { + tail: string; + documentId: DocumentId; + heads?: UrlHeads; +}; const isNested = (value: unknown): value is Record => !!value && @@ -431,7 +472,7 @@ const LIST_CONCURRENCY = 8; export async function listFiles( repo: Repo, path: string, - options: WalkOptions & { findFile?: WalkOptions["find"] } + options: WalkOptions & { findFile?: WalkOptions["find"]; only?: string[] } ): Promise<{ prefix: string; files: ListedFile[]; missing: string[] } | undefined> { const end = await walkPath(repo, path, options); if (!end?.canonical || !end.container) return undefined; @@ -444,17 +485,25 @@ export async function listFiles( const found = new Map(); const missing: string[] = []; const tailOf = (parts: string[]) => parts.map(encodeURIComponent).join("/"); + const wanted = (parts: string[]) => !options.only || options.only.includes(tailOf(parts)); const add = async ( parts: string[], keys: number[], node: unknown, - holder: DocumentId + holder: DocumentId, + heads?: UrlHeads ) => { + if (!wanted(parts)) return; const tail = tailOf(parts); const existing = found.get(tail); if (existing && !walkPrefers(keys, existing.keys)) return; const read = await readNode(repo, node); - if (read) found.set(tail, { keys, file: { ...read, tail, documentId: holder } }); + if (read) { + found.set(tail, { + keys, + file: { ...read, tail, documentId: holder, ...(heads ? { heads } : {}) }, + }); + } }; // Values are read in place; links are gathered and read below. @@ -472,7 +521,7 @@ export async function listFiles( if (provider === directoryProvider && isNested(child.value)) { await visit(child.value, childParts, childKeys); } - } else if ("link" in child) { + } else if ("link" in child && wanted(childParts)) { links.push({ parts: childParts, keys: childKeys, link: child.link }); } } @@ -490,9 +539,10 @@ export async function listFiles( stringifyAutomergeUrl({ documentId: parsed.documentId }), parsed.heads ); - const doc = live.view(parsed.heads ?? live.heads()).doc(); + const heads = parsed.heads ?? live.heads(); + const doc = live.view(heads).doc(); if (doc === undefined || providerFor(doc)) continue; - await add(parts, keys, doc, parsed.documentId); + await add(parts, keys, doc, parsed.documentId, heads); } catch { missing.push(tailOf(parts)); } diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts new file mode 100644 index 000000000..f00b4a425 --- /dev/null +++ b/core/filesystem/test/file-list.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect } from "vitest"; +import { parseAutomergeUrl } from "@automerge/automerge-repo"; +import { formatHeads, livePath, parseHeads } from "../src/paths.js"; +import { readPath } from "../src/walk.js"; +import { contentHash, toBytes } from "../src/content-hash.js"; +import { buildFileList, canonicalJson, parseFileList } from "../src/file-list.js"; +import { chain, fixture } from "./namespace-fixture.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +describe("canonicalJson", () => { + it("sorts keys at every level and leaves no whitespace", () => { + expect(canonicalJson({ b: 1, a: { d: 1, c: [2, { f: 1, e: 2 }] } })).toBe( + '{"a":{"c":[2,{"e":2,"f":1}],"d":1},"b":1}' + ); + }); +}); + +describe("buildFileList", () => { + it("is canonical: building it twice gives the same bytes and hash", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const first = await buildFileList(repo, prefix!, { accounts }); + const second = await buildFileList(repo, prefix!, { accounts }); + expect(first?.bytes).toEqual(second?.bytes); + expect(first?.listHash).toBe(await contentHash(first!.bytes)); + expect(parseFileList(first!.bytes)).toEqual(first?.list); + }); + + it("records each file's hash, type and heads, and no document ids", async () => { + const { repo, a, accounts, minePkg, mine, account } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const fileUrl = (minePkg.doc() as any)["dist/index.js"] as string; + const file = await repo.find(fileUrl as any); + expect(built?.list.files["dist/index.js"]).toEqual({ + sha256: await contentHash(toBytes("mine")), + type: "text/plain", + heads: formatHeads(file.heads()), + store: true, + }); + const text = new TextDecoder().decode(built!.bytes); + for (const doc of [minePkg, mine, account, file]) { + expect(text).not.toContain(doc.documentId); + } + expect(text).not.toContain(parseAutomergeUrl(fileUrl as any).documentId); + }); + + it("lists only what it's asked to", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts, only: ["dist/index.js"] }); + expect(Object.keys(built!.list.files)).toEqual(["dist/index.js"]); + }); + + it("stores protected content only through /packages", async () => { + const { repo, a, accounts } = await fixture(); + const isProtected = () => true; + const projects = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const pkg = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const userList = await buildFileList(repo, projects!, { accounts, isProtected }); + const pkgList = await buildFileList(repo, pkg!, { accounts, isProtected }); + expect(userList?.list.files.notes.store).toBe(false); + expect(pkgList?.list.files["dist/index.js"].store).toBe(true); + }); + + it("reads a file at the heads its list recorded, after it changed", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dist/index.js"]; + const file = await repo.find((minePkg.doc() as any)["dist/index.js"]); + (file as any).change((d: any) => (d.content = "changed")); + + const now = await readPath(repo, `${prefix}/dist/index.js`, { accounts }); + expect(decode(now!.content)).toBe("changed"); + const then = await readPath(repo, `${prefix}/dist/index.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("mine"); + }); +}); diff --git a/core/filesystem/test/files.test.ts b/core/filesystem/test/files.test.ts index 53bf3b81e..b0d42e3a5 100644 --- a/core/filesystem/test/files.test.ts +++ b/core/filesystem/test/files.test.ts @@ -1,8 +1,8 @@ import { describe, it, expect } from "vitest"; import { parseAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; import { livePath } from "../src/paths.js"; -import { listFiles, pinnedPath, readPath, type ListedFile } from "../src/walk.js"; -import { file, fixture, folder } from "./namespace-fixture.js"; +import { listFiles, readPath, type ListedFile } from "../src/walk.js"; +import { chain, file, fixture, folder } from "./namespace-fixture.js"; const decode = (content: string | Uint8Array) => typeof content === "string" ? content : new TextDecoder().decode(content); @@ -23,7 +23,7 @@ describe("readPath", () => { expect(read && decode(read.content)).toBe("mine"); expect(read?.type).toBe("text/plain"); expect(read?.canonical).toBe(false); - expect(read?.pinned).toBe(await pinnedPath(repo, path, { accounts })); + expect(read?.pinned).toBe(await chain(repo, path, accounts)); }); it("reads nothing for a path that doesn't resolve", async () => { @@ -37,7 +37,7 @@ describe("readPath", () => { describe("listFiles", () => { it("lists a directory package's files", async () => { const { repo, a, accounts, minePkg } = await fixture(); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const listed = await listFiles(repo, prefix!, { accounts }); expect(listed?.prefix).toBe(prefix); const files = await collect(listed!.files); @@ -51,7 +51,7 @@ describe("listFiles", () => { const { repo, a, accounts, foo } = await fixture(); const sub = folder(repo, { inner: file(repo, "inner").url }); foo.change((d: any) => d.docs.push({ name: "sub", type: "folder", url: sub.url })); - const prefix = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); expect(files.map((f) => f.tail)).toEqual(["", "notes"]); }); @@ -59,7 +59,7 @@ describe("listFiles", () => { it("lists nested values inside a directory doc", async () => { const { repo, a, accounts, minePkg } = await fixture(); minePkg.change((d: any) => (d.dist = { "a.js": "x" })); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); expect(files.map((f) => f.tail)).toContain("dist/a.js"); expect(files.find((f) => f.tail === "dist/a.js")?.documentId).toBe(minePkg.documentId); @@ -68,7 +68,7 @@ describe("listFiles", () => { it("encodes names in tails", async () => { const { repo, a, accounts, minePkg } = await fixture(); minePkg.change((d: any) => (d["my file@1.js"] = file(repo, "spaced").url)); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); expect(files.map((f) => f.tail)).toContain("my%20file%401.js"); }); @@ -79,7 +79,7 @@ describe("listFiles", () => { d["x/y"] = file(repo, "flat").url; d.x = { y: "nested" }; }); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); const xy = files.filter((f) => f.tail === "x/y"); expect(xy).toHaveLength(1); @@ -94,7 +94,7 @@ describe("listFiles", () => { d.x = { y: "nested" }; }); for (const names of [["packages", "foo"], ["Projects", "foo"]]) { - const prefix = await pinnedPath(repo, livePath(a, names), { accounts }); + const prefix = await chain(repo, livePath(a, names), accounts); const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); expect(files.length).toBeGreaterThan(1); for (const f of files) { @@ -110,7 +110,7 @@ describe("listFiles", () => { it("lists the package a file path is in", async () => { const { repo, a, accounts } = await fixture(); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const listed = await listFiles(repo, `${prefix}/dist/index.js`, { accounts }); expect(listed?.prefix).toBe(prefix); expect((await collect(listed!.files)).map((f) => f.tail)).toContain("dist/index.js"); @@ -119,7 +119,7 @@ describe("listFiles", () => { it("lists the package for a path that ends inside it", async () => { const { repo, a, accounts, minePkg } = await fixture(); minePkg.change((d: any) => (d.dist = { "a.js": "x" })); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); expect((await listFiles(repo, `${prefix}/dist`, { accounts }))?.prefix).toBe(prefix); }); @@ -128,7 +128,7 @@ describe("listFiles", () => { const bad = (minePkg.doc() as any)["package.json"] as string; const find = (url: AutomergeUrl) => url === bad ? Promise.reject(new Error("unavailable")) : repo.find(url); - const prefix = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); const listed = await listFiles(repo, prefix!, { accounts, find }); expect(listed?.missing).toEqual(["package.json"]); expect(listed?.files.map((f) => f.tail)).not.toContain("package.json"); diff --git a/core/filesystem/test/namespace-fixture.ts b/core/filesystem/test/namespace-fixture.ts index 869612c7f..1f4da5603 100644 --- a/core/filesystem/test/namespace-fixture.ts +++ b/core/filesystem/test/namespace-fixture.ts @@ -1,6 +1,7 @@ // A small account to walk: / is a folder of Projects, /packages a union of two folders. import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; import { docStep, idhash } from "../src/paths.js"; +import { walkPath } from "../src/walk.js"; export const step = async (handle: DocHandle) => docStep(await idhash(handle.documentId), handle.heads()); @@ -70,3 +71,10 @@ export async function fixture() { } export const text = (handle: DocHandle | undefined) => handle?.doc()?.content; + +/** A path's chain of doc steps, without file lists: what the walk writes. */ +export const chain = async ( + repo: Repo, + path: string, + accounts: ReadonlyMap +) => (await walkPath(repo, path, { accounts }))?.pinned; diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts index 886bf27ac..5437c087b 100644 --- a/core/filesystem/test/walk.test.ts +++ b/core/filesystem/test/walk.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; import { docStep, idhash, livePath } from "../src/paths.js"; -import { file, fixture, step, text } from "./namespace-fixture.js"; +import { chain, file, fixture, step, text } from "./namespace-fixture.js"; +import { contentHash } from "../src/content-hash.js"; import { listChildren, listUnion, @@ -44,7 +45,7 @@ describe("resolveLivePath", () => { it("a pinned path keeps resolving to the same version after a rename", async () => { const { repo, a, accounts, projects, foo } = await fixture(); - const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const pinned = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); const before = foo.heads(); projects.change((d: any) => (d.docs[0].name = "bar")); @@ -182,26 +183,33 @@ describe("resolveLivePath", () => { }); describe("pinnedPath", () => { - it("pins every container the walk enters, and keeps the tail after the last one as names", async () => { + const listHashOf = async (list: Uint8Array | undefined) => contentHash(list!); + + it("pins every container the walk enters, lists the last, and keeps the tail as names", async () => { const { repo, a, accounts, account, mine, minePkg } = await fixture(); const pinned = await pinnedPath( repo, livePath(a, ["packages", "foo", "dist", "index.js"]), { accounts } ); - expect(pinned).toBe( - `${docStep(a, account.heads())}/packages/${await step(mine)}/${await step(minePkg)}/dist/index.js` + const listed = docStep( + await idhash(minePkg.documentId), + minePkg.heads(), + await listHashOf(pinned?.list) ); - const index = await resolveLivePath(repo, pinned!, { accounts }); + expect(pinned?.path).toBe( + `${docStep(a, account.heads())}/packages/${await step(mine)}/${listed}/dist/index.js` + ); + const index = await resolveLivePath(repo, pinned!.path, { accounts }); expect(text(index)).toBe("mine"); }); it("writes the / binding as a doc step with no name", async () => { const { repo, a, accounts, account, root, projects, foo } = await fixture(); - expect( - await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }) - ).toBe( - [docStep(a, account.heads()), await step(root), await step(projects), await step(foo)].join("/") + const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const listed = docStep(await idhash(foo.documentId), foo.heads(), await listHashOf(pinned?.list)); + expect(pinned?.path).toBe( + [docStep(a, account.heads()), await step(root), await step(projects), listed].join("/") ); }); @@ -212,10 +220,15 @@ describe("pinnedPath", () => { livePath(a, ["packages", "branchy", "dist", "index.js"]), { accounts } ); - expect(pinned).toBe( - `${docStep(a, account.heads())}/packages/${await step(team)}/${await step(branchy)}/${await step(branchyPkg)}/dist/index.js` + const listed = docStep( + await idhash(branchyPkg.documentId), + branchyPkg.heads(), + await listHashOf(pinned?.list) + ); + expect(pinned?.path).toBe( + `${docStep(a, account.heads())}/packages/${await step(team)}/${await step(branchy)}/${listed}/dist/index.js` ); - expect(text(await resolveLivePath(repo, pinned!, { accounts }))).toBe("branchy"); + expect(text(await resolveLivePath(repo, pinned!.path, { accounts }))).toBe("branchy"); // a path that ends at a branches doc stands for its default branch const end = await resolveLivePath(repo, livePath(a, ["packages", "branchy"]), { accounts }); expect(end?.url).toBe(branchyPkg.url); @@ -228,26 +241,7 @@ describe("pinnedPath", () => { livePath(a, ["packages", "bar", "dist", "index.js"]), { accounts } ); - expect(await pinnedPath(repo, pinned!, { accounts })).toBe(pinned); - }); -}); - -describe("file list hashes in a walk", () => { - it("keeps a container's list hash and says which tail follows it", async () => { - const { repo, a, accounts } = await fixture(); - const pinned = await pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts }); - const end = await walkPath(repo, `${pinned}~L/dist/index.js`, { accounts }); - expect(end?.container?.listHash).toBe("L"); - expect(end?.container?.pinned).toBe(pinned); - expect(end?.tail).toBe("dist/index.js"); - expect(end?.pinned).toBe(`${pinned}~L/dist/index.js`); - expect(end?.canonical).toBe(true); - }); - - it("says the tail is empty at the container itself", async () => { - const { repo, a, accounts } = await fixture(); - const end = await walkPath(repo, livePath(a, ["packages", "foo"]), { accounts }); - expect(end?.tail).toBe(""); + expect((await pinnedPath(repo, pinned!.path, { accounts }))?.path).toBe(pinned?.path); }); }); @@ -256,8 +250,8 @@ describe("pinPath", () => { vi.stubGlobal("location", { origin: "https://patchwork.test" }); const { repo, a, accounts } = await fixture(); const pinned = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); - expect(await pinPath(repo, livePath(a, ["Projects"]), { accounts })).toBe( - `https://patchwork.test/@/${pinned}` + expect((await pinPath(repo, livePath(a, ["Projects"]), { accounts }))?.url).toBe( + `https://patchwork.test/@/${pinned?.path}` ); }); }); diff --git a/core/plugins/src/keyhive.ts b/core/plugins/src/keyhive.ts index c95700af3..029cfafeb 100644 --- a/core/plugins/src/keyhive.ts +++ b/core/plugins/src/keyhive.ts @@ -2,14 +2,11 @@ import { parseAutomergeUrl, type AutomergeUrl, } from "@automerge/automerge-repo/slim"; +import { isKeyhiveDocumentId } from "@inkandswitch/patchwork-filesystem"; export function isKeyhiveDoc(url: AutomergeUrl): boolean { try { - const { binaryDocumentId } = parseAutomergeUrl(url); - return ( - binaryDocumentId.length >= 32 && - binaryDocumentId.subarray(16, 32).some((byte) => byte !== 0) - ); + return isKeyhiveDocumentId(parseAutomergeUrl(url).documentId); } catch { return false; } From d1a6b35a68394292a756bd3e61f61f262f8a2a17 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:53:58 +0000 Subject: [PATCH 28/81] bootloader: check pinned URLs against their own file list Removes the manifest, the index and the authenticated answer channel; the service worker and the automerge worker are rewired next. Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/index-channel.ts | 115 ------ core/bootloader/src/manifest.ts | 144 -------- core/bootloader/src/pinned.ts | 344 ++++++++---------- core/bootloader/src/types.ts | 51 +-- core/bootloader/test/index-channel.test.ts | 99 ----- core/bootloader/test/manifest.test.ts | 65 ---- core/bootloader/test/pinned.test.ts | 330 ++++++++--------- core/bootloader/test/worker-manifests.test.ts | 104 ------ 8 files changed, 312 insertions(+), 940 deletions(-) delete mode 100644 core/bootloader/src/index-channel.ts delete mode 100644 core/bootloader/src/manifest.ts delete mode 100644 core/bootloader/test/index-channel.test.ts delete mode 100644 core/bootloader/test/manifest.test.ts delete mode 100644 core/bootloader/test/worker-manifests.test.ts diff --git a/core/bootloader/src/index-channel.ts b/core/bootloader/src/index-channel.ts deleted file mode 100644 index bff21d843..000000000 --- a/core/bootloader/src/index-channel.ts +++ /dev/null @@ -1,115 +0,0 @@ -// The automerge worker's answers to the service worker, and how the service worker tells they -// really are its. A BroadcastChannel can't: anything on the origin can listen and answer first. -// Messages on a port carry no sender at all. A fetch() carries its client: the service worker -// looks `clientId` up and checks the client is a SharedWorker running the worker's script. -// That's the only reply path every engine supports; Chromium has no navigator.serviceWorker in -// workers (see the spec's spike results). -import { - AUTOMERGE_WORKER_PATH, - INDEX_ANSWER_PATH, - type IndexAnswer, -} from "./types.js"; - -const ACCEPTED_HEADER = "x-patchwork-index"; - -/** - * True only for the automerge worker: a SharedWorker on this origin running its script, whatever - * the query. Any page can start a dedicated Worker from the same URL, which reports the same url - * with type "worker", so the type matters as much as the url. - */ -export function isAutomergeWorker( - client: Pick | undefined, - origin: string -): boolean { - if (client?.type !== "sharedworker") return false; - try { - const url = new URL(client.url); - return url.origin === origin && url.pathname === AUTOMERGE_WORKER_PATH; - } catch { - return false; - } -} - -const isAnswer = (value: unknown): value is IndexAnswer => - !!value && - typeof value === "object" && - typeof (value as IndexAnswer).id === "string"; - -/** The service worker's side of a POST to INDEX_ANSWER_PATH. */ -export async function acceptIndexAnswer( - request: Request, - client: Pick | undefined, - origin: string, - onAnswer: (answer: IndexAnswer) => void -): Promise { - if (!isAutomergeWorker(client, origin)) { - console.warn( - `refusing an index answer from ${client?.type ?? "an unknown client"} at ${client?.url ?? "?"}` - ); - return new Response("only the automerge worker answers here", { status: 403 }); - } - const body: unknown = await request.json().catch(() => undefined); - if (!isAnswer(body)) return new Response("not an index answer", { status: 400 }); - onAnswer(body); - return new Response(null, { status: 204, headers: { [ACCEPTED_HEADER]: "accepted" } }); -} - -/** - * The service worker's answer to a GET of INDEX_ANSWER_PATH: nothing to get, but the header says - * the request reached it rather than the network. - */ -export function indexChannelProbe(): Response { - return new Response(null, { - status: 405, - headers: { [ACCEPTED_HEADER]: "present", allow: "POST" }, - }); -} - -const TRIES = 5; -const RETRY_MS = 200; - -const pause = () => new Promise((resolve) => setTimeout(resolve, RETRY_MS)); - -// Fetchers whose requests have been seen to reach the service worker. -const reachesServiceWorker = new WeakSet(); - -/** - * Whether this worker's fetches reach the service worker. Firefox sent a worker's first fetch - * straight to the network in the spike, and an answer that does that hands the origin server - * the account's file names; so probe with a GET that carries nothing first. - */ -async function probe(fetcher: typeof fetch): Promise { - if (reachesServiceWorker.has(fetcher)) return true; - for (let i = 0; i < TRIES; i++) { - if (i) await pause(); - const response = await fetcher(INDEX_ANSWER_PATH).catch(() => undefined); - if (response?.headers.get(ACCEPTED_HEADER) === "present") { - reachesServiceWorker.add(fetcher); - return true; - } - } - return false; -} - -/** - * The automerge worker's side: POST an answer, once a probe says this worker's fetches reach the - * service worker. A reply without the accepted header didn't get there, so try again a few - * times. False if it never got through. - */ -export async function postIndexAnswer( - answer: IndexAnswer, - fetcher: typeof fetch = fetch -): Promise { - if (!(await probe(fetcher))) return false; - const body = JSON.stringify(answer); - for (let i = 0; i < TRIES; i++) { - if (i) await pause(); - const response = await fetcher(INDEX_ANSWER_PATH, { - method: "POST", - headers: { "content-type": "application/json" }, - body, - }).catch(() => undefined); - if (response?.headers.get(ACCEPTED_HEADER) === "accepted") return true; - } - return false; -} diff --git a/core/bootloader/src/manifest.ts b/core/bootloader/src/manifest.ts deleted file mode 100644 index 3b55d8cdc..000000000 --- a/core/bootloader/src/manifest.ts +++ /dev/null @@ -1,144 +0,0 @@ -// The automerge worker's side of the index: what every file under a pinned document hashes to, -// and whether the service worker may keep it. -import { - stringifyAutomergeUrl, - type AutomergeUrl, - type DocumentId, - type Repo, -} from "@automerge/automerge-repo/slim"; -import { - listFiles, - parsePath, - walkPath, - type WalkOptions, -} from "@inkandswitch/patchwork-filesystem"; -import { isKeyhiveDoc } from "@inkandswitch/patchwork-plugins"; -import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; -import { postIndexAnswer } from "./index-channel.js"; -import type { - IndexAnswer, - IndexEntry, - Manifest, - ManifestRequestMessage, -} from "./types.js"; - -const isProtectedDoc = (documentId: DocumentId) => - isKeyhiveDoc(stringifyAutomergeUrl({ documentId })); - -/** - * The manifest covering canonical pinned `path`, or undefined if it isn't one. - * - * What may be stored: content from unprotected documents, and keyhive-protected content only - * when it's reached through `/packages`, since package code needs to load fast at boot and user - * documents don't. Nothing under a share link. - */ -export async function buildManifest( - repo: Repo, - path: string, - options: WalkOptions & { - findFile?: WalkOptions["find"]; - isProtected?: (documentId: DocumentId) => boolean; - } -): Promise<{ manifest: Manifest; missing: string[] } | undefined> { - const listed = await listFiles(repo, path, options); - if (!listed) return undefined; - const { root, segments } = parsePath(listed.prefix); - const first = segments[0]; - const throughPackages = first?.kind === "name" && first.name === "packages"; - const isProtected = options.isProtected ?? isProtectedDoc; - - const entries: Record = {}; - for (const file of listed.files) { - entries[file.tail] = { - sha256: await contentHash(toBytes(file.content)), - contentType: file.type, - store: - root.kind === "account" && - (throughPackages || !isProtected(file.documentId)), - }; - } - return { manifest: { prefix: listed.prefix, entries }, missing: listed.missing }; -} - -const REMEMBERED_MANIFESTS = 64; - -/** - * The automerge worker's answer to the service worker's manifest requests. A request names a - * file; its manifest covers the container the file is in, so every file of a package shares one. - * Manifests are remembered by that container's pinned path, which never changes meaning, and - * concurrent requests for one share a build. - */ -export function createManifestHandler( - getRepo: () => Promise, - accounts: () => ReadonlyMap, - options: { - post?: (answer: IndexAnswer) => Promise; - /** The walk's find, per request: one that waits for heads, in the worker. */ - find?: (repo: Repo) => WalkOptions["find"]; - /** The listing's find for file docs, per build, with a budget of its own. */ - findFile?: (repo: Repo) => WalkOptions["find"]; - log?: (...args: unknown[]) => void; - } = {} -): (message: ManifestRequestMessage) => Promise { - const post = options.post ?? ((answer) => postIndexAnswer(answer)); - const remembered = new Map(); - const building = new Map< - string, - Promise<{ manifest: Manifest; missing: string[] } | undefined> - >(); - - const manifestFor = async (path: string): Promise => { - const repo = await getRepo(); - const walk: WalkOptions = { accounts: accounts(), find: options.find?.(repo) }; - const end = await walkPath(repo, path, walk); - const prefix = end?.canonical ? end.container?.pinned : undefined; - if (!prefix) throw new Error(`${path} isn't a pinned path this worker can walk`); - - const known = remembered.get(prefix); - if (known) { - remembered.delete(prefix); - remembered.set(prefix, known); - return known; - } - let build = building.get(prefix); - if (!build) { - build = buildManifest(repo, prefix, { - ...walk, - findFile: options.findFile?.(repo), - }).finally(() => building.delete(prefix)); - building.set(prefix, build); - } - const built = await build; - if (!built) throw new Error(`couldn't list the files under ${prefix}`); - // A partial manifest still checks what it lists, but isn't kept: the next - // request tries the missing files again. - if (built.missing.length) { - options.log?.( - `manifest for ${prefix} is missing ${built.missing.length} file(s); not keeping it` - ); - return built.manifest; - } - remembered.set(prefix, built.manifest); - if (remembered.size > REMEMBERED_MANIFESTS) { - remembered.delete(remembered.keys().next().value!); - } - return built.manifest; - }; - - return async ({ id, path }) => { - let answer: IndexAnswer; - try { - answer = { id, path, manifest: await manifestFor(path) }; - } catch (error) { - answer = { - id, - path, - error: error instanceof Error ? error.message : String(error), - }; - } - options.log?.(`manifest ${id} for ${path}: ${answer.error ?? "built"}`); - if (!(await post(answer))) { - console.warn(`couldn't get manifest ${id} to the service worker`); - } - }; -} diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index f110416da..75852073d 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -1,32 +1,37 @@ -// Serving a pinned `/@/` URL: find what its bytes must hash to, read them from the content -// store or a resolver, check them, and decide whether to keep them. Only the automerge worker's -// manifests say what a URL's bytes must hash to (see index-channel.ts); a store entry or a -// resolver's answer that doesn't match is never served. +// Serving a pinned `/@/` URL. Its last container step names a file list by hash +// (`@~`), so the URL vouches for the list and the list for each file's +// bytes. Lists, store entries and resolvers' answers are all checked against those hashes; none +// is trusted for who made it (spec, section 5). +import { + parseFileList, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; import type { ContentStore } from "./content-store.js"; -import type { - HandoffRequestMessage, - IndexAnswer, - IndexEntry, - Manifest, - ManifestRequestMessage, -} from "./types.js"; +import type { HandoffRequestMessage } from "./types.js"; const BASE58 = "[1-9A-HJ-NP-Za-km-z]+"; -const DOC_STEP = new RegExp(`^${BASE58}@${BASE58}(\\.${BASE58})*$`); -const ACCOUNT_ROOT = new RegExp(`^(${BASE58})(@${BASE58}(\\.${BASE58})*)?$`); +const LIST = "~[A-Za-z0-9_-]+"; +const DOC_STEP = new RegExp(`^${BASE58}@${BASE58}(?:\\.${BASE58})*(?:${LIST})?$`); +const ACCOUNT_ROOT = new RegExp(`^(${BASE58})(@${BASE58}(?:\\.${BASE58})*(?:${LIST})?)?$`); const SHARE_PREFIX = "automerge:"; const reencode = (segment: string) => encodeURIComponent(decodeURIComponent(segment)); +const listHashOf = (step: string) => /~([A-Za-z0-9_-]+)$/.exec(step)?.[1]; +const withoutList = (step: string) => step.replace(/~[A-Za-z0-9_-]+$/, ""); + +/** + * One reading of a pinned path: the container's pinned path (without its own `~listHash`), the + * names after it, and the list hash its step carries, if any. + */ +export type Candidate = { prefix: string; tail: string; listHash?: string }; export type NormalizedPath = { /** - * The keys this path may be indexed under, most likely first. A segment shaped like a doc step - * (`react@18.3.1`) may be a name, and only the worker's manifest can say, so there's a key for - * each place the doc steps could end; every name after that point is re-encoded. + * The readings of this path, most likely first. A segment shaped like a doc step + * (`react@18.3.1`) may be a name, so there's a reading for each place the container could be; + * every segment after it is a name, re-encoded. */ - keys: string[]; - /** The first key up to and including its last doc step: the document its manifest covers. */ - prefix: string; + candidates: Candidate[]; acctHash?: string; /** False for a live path, whose names resolve at current heads. */ pinned: boolean; @@ -35,93 +40,117 @@ export type NormalizedPath = { }; /** - * The path after `/@/`, normalized so every spelling of one file has one key among its - * candidates: segments are percent-decoded and re-encoded, and empty ones dropped. Undefined for - * a malformed escape or a root that isn't an account hash or a share link. + * The path after `/@/`, normalized so every spelling of one file has the same candidates: + * segments are percent-decoded and re-encoded, and empty ones dropped. Undefined for a malformed + * escape or a root that isn't an account hash or a share link. */ export function normalizePinned(path: string): NormalizedPath | undefined { const [root, ...rest] = path.split("/").filter(Boolean); if (!root) return undefined; try { const decodedRoot = decodeURIComponent(root); - let head: string; - let acctHash: string | undefined; - let pinned: boolean; const share = decodedRoot.startsWith(SHARE_PREFIX); - if (share) { - head = decodedRoot; - pinned = true; - } else { + let acctHash: string | undefined; + let pinned = true; + if (!share) { const match = ACCOUNT_ROOT.exec(root); if (!match) return undefined; - head = root; acctHash = match[1]; pinned = !!match[2]; } + const head = share ? decodedRoot : root; + + // The container is the segment at `split`, or the root at -1. + const reading = (split: number): Candidate => { + const container = split < 0 ? head : rest[split]; + const before = + split < 0 + ? [] + : [head, ...rest.slice(0, split).map((s) => (DOC_STEP.test(s) ? s : reencode(s)))]; + const listHash = listHashOf(container); + return { + prefix: [...before, withoutList(container)].join("/"), + tail: rest.slice(split + 1).map(reencode).join("/"), + ...(listHash ? { listHash } : {}), + }; + }; - // Doc-step-shaped segments after the root, latest first. + // Doc-step-shaped segments after the root, latest first; then the root, when it's a + // container step itself (a share link, or a root that names a list), or when nothing else is. const splits = rest .map((segment, i) => (DOC_STEP.test(segment) ? i : -1)) .filter((i) => i >= 0) .reverse(); - const keyAt = (split: number) => - [ - head, - ...rest.map((segment, i) => - i <= split && DOC_STEP.test(segment) ? segment : reencode(segment) - ), - ].join("/"); - const keys = splits.length ? splits.map(keyAt) : [keyAt(-1)]; - // Encoded segments hold no "/", so the first key splits back into its segments. - const prefix = keys[0] - .split("/") - .slice(0, (splits[0] ?? -1) + 2) - .join("/"); - return { keys, prefix, acctHash, pinned, share }; + if (share || listHashOf(head) || !splits.length) splits.push(-1); + return { candidates: splits.map(reading), acctHash, pinned, share }; } catch { return undefined; } } -/** Add a manifest's entries to the index, under `prefix/tail`. */ -export function indexManifest( - index: Map, - manifest: Manifest -): void { - for (const [tail, entry] of Object.entries(manifest.entries)) { - index.set(tail ? `${manifest.prefix}/${tail}` : manifest.prefix, entry); - } -} - /** Thrown to fail the request as a network error: nothing that can be memoized. */ export class PinnedAbort extends Error {} export type PinnedDeps = { - index: Map; + /** File lists held in memory, by list hash: ones that may not be stored, and a cache. */ + lists: Map; /** Undefined: nothing is read or written (a share link). */ store: ContentStore | undefined; hash(bytes: Uint8Array): Promise; - /** Ask the automerge worker for the manifest covering `key`; undefined if it can't answer. */ - manifest(key: string): Promise; + /** Ask a resolver for the full list of the container at `prefix`: its bytes, or undefined. */ + buildList(prefix: string): Promise; /** - * Ask a tab or the automerge worker for `key`'s bytes: undefined if it doesn't resolve, a - * Response for any answer that isn't bytes (a redirect for a path that isn't canonical). + * Ask a tab or the automerge worker for the file at `path`, read at `heads` if given: its + * bytes, a Response for any other answer (a redirect, a 404), or undefined. */ resolve( - key: string + path: string, + heads?: string ): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; warn(message: string): void; /** Keep the service worker alive for work that outlives the response. */ keep(work: Promise): void; }; -const lookup = (index: Map, keys: string[]) => { - for (const key of keys) { - const entry = index.get(key); - if (entry) return entry; +/** + * Keep a list if `bytes` hash to `listHash`: in memory always, and in the store too when every + * file in it may be stored. False if they don't match, or aren't a list. + */ +export async function keepFileList( + bytes: Uint8Array, + listHash: string, + deps: Pick +): Promise { + if ((await deps.hash(bytes)) !== listHash) return false; + const list = parseFileList(bytes); + if (!list) return false; + deps.lists.set(listHash, list); + if (deps.store && Object.values(list.files).every((entry) => entry.store)) { + deps.keep(deps.store.put(listHash, bytes)); + } + return true; +} + +/** The list a URL names: from memory, from the store (checked), or rebuilt (checked). */ +async function loadList( + { prefix, listHash }: { prefix: string; listHash: string }, + deps: PinnedDeps +): Promise { + const known = deps.lists.get(listHash); + if (known) return known; + const stored = await deps.store?.get(listHash); + if (stored) { + // Already in the store: kept in memory only. + if (await keepFileList(stored, listHash, { ...deps, store: undefined })) { + return deps.lists.get(listHash); + } + deps.warn(`content store entry ${listHash} isn't the file list it's named for; deleting it`); + await deps.store!.delete(listHash); } + const built = await deps.buildList(prefix); + if (built && (await keepFileList(built, listHash, deps))) return deps.lists.get(listHash); return undefined; -}; +} const respond = (bytes: Uint8Array, contentType: string, immutable: boolean) => new Response(bytes as BlobPart, { @@ -131,83 +160,84 @@ const respond = (bytes: Uint8Array, contentType: string, immutable: boolean) => }, }); +const join = (prefix: string, tail: string) => (tail ? `${prefix}/${tail}` : prefix); + /** - * Serve a pinned URL, given its candidate keys (see {@link normalizePinned}): + * Serve a pinned URL (see the spec's *Request flow for a pinned URL*): * - * 1. Find its index entry, asking for a manifest on a miss. - * 2. With an entry, read the store and hash what's there: a match is served; a mismatch is - * deleted, warned about, and treated as a miss. - * 3. Ask a resolver and hash its bytes. With an entry, a match is served (and stored, if the - * entry allows) and a mismatch fails as a network error. Without one, because no worker - * answered, the bytes are served as they are and kept nowhere. + * 1. Load the list its container step names; none to be had fails as a network error. + * 2. Look the file up in it; not there, 404. + * 3. Read the store by the file's hash and check it; a mismatch is deleted. + * 4. Ask a resolver for the file at the heads the list recorded and check that; a match is served + * (and stored, if the list allows), anything else fails as a network error. + * + * A path with no list anywhere is served live from a resolver: `no-store`, unchecked. */ export async function servePinned( - keys: string[], + normalized: Pick, deps: PinnedDeps ): Promise { - let entry = lookup(deps.index, keys); + const listed = normalized.candidates.find( + (candidate): candidate is Candidate & { listHash: string } => !!candidate.listHash + ); + if (!listed) { + const [first] = normalized.candidates; + const path = join(first.prefix, first.tail); + const answer = await deps.resolve(path, undefined); + if (answer instanceof Response) return answer; + if (!answer) return new Response(`nothing at /@/${path}`, { status: 404 }); + return respond(answer.body, answer.contentType, false); + } + + const path = join(listed.prefix, listed.tail); + const list = await loadList(listed, deps); + if (!list) { + throw new PinnedAbort(`no file list ${listed.listHash} to check /@/${path} against`); + } + const entry = list.files[listed.tail]; if (!entry) { - const manifest = await deps.manifest(keys[0]); - if (manifest) indexManifest(deps.index, manifest); - entry = lookup(deps.index, keys); + return new Response(`/@/${path} isn't in its file list`, { + status: 404, + headers: { "content-type": "text/plain", "cache-control": "no-store" }, + }); } - if (entry && deps.store) { + if (deps.store) { const stored = await deps.store.get(entry.sha256); if (stored) { if ((await deps.hash(stored)) === entry.sha256) { - return respond(stored, entry.contentType, true); + return respond(stored, entry.type, true); } deps.warn( - `content store entry ${entry.sha256} for ${keys[0]} doesn't hash to its key; deleting it` + `content store entry ${entry.sha256} for /@/${path} doesn't hash to its key; deleting it` ); await deps.store.delete(entry.sha256); } } - const answer = await deps.resolve(keys[0]); - if (!answer) { - return new Response(`nothing at ${keys[0]}`, { - status: 404, - headers: { "content-type": "text/plain", "cache-control": "no-store" }, - }); - } - if (answer instanceof Response) { - if (entry) { - throw new PinnedAbort( - `${keys[0]} is in the automerge worker's manifest, but a resolver answered ${answer.status}` - ); - } - return answer; - } - if (!entry) return respond(answer.body, answer.contentType, false); - - if ((await deps.hash(answer.body)) !== entry.sha256) { + const answer = await deps.resolve(path, entry.heads); + if (!answer || answer instanceof Response) { throw new PinnedAbort( - `${keys[0]} resolved to bytes that don't match the automerge worker's manifest` + `/@/${path} is in its file list, but a resolver answered ${answer?.status ?? "nothing"}` ); } - if (entry.store && deps.store) { - deps.keep(deps.store.put(entry.sha256, answer.body)); + if ((await deps.hash(answer.body)) !== entry.sha256) { + throw new PinnedAbort(`/@/${path} resolved to bytes that don't match its file list`); } - return respond(answer.body, entry.contentType, true); + if (entry.store && deps.store) deps.keep(deps.store.put(entry.sha256, answer.body)); + return respond(answer.body, entry.type, true); } -/** - * Wrap a manifest fetcher so concurrent calls for one prefix share a request. A settled request - * is forgotten: its entries are in the index by then. - */ -export function sharedManifests( - fetch: (key: string) => Promise, - prefixOf: (key: string) => string -): (key: string) => Promise { - const inFlight = new Map>(); +/** Wrap `fn` so concurrent calls with one key share a request. A settled request is forgotten. */ +export function sharedRequests( + fn: (key: string) => Promise +): (key: string) => Promise { + const inFlight = new Map>(); return (key) => { - const prefix = prefixOf(key); - let request = inFlight.get(prefix); + let request = inFlight.get(key); if (!request) { - request = fetch(key).finally(() => inFlight.delete(prefix)); - inFlight.set(prefix, request); + request = fn(key).finally(() => inFlight.delete(key)); + inFlight.set(key, request); } return request; }; @@ -215,22 +245,31 @@ export function sharedManifests( /** * The handoff the service worker sends tabs and the automerge worker: `to` is a legacy special - * URL (`automerge:…`), or a `/@/` path. A path's `handoffURL` is `patchwork-path:` and the path, - * which resolvers from before paths parse and then decline. + * URL (`automerge:…`), or a `/@/` path, with the heads to read its file at, or a request for + * its container's file list. A path's `handoffURL` is `patchwork-path:` and the path, which + * resolvers from before paths parse and then decline. */ export function handoffRequest( request: Request, cachename: string, - to: URL | string + to: URL | { path: string; heads?: string; list?: boolean } ): HandoffRequestMessage { + const target = + to instanceof URL + ? { handoffURL: to.href } + : { + handoffURL: `patchwork-path:${to.path}`, + path: to.path, + ...(to.heads ? { heads: to.heads } : {}), + ...(to.list ? { list: true } : {}), + }; return { id: crypto.randomUUID(), type: "request", cachename, request: { url: request.url, - handoffURL: typeof to === "string" ? `patchwork-path:${to}` : to.href, - ...(typeof to === "string" ? { path: to } : {}), + ...target, headers: Object.fromEntries(request.headers.entries()), method: request.method, destination: request.destination, @@ -238,66 +277,3 @@ export function handoffRequest( }, }; } - -/** - * The service worker's manifest requests to the automerge worker: sent with `send`, settled by - * an answer for the same id *and* path (requests are broadcast, so a page can get the worker to - * answer a copy with another path), or given up on after `timeoutMs`. Once one times out, the - * worker is taken to be away and later requests wait only `quickTimeoutMs`, for `quietMs` or - * until it answers again, so a missing worker costs each request a moment rather than the - * whole timeout. - */ -export function manifestRequests(options: { - send(message: ManifestRequestMessage): void; - timeoutMs: number; - quickTimeoutMs: number; - quietMs: number; - onTimeout(path: string): void; -}) { - const pending = new Map< - string, - { message: ManifestRequestMessage; finish(manifest: Manifest | undefined): void } - >(); - let quietUntil = 0; - - return { - request(path: string): Promise { - const message: ManifestRequestMessage = { - type: "manifest", - id: crypto.randomUUID(), - path, - }; - const quiet = Date.now() < quietUntil; - return new Promise((resolve) => { - const timer = setTimeout( - () => { - if (!quiet) { - quietUntil = Date.now() + options.quietMs; - options.onTimeout(path); - } - finish(undefined); - }, - quiet ? options.quickTimeoutMs : options.timeoutMs - ); - const finish = (manifest: Manifest | undefined) => { - clearTimeout(timer); - pending.delete(message.id); - resolve(manifest); - }; - pending.set(message.id, { message, finish }); - options.send(message); - }); - }, - /** False if `answer` settles nothing: an unknown id, or another path. */ - settle(answer: IndexAnswer): boolean { - const request = pending.get(answer.id); - if (!request || answer.path !== request.message.path) return false; - quietUntil = 0; - request.finish(answer.manifest); - return true; - }, - has: (id: string) => pending.has(id), - /** What's still waiting, to send again to a worker that restarted. */ - pending: () => [...pending.values()].map(({ message }) => message), - }; -} diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 5cd8047c8..529da4db8 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -48,9 +48,14 @@ export interface HandoffRequest { handoffURL: string; /** * For a `/@/` URL, the path after `/@/`: live (answered with a redirect to - * its pinned form) or pinned (answered with its bytes, inline). + * its pinned form, whose body is the file list it names) or pinned (answered + * with its bytes, inline). */ path?: string; + /** Read the file `path` names at these heads, as its file list recorded. */ + heads?: string; + /** Answer the file list of the container `path` names, not a file. */ + list?: boolean; /** * @deprecated A briefly-deployed shape put the special URL in `url` and * the cache key here. Only read, never sent. @@ -204,51 +209,11 @@ export type SetupServiceWorkerResult = { }; /** - * What the service worker knows about one pinned URL: the hash its bytes must have, the type to - * serve them as, and whether they may be kept in the content store. - */ -export type IndexEntry = { sha256: string; contentType: string; store: boolean }; - -/** - * Automerge worker → service worker: every file under the last document a pinned URL enters, at - * its heads. `prefix` is the canonical pinned path of that document; each entry's key is the tail - * below it ("" for the document itself). - */ -export type Manifest = { prefix: string; entries: Record }; - -/** - * Where the automerge worker's script is served. The vite plugin keeps this name from one build - * to the next (it's started by URL), and the service worker trusts index answers only from a - * SharedWorker running it. + * Where the automerge worker's script is served, by default. The vite plugin keeps this name + * from one build to the next, since it's started by URL. */ export const AUTOMERGE_WORKER_PATH = "/automerge-protocol-handler-worker.js"; -/** - * Where the automerge worker POSTs its answers to the service worker. Under the reserved `/@/` - * prefix, and `.` isn't base58, so no account path can collide with it. - */ -export const INDEX_ANSWER_PATH = "/@/.index"; - -/** Service worker → automerge worker, on HANDOFF_CHANNEL: build the manifest covering `path`. */ -export interface ManifestRequestMessage { - type: "manifest"; - id: string; - path: string; -} - -/** Automerge worker → service worker, as the body of a POST to INDEX_ANSWER_PATH. */ -export type IndexAnswer = { - id: string; - /** - * The path the request named. Requests are broadcast, so a page can send - * the worker a copy with the same id and another path; the service worker - * takes an answer only for the path it asked about. - */ - path?: string; - manifest?: Manifest; - error?: string; -}; - /** Tab → automerge worker: resolve paths under this account. */ export interface RegisterAccountMessage { type: "register-account"; diff --git a/core/bootloader/test/index-channel.test.ts b/core/bootloader/test/index-channel.test.ts deleted file mode 100644 index 49ca52c44..000000000 --- a/core/bootloader/test/index-channel.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { - acceptIndexAnswer, - indexChannelProbe, - isAutomergeWorker, - postIndexAnswer, -} from "../src/index-channel.js"; - -const o = "https://p.test"; -const worker = { type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js` } as const; - -describe("isAutomergeWorker", () => { - it.each([ - [worker, true], - [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js?x=1` }, true], - // what any page can start from the same url: a dedicated worker - [{ type: "worker", url: `${o}/automerge-protocol-handler-worker.js` }, false], - [{ type: "window", url: `${o}/` }, false], - [{ type: "sharedworker", url: `${o}/@/acct@h/evil.js` }, false], - [{ type: "sharedworker", url: `https://evil.test/automerge-protocol-handler-worker.js` }, false], - [undefined, false], - ])("%j → %s", (client, ok) => { - expect(isAutomergeWorker(client as any, o)).toBe(ok); - }); -}); - -const answerRequest = (body: unknown) => - new Request(`${o}/@/.index`, { method: "POST", body: JSON.stringify(body) }); - -describe("acceptIndexAnswer", () => { - it("refuses a window with 403 and hands nothing on", async () => { - const onAnswer = vi.fn(); - const response = await acceptIndexAnswer( - answerRequest({ id: "1" }), - { type: "window", url: `${o}/` }, - o, - onAnswer - ); - expect(response.status).toBe(403); - expect(onAnswer).not.toHaveBeenCalled(); - }); - - it("accepts the worker with 204 and the header", async () => { - const onAnswer = vi.fn(); - const answer = { id: "1", manifest: { prefix: "a@h", entries: {} } }; - const response = await acceptIndexAnswer(answerRequest(answer), worker, o, onAnswer); - expect(response.status).toBe(204); - expect(response.headers.get("x-patchwork-index")).toBe("accepted"); - expect(onAnswer).toHaveBeenCalledWith(answer); - }); - - it("refuses a body that isn't an answer", async () => { - const onAnswer = vi.fn(); - const response = await acceptIndexAnswer(answerRequest("nope"), worker, o, onAnswer); - expect(response.status).toBe(400); - expect(onAnswer).not.toHaveBeenCalled(); - }); -}); - -describe("indexChannelProbe", () => { - it("says the service worker is there, and takes nothing", () => { - const response = indexChannelProbe(); - expect(response.status).toBe(405); - expect(response.headers.get("x-patchwork-index")).toBe("present"); - }); -}); - -const present = () => - new Response(null, { status: 405, headers: { "x-patchwork-index": "present" } }); -const accepted = () => - new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } }); - -describe("postIndexAnswer", () => { - it("probes before posting, so an answer never reaches the network", async () => { - const replies = [new Response("", { status: 404 }), present(), accepted()]; - const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => replies.shift()!); - expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(true); - expect(fetcher.mock.calls.map(([, init]) => init?.method ?? "GET")).toEqual([ - "GET", - "GET", - "POST", - ]); - }); - - it("never posts when the probe never reaches the service worker", async () => { - const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => new Response("", { status: 404 })); - expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(false); - expect(fetcher.mock.calls.every(([, init]) => (init?.method ?? "GET") === "GET")).toBe(true); - expect(fetcher).toHaveBeenCalledTimes(5); - }); - - it("probes once per fetcher, then retries a post that didn't get through", async () => { - const replies = [present(), new Response("", { status: 404 }), accepted(), accepted()]; - const fetcher = vi.fn(async (_url: unknown, _init?: RequestInit) => replies.shift()!); - expect(await postIndexAnswer({ id: "1" }, fetcher as typeof fetch)).toBe(true); - expect(await postIndexAnswer({ id: "2" }, fetcher as typeof fetch)).toBe(true); - expect(fetcher).toHaveBeenCalledTimes(4); - }); -}); diff --git a/core/bootloader/test/manifest.test.ts b/core/bootloader/test/manifest.test.ts deleted file mode 100644 index c1b70e66b..000000000 --- a/core/bootloader/test/manifest.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; -import { formatHeads, idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; -import { buildManifest } from "../src/manifest.js"; -import { contentHash } from "@inkandswitch/patchwork-filesystem"; - -const file = (repo: Repo, content: string) => - repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; -const folder = (repo: Repo, entries: Record) => - repo.create({ - title: "folder", - docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), - }); - -async function fixture() { - const repo = new Repo(); - const projects = folder(repo, { note: file(repo, "héllo ✓") }); - const root = folder(repo, { Projects: projects.url }); - const pkg = repo.create({ - "@patchwork": { type: "directory" }, - "index.js": file(repo, "export default 1"), - }); - const packages = folder(repo, { pkg: pkg.url }); - const account = repo.create({ - "@patchwork": { type: "account" }, - namespace: { "/": [{ url: root.url }], "/packages": [{ url: packages.url }] }, - }); - const a = await idhash(account.documentId); - return { repo, a, accounts: new Map([[a, account.url]]), projects }; -} - -describe("buildManifest", () => { - it("hashes each file as the service worker will", async () => { - const { repo, a, accounts } = await fixture(); - const prefix = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); - const manifest = (await buildManifest(repo, prefix!, { accounts }))?.manifest; - expect(manifest?.prefix).toBe(prefix); - expect(manifest?.entries.note).toEqual({ - sha256: await contentHash(new TextEncoder().encode("héllo ✓")), - contentType: "text/plain", - store: true, - }); - }); - - it("stores protected content only through /packages", async () => { - const { repo, a, accounts } = await fixture(); - const isProtected = () => true; - const projects = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); - const pkg = await pinnedPath(repo, livePath(a, ["packages", "pkg"]), { accounts }); - expect((await buildManifest(repo, projects!, { accounts, isProtected }))?.manifest.entries.note.store).toBe(false); - expect((await buildManifest(repo, pkg!, { accounts, isProtected }))?.manifest.entries["index.js"].store).toBe(true); - }); - - it("never stores under a share root", async () => { - const { repo, accounts, projects } = await fixture(); - const share = `automerge:${projects.documentId}@${formatHeads(projects.heads())}`; - const built = await buildManifest(repo, share, { accounts }); - expect(built?.manifest.entries.note.store).toBe(false); - }); - - it("returns nothing for a live path", async () => { - const { repo, a, accounts } = await fixture(); - expect(await buildManifest(repo, livePath(a, ["Projects"]), { accounts })).toBeUndefined(); - }); -}); diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index baaabc794..3f78ff076 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -1,67 +1,54 @@ import { describe, it, expect, vi } from "vitest"; -import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; +import { + canonicalJson, + contentHash, + toBytes, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; import type { ContentStore } from "../src/content-store.js"; import { PinnedAbort, handoffRequest, - manifestRequests, + keepFileList, normalizePinned, servePinned, - sharedManifests, + sharedRequests, type PinnedDeps, } from "../src/pinned.js"; -import type { - IndexEntry, - Manifest, - ManifestRequestMessage, -} from "../src/types.js"; -const keysOf = (path: string) => normalizePinned(path)!.keys; +const candidatesOf = (path: string) => normalizePinned(path)!.candidates; describe("normalizePinned", () => { - it("gives every spelling of a file one key among its candidates", () => { - const canonical = keysOf("acct@h/x@h/a%40b.js")[0]; - expect(canonical).toBe("acct@h/x@h/a%40b.js"); - for (const spelling of [ - "acct@h/x@h/a@b.js", - "/acct@h//x@h/a%40b.js/", - "acct@h/x@h/a%40b.js", - ]) { - expect(keysOf(spelling)).toContain(canonical); + it("gives every spelling of a file one candidate", () => { + const [canonical] = candidatesOf("acct@h/x@h~L/a%40b.js"); + expect(canonical).toEqual({ prefix: "acct@h/x@h", tail: "a%40b.js", listHash: "L" }); + for (const spelling of ["acct@h/x@h~L/a@b.js", "/acct@h//x@h~L/a%40b.js/"]) { + expect(candidatesOf(spelling)).toContainEqual(canonical); } - expect(keysOf("acct@h/x@h/日本")).toEqual( - keysOf("acct@h/x@h/%E6%97%A5%E6%9C%AC") - ); + expect(candidatesOf("acct@h/x@h~L/日本")).toEqual(candidatesOf("acct@h/x@h~L/%E6%97%A5%E6%9C%AC")); }); it("tries the latest doc step first, then each earlier one", () => { - expect(keysOf("acct@h/packages/m@h/f@h/vendor/react@18.3.1/index.js")).toEqual([ - "acct@h/packages/m@h/f@h/vendor/react@18.3.1/index.js", - "acct@h/packages/m@h/f@h/vendor/react%4018.3.1/index.js", - "acct@h/packages/m@h/f%40h/vendor/react%4018.3.1/index.js", + expect(candidatesOf("acct@h/packages/m@h/f@h~L/vendor/react@18.3.1/index.js")).toEqual([ + { prefix: "acct@h/packages/m@h/f@h~L/vendor/react@18.3.1", tail: "index.js" }, + { prefix: "acct@h/packages/m@h/f@h", tail: "vendor/react%4018.3.1/index.js", listHash: "L" }, + { prefix: "acct@h/packages/m@h", tail: "f%40h~L/vendor/react%4018.3.1/index.js" }, ]); }); - it("splits the first key at its last doc step", () => { - expect(normalizePinned("acct@h/packages/m@h/f@h/dist/index.js")).toMatchObject({ - prefix: "acct@h/packages/m@h/f@h", - acctHash: "acct", - pinned: true, - share: false, - }); - }); - it("says a live path is live", () => { - expect(normalizePinned("acct/Projects")).toMatchObject({ - pinned: false, - acctHash: "acct", - }); + expect(normalizePinned("acct/Projects")).toMatchObject({ pinned: false, acctHash: "acct" }); }); - it("reads a share root", () => { - const n = normalizePinned("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h/x"); + it("reads a share root, and a doc-step-shaped name under it", () => { + const id = "2j9knpCseyhnK8izDmLpGP5WMdZQ"; + const n = normalizePinned(`automerge%3A${id}@h~L/react@18.3.1/x.js`)!; expect(n).toMatchObject({ pinned: true, share: true, acctHash: undefined }); - expect(n!.keys[0]).toBe("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h/x"); + expect(n.candidates).toContainEqual({ + prefix: `automerge:${id}@h`, + tail: "react%4018.3.1/x.js", + listHash: "L", + }); }); it("refuses a malformed escape or a bad root", () => { @@ -84,171 +71,187 @@ class MemoryStore implements ContentStore { } } -const KEY = "acct@h/pkg@h/index.js"; const REAL = toBytes("export default 42"); +const FORGED = toBytes("export default 666"); -async function setup(options: { store?: boolean; manifest?: boolean } = {}) { +async function setup(options: { store?: boolean } = {}) { const sha256 = await contentHash(REAL); - const entry: IndexEntry = { - sha256, - contentType: "text/javascript", - store: options.store ?? true, - }; - const manifest: Manifest = { - prefix: "acct@h/pkg@h", - entries: { "index.js": entry }, + const list: FileList = { + v: 1, + files: { + "index.js": { sha256, type: "text/javascript", heads: "fh", store: options.store ?? true }, + }, }; + const listBytes = new TextEncoder().encode(canonicalJson(list)); + const listHash = await contentHash(listBytes); + const path = `acct@h/pkg@h~${listHash}/index.js`; const kept: Promise[] = []; const deps = { - index: new Map(), + lists: new Map(), store: new MemoryStore(), hash: contentHash, - manifest: vi.fn(async () => (options.manifest === false ? undefined : manifest)), - resolve: vi.fn(async () => ({ body: REAL, contentType: "text/javascript" })), + buildList: vi.fn(async (_prefix: string) => listBytes as Uint8Array | undefined), + resolve: vi.fn(async (_path: string, _heads?: string) => ({ + body: REAL, + contentType: "text/javascript", + }) as { body: Uint8Array; contentType: string } | Response | undefined), warn: vi.fn(), keep: (work: Promise) => void kept.push(work), } satisfies PinnedDeps; - const settle = () => Promise.all(kept); - return { deps, sha256, entry, manifest, settle }; + return { + deps, + sha256, + list, + listBytes, + listHash, + normalized: normalizePinned(path)!, + settle: () => Promise.all(kept), + }; } const text = (response: Response) => response.text(); describe("servePinned", () => { - it("serves a store hit whose bytes match, without asking a resolver", async () => { - const { deps, sha256 } = await setup(); + it("serves a file whose list and bytes are in the store, asking no resolver", async () => { + const { deps, sha256, listBytes, listHash, normalized } = await setup(); + await deps.store.put(listHash, listBytes); await deps.store.put(sha256, REAL); - const response = await servePinned([KEY], deps); + const response = await servePinned(normalized, deps); expect(await text(response)).toBe("export default 42"); expect(response.headers.get("content-type")).toBe("text/javascript"); expect(response.headers.get("cache-control")).toContain("immutable"); expect(deps.resolve).not.toHaveBeenCalled(); + expect(deps.buildList).not.toHaveBeenCalled(); }); - it("deletes a forged store entry, warns, and serves the resolver's real bytes", async () => { - const { deps, sha256, settle } = await setup(); - await deps.store.put(sha256, toBytes("export default 666")); - const response = await servePinned([KEY], deps); + it("deletes a forged list, rebuilds it, and serves", async () => { + const { deps, listHash, listBytes, normalized, settle } = await setup(); + await deps.store.put(listHash, toBytes('{"v":1,"files":{}}')); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); await settle(); - expect(await text(response)).toBe("export default 42"); expect(deps.warn).toHaveBeenCalled(); - expect(deps.store.bytes.get(sha256)).toEqual(REAL); + expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h"); + expect(deps.store.bytes.get(listHash)).toEqual(listBytes); }); - it("asks for one manifest on an index miss, then serves", async () => { - const { deps } = await setup(); - await servePinned([KEY], deps); - await servePinned([KEY], deps); - expect(deps.manifest).toHaveBeenCalledTimes(1); + it("fails when the rebuilt list doesn't match the URL", async () => { + const { deps, normalized } = await setup(); + deps.buildList.mockResolvedValue(toBytes('{"files":{},"v":1}')); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); }); - it("finds the entry under a later candidate key", async () => { - const { deps } = await setup(); - const response = await servePinned(["acct@h/pkg@h/index@x.js", KEY], deps); - expect(response.headers.get("cache-control")).toContain("immutable"); - }); - - it("stores resolver bytes that match an entry that allows it", async () => { - const { deps, sha256, settle } = await setup(); - await servePinned([KEY], deps); + it("deletes a forged file, rereads it at the list's heads, and serves", async () => { + const { deps, sha256, normalized, settle } = await setup(); + await deps.store.put(sha256, FORGED); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); await settle(); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh"); + expect(deps.warn).toHaveBeenCalled(); expect(deps.store.bytes.get(sha256)).toEqual(REAL); }); - it("never stores when the entry says not to", async () => { - const { deps, settle } = await setup({ store: false }); - expect(await text(await servePinned([KEY], deps))).toBe("export default 42"); + it("fails on a resolver's bytes that don't match, storing nothing", async () => { + const { deps, sha256, normalized, settle } = await setup(); + deps.resolve.mockResolvedValue({ body: FORGED, contentType: "text/javascript" }); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); await settle(); - expect(deps.store.bytes.size).toBe(0); + expect(deps.store.bytes.has(sha256)).toBe(false); }); - it("fails as a network error when resolver bytes don't match, storing nothing", async () => { - const { deps, settle } = await setup(); - deps.resolve.mockResolvedValue({ - body: toBytes("export default 666"), - contentType: "text/javascript", - }); - await expect(servePinned([KEY], deps)).rejects.toBeInstanceOf(PinnedAbort); - await settle(); - expect(deps.store.bytes.size).toBe(0); + it("fails when a resolver answers anything but the bytes", async () => { + const { deps, normalized } = await setup(); + deps.resolve.mockResolvedValue(new Response("nope", { status: 404 })); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); + }); + + it("answers 404 for a tail the list doesn't cover", async () => { + const { deps, listHash } = await setup(); + const response = await servePinned(normalizePinned(`acct@h/pkg@h~${listHash}/other.js`)!, deps); + expect(response.status).toBe(404); + expect(deps.resolve).not.toHaveBeenCalled(); }); - it("serves inline and stores nothing with no manifest", async () => { - const { deps, settle } = await setup({ manifest: false }); - const response = await servePinned([KEY], deps); + it("never stores bytes the list says not to", async () => { + const { deps, sha256, normalized, settle } = await setup({ store: false }); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); await settle(); + expect(deps.store.bytes.has(sha256)).toBe(false); + }); + + it("reads and writes no store for a share link", async () => { + const { deps, normalized } = await setup(); + const response = await servePinned(normalized, { ...deps, store: undefined }); expect(await text(response)).toBe("export default 42"); - expect(response.headers.get("cache-control")).toBe("no-store"); - expect(deps.store.bytes.size).toBe(0); - expect(deps.index.size).toBe(0); }); - it("answers 404 when nothing resolves the path", async () => { - const { deps } = await setup({ manifest: false }); - deps.resolve.mockResolvedValue(undefined as never); - expect((await servePinned([KEY], deps)).status).toBe(404); + it("serves a path with no list live and unchecked", async () => { + const { deps } = await setup(); + const response = await servePinned(normalizePinned("acct@h/pkg@h/index.js")!, deps); + expect(await text(response)).toBe("export default 42"); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", undefined); }); - it("passes a resolver's other answer through when there's no entry", async () => { - const { deps } = await setup({ manifest: false }); + it("passes a resolver's redirect through for a path with no list", async () => { + const { deps } = await setup(); deps.resolve.mockResolvedValue( - new Response(null, { status: 307, headers: { location: "/@/acct@h/x@h" } }) as never + new Response(null, { status: 307, headers: { location: "/@/acct@h/pkg@h~L" } }) ); - const response = await servePinned([KEY], deps); + const response = await servePinned(normalizePinned("acct@h/pkg@h")!, deps); expect(response.status).toBe(307); - expect(response.headers.get("location")).toBe("/@/acct@h/x@h"); }); +}); - it("fails a resolver's other answer as a network error when there's an entry", async () => { - const { deps } = await setup(); - deps.resolve.mockResolvedValue(new Response("nope", { status: 404 }) as never); - await expect(servePinned([KEY], deps)).rejects.toBeInstanceOf(PinnedAbort); - }); +describe("keepFileList", () => { + it("keeps a list in memory, and stores it only if every entry may be stored", async () => { + const stored = await setup(); + expect(await keepFileList(stored.listBytes, stored.listHash, stored.deps)).toBe(true); + await stored.settle(); + expect(stored.deps.lists.get(stored.listHash)).toEqual(stored.list); + expect(stored.deps.store.bytes.has(stored.listHash)).toBe(true); - it("rebuilds the index after a restart and reuses the store", async () => { - const first = await setup(); - await servePinned([KEY], first.deps); - await first.settle(); - const second = await setup(); - second.deps.store = first.deps.store; - expect(await text(await servePinned([KEY], second.deps))).toBe("export default 42"); - expect(second.deps.manifest).toHaveBeenCalledTimes(1); - expect(second.deps.resolve).not.toHaveBeenCalled(); + const memory = await setup({ store: false }); + expect(await keepFileList(memory.listBytes, memory.listHash, memory.deps)).toBe(true); + await memory.settle(); + expect(memory.deps.lists.has(memory.listHash)).toBe(true); + expect(memory.deps.store.bytes.has(memory.listHash)).toBe(false); }); - it("reads and writes nothing without a store", async () => { - const { deps } = await setup(); - const response = await servePinned([KEY], { ...deps, store: undefined }); - expect(await text(response)).toBe("export default 42"); + it("refuses a list that doesn't hash to its name", async () => { + const { deps, listBytes } = await setup(); + expect(await keepFileList(listBytes, "not-its-hash", deps)).toBe(false); + expect(deps.lists.size).toBe(0); }); }); -describe("sharedManifests", () => { - it("shares one manifest request among concurrent requests for a package", async () => { - const { deps, manifest } = await setup(); +describe("sharedRequests", () => { + it("shares one request among concurrent calls for a key, and forgets it once settled", async () => { let release!: () => void; const gate = new Promise((resolve) => (release = resolve)); - const fetcher = vi.fn(async () => (await gate, manifest)); - const shared = sharedManifests(fetcher, (key) => normalizePinned(key)!.prefix); - const all = ["a.js", "b.js", "c.js", "d.js", "index.js"].map((file) => - shared(`acct@h/pkg@h/${file}`) - ); + const fn = vi.fn(async (key: string) => (await gate, key)); + const shared = sharedRequests(fn); + const all = [shared("a"), shared("a"), shared("a"), shared("b")]; release(); - await Promise.all(all); - expect(fetcher).toHaveBeenCalledTimes(1); - // settled requests aren't remembered: the next asks again - await shared(KEY); - expect(fetcher).toHaveBeenCalledTimes(2); - void deps; + expect(await Promise.all(all)).toEqual(["a", "a", "a", "b"]); + expect(fn).toHaveBeenCalledTimes(2); + await shared("a"); + expect(fn).toHaveBeenCalledTimes(3); }); }); describe("handoffRequest", () => { it("builds a path handoff that old resolvers decline", () => { - const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "patchwork", "acct/Projects"); + const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "patchwork", { + path: "acct/Projects", + }); expect(m.request.path).toBe("acct/Projects"); expect(new URL(m.request.handoffURL).protocol).not.toBe("automerge:"); - expect(m.request.url).toBe("https://p.test/@/acct/Projects"); + }); + + it("carries the heads to read at, or a request for the list", () => { + const req = new Request("https://p.test/@/x"); + expect(handoffRequest(req, "patchwork", { path: "a@h/x", heads: "fh" }).request.heads).toBe("fh"); + expect(handoffRequest(req, "patchwork", { path: "a@h", list: true }).request.list).toBe(true); }); it("builds a legacy handoff with no path", () => { @@ -258,48 +261,3 @@ describe("handoffRequest", () => { expect(m.request.path).toBeUndefined(); }); }); - -describe("manifestRequests", () => { - const manifest: Manifest = { prefix: "acct@h/p@h", entries: {} }; - const make = (timeouts = { timeoutMs: 1000, quickTimeoutMs: 20 }) => { - const sent: ManifestRequestMessage[] = []; - const requests = manifestRequests({ - send: (message) => sent.push(message), - ...timeouts, - quietMs: 60_000, - onTimeout: () => {}, - }); - return { sent, requests }; - }; - - it("ignores an answer for a different path and keeps waiting", async () => { - const { sent, requests } = make(); - const pending = requests.request("acct@h/p@h/a.js"); - requests.settle({ id: sent[0].id, path: "garbage", error: "unparseable" }); - expect(requests.has(sent[0].id)).toBe(true); - requests.settle({ id: sent[0].id, path: sent[0].path, manifest }); - expect(await pending).toEqual(manifest); - }); - - it("gives up quickly after a timeout, until the worker answers again", async () => { - const { sent, requests } = make({ timeoutMs: 150, quickTimeoutMs: 10 }); - const elapsed = async (path: string) => { - const start = performance.now(); - await requests.request(path); - return performance.now() - start; - }; - expect(await elapsed("acct@h/p@h/a.js")).toBeGreaterThanOrEqual(140); - expect(await elapsed("acct@h/p@h/b.js")).toBeLessThan(100); - const answered = requests.request("acct@h/p@h/c.js"); - const last = sent.at(-1)!; - requests.settle({ id: last.id, path: last.path, manifest }); - await answered; - expect(await elapsed("acct@h/p@h/d.js")).toBeGreaterThanOrEqual(140); - }); - - it("lists what's pending, for re-sending to a restarted worker", () => { - const { sent, requests } = make(); - void requests.request("acct@h/p@h/a.js"); - expect(requests.pending()).toEqual(sent); - }); -}); diff --git a/core/bootloader/test/worker-manifests.test.ts b/core/bootloader/test/worker-manifests.test.ts deleted file mode 100644 index 6dc6101b3..000000000 --- a/core/bootloader/test/worker-manifests.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; -import { idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; -import { createManifestHandler } from "../src/manifest.js"; -import type { IndexAnswer } from "../src/types.js"; - -const file = (repo: Repo, content: string) => - repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; - -async function fixture(packages = 1) { - const repo = new Repo(); - const pkgs = Array.from({ length: packages }, (_, i) => - repo.create({ - "@patchwork": { type: "directory" }, - "a.js": file(repo, `a${i}`), - "b.js": file(repo, `b${i}`), - }) - ); - const folder = repo.create({ - title: "packages", - docs: pkgs.map((pkg, i) => ({ name: `p${i}`, type: "x", url: pkg.url })), - }); - const account = repo.create({ - "@patchwork": { type: "account" }, - namespace: { "/packages": [{ url: folder.url as AutomergeUrl }] }, - }); - const a = await idhash(account.documentId); - const accounts = new Map([[a, account.url]]); - const prefixes = await Promise.all( - pkgs.map((_, i) => pinnedPath(repo, livePath(a, ["packages", `p${i}`]), { accounts })) - ); - return { repo, accounts, pkgs, prefixes: prefixes as string[] }; -} - -const setup = async (packages?: number, unreadable?: (url: string) => boolean) => { - const f = await fixture(packages); - const posted: IndexAnswer[] = []; - const post = vi.fn(async (answer: IndexAnswer) => (posted.push(answer), true)); - const finds: string[] = []; - const find = (repo: Repo) => (url: AutomergeUrl) => { - finds.push(url); - return unreadable?.(url) ? Promise.reject(new Error("unavailable")) : repo.find(url); - }; - const handle = createManifestHandler(() => Promise.resolve(f.repo), () => f.accounts, { - post, - find, - }); - return { ...f, handle, posted, finds }; -}; - -describe("createManifestHandler", () => { - it("builds a package's manifest once and answers every file's request with it", async () => { - const { handle, prefixes, posted, finds } = await setup(); - await Promise.all([ - handle({ type: "manifest", id: "1", path: `${prefixes[0]}/a.js` }), - handle({ type: "manifest", id: "2", path: `${prefixes[0]}/b.js` }), - ]); - await handle({ type: "manifest", id: "3", path: `${prefixes[0]}/a.js` }); - expect(posted.map((p) => p.id).sort()).toEqual(["1", "2", "3"]); - expect(posted[0].manifest).toEqual(posted[1].manifest); - expect(Object.keys(posted[0].manifest!.entries).sort()).toEqual(["", "a.js", "b.js"]); - // a request for a package already built walks to it and lists nothing again - const fileFinds = finds.length; - await handle({ type: "manifest", id: "4", path: `${prefixes[0]}/b.js` }); - expect(finds.length - fileFinds).toBeLessThanOrEqual(4); // a walk, no relisting - }); - - it("echoes the path it answers for", async () => { - const { handle, prefixes, posted } = await setup(); - await handle({ type: "manifest", id: "1", path: `${prefixes[0]}/a.js` }); - expect(posted[0].path).toBe(`${prefixes[0]}/a.js`); - }); - - it("doesn't remember a manifest that's missing files", async () => { - let bUrl = ""; - const { handle, prefixes, posted, finds, pkgs } = await setup(1, (url) => url === bUrl); - bUrl = String((pkgs[0].doc() as Record)["b.js"]); - await handle({ type: "manifest", id: "1", path: prefixes[0] }); - expect(Object.keys(posted[0].manifest!.entries)).not.toContain("b.js"); - await handle({ type: "manifest", id: "2", path: prefixes[0] }); - expect(finds.filter((url) => url === bUrl)).toHaveLength(2); - }); - - it("posts an error for a path it can't list", async () => { - const { handle, posted } = await setup(); - await handle({ type: "manifest", id: "x", path: "unknown@h/a.js" }); - expect(posted[0]).toMatchObject({ id: "x", error: expect.any(String) }); - expect(posted[0].manifest).toBeUndefined(); - }); - - it("evicts the oldest manifest after 64", async () => { - const { handle, prefixes, posted, finds } = await setup(65); - for (const [i, prefix] of prefixes.entries()) { - await handle({ type: "manifest", id: `${i}`, path: prefix }); - } - const before = finds.length; - await handle({ type: "manifest", id: "again", path: prefixes[64] }); - const cached = finds.length - before; - await handle({ type: "manifest", id: "evicted", path: prefixes[0] }); - const rebuilt = finds.length - before - cached; - expect(rebuilt).toBeGreaterThan(cached); - expect(posted.at(-1)?.manifest?.prefix).toBe(prefixes[0]); - }); -}); From 9cc8ca0f1530b143b3fff227dded0952c3fff54a Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:56:14 +0000 Subject: [PATCH 29/81] bootloader: resolvers answer file lists and reads at heads A live path's 307 carries the one-file list it names as its body. The automerge worker no longer builds manifests. Co-Authored-By: Claude Opus 5.5 --- .../src/automerge-protocol-handler.ts | 37 +---------- core/bootloader/src/resolver.ts | 64 ++++++++++++++----- core/bootloader/test/resolver.test.ts | 60 ++++++++++++++--- 3 files changed, 103 insertions(+), 58 deletions(-) diff --git a/core/bootloader/src/automerge-protocol-handler.ts b/core/bootloader/src/automerge-protocol-handler.ts index 39169679b..f11c25df2 100644 --- a/core/bootloader/src/automerge-protocol-handler.ts +++ b/core/bootloader/src/automerge-protocol-handler.ts @@ -34,14 +34,12 @@ import { loadOrCreateSigner } from "./signer.js"; import { keyhiveStorageName, storagePrefix } from "./storage.js"; import { startWorkerControl } from "./worker-control.js"; import { AUTOMERGE_WASM, SUBDUCTION_WASM } from "./emitted.js"; -import { createManifestHandler } from "./manifest.js"; -import { createResolver, waitingFind } from "./resolver.js"; +import { createResolver } from "./resolver.js"; import { AUTOMERGE_WORKER_LOCK, HANDOFF_CHANNEL, type HandoffOnlineMessage, type HandoffRequestMessage, - type ManifestRequestMessage, type RegisterAccountMessage, } from "./types.js"; @@ -195,27 +193,6 @@ async function buildKeyhiveRepo( // its own Repo). const resolver = createResolver(getRepo, log, () => accounts); - -// The service worker stops waiting for a manifest after 10s; walking to it -// gets no longer than that. -const MANIFEST_WALK_TIMEOUT_MS = 10_000; -const MANIFEST_LIST_TIMEOUT_MS = 30_000; - -// Manifests are what the service worker checks every pinned `/@/` URL -// against. Only this worker answers them: tabs run package code. -const answerManifest = createManifestHandler(getRepo, () => accounts, { - find: (repo) => - waitingFind(repo, AbortSignal.timeout(MANIFEST_WALK_TIMEOUT_MS), new Set()), - // Listing a package reads every file doc in it. Each is read as this node - // has it, without waiting for peers, and the listing gets its own budget: a - // manifest finished after the service worker stopped waiting is still kept - // for the next request. - findFile: (repo) => - waitingFind(repo, AbortSignal.timeout(MANIFEST_LIST_TIMEOUT_MS), new Set(), { - catchUp: false, - }), - log, -}); const handoffChannel = new BroadcastChannel(HANDOFF_CHANNEL); const EVICT_IDLE_MS = 5_000; @@ -358,17 +335,9 @@ async function drainForRestart(): Promise { } handoffChannel.addEventListener("message", (event) => { - const type = event.data?.type; - if (type !== "request" && type !== "manifest") return; + if (event.data?.type !== "request") return; if (draining) { - log(`ignoring ${type} ${event.data.id} while restarting`); - return; - } - if (type === "manifest") { - handoffStarted(); - void answerManifest(event.data as ManifestRequestMessage).finally( - handoffFinished - ); + log(`ignoring handoff ${event.data.id} while restarting`); return; } void handleHandoffRequest(event.data as HandoffRequestMessage); diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 2a358a09a..7d8e2cf8a 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -17,14 +17,18 @@ import { type UrlHeads, } from "@automerge/automerge-repo/slim"; import { + buildFileList, getDefaultBranch, + parseHeads, parsePath, + pinnedPath, readNode, + readPath, resolvePath, + toBytes, walkPath, type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; -import { toBytes } from "@inkandswitch/patchwork-filesystem"; import type { HandoffReplyMessage, @@ -211,15 +215,23 @@ export function waitingFind( } /** - * A `/@/` path, answered inline: a 307 to the pinned form for a live path (or a pinned-looking - * one that isn't canonical), the bytes for a canonical pinned one, 404 for one that doesn't - * resolve. Pinned heads that never arrive throw {@link AbortHandoff}. + * A `/@/` path, answered inline: + * + * - `list`: the file list of the container the path names (canonical), as its bytes. + * - A live path, or a pinned-looking one that isn't canonical: a 307 to its pinned form, whose + * list covers just the file asked for; the body is that list, for the service worker to check + * and keep. + * - A canonical pinned path: the bytes, read at `heads` if given (what its file list recorded). + * - One that doesn't resolve: 404. + * + * Pinned heads that never arrive throw {@link AbortHandoff}. */ async function resolveWalkPath( repo: Repo, path: string, accounts: ReadonlyMap, - signal: AbortSignal + signal: AbortSignal, + { heads, list }: { heads?: string; list?: boolean } = {} ): Promise { try { parsePath(path); @@ -227,11 +239,8 @@ async function resolveWalkPath( return new Response(String(error), { status: 400 }); } const missing = new Set(); - const end = await walkPath(repo, path, { - accounts, - find: waitingFind(repo, signal, missing), - }); - if (!end) { + const options = { accounts, find: waitingFind(repo, signal, missing) }; + const notFound = () => { if (missing.size) { throw new AbortHandoff(`heads not found for ${[...missing].join(", ")}`); } @@ -239,15 +248,40 @@ async function resolveWalkPath( status: 404, headers: { "cache-control": "no-store" }, }); + }; + + if (list) { + const built = await buildFileList(repo, path, { + ...options, + // Each file as this node has it, without waiting on peers for every one. + findFile: waitingFind(repo, signal, missing, { catchUp: false }), + }); + if (!built) return notFound(); + return new Response(built.bytes as BlobPart, { + status: 200, + headers: { "content-type": "application/json", "cache-control": "no-store" }, + }); } + + const end = await walkPath(repo, path, options); + if (!end) return notFound(); if (!end.canonical) { - return new Response(null, { + const pinned = await pinnedPath(repo, path, { ...options, only: "end" }); + if (!pinned) return notFound(); + return new Response((pinned.list ?? null) as BlobPart | null, { status: 307, - headers: { location: `/@/${end.pinned}`, "cache-control": "no-store" }, + headers: { + location: `/@/${pinned.path}`, + "content-type": "application/json", + "cache-control": "no-store", + }, }); } - const read = await readNode(repo, end.node); - if (!read) return new Response(`can't read ${path}`, { status: 404 }); + + const read = heads + ? await readPath(repo, path, { ...options, heads: parseHeads(heads) }) + : await readNode(repo, end.node); + if (!read) return notFound(); return new Response(toBytes(read.content) as BlobPart, { status: 200, headers: { "content-type": read.type }, @@ -361,7 +395,7 @@ export function createResolver( const limit = within(signal); log(`resolving handoff ${id} for /@/${request.path}`); response = await Promise.race([ - resolveWalkPath(repo, request.path, accounts(), limit), + resolveWalkPath(repo, request.path, accounts(), limit, request), new Promise((_, reject) => limit.addEventListener("abort", () => reject(new AbortHandoff(`resolving /@/${request.path} timed out`)) diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index 520fe736f..bbc35f350 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -5,7 +5,15 @@ import { stringifyAutomergeUrl, type AutomergeUrl, } from "@automerge/automerge-repo"; -import { idhash, livePath, pinnedPath } from "@inkandswitch/patchwork-filesystem"; +import { + contentHash, + formatHeads, + idhash, + livePath, + parseFileList, + pinnedPath, + walkPath, +} from "@inkandswitch/patchwork-filesystem"; import { createResolver } from "../src/resolver.js"; import { registerAccount, resolveHandoffsWith, handleResolveRequest } from "../src/tab-resolver.js"; import type { HandoffRequestMessage, HandoffResponseMessage } from "../src/types.js"; @@ -22,7 +30,8 @@ const folder = (repo: Repo, entries: Record) => async function fixture() { const repo = new Repo(); - const foo = folder(repo, { notes: file(repo, "hi") }); + const notes = file(repo, "hi"); + const foo = folder(repo, { notes }); const root = folder(repo, { Projects: folder(repo, { foo: foo.url }).url }); const account = repo.create({ "@patchwork": { type: "account" }, @@ -31,9 +40,21 @@ async function fixture() { const a = await idhash(account.documentId); const accounts = new Map([[a, account.url]]); const resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts); - return { repo, a, accounts, account, resolver }; + return { repo, a, accounts, account, resolver, notes }; } +/** A path's chain of doc steps, without file lists: what a resolver is asked to read. */ +const chain = async (repo: Repo, path: string, accounts: ReadonlyMap) => + (await walkPath(repo, path, { accounts }))!.pinned; + +const withRequest = ( + path: string, + extra: { heads?: string; list?: boolean } +): HandoffRequestMessage => { + const message = pathRequest(path); + return { ...message, request: { ...message.request, ...extra } }; +}; + const pathRequest = (path: string): HandoffRequestMessage => ({ id: "1", type: "request", @@ -56,22 +77,43 @@ const body = (reply: unknown) => { }; describe("resolver: /@/ paths", () => { - it("redirects a live path to its pinned form", async () => { + it("answers a live path with a 307 whose body is the list it names", async () => { const { repo, a, accounts, resolver } = await fixture(); const live = livePath(a, ["Projects", "foo", "notes"]); const reply = await resolver.answer(pathRequest(live)); expect(reply?.type).toBe("response"); expect(inline(reply).status).toBe(307); - expect(inline(reply).headers?.location).toBe( - `/@/${await pinnedPath(repo, live, { accounts })}` - ); + const pinned = await pinnedPath(repo, live, { accounts, only: "end" }); + expect(inline(reply).headers?.location).toBe(`/@/${pinned!.path}`); expect(inline(reply).headers?.["cache-control"]).toBe("no-store"); + const list = inline(reply).body as Uint8Array; + expect(pinned!.path).toContain(`~${await contentHash(list)}/notes`); + expect(Object.keys(parseFileList(list)!.files)).toEqual(["notes"]); + }); + + it("answers a container's full list", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const foo = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const reply = await resolver.answer(withRequest(foo, { list: true })); + expect(inline(reply).status).toBe(200); + const pinned = await pinnedPath(repo, foo, { accounts }); + expect(inline(reply).body).toEqual(pinned!.list); + }); + + it("reads a file at given heads, after it changed", async () => { + const { repo, a, accounts, resolver, notes } = await fixture(); + const path = await chain(repo, livePath(a, ["Projects", "foo", "notes"]), accounts); + const handle = await repo.find<{ content: string }>(notes); + const heads = formatHeads(handle.heads()); + handle.change((d) => (d.content = "changed")); + expect(body(await resolver.answer(pathRequest(path)))).toBe("changed"); + expect(body(await resolver.answer(withRequest(path, { heads })))).toBe("hi"); }); it("answers a pinned path with its bytes inline", async () => { const { repo, a, accounts, resolver } = await fixture(); - const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo", "notes"]), { accounts }); - const reply = await resolver.answer(pathRequest(pinned!)); + const pinned = await chain(repo, livePath(a, ["Projects", "foo", "notes"]), accounts); + const reply = await resolver.answer(pathRequest(pinned)); expect(inline(reply).status).toBe(200); expect(inline(reply).headers?.["content-type"]).toBe("text/plain"); expect(body(reply)).toBe("hi"); From abea6f6126e767500108ed2ae9f0ef6a759c3f12 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:58:34 +0000 Subject: [PATCH 30/81] service worker: serve pinned paths checked against their file list The service worker keeps the one-file list a live redirect carries, and otherwise loads a pinned URL's list from memory, the store, or a resolver's rebuild, checking it against the URL each time. The index, the manifest requests and the /@/.index route are gone. Tested end to end through the harness: a pinned URL keeps serving its old bytes after the file changes, and neither a forged store entry nor a tab's forged answer is ever served. Co-Authored-By: Claude Opus 5.5 --- core/bootloader/src/pinned.ts | 15 +- core/bootloader/src/service-worker.ts | 202 +++++++++----------- core/bootloader/test/service-worker.test.ts | 124 ++++++++++-- 3 files changed, 206 insertions(+), 135 deletions(-) diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index 75852073d..1c2ca165e 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -228,15 +228,18 @@ export async function servePinned( return respond(answer.body, entry.type, true); } -/** Wrap `fn` so concurrent calls with one key share a request. A settled request is forgotten. */ -export function sharedRequests( - fn: (key: string) => Promise -): (key: string) => Promise { +/** + * Wrap `fn` so concurrent calls with one key share a request, made with the first call's other + * arguments. A settled request is forgotten. + */ +export function sharedRequests( + fn: (key: string, ...args: A) => Promise +): (key: string, ...args: A) => Promise { const inFlight = new Map>(); - return (key) => { + return (key, ...args) => { let request = inFlight.get(key); if (!request) { - request = fn(key).finally(() => inFlight.delete(key)); + request = fn(key, ...args).finally(() => inFlight.delete(key)); inFlight.set(key, request); } return request; diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 88e6e123e..3b6322665 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -3,32 +3,31 @@ import { AUTOMERGE_WORKER_LOCK, HANDOFF_CHANNEL, - INDEX_ANSWER_PATH, type AutomergeWorkerMissingMessage, type HandoffReplyMessage, type HandoffRequestMessage, type HandoffResponse, - type IndexAnswer, - type IndexEntry, - type Manifest, - type ManifestRequestMessage, type ResolveAcceptedMessage, type ResolveRequestMessage, } from "./types.js"; -import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; +import { + contentHash, + toBytes, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; import { CONTENT_CACHE_PREFIX, cacheStore, contentCacheName, } from "./content-store.js"; -import { acceptIndexAnswer, indexChannelProbe } from "./index-channel.js"; import { PinnedAbort, handoffRequest, - manifestRequests, + keepFileList, normalizePinned, servePinned, - sharedManifests, + sharedRequests, + type PinnedDeps, } from "./pinned.js"; // Referenced so this script's bytes change with every build. The browser only // installs a replacement service worker when they do, and the controllerchange @@ -200,71 +199,19 @@ handoffChannel.addEventListener("message", (event) => { if (data?.type !== "online") return; // The automerge worker (re)started — re-broadcast anything still in flight so // requests that raced its boot aren't stranded. - const stranded = [ - ...[...pendingHandoffs.values()].map(({ message }) => message), - ...manifestRequestsInFlight.pending(), - ]; + const stranded = [...pendingHandoffs.values()]; if (stranded.length === 0) return; lifecycle( "info", - `automerge worker (re)started; re-broadcasting ${stranded.length} in-flight request(s)` + `automerge worker (re)started; re-broadcasting ${stranded.length} in-flight asset handoff(s)` ); - for (const message of stranded) handoffChannel.postMessage(message); + for (const { message } of stranded) handoffChannel.postMessage(message); }); -// ── The content index ────────────────────────────────────────────────── -// Pinned `/@/` URL → what its bytes must hash to. Held only here, in memory, -// and filled only from the automerge worker's manifests, whose answers arrive -// as fetches this worker checks the sender of (see index-channel.ts). Lost -// whenever this worker stops, and rebuilt from the automerge worker on demand. - -const contentIndex = new Map(); - -// The automerge worker answers within its own 10s; past that, serve unchecked. -const MANIFEST_TIMEOUT_MS = 10_000; -// After one goes unanswered, how long later requests wait, and for how long. -const MANIFEST_QUICK_TIMEOUT_MS = 250; -const MANIFEST_QUIET_MS = 60_000; - -const manifestRequestsInFlight = manifestRequests({ - send(message) { - handoffChannel.postMessage(message); - void startWorkerIfMissing(message.id); - }, - timeoutMs: MANIFEST_TIMEOUT_MS, - quickTimeoutMs: MANIFEST_QUICK_TIMEOUT_MS, - quietMs: MANIFEST_QUIET_MS, - onTimeout(path) { - lifecycle( - "warn", - `no manifest for /@/${path} after ${MANIFEST_TIMEOUT_MS}ms; serving /@/ unchecked, and waiting only ${MANIFEST_QUICK_TIMEOUT_MS}ms for manifests, for ${MANIFEST_QUIET_MS / 1000}s or until the automerge worker answers` - ); - }, -}); - -const manifests = sharedManifests( - (path) => manifestRequestsInFlight.request(path), - (key) => normalizePinned(key)?.prefix ?? key -); - -function settleIndexAnswer(answer: IndexAnswer) { - if (answer.error) log(`no manifest for /@/${answer.path}: ${answer.error}`); - if (!manifestRequestsInFlight.settle(answer)) { - log(`no pending manifest ${answer.id} for /@/${answer.path}`); - } -} - -async function answerIndex(fetchEvent: FetchEvent): Promise { - const client = fetchEvent.clientId - ? await self.clients.get(fetchEvent.clientId) - : undefined; - return acceptIndexAnswer( - fetchEvent.request, - client, - self.location.origin, - settleIndexAnswer - ); -} +// File lists this worker holds, by list hash: ones that may not be stored, and +// ones read from the store. Lost when it stops; the store, or a resolver +// rebuilding a list from its pinned heads, brings them back. +const fileLists = new Map(); const MISSING_WORKER_NUDGE_MS = 5_000; // A worker that's been spawned only asks for the lock once its imports have @@ -290,7 +237,7 @@ async function startWorkerIfMissing(id: string) { if (!navigator.locks) return; if (await automergeWorkerExists()) return; await new Promise((resolve) => setTimeout(resolve, MISSING_WORKER_GRACE_MS)); - if (!pendingHandoffs.has(id) && !manifestRequestsInFlight.has(id)) return; + if (!pendingHandoffs.has(id)) return; if (await automergeWorkerExists()) return; // Check and claim the throttle with no await in between: every handoff that // raced the same cold start comes through here at once, and would all pass a @@ -406,7 +353,7 @@ function askTab( */ async function handoff( fetchEvent: FetchEvent, - to: URL | string + to: URL | { path: string; heads?: string; list?: boolean } ): Promise { const message = handoffRequest(fetchEvent.request, cachename, to); const deadline = Date.now() + HANDOFF_TIMEOUT_MS; @@ -591,8 +538,10 @@ function isImmutable(response: Response): boolean { } /** - * A `/@/` path: a live one is handed off and its redirect served as is; a - * pinned one is served from the content store, checked against the index. + * A `/@/` path. A live one is handed off, and the redirect it answers with is + * served without its body, the file list the pinned URL names, which is kept + * once it checks out. A pinned one is served checked against its own list + * (pinned.ts). */ async function servePath(fetchEvent: FetchEvent, path: string): Promise { const normalized = normalizePinned(path); @@ -603,11 +552,11 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise { - const reply = await handoff(fetchEvent, key); + const ask: Ask = async (to) => { + const reply = await handoff(fetchEvent, to); if (reply.type === "abort") throw new HandoffAborted(reply.reason); if (reply.type !== "response") { - return new Response(`unexpected ${reply.type} reply for /@/${key}`, { + return new Response(`unexpected ${reply.type} reply for /@/${to.path}`, { status: 502, }); } @@ -619,38 +568,34 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise buildList(prefix, ask), + resolve: (target, heads) => ask({ path: target, heads }), + warn: (message) => lifecycle("warn", message), + keep: (work) => + fetchEvent.waitUntil( + work.catch((error) => { + // Always loud, as for the page cache: a QuotaExceededError here + // is the first sign the origin is under storage pressure. + console.warn(`error keeping /@/${path}`, error); + }) + ), + }; + try { - if (!normalized.pinned) { - const answer = await resolve(normalized.keys[0]); - return answer instanceof Response - ? answer - : new Response(answer.body as BlobPart, { - headers: { "content-type": answer.contentType, "cache-control": "no-store" }, - }); - } - const store = - normalized.acctHash && !normalized.share - ? cacheStore( - await caches.open(contentCacheName(normalized.acctHash)), - self.location.origin - ) - : undefined; - return await servePinned(normalized.keys, { - index: contentIndex, - store, - hash: contentHash, - manifest: manifests, - resolve, - warn: (message) => lifecycle("warn", message), - keep: (work) => - fetchEvent.waitUntil( - work.catch((error) => { - // Always loud, as for the page cache: a QuotaExceededError here - // is the first sign the origin is under storage pressure. - console.warn(`error keeping /@/${path}`, error); - }) - ), - }); + const response = await servePinned(normalized, deps); + if (response.status === 307) return keepRedirectedList(response, deps); + return response; } catch (error) { // Deliberate: fail the request as a network error, with no response. if (error instanceof PinnedAbort || error instanceof HandoffAborted) { @@ -665,6 +610,44 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise Promise<{ body: Uint8Array; contentType: string } | Response>; + +// Concurrent requests for files of one package share a rebuild of its list. +const buildList = sharedRequests( + async (prefix: string, ask: Ask): Promise => { + const answer = await ask({ path: prefix, list: true }); + return answer instanceof Response ? undefined : answer.body; + } +); + +/** + * A resolver's redirect from a live path carries, as its body, the file list + * the pinned URL names. Keep it if it hashes to the `~listHash` in `location`, + * and serve the redirect without it. + */ +async function keepRedirectedList( + response: Response, + deps: PinnedDeps +): Promise { + const location = response.headers.get("location") ?? ""; + const listHash = normalizePinned(location.replace(/^\/@\//, ""))?.candidates.find( + (candidate) => candidate.listHash + )?.listHash; + const body = new Uint8Array(await response.arrayBuffer()); + if (listHash && body.length && !(await keepFileList(body, listHash, deps))) { + lifecycle("warn", `the file list with the redirect to ${location} isn't the one it names`); + } + return new Response(null, { + status: response.status, + headers: { location, "cache-control": "no-store" }, + }); +} + async function servePassthrough( fetchEvent: FetchEvent, cache: Cache, @@ -742,13 +725,6 @@ self.addEventListener("fetch", (fetchEvent: FetchEvent) => { const url = new URL(request.url); const ours = url.origin === self.location.origin; - if (ours && url.pathname === INDEX_ANSWER_PATH) { - fetchEvent.respondWith( - request.method === "POST" ? answerIndex(fetchEvent) : indexChannelProbe() - ); - return; - } - if (request.method !== "GET") return; if (ours && url.pathname.startsWith("/@/")) { diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts index 0ca80d8a5..ac8da4fea 100644 --- a/core/bootloader/test/service-worker.test.ts +++ b/core/bootloader/test/service-worker.test.ts @@ -1,6 +1,11 @@ import { afterEach, describe, it, expect } from "vitest"; -import { Repo, stringifyAutomergeUrl } from "@automerge/automerge-repo"; -import { handleResolveRequest, resolveHandoffsWith } from "../src/tab-resolver.js"; +import { Repo, stringifyAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; +import { contentHash, livePath, toBytes } from "@inkandswitch/patchwork-filesystem"; +import { + handleResolveRequest, + registerAccount, + resolveHandoffsWith, +} from "../src/tab-resolver.js"; import { ORIGIN, navigation, @@ -56,24 +61,111 @@ describe("service worker: legacy automerge URLs", () => { }); }); -describe("service worker: the index channel", () => { - it("refuses an answer posted by a page", async () => { +/** An account whose / is a folder Projects/foo holding one file, `notes`. */ +async function account(repo: Repo) { + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + const folder = (entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + const notes = file("hi"); + const foo = folder({ notes: notes.url }); + const root = folder({ Projects: folder({ foo: foo.url }).url }); + const acct = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await registerAccount(acct.url); + return { a, notes }; +} + +/** Follow a live /@/ URL's redirect, as fetch() would: the pinned URL it names. */ +async function pin(harness: ServiceWorkerHarness, live: string) { + const response = await harness.fetch(new Request(`${ORIGIN}/@/${live}`), "tab").response()!; + expect(response.status).toBe(307); + expect(response.body).toBeNull(); + return `${ORIGIN}${response.headers.get("location")}`; +} + +describe("service worker: /@/ paths", () => { + it("redirects a live path, then serves the pinned file from its list", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + const event = sw.fetch(new Request(pinned), "tab"); + const response = await event.response()!; + expect(await response.text()).toBe("hi"); + expect(response.headers.get("cache-control")).toContain("immutable"); + await event.settled(); + // the list and the file's bytes, each under its own hash + const stored = sw.caches.named.get(`patchwork-content-${a}`)!; + expect(stored.entries.size).toBe(2); + expect(await stored.match(`${ORIGIN}/.content/${await contentHash(toBytes("hi"))}`)).toBeDefined(); + }); + + it("serves the old bytes for a pinned URL after the file changed", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a, notes } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + notes.change((d: any) => (d.content = "changed")); + // nothing stored: the bytes come from a resolver, read at the heads the list recorded + await sw.caches.delete(`patchwork-content-${a}`); + expect(await (await sw.fetch(new Request(pinned), "tab").response()!).text()).toBe("hi"); + }); + + it("never serves a forged store entry", async () => { sw = await startServiceWorker(); - sw.clients.set("page", { id: "page", type: "window", url: `${ORIGIN}/`, postMessage() {} }); - const event = sw.fetch( - new Request(`${ORIGIN}/@/.index`, { - method: "POST", - body: JSON.stringify({ id: "x", path: "a@h", manifest: { prefix: "a@h", entries: {} } }), - }), - "page" - ); - expect((await event.response()!).status).toBe(403); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + const first = sw.fetch(new Request(pinned), "tab"); + await (await first.response()!).text(); + await first.settled(); + + const key = `${ORIGIN}/.content/${await contentHash(toBytes("hi"))}`; + const store = sw.caches.named.get(`patchwork-content-${a}`)!; + await store.put(key, new Response("forged")); + const again = sw.fetch(new Request(pinned), "tab"); + expect(await (await again.response()!).text()).toBe("hi"); + await again.settled(); + expect(await (await store.match(key))!.text()).toBe("hi"); }); - it("answers a probe so the worker knows its fetches arrive", async () => { + it("fails, rather than serving, when a tab answers with other bytes", async () => { sw = await startServiceWorker(); - const response = await sw.fetch(new Request(`${ORIGIN}/@/.index`)).response()!; - expect(response.headers.get("x-patchwork-index")).toBe("present"); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + // a tab that answers everything with bytes of its own + sw.clients.set("tab", { + id: "tab", + type: "window", + url: `${ORIGIN}/`, + focused: true, + visibilityState: "visible", + postMessage(message: any, transfer) { + const [port] = (transfer ?? []) as MessagePort[]; + if (!port) return; // not a request to resolve + port.postMessage({ type: "accepted" }); + port.postMessage({ + id: message.id, + type: "response", + response: { status: 200, body: toBytes("forged"), headers: { "content-type": "text/plain" } }, + }); + }, + }); + await expect(sw.fetch(new Request(pinned), "tab").response()!).rejects.toThrow(); }); }); From 399a4180db0efa611a96ed24415353f75bf04727 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 13:59:50 +0000 Subject: [PATCH 31/81] content store: changesets and e2e for file lists Co-Authored-By: Claude Opus 5.5 --- .changeset/paths-content-store.md | 2 +- .changeset/paths-grammar-providers.md | 2 +- packages/e2e/tests/content-store.spec.ts | 23 +++-------------------- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/.changeset/paths-content-store.md b/.changeset/paths-content-store.md index e59ff16d1..eeeb40a1f 100644 --- a/.changeset/paths-content-store.md +++ b/.changeset/paths-content-store.md @@ -5,4 +5,4 @@ "@inkandswitch/patchwork-e2e": patch --- -The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one is served from a content-addressed store (`patchwork-content-`) whose every read is hashed and checked against an index that only the automerge worker fills, over `fetch()` requests the service worker checks are really from it. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `readPath`, `listFiles`, `readNode`, `WalkOptions.find` and `WalkEnd.container`. The e2e suite gains tests that a forged store entry is never served and a page can't answer for the automerge worker. +The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one names its folder's file list by hash (`…/@~/x.js`): the list records each file's content hash and the heads it was read at, so the service worker checks the list against the URL and the bytes against the list, and serves a pinned URL's old bytes even after the file changes. Lists and bytes are kept in a content-addressed store (`patchwork-content-`) that anything on the origin can write and that is checked on every read. Links no longer need heads to pin what they point at, and no document id appears in a URL or in the store. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `buildFileList`, `canonicalJson`, `parseFileList`, `contentHash`, `toBytes`, `readPath` (optionally at given heads), `listFiles`, `readNode`, `WalkOptions.find`, `WalkEnd.container` and `WalkEnd.tail`. The e2e suite gains a test that a forged store entry is never served. diff --git a/.changeset/paths-grammar-providers.md b/.changeset/paths-grammar-providers.md index 327ee9ba2..6f469dfa6 100644 --- a/.changeset/paths-grammar-providers.md +++ b/.changeset/paths-grammar-providers.md @@ -3,4 +3,4 @@ "@inkandswitch/patchwork-plugins": patch --- -Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@/…`). `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is now a deprecated alias of `PathProvider`. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. +Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@~/…`), where a container's step may name a hash of its file list. `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path; `pinnedPath` and `pinPath` return the pinned path or URL together with the file list it names. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is now a deprecated alias of `PathProvider`. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. `isKeyhiveDoc` in `@inkandswitch/patchwork-plugins` now goes through filesystem's new `isKeyhiveDocumentId`, so the rule lives in one place. diff --git a/packages/e2e/tests/content-store.spec.ts b/packages/e2e/tests/content-store.spec.ts index 8bdde13db..a63f47bd6 100644 --- a/packages/e2e/tests/content-store.spec.ts +++ b/packages/e2e/tests/content-store.spec.ts @@ -1,11 +1,11 @@ import { expect, test } from "@playwright/test"; -import { waitForRepoReady, waitForServiceWorkerActive } from "./helpers.js"; +import { waitForRepoReady } from "./helpers.js"; /** * The service worker's content store for pinned `/@/` URLs. Anything on the * origin can write CacheStorage, so the store is untrusted: every read is - * hashed and checked against an index only the automerge worker can fill. - * These tests play the page that tries to get around that. + * hashed and checked against the file list the pinned URL names by hash. + * This plays the page that tries to get around that. */ declare global { @@ -98,20 +98,3 @@ test("a forged store entry is never served", async ({ page, browserName }) => { ); expect(value).toBe(42); }); - -test("an index answer from a page is refused", async ({ page }) => { - await page.goto("/"); - await waitForServiceWorkerActive(page); - const status = await page.evaluate(async () => { - const response = await fetch("/@/.index", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - id: "forged", - manifest: { prefix: "x@y", entries: {} }, - }), - }); - return response.status; - }); - expect(status).toBe(403); -}); From caae9f91540d5082aec9c4744f8fb2ca4379b54a Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 14:13:05 +0000 Subject: [PATCH 32/81] content store: rebuild a redirect's one-file list; refuse partial lists A pinned URL from a live redirect names a one-file list. Rebuilding it asked for the whole folder's list, which never matched, so reloading a page you'd navigated to failed once the service worker had slept, after reading the folder for nothing. Rebuilds now try the one-file list first. pinPath refused nothing: a file it couldn't read was left out of the list and answered 404 under that URL for good. It now throws IncompleteFileListError, and the resolver answers a redirect it can't pin yet as a network error rather than a 404. Also corrects comments that still spoke of the index, and the changeset's claim that the store holds no document ids. Co-Authored-By: Claude Opus 5.5 --- .changeset/paths-content-store.md | 2 +- core/bootloader/src/content-store.ts | 7 +++--- core/bootloader/src/pinned.ts | 24 ++++++++++++------ core/bootloader/src/resolver.ts | 15 ++++++++--- core/bootloader/src/service-worker.ts | 16 +++++++++--- core/bootloader/src/types.ts | 2 ++ core/bootloader/test/pinned.test.ts | 25 +++++++++++++++++-- core/bootloader/test/resolver.test.ts | 11 +++++++- core/bootloader/test/service-worker.test.ts | 14 +++++++++++ core/filesystem/src/walk.ts | 15 ++++++++++- core/filesystem/test/file-list.test.ts | 14 ++++++++++- ...28-paths-namespace-content-cache-design.md | 6 ++--- 12 files changed, 125 insertions(+), 26 deletions(-) diff --git a/.changeset/paths-content-store.md b/.changeset/paths-content-store.md index eeeb40a1f..7801c9c95 100644 --- a/.changeset/paths-content-store.md +++ b/.changeset/paths-content-store.md @@ -5,4 +5,4 @@ "@inkandswitch/patchwork-e2e": patch --- -The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one names its folder's file list by hash (`…/@~/x.js`): the list records each file's content hash and the heads it was read at, so the service worker checks the list against the URL and the bytes against the list, and serves a pinned URL's old bytes even after the file changes. Lists and bytes are kept in a content-addressed store (`patchwork-content-`) that anything on the origin can write and that is checked on every read. Links no longer need heads to pin what they point at, and no document id appears in a URL or in the store. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `buildFileList`, `canonicalJson`, `parseFileList`, `contentHash`, `toBytes`, `readPath` (optionally at given heads), `listFiles`, `readNode`, `WalkOptions.find`, `WalkEnd.container` and `WalkEnd.tail`. The e2e suite gains a test that a forged store entry is never served. +The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one names its folder's file list by hash (`…/@~/x.js`): the list records each file's content hash and the heads it was read at, so the service worker checks the list against the URL and the bytes against the list, and serves a pinned URL's old bytes even after the file changes. Lists and bytes are kept in a content-addressed store (`patchwork-content-`) that anything on the origin can write and that is checked on every read. Links no longer need heads to pin what they point at, and no document id appears in a URL, a file list or a store key (a stored folder's own JSON still holds its children's URLs). A pinned URL whose list has been lost is rebuilt from its pinned heads, trying the one-file list a redirect hands out before the whole folder's; `pinPath` refuses to pin a list it couldn't read every file for (`IncompleteFileListError`). Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `buildFileList`, `canonicalJson`, `parseFileList`, `contentHash`, `toBytes`, `readPath` (optionally at given heads), `listFiles`, `readNode`, `WalkOptions.find`, `WalkEnd.container` and `WalkEnd.tail`. The e2e suite gains a test that a forged store entry is never served. diff --git a/core/bootloader/src/content-store.ts b/core/bootloader/src/content-store.ts index bd3135495..6c832d82f 100644 --- a/core/bootloader/src/content-store.ts +++ b/core/bootloader/src/content-store.ts @@ -1,6 +1,6 @@ // Bytes by their hash, in CacheStorage. Nothing here is trusted: any code on the origin can -// write these caches, so every read is checked against the service worker's index before it's -// served (see pinned.ts). Reads and writes go through this one interface so the bytes can be +// write these caches, so every read is hashed and checked against the hash a pinned URL vouches +// for, through its file list, before it's served (see pinned.ts). Reads and writes go through this one interface so the bytes can be // encrypted at rest later. export interface ContentStore { @@ -34,7 +34,8 @@ export function cacheStore(cache: Cache, origin: string): ContentStore { /** * Forget everything stored for an account. There's no sign-out flow; a site calls this from its - * own. The service worker's index may still name these hashes, which then read as misses. + * own. File lists the service worker holds in memory may still name these hashes, which then + * read as misses. */ export function clearAccountStore(acctHash: string): Promise { return caches.delete(contentCacheName(acctHash)); diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index 1c2ca165e..716ed4944 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -97,8 +97,11 @@ export type PinnedDeps = { /** Undefined: nothing is read or written (a share link). */ store: ContentStore | undefined; hash(bytes: Uint8Array): Promise; - /** Ask a resolver for the full list of the container at `prefix`: its bytes, or undefined. */ - buildList(prefix: string): Promise; + /** + * Ask a resolver for the list of the container at `prefix`, covering only the tail `only` if + * given, else every file: its bytes, or undefined. + */ + buildList(prefix: string, only?: string): Promise; /** * Ask a tab or the automerge worker for the file at `path`, read at `heads` if given: its * bytes, a Response for any other answer (a redirect, a 404), or undefined. @@ -131,9 +134,13 @@ export async function keepFileList( return true; } -/** The list a URL names: from memory, from the store (checked), or rebuilt (checked). */ +/** + * The list a URL names: from memory, from the store (checked), or rebuilt (checked). A rebuild + * tries the one-file list a redirect hands out first, which costs one file read, then the whole + * container's. + */ async function loadList( - { prefix, listHash }: { prefix: string; listHash: string }, + { prefix, tail, listHash }: { prefix: string; tail: string; listHash: string }, deps: PinnedDeps ): Promise { const known = deps.lists.get(listHash); @@ -147,8 +154,10 @@ async function loadList( deps.warn(`content store entry ${listHash} isn't the file list it's named for; deleting it`); await deps.store!.delete(listHash); } - const built = await deps.buildList(prefix); - if (built && (await keepFileList(built, listHash, deps))) return deps.lists.get(listHash); + for (const only of [tail, undefined]) { + const built = await deps.buildList(prefix, only); + if (built && (await keepFileList(built, listHash, deps))) return deps.lists.get(listHash); + } return undefined; } @@ -255,7 +264,7 @@ export function sharedRequests( export function handoffRequest( request: Request, cachename: string, - to: URL | { path: string; heads?: string; list?: boolean } + to: URL | { path: string; heads?: string; list?: boolean; only?: string } ): HandoffRequestMessage { const target = to instanceof URL @@ -265,6 +274,7 @@ export function handoffRequest( path: to.path, ...(to.heads ? { heads: to.heads } : {}), ...(to.list ? { list: true } : {}), + ...(to.only !== undefined ? { only: to.only } : {}), }; return { id: crypto.randomUUID(), diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 7d8e2cf8a..6a6040ca9 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -19,6 +19,7 @@ import { import { buildFileList, getDefaultBranch, + IncompleteFileListError, parseHeads, parsePath, pinnedPath, @@ -231,7 +232,7 @@ async function resolveWalkPath( path: string, accounts: ReadonlyMap, signal: AbortSignal, - { heads, list }: { heads?: string; list?: boolean } = {} + { heads, list, only }: { heads?: string; list?: boolean; only?: string } = {} ): Promise { try { parsePath(path); @@ -253,6 +254,7 @@ async function resolveWalkPath( if (list) { const built = await buildFileList(repo, path, { ...options, + only: only === undefined ? undefined : [only], // Each file as this node has it, without waiting on peers for every one. findFile: waitingFind(repo, signal, missing, { catchUp: false }), }); @@ -266,7 +268,14 @@ async function resolveWalkPath( const end = await walkPath(repo, path, options); if (!end) return notFound(); if (!end.canonical) { - const pinned = await pinnedPath(repo, path, { ...options, only: "end" }); + let pinned: Awaited>; + try { + pinned = await pinnedPath(repo, path, { ...options, only: "end" }); + } catch (error) { + // The file isn't readable here yet: not a 404, which would be memoized. + if (error instanceof IncompleteFileListError) throw new AbortHandoff(error.message); + throw error; + } if (!pinned) return notFound(); return new Response((pinned.list ?? null) as BlobPart | null, { status: 307, @@ -430,7 +439,7 @@ export function createResolver( try { // Everything goes back inline. The service worker keeps what's - // immutable (see its serveHandoff), or checks it against the index. + // immutable (see its serveHandoff), or checks it against a file list. log(`responding inline to ${request.url} with ${response.status}`); return await reply(id, response); } catch (error) { diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 3b6322665..27d838c8b 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -579,7 +579,7 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise buildList(prefix, ask), + buildList: (prefix, only) => buildList(prefix, only, ask), resolve: (target, heads) => ask({ path: target, heads }), warn: (message) => lifecycle("warn", message), keep: (work) => @@ -615,15 +615,23 @@ type Ask = (to: { path: string; heads?: string; list?: boolean; + only?: string; }) => Promise<{ body: Uint8Array; contentType: string } | Response>; // Concurrent requests for files of one package share a rebuild of its list. -const buildList = sharedRequests( - async (prefix: string, ask: Ask): Promise => { - const answer = await ask({ path: prefix, list: true }); +const sharedListBuilds = sharedRequests( + async ( + _key: string, + ask: Ask, + prefix: string, + only: string | undefined + ): Promise => { + const answer = await ask({ path: prefix, list: true, only }); return answer instanceof Response ? undefined : answer.body; } ); +const buildList = (prefix: string, only: string | undefined, ask: Ask) => + sharedListBuilds(`${prefix}\n${only ?? ""}`, ask, prefix, only); /** * A resolver's redirect from a live path carries, as its body, the file list diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 529da4db8..2990ed290 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -56,6 +56,8 @@ export interface HandoffRequest { heads?: string; /** Answer the file list of the container `path` names, not a file. */ list?: boolean; + /** With `list`: list only this tail, as the one-file list a redirect hands out. */ + only?: string; /** * @deprecated A briefly-deployed shape put the special URL in `url` and * the cache key here. Only read, never sent. diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index 3f78ff076..61ec0af4a 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -90,7 +90,9 @@ async function setup(options: { store?: boolean } = {}) { lists: new Map(), store: new MemoryStore(), hash: contentHash, - buildList: vi.fn(async (_prefix: string) => listBytes as Uint8Array | undefined), + buildList: vi.fn( + async (_prefix: string, _only?: string) => listBytes as Uint8Array | undefined + ), resolve: vi.fn(async (_path: string, _heads?: string) => ({ body: REAL, contentType: "text/javascript", @@ -130,10 +132,29 @@ describe("servePinned", () => { expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); await settle(); expect(deps.warn).toHaveBeenCalled(); - expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h"); + expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h", "index.js"); expect(deps.store.bytes.get(listHash)).toEqual(listBytes); }); + it("rebuilds a one-file list, as a redirect hands out, before a whole one", async () => { + const { deps, listBytes, normalized } = await setup(); + deps.buildList.mockImplementation(async (_prefix: string, only?: string) => + only === "index.js" ? listBytes : toBytes('{"files":{},"v":1}') + ); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.buildList).toHaveBeenCalledTimes(1); + expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h", "index.js"); + }); + + it("falls back to the whole list when the one-file rebuild doesn't match", async () => { + const { deps, listBytes, normalized } = await setup(); + deps.buildList.mockImplementation(async (_prefix: string, only?: string) => + only === undefined ? listBytes : toBytes('{"files":{},"v":1}') + ); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.buildList).toHaveBeenLastCalledWith("acct@h/pkg@h", undefined); + }); + it("fails when the rebuilt list doesn't match the URL", async () => { const { deps, normalized } = await setup(); deps.buildList.mockResolvedValue(toBytes('{"files":{},"v":1}')); diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index bbc35f350..be447f51b 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -49,7 +49,7 @@ const chain = async (repo: Repo, path: string, accounts: ReadonlyMap { const message = pathRequest(path); return { ...message, request: { ...message.request, ...extra } }; @@ -100,6 +100,15 @@ describe("resolver: /@/ paths", () => { expect(inline(reply).body).toEqual(pinned!.list); }); + it("answers a one-file list, the same a redirect hands out", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const live = livePath(a, ["Projects", "foo", "notes"]); + const foo = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const reply = await resolver.answer(withRequest(foo, { list: true, only: "notes" })); + const pinned = await pinnedPath(repo, live, { accounts, only: "end" }); + expect(inline(reply).body).toEqual(pinned!.list); + }); + it("reads a file at given heads, after it changed", async () => { const { repo, a, accounts, resolver, notes } = await fixture(); const path = await chain(repo, livePath(a, ["Projects", "foo", "notes"]), accounts); diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts index ac8da4fea..4f58aa1ae 100644 --- a/core/bootloader/test/service-worker.test.ts +++ b/core/bootloader/test/service-worker.test.ts @@ -121,6 +121,20 @@ describe("service worker: /@/ paths", () => { expect(await (await sw.fetch(new Request(pinned), "tab").response()!).text()).toBe("hi"); }); + it("serves a redirect's pinned URL again after the service worker restarts", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + // a fresh service worker: no lists in memory, and nothing stored + sw.close(); + sw = await startServiceWorker(); + addTab(sw, repo); + expect(await (await sw.fetch(new Request(pinned), "tab").response()!).text()).toBe("hi"); + }); + it("never serves a forged store entry", async () => { sw = await startServiceWorker(); const repo = new Repo(); diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index e98e95fe0..4da3f88a7 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -321,11 +321,23 @@ export async function resolveLivePath( return repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); } +/** + * A file list couldn't read every file it covers. A pinned URL never changes meaning, so a + * list missing a file would make it answer 404 for good; pin again once they can be read. + */ +export class IncompleteFileListError extends Error { + name = "IncompleteFileListError"; + constructor(readonly missing: string[]) { + super(`couldn't read ${missing.join(", ")} to pin them`); + } +} + /** * The pinned form of a path, with a file list on the last container it enters: every file * under that container, or with `only: "end"`, only the one the path ends at (what a live → * pinned redirect hands out). Returns the path and the list's bytes, which whoever pins hands to - * the service worker. No list if the path enters no container. + * the service worker. No list if the path enters no container. Throws + * {@link IncompleteFileListError} rather than pin a list missing a file it should cover. */ export async function pinnedPath( repo: Repo, @@ -344,6 +356,7 @@ export async function pinnedPath( only: options.only === "end" ? [end.tail] : undefined, }); if (!built) return { path: end.pinned }; + if (built.missing.length) throw new IncompleteFileListError(built.missing); const listed = `${end.container.pinned}~${built.listHash}`; return { path: end.tail ? `${listed}/${end.tail}` : listed, list: built.bytes }; } diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index f00b4a425..ea13684ed 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import { parseAutomergeUrl } from "@automerge/automerge-repo"; import { formatHeads, livePath, parseHeads } from "../src/paths.js"; -import { readPath } from "../src/walk.js"; +import { IncompleteFileListError, pinnedPath, readPath } from "../src/walk.js"; import { contentHash, toBytes } from "../src/content-hash.js"; import { buildFileList, canonicalJson, parseFileList } from "../src/file-list.js"; import { chain, fixture } from "./namespace-fixture.js"; @@ -82,3 +82,15 @@ describe("buildFileList", () => { expect(decode(then!.content)).toBe("mine"); }); }); + +describe("pinnedPath", () => { + it("refuses to pin a list that's missing files", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const bad = (minePkg.doc() as any)["package.json"] as string; + const find = (url: any) => + url === bad ? Promise.reject(new Error("unavailable")) : repo.find(url); + const pinning = pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts, find }); + await expect(pinning).rejects.toBeInstanceOf(IncompleteFileListError); + await expect(pinning).rejects.toMatchObject({ missing: ["package.json"] }); + }); +}); diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 609f844fc..cf182847b 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -388,9 +388,9 @@ These replace the deleted `package-load-gate.md` note. - A pinned URL can't be served if its file list is gone (evicted, cleared, or held only in a service worker that stopped) and no resolver can rebuild it. A rebuild reads each file at its current heads, so it matches - only if nothing changed since, and only for a list that covers the whole - container (not a redirect's one-file list, which a fresh request to the - live URL replaces). The ModuleWatcher pins again on boot, so + only if nothing changed since. It tries the one-file list a redirect hands + out first, then the whole container's, so a reloaded page rebuilds as + cheaply as it was pinned. The ModuleWatcher pins again on boot, so imports recover; a hand-made share link doesn't. - Legacy `/automerge%3A…` URLs stay keyed by URL and unchecked. - Protected package code is stored in plaintext. Today keyhive's secrets sit From 50075b7d21b8508d8059fa4bae9ec09de304fae9 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 15:11:04 +0000 Subject: [PATCH 33/81] plan: paths phase 3 (routing) Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-10-28-paths-phase-3-routing.md | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md b/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md new file mode 100644 index 000000000..20142fe57 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md @@ -0,0 +1,119 @@ +# Paths phase 3: routing — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Clicking into a document can say where it was opened from: the live path lands in `&path=` in the hash, a hash with only `&path=` opens what the path names, and a `doc=` that disagrees with its `path=` wins. + +**Architecture:** Bring in the `open-document-path` branch (48f4ab98), which threads `path` through the `patchwork:open-document` event, `patchwork.open` and ``. Then redefine that `path` as the **live path** (`/Projects/foo`), not a service worker path. The router reads and writes `path` raw, since its segments are already percent-encoded, and resolves it with `resolveLivePath` from phase 1. + +**Tech Stack:** TypeScript; `core/patchwork/src/router.ts`; `@inkandswitch/patchwork-filesystem` (`idhash`, `resolveLivePath`); vitest in node against a fake `window`, `location` and root element. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 3, *Hash routing* and *Opening at a path*. + +## Global Constraints + +- `path` joins `HASH_KEY_ORDER` right after `doc`. Each segment is `encodeURIComponent`-encoded and the slashes stay literal: the value goes into the hash exactly as `livePath` wrote it. +- `doc=` says what is open; `path=` says where it was opened from. +- A hash with `path=` and no `doc=` resolves the path and opens the result. +- If `doc=` and `path=` disagree, `doc=` wins and `path=` is dropped with a console warning. +- `patchwork:open-document`'s `path` holds the live path, and the router writes it to `&path=`. That reverses the branch's choice to keep it out of the hash. +- `` hands the path to the tool without remounting, as on the branch. + +## Decisions this plan makes + +1. **`path` is read raw from the hash.** `URLSearchParams` decodes `%2F` to `/`, which would split a name containing a slash into two. A single `parseHashParams(hash)` reads every key through `URLSearchParams` and then puts `path`'s raw text back. Every place the router reads the hash uses it. +2. **The router resolves only the signed-in account's paths.** Its `accounts` map is the account doc's own acctHash. A path under any other account doesn't resolve: it's dropped with a warning if there's a `doc=`, or ignored if there isn't. +3. **The agreement check runs after the document is opened**, so a slow path lookup never delays opening it. If the path names a different document, or none at all, the router rewrites the hash without `path=`. +4. **The router is tested in node.** `@inkandswitch/patchwork-elements` is mocked down to an `openDocument` that dispatches the event, and the DOM is replaced with small fakes: an `EventTarget` with attributes, plus `window`, `location` and `document`. + +## Review Focus + +- **Reserved characters in a path segment** (`a%2Fb`, `%26`, `%23`, a space as `%20`, `+`) survive a round trip through the hash, including through an open-document event that re-serializes the other params. Pinned in Task 2. +- **Path only, unresolvable** (a typo, another account): no exception, a warning, and no document opened. Pinned in Task 3. +- **The same `path=` arriving again on every hashchange** doesn't re-open anything or loop. Only a hash without `doc=` opens from the path. Pinned in Task 3. + +--- + +### Task 1: Bring in `open-document-path`, as a live path + +**Files:** cherry-pick 48f4ab98 (`core/elements/src/{events,index,patchwork-view}.ts`, `core/patchwork/src/{index,types}.ts`, `.changeset/open-document-sw-path.md`). Then edit the doc comments and the changeset. + +- [ ] **Step 1:** `git cherry-pick 48f4ab98`. Expected: it applies cleanly, since those files aren't touched on `paths-design`. Verify with `git diff HEAD~1 --stat`. +- [ ] **Step 2: Redefine `path`.** + - `OpenDocumentEventDetail.path` becomes: "The live path the document was opened at, like a cwd: `/Projects/foo`, each segment `encodeURIComponent`-encoded. The router writes it to `&path=`." + - Point the `OpenOptions.path` and `PatchworkViewElementBase.path` comments at it. + - Rewrite the changeset (rename it `open-document-path.md`) to say the same, listing `@inkandswitch/patchwork-elements` and `@inkandswitch/patchwork`. +- [ ] **Step 3: Typecheck:** `pnpm --filter @inkandswitch/patchwork-elements build && pnpm --filter @inkandswitch/patchwork exec tsc --noEmit`. Expected: no errors. +- [ ] **Step 4: Commit** `elements: open-document's path is the live path it was opened at`. + +### Task 2: Hash params keep `path` raw + +**Files:** create `core/patchwork/src/hash-params.ts` (moving `serializeHashParams`, `HASH_KEY_ORDER`, `RAW_HASH_KEYS` and `docParamToUrl` out of `router.ts`); modify `router.ts` to import them. Test `core/patchwork/test/hash-params.test.ts`. + +**Interfaces:** +```ts +export const HASH_KEY_ORDER = ["doc", "path", "tool", "type", "title", "frame", "draft"]; +/** `URLSearchParams` over a hash, except that `path`'s value is its raw, still-encoded text. */ +export function parseHashParams(hash: string): URLSearchParams; +export function serializeHashParams(params: URLSearchParams): string; // doc, draft and path emitted raw +export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined; // moved unchanged +``` + +- [ ] **Step 1: Failing tests** + +```ts +it("keeps a path's segments as they're encoded", () => { + const hash = "doc=automerge:abc&path=acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"; + expect(parseHashParams(hash).get("path")).toBe("acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"); + expect(serializeHashParams(parseHashParams(hash))).toBe(hash); +}); +it("puts path right after doc", () => { + const params = new URLSearchParams({ tool: "t", doc: "automerge:abc" }); + params.set("path", "acct/x"); + expect(serializeHashParams(params)).toBe("doc=automerge:abc&path=acct/x&tool=t"); +}); +it("reads a hash with no path as before", () => { + expect(parseHashParams("doc=automerge:abc&title=a%20b").get("title")).toBe("a b"); +}); +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork exec vitest run test/hash-params.test.ts`. +- [ ] **Step 3: Implement.** Use `parseHashParams` in place of `new URLSearchParams(hash)` everywhere in `router.ts`. +- [ ] **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `patchwork: hash params keep a path's encoding`. + +### Task 3: The router writes, opens from, and checks `path=` + +**Files:** `core/patchwork/src/router.ts`; tests `core/patchwork/test/router.test.ts`. + +**Interfaces:** +- `createRouter` is unchanged on the outside. Internally it: + - computes `accounts = new Map([[await idhash(accountDocHandle.documentId), accountDocHandle.url]])` once; + - in the open-document listener, sets `path` from `detail.path`, or deletes it if there's none; + - in `route()`, for a hash with `path` and no `doc`: `resolveLivePath(repo, path, { accounts })`. If that finds a document, it dispatches `patchwork:open-document` with `{ url: handle.url, path }`; if not, it warns and rewrites the hash without `path`; + - in `route()`, for a hash with both: dispatches open-document as now, with `path` in its detail. Then it resolves the path, and if the result's `documentId` differs from `doc`'s, or nothing resolves, it warns and rewrites the hash without `path`. + +- [ ] **Step 1: Failing tests** (fake DOM, real in-memory `Repo`, an account whose `/` holds `Projects/foo/notes`) + +```ts +it("writes an opened document's path to the hash", ...) + // dispatch open-document {url: notes.url, path: livePath(a, ["Projects","foo","notes"])} → location.hash has doc=…&path= +it("drops path from the hash when a document is opened without one", ...) +it("opens what a path names when the hash has no doc", ...) + // location.hash = `path=${live}`; await router.route() → an open-document event with url === notes.url and path === live +it("warns and drops a path that names nothing, opening nothing", ...) +it("keeps a path that agrees with doc", ...) +it("drops a path that disagrees with doc, keeping doc", ...) // console.warn called; hash keeps doc, loses path +it("doesn't open anything twice when the same path arrives with its doc", ...) + // hash doc=notes&path=live → route() → exactly one open-document event +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork exec vitest run test/router.test.ts`. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run the patchwork suite, expect PASS:** `pnpm --filter @inkandswitch/patchwork test`, plus `pnpm --filter @inkandswitch/patchwork exec tsc --noEmit`. +- [ ] **Step 5: Commit** `patchwork: route &path= — write it, open from it, drop it when it disagrees`. + +### Task 4: Changeset + +- [ ] Add `.changeset/paths-routing.md` (`@inkandswitch/patchwork: patch`) describing `&path=`: written when a document is opened at a path, opened from when there's no `doc=`, and dropped with a warning when it disagrees. The e2e test the spec lists ("open a document with `&path=` alone") needs a full-UI boot. Add it to `packages/e2e/tests/content-store.spec.ts`'s Chromium-only pattern as `path-routing.spec.ts`, typecheck it, and let CI run it. +- [ ] Commit `changeset: path routing; e2e: open with &path= alone`. From 6ae06c393b155d73b43af7fe251b1953ec4c8c2b Mon Sep 17 00:00:00 2001 From: chee Date: Tue, 27 Oct 2026 05:04:31 +0000 Subject: [PATCH 34/81] add optional path to patchwork:open-document openDocument / patchwork.open / carry a cwd-like service-worker path (///) on the event detail, without writing it to the hash or remounting the tool. Co-Authored-By: Claude Opus 5.5 --- .changeset/open-document-sw-path.md | 8 ++++++++ core/elements/src/events.ts | 11 +++++++++-- core/elements/src/index.ts | 2 ++ core/elements/src/patchwork-view.ts | 15 +++++++++++++++ core/patchwork/src/index.ts | 1 + core/patchwork/src/types.ts | 2 ++ 6 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .changeset/open-document-sw-path.md diff --git a/.changeset/open-document-sw-path.md b/.changeset/open-document-sw-path.md new file mode 100644 index 000000000..5e5e77b24 --- /dev/null +++ b/.changeset/open-document-sw-path.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-elements": patch +--- + +`patchwork:open-document` takes an optional `path`: where the document was opened from, like a cwd. It is shaped like a service-worker path, `///` (e.g. `/automerge%3A/src/`). Pass it via `openDocument(element, url, toolId, path)` or `patchwork.open(url, { path })`; it travels on the event detail only and is not written to the hash. + +`` has a matching `path` attribute and property, so a frame can hand the opened tool its cwd. Changing it does not remount the tool. diff --git a/core/elements/src/events.ts b/core/elements/src/events.ts index 7a29f6fda..93bc01a4a 100644 --- a/core/elements/src/events.ts +++ b/core/elements/src/events.ts @@ -5,6 +5,12 @@ export interface OpenDocumentEventDetail { toolId?: string; title?: string; type?: string; + /** + * Where the document was opened from, like a cwd. Same shape as a + * service-worker path: `//`, e.g. + * `/automerge%3A/src/components`. + */ + path?: string; } export class OpenDocumentEvent extends CustomEvent { @@ -58,7 +64,8 @@ declare global { export const openDocument = ( element: HTMLElement | ShadowRoot, url: AutomergeUrl, - toolId?: string + toolId?: string, + path?: string ) => { - element.dispatchEvent(new OpenDocumentEvent({ url, toolId })); + element.dispatchEvent(new OpenDocumentEvent({ url, toolId, path })); }; diff --git a/core/elements/src/index.ts b/core/elements/src/index.ts index dff235b4b..b15983098 100644 --- a/core/elements/src/index.ts +++ b/core/elements/src/index.ts @@ -11,6 +11,7 @@ import type {} from "solid-js"; type LegacyPatchworkViewAttrs = { "doc-url"?: string; "tool-id"?: string | null; + path?: string | null; component?: never; url?: never; }; @@ -20,6 +21,7 @@ type PatchworkViewComponentAttrs = { url?: string; "doc-url"?: string; "tool-id"?: string | null; + path?: string | null; }; type PatchworkViewAttrs = diff --git a/core/elements/src/patchwork-view.ts b/core/elements/src/patchwork-view.ts index 51a66b307..18009a959 100644 --- a/core/elements/src/patchwork-view.ts +++ b/core/elements/src/patchwork-view.ts @@ -60,6 +60,7 @@ const ATTRS = { url: "url", docUrl: "doc-url", toolId: "tool-id", + path: "path", } as const; export type RegisterPatchworkViewElementParams = { @@ -87,6 +88,8 @@ export type PatchworkViewElementBase = HTMLElement & { repo: Repo; docUrl?: AutomergeUrl | null; toolId?: string | null; + /** Where the doc was opened from; see `OpenDocumentEventDetail.path`. */ + path?: string | null; }; export type PatchworkViewElement = PatchworkViewElementBase & { @@ -189,6 +192,18 @@ export function registerPatchworkViewElement( else this.removeAttribute(ATTRS.toolId); } + // Not observed: the mounted tool reads it, but a new path for the same + // doc shouldn't remount the tool. + get path(): string | null { + return this.getAttribute(ATTRS.path); + } + + set path(path: string | null) { + if (this.path === path) return; + if (path) this.setAttribute(ATTRS.path, path); + else this.removeAttribute(ATTRS.path); + } + static get observedAttributes() { return [ATTRS.component, ATTRS.url, ATTRS.docUrl, ATTRS.toolId]; } diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index 617477829..6047f16fb 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -270,6 +270,7 @@ async function doSetup(options: PatchworkOptions): Promise { toolId: openOptions.tool, type: openOptions.type, title: openOptions.title, + path: openOptions.path, }, }) ); diff --git a/core/patchwork/src/types.ts b/core/patchwork/src/types.ts index 1ecc33f32..06f5abd52 100644 --- a/core/patchwork/src/types.ts +++ b/core/patchwork/src/types.ts @@ -39,6 +39,8 @@ export interface OpenOptions { tool?: string; type?: string; title?: string; + /** Where the document was opened from; see `OpenDocumentEventDetail.path`. */ + path?: string; } export interface PatchworkOptions { From 1767c14b91a7408b2c1cb3d01174d4361b21ec1d Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 15:11:23 +0000 Subject: [PATCH 35/81] elements: open-document's path is the live path it was opened at Co-Authored-By: Claude Opus 5.5 --- .changeset/open-document-path.md | 6 ++++++ .changeset/open-document-sw-path.md | 8 -------- core/elements/src/events.ts | 7 ++++--- core/elements/src/patchwork-view.ts | 2 +- core/patchwork/src/types.ts | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 .changeset/open-document-path.md delete mode 100644 .changeset/open-document-sw-path.md diff --git a/.changeset/open-document-path.md b/.changeset/open-document-path.md new file mode 100644 index 000000000..19451f1c0 --- /dev/null +++ b/.changeset/open-document-path.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork-elements": patch +"@inkandswitch/patchwork": patch +--- + +`patchwork:open-document` (and `openDocument`, `patchwork.open`, ``) can say where a document was opened from: `path` is the live path it was opened at, like a cwd, e.g. `/Projects/foo` with each segment `encodeURIComponent`-encoded. The router writes it to `&path=`. A `` hands a new path to its tool without remounting it. diff --git a/.changeset/open-document-sw-path.md b/.changeset/open-document-sw-path.md deleted file mode 100644 index 5e5e77b24..000000000 --- a/.changeset/open-document-sw-path.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -"@inkandswitch/patchwork": patch -"@inkandswitch/patchwork-elements": patch ---- - -`patchwork:open-document` takes an optional `path`: where the document was opened from, like a cwd. It is shaped like a service-worker path, `///` (e.g. `/automerge%3A/src/`). Pass it via `openDocument(element, url, toolId, path)` or `patchwork.open(url, { path })`; it travels on the event detail only and is not written to the hash. - -`` has a matching `path` attribute and property, so a frame can hand the opened tool its cwd. Changing it does not remount the tool. diff --git a/core/elements/src/events.ts b/core/elements/src/events.ts index 93bc01a4a..f9bc99238 100644 --- a/core/elements/src/events.ts +++ b/core/elements/src/events.ts @@ -6,9 +6,10 @@ export interface OpenDocumentEventDetail { title?: string; type?: string; /** - * Where the document was opened from, like a cwd. Same shape as a - * service-worker path: `//`, e.g. - * `/automerge%3A/src/components`. + * The live path the document was opened at, like a cwd: + * `/Projects/foo`, each segment `encodeURIComponent`-encoded + * (see `livePath` in `@inkandswitch/patchwork-filesystem`). The router + * writes it to `&path=`. */ path?: string; } diff --git a/core/elements/src/patchwork-view.ts b/core/elements/src/patchwork-view.ts index 18009a959..025e4259f 100644 --- a/core/elements/src/patchwork-view.ts +++ b/core/elements/src/patchwork-view.ts @@ -88,7 +88,7 @@ export type PatchworkViewElementBase = HTMLElement & { repo: Repo; docUrl?: AutomergeUrl | null; toolId?: string | null; - /** Where the doc was opened from; see `OpenDocumentEventDetail.path`. */ + /** The live path the doc was opened at; see `OpenDocumentEventDetail.path`. */ path?: string | null; }; diff --git a/core/patchwork/src/types.ts b/core/patchwork/src/types.ts index 06f5abd52..829b31129 100644 --- a/core/patchwork/src/types.ts +++ b/core/patchwork/src/types.ts @@ -39,7 +39,7 @@ export interface OpenOptions { tool?: string; type?: string; title?: string; - /** Where the document was opened from; see `OpenDocumentEventDetail.path`. */ + /** The live path the document was opened at; see `OpenDocumentEventDetail.path`. */ path?: string; } From 472ef6610b4fcda5e885717f711f5cfcc3d0f347 Mon Sep 17 00:00:00 2001 From: chee Date: Wed, 28 Oct 2026 15:11:58 +0000 Subject: [PATCH 36/81] patchwork: hash params keep a path's encoding Co-Authored-By: Claude Opus 5.5 --- core/patchwork/src/hash-params.ts | 59 +++++++++++++++++++++++++ core/patchwork/src/router.ts | 48 ++++---------------- core/patchwork/test/hash-params.test.ts | 24 ++++++++++ 3 files changed, 92 insertions(+), 39 deletions(-) create mode 100644 core/patchwork/src/hash-params.ts create mode 100644 core/patchwork/test/hash-params.test.ts diff --git a/core/patchwork/src/hash-params.ts b/core/patchwork/src/hash-params.ts new file mode 100644 index 000000000..dfe5a6361 --- /dev/null +++ b/core/patchwork/src/hash-params.ts @@ -0,0 +1,59 @@ +// The router's hash: `#doc=…&path=…&tool=…`, read and written the same way +// everywhere so re-serializing the same params is byte-identical. +import { + isValidAutomergeUrl, + isValidDocumentId, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocumentId, +} from "@automerge/vanillajs/slim"; + +// `doc=` and `draft=` are automerge URLs, kept literal rather than +// percent-encoded so links stay readable. `path=` is a live path whose +// segments are already encoded, with literal slashes between them. +const RAW_HASH_KEYS = new Set(["doc", "path", "draft"]); +// A stable order means re-serializing the same logical params is +// byte-identical, avoiding spurious `hashchange` round-trips. +export const HASH_KEY_ORDER = ["doc", "path", "tool", "type", "title", "frame", "draft"]; + +/** + * The hash's params. `URLSearchParams` decodes every value, which would turn + * an encoded slash in a path segment (`a%2Fb`) into a real one, so `path` is + * put back as its raw text. + */ +export function parseHashParams(hash: string): URLSearchParams { + const params = new URLSearchParams(hash); + const raw = /(?:^|&)path=([^&]*)/.exec(hash)?.[1]; + if (raw !== undefined) params.set("path", raw); + return params; +} + +export function serializeHashParams(params: URLSearchParams): string { + const keys = [...HASH_KEY_ORDER, ...params.keys()]; + const parts: string[] = []; + const emitted = new Set(); + for (const key of keys) { + if (emitted.has(key)) continue; + const value = params.get(key); + if (!value) continue; + emitted.add(key); + parts.push( + `${key}=${RAW_HASH_KEYS.has(key) ? value : encodeURIComponent(value)}` + ); + } + return parts.join("&"); +} + +/** + * Coerce a `doc=` hash param to a full automerge URL. Accepts a full URL + * (`automerge:[#heads]`) or a bare document id, for older links. + */ +export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined { + if (!docParam) return undefined; + if (isValidAutomergeUrl(docParam as AutomergeUrl)) { + return docParam as AutomergeUrl; + } + const documentId = docParam.replace(/^automerge:/, ""); + if (!isValidDocumentId(documentId)) return undefined; + return stringifyAutomergeUrl({ documentId: documentId as DocumentId }); +} diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index e425696f4..4c6304dcd 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -9,6 +9,11 @@ import { type Repo, } from "@automerge/vanillajs/slim"; import { openDocument } from "@inkandswitch/patchwork-elements"; +import { + docParamToUrl, + parseHashParams, + serializeHashParams, +} from "./hash-params.js"; import { type AccountDoc, type DatatypeDescription, @@ -24,42 +29,7 @@ import { const BIG_PATCHWORK_HASH_REGEX = /^(?[^=&?/#]*)--(?<docId>[1-9A-HJ-NP-Za-km-z]+)/; -// The `doc=` and `draft=` values are automerge URLs, kept literal rather than -// percent-encoded so links stay readable. -const RAW_HASH_KEYS = new Set(["doc", "draft"]); -// A stable order means re-serializing the same logical params is -// byte-identical, avoiding spurious `hashchange` round-trips. -const HASH_KEY_ORDER = ["doc", "tool", "type", "title", "frame", "draft"]; - -function serializeHashParams(params: URLSearchParams): string { - const keys = [...HASH_KEY_ORDER, ...params.keys()]; - const parts: string[] = []; - const emitted = new Set<string>(); - for (const key of keys) { - if (emitted.has(key)) continue; - const value = params.get(key); - if (!value) continue; - emitted.add(key); - parts.push( - `${key}=${RAW_HASH_KEYS.has(key) ? value : encodeURIComponent(value)}` - ); - } - return parts.join("&"); -} - -/** - * Coerce a `doc=` hash param to a full automerge URL. Accepts a full URL - * (`automerge:<id>[#heads]`) or a bare document id, for older links. - */ -export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined { - if (!docParam) return undefined; - if (isValidAutomergeUrl(docParam as AutomergeUrl)) { - return docParam as AutomergeUrl; - } - const documentId = docParam.replace(/^automerge:/, ""); - if (!isValidDocumentId(documentId)) return undefined; - return stringifyAutomergeUrl({ documentId: documentId as DocumentId }); -} +export { docParamToUrl } from "./hash-params.js"; export interface RouterParams { rootElement: HTMLElement; @@ -91,7 +61,7 @@ export function createRouter({ // The first call seeds the root view's tool/doc so it can mount; later // calls reconcile the mounted view with the hash. if (!rootElement.hasAttribute("tool-id")) { - const params = new URLSearchParams(location.hash.slice(1)); + const params = parseHashParams(location.hash.slice(1)); const frame = params.get("frame"); const toolId = frame ?? @@ -131,7 +101,7 @@ export function createRouter({ return; } - const params = new URLSearchParams(hash); + const params = parseHashParams(hash); const docUrl = docParamToUrl(params.get("doc")); const frame = params.get("frame"); @@ -168,7 +138,7 @@ export function createRouter({ title?: string; }; - const params = new URLSearchParams(window.location.hash.slice(1)); + const params = parseHashParams(window.location.hash.slice(1)); // `doc` is the full automerge URL, so heads live inside it and the separate // `heads=` param is gone. params.delete("heads"); diff --git a/core/patchwork/test/hash-params.test.ts b/core/patchwork/test/hash-params.test.ts new file mode 100644 index 000000000..d5bf4bafe --- /dev/null +++ b/core/patchwork/test/hash-params.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { parseHashParams, serializeHashParams } from "../src/hash-params.js"; + +describe("hash params", () => { + it("keeps a path's segments as they're encoded", () => { + const hash = "doc=automerge:abc&path=acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"; + expect(parseHashParams(hash).get("path")).toBe( + "acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1" + ); + expect(serializeHashParams(parseHashParams(hash))).toBe(hash); + }); + + it("puts path right after doc", () => { + const params = new URLSearchParams({ tool: "t", doc: "automerge:abc" }); + params.set("path", "acct/x"); + expect(serializeHashParams(params)).toBe("doc=automerge:abc&path=acct/x&tool=t"); + }); + + it("reads a hash with no path as before", () => { + const params = parseHashParams("doc=automerge:abc&title=a%20b"); + expect(params.get("title")).toBe("a b"); + expect(params.has("path")).toBe(false); + }); +}); From a7639533ea8406b130130153121c1dc2aa307fc2 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:13:30 +0000 Subject: [PATCH 37/81] =?UTF-8?q?patchwork:=20route=20&path=3D=20=E2=80=94?= =?UTF-8?q?=20write=20it,=20open=20from=20it,=20drop=20it=20when=20it=20di?= =?UTF-8?q?sagrees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/patchwork/src/router.ts | 48 +++++++++- core/patchwork/test/router.test.ts | 146 +++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 core/patchwork/test/router.test.ts diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index 4c6304dcd..3a357b31e 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -9,6 +9,7 @@ import { type Repo, } from "@automerge/vanillajs/slim"; import { openDocument } from "@inkandswitch/patchwork-elements"; +import { idhash, resolveLivePath } from "@inkandswitch/patchwork-filesystem"; import { docParamToUrl, parseHashParams, @@ -57,6 +58,34 @@ export function createRouter({ siteTitle, frameToolId, }: RouterParams): Router { + // A path in the hash resolves only under this site's own account. + const accounts = idhash(accountDocHandle.documentId).then( + (acctHash) => new Map([[acctHash, accountDocHandle.url]]) + ); + const resolvePathParam = async (path: string) => { + try { + return await resolveLivePath(repo, path, { accounts: await accounts }); + } catch { + return undefined; // a path that doesn't parse names nothing + } + }; + /** Take `path=` out of the hash, if it's still the one that was checked. */ + const dropPath = (path: string, reason: string) => { + const params = parseHashParams(window.location.hash.slice(1)); + if (params.get("path") !== path) return; + console.warn(`patchwork: dropping path=${path} from the URL: ${reason}`); + params.delete("path"); + window.location.hash = serializeHashParams(params); + }; + /** `doc=` says what is open; when `path=` names something else, `doc=` wins. */ + const checkPath = async (docUrl: AutomergeUrl, path: string) => { + const handle = await resolvePathParam(path); + if (!handle) return dropPath(path, "it names nothing"); + if (handle.documentId !== parseAutomergeUrl(docUrl).documentId) { + dropPath(path, `it names ${handle.url}, not ${docUrl}`); + } + }; + const route = async () => { // The first call seeds the root view's tool/doc so it can mount; later // calls reconcile the mounted view with the hash. @@ -103,6 +132,7 @@ export function createRouter({ const params = parseHashParams(hash); const docUrl = docParamToUrl(params.get("doc")); + const path = params.get("path") ?? undefined; const frame = params.get("frame"); if (frame) { @@ -116,26 +146,37 @@ export function createRouter({ } } - if (docUrl) { + const open = (url: AutomergeUrl) => rootElement.dispatchEvent( new CustomEvent("patchwork:open-document", { detail: { - url: docUrl, + url, toolId: params.get("tool"), title: params.get("title"), type: params.get("type"), + path, }, }) ); + + if (docUrl) { + open(docUrl); + if (path) void checkPath(docUrl, path); + } else if (path) { + // Just a path: open what it names. + const handle = await resolvePathParam(path); + if (handle) open(handle.url); + else dropPath(path, "it names nothing"); } }; rootElement.addEventListener("patchwork:open-document", async (event) => { - const { url, toolId, type, title } = event.detail as { + const { url, toolId, type, title, path } = event.detail as { url: AutomergeUrl; toolId?: string; type?: string; title?: string; + path?: string; }; const params = parseHashParams(window.location.hash.slice(1)); @@ -154,6 +195,7 @@ export function createRouter({ } params.set("doc", url); for (const [key, value] of [ + ["path", path], ["tool", toolId], ["title", title], ["type", type], diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts new file mode 100644 index 000000000..8e786529c --- /dev/null +++ b/core/patchwork/test/router.test.ts @@ -0,0 +1,146 @@ +// The router in node: a fake window, location and root element, a real +// in-memory Repo, and an account whose / holds Projects/foo/notes. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { livePath } from "@inkandswitch/patchwork-filesystem"; + +vi.mock("@inkandswitch/patchwork-elements", () => ({ + openDocument: (element: EventTarget, url: string) => + element.dispatchEvent(new CustomEvent("patchwork:open-document", { detail: { url } })), +})); + +const { createRouter } = await import("../src/router.js"); +const { idhash } = await import("@inkandswitch/patchwork-filesystem"); + +class FakeElement extends EventTarget { + attrs = new Map<string, string>(); + setAttribute(name: string, value: string) { + this.attrs.set(name, value); + } + getAttribute(name: string) { + return this.attrs.get(name) ?? null; + } + hasAttribute(name: string) { + return this.attrs.has(name); + } +} + +/** A location whose hash reads back with its "#", as a browser's does. */ +function fakeLocation(initial: string) { + let hash = initial; + return { + get hash() { + return hash ? `#${hash}` : ""; + }, + set hash(value: string) { + hash = value.replace(/^#/, ""); + }, + }; +} + +async function setup(hash: (a: string, notes: AutomergeUrl) => string) { + const repo = new Repo(); + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + const folder = (entries: Record<string, AutomergeUrl>) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + const notes = file("hi"); + const other = file("other"); + const root = folder({ Projects: folder({ foo: folder({ notes: notes.url }).url }).url }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await idhash(account.documentId); + + const location = fakeLocation(hash(a, notes.url)); + vi.stubGlobal("location", location); + vi.stubGlobal("window", { location, addEventListener() {} }); + vi.stubGlobal("document", { title: "" }); + + const rootElement = new FakeElement(); + rootElement.setAttribute("tool-id", "frame"); + const opened: Array<{ url: string; path?: string }> = []; + rootElement.addEventListener("patchwork:open-document", (event) => + opened.push((event as CustomEvent).detail) + ); + const router = createRouter({ + rootElement: rootElement as unknown as HTMLElement, + repo, + accountDocHandle: account as unknown as DocHandle<any>, + siteTitle: "test", + }); + const live = livePath(a, ["Projects", "foo", "notes"]); + return { a, live, notes, other, location, rootElement, opened, router }; +} + +const hashParam = (location: { hash: string }, key: string) => + new RegExp(`(?:^|[#&])${key}=([^&]*)`).exec(location.hash)?.[1]; + +// A document this repo doesn't have, for a doc= the path disagrees with. +const OTHER = "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("router: path=", () => { + it("writes an opened document's path to the hash", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + expect(hashParam(location, "doc")).toBe(notes.url); + expect(hashParam(location, "path")).toBe(live); + }); + + it("drops path from the hash when a document is opened without one", async () => { + const { other, location, rootElement } = await setup( + (_, n) => `doc=${n}&path=${livePath("x", ["y"])}` + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: other.url } }) + ); + expect(hashParam(location, "path")).toBeUndefined(); + }); + + it("opens what a path names when the hash has no doc", async () => { + const { live, notes, opened, location } = await setup((a) => + `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(opened).toHaveLength(1)); + expect(opened[0]).toMatchObject({ url: notes.url, path: live }); + expect(hashParam(location, "doc")).toBe(notes.url); + }); + + it("warns and drops a path that names nothing, opening nothing", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { opened, location } = await setup((a) => `path=${livePath(a, ["nope"])}`); + await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined()); + expect(opened).toHaveLength(0); + expect(warn).toHaveBeenCalled(); + }); + + it("keeps a path that agrees with doc, opening once", async () => { + const { live, opened, location, notes } = await setup( + (a, n) => `doc=${n}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(opened).toHaveLength(1)); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(opened).toHaveLength(1); + expect(opened[0]).toMatchObject({ url: notes.url, path: live }); + expect(hashParam(location, "path")).toBe(live); + }); + + it("drops a path that disagrees with doc, keeping doc", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { other, location } = await setup( + (a) => `doc=${OTHER}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + void other; + await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined()); + expect(hashParam(location, "doc")).toBe(OTHER); + expect(warn).toHaveBeenCalled(); + }); +}); From 0966c927379cc3098027bf72dcb2cd49d377be3b Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:13:48 +0000 Subject: [PATCH 38/81] changeset: path routing; e2e: open with &path= alone Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/paths-routing.md | 6 +++ packages/e2e/tests/path-routing.spec.ts | 72 +++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 .changeset/paths-routing.md create mode 100644 packages/e2e/tests/path-routing.spec.ts diff --git a/.changeset/paths-routing.md b/.changeset/paths-routing.md new file mode 100644 index 000000000..0f557825d --- /dev/null +++ b/.changeset/paths-routing.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The router keeps `&path=` in the hash: the live path a document was opened at (`<acctHash>/Projects/foo`), written whenever `patchwork:open-document` carries one and its segments kept exactly as encoded. A hash with `path=` and no `doc=` opens what the path names. When `doc=` and `path=` disagree, because something moved or was renamed, `doc=` wins and `path=` is dropped with a console warning. Only paths under the site's own account resolve. The e2e suite gains a test that opens a document from `&path=` alone. diff --git a/packages/e2e/tests/path-routing.spec.ts b/packages/e2e/tests/path-routing.spec.ts new file mode 100644 index 000000000..1d84254b0 --- /dev/null +++ b/packages/e2e/tests/path-routing.spec.ts @@ -0,0 +1,72 @@ +import { expect, test } from "@playwright/test"; + +/** + * `&path=` in the hash: a live path rooted at the account. A hash with only a + * path opens what it names, and the router writes the document's url beside + * it as `doc=`. + */ + +declare global { + interface Window { + patchwork?: { + account: { + documentId: string; + doc(): { rootFolderUrl?: string }; + }; + }; + } +} + +test("a hash with only path= opens what it names", async ({ page, browserName }) => { + test.skip( + browserName !== "chromium", + "needs the full-UI boot for an account, which is only reliable on chromium (see install-tool.spec.ts)" + ); + test.setTimeout(300_000); + + await page.goto("/", { timeout: 120_000 }); + await page.waitForFunction(() => !!window.patchwork?.account, null, { + timeout: 180_000, + }); + + const { url, path } = await page.evaluate(async () => { + const ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + const base58 = (bytes: Uint8Array) => { + let n = 0n; + for (const byte of bytes) n = n * 256n + BigInt(byte); + let out = ""; + while (n > 0n) { + out = ALPHABET[Number(n % 58n)] + out; + n /= 58n; + } + for (const byte of bytes) { + if (byte !== 0) break; + out = "1" + out; + } + return out; + }; + const account = window.patchwork!.account; + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(account.documentId)) + ); + const acctHash = base58(digest.slice(0, 12)); + + const file = window.repo.create<Record<string, unknown>>(); + file.change((d) => { + d["@patchwork"] = { type: "file" }; + d.content = "opened by path"; + d.mimeType = "text/plain"; + }); + const root = await window.repo.find<{ docs: unknown[] }>(account.doc().rootFolderUrl!); + root.change((d) => { + d.docs.push({ name: "path test", type: "file", url: file.url }); + }); + return { url: file.url, path: `${acctHash}/${encodeURIComponent("path test")}` }; + }); + + await page.evaluate((path) => (location.hash = `path=${path}`), path); + await expect + .poll(() => page.evaluate(() => location.hash), { timeout: 60_000 }) + .toContain(`doc=${url}`); + expect(await page.evaluate(() => location.hash)).toContain(`path=${path}`); +}); From 274dc28de8782f27383349d09a3f0d6ce6dd0ee1 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:16:45 +0000 Subject: [PATCH 39/81] spec: how phase 4 looks packages up, and keeps module settings for now Package lists are deep (a container with package.json lists everything under it), a redirect to a container carries its whole list, and the watcher looks a package up through its union member. The Packages UI still writes the module-settings doc, so it stays a source beside /packages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...28-paths-namespace-content-cache-design.md | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index cf182847b..402c11454 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -238,16 +238,34 @@ Builds on the `open-document-path` branch (48f4ab98): packages with the same name both load; the name only decides which one a path lookup finds. - Packages are tracked by document id, so a rename doesn't reload anything. - - When a package's own heads move, the watcher builds its pinned chain with - `pinPath`, with a list on every container from the package's own step - down, and imports it. A push that changes only files behind links - without heads must still touch the package doc; pushwork already bumps - `lastSyncAt` for that. Other packages keep their existing URLs: a + - When a package's own heads move, the watcher looks it up again, gets a + new pinned URL, and imports that. A push that changes only files behind + links without heads must still touch the package doc; pushwork already + bumps `lastSyncAt` for that. + - The lookup is a live URL through the union member that holds the + package: `/@/<acctHash>/packages/<memberIdhash>@<heads>/<name>`. That + finds a shadowed package too. A resolver answers with a 307 to the + package's pinned URL. The watcher imports the package as an http + package from that URL: `package.json`, then its entry, both checked + against the list. + - A package's list is **deep**: a container with a `package.json` file + lists every file under it, through nested folders. A folder package's + `./sub/x.js` is reached by name, not by a doc step, so it has to be in + the package's own list. Other containers list only their own files. + - A live → pinned redirect to a **container** carries that container's + whole list (deep, for a package). A redirect to a file carries only that + file's. A file in a nested folder is read through a link without heads, + so a resolver reads at a list's `heads` whether or not the path is + canonical. Other packages keep their existing URLs: a pinned URL stays valid forever, so a change elsewhere in the chain doesn't reload them. - `PackageTracker.findLatest` keeps asking the service worker, in the new grammar. - The site's `system` HTTP manifest stays a separate source. +- So does the module-settings doc, for now. The Packages UI (an installed + tool, not in this repo) still installs into it, so the watcher loads both + it and `/packages`, deduplicated by document id. A package in both is + imported through `/packages`. - A link under `/packages` that isn't a package (no `package.json`) is warned about and skipped. @@ -259,8 +277,9 @@ Builds on the `open-document-path` branch (48f4ab98): `/packages` binding, create a folder, add a link for each automerge entry (named by `package.json` `name`, falling back to the doc title), and bind it at `/packages`. The module-settings doc is left untouched. -- Builds older than this one keep reading the frozen module-settings list, so - packages installed afterwards won't show up there. +- Packages installed through the old Packages UI after the migration still + land in the module-settings doc, which this build keeps reading, but not + in `/packages`. Builds older than this one never see `/packages`. - `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding. From e464e19ff10295289b9ee3b13af20fa585f70a03 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:17:37 +0000 Subject: [PATCH 40/81] plan: paths phase 4 (packages via /packages) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../2026-10-28-paths-phase-4-packages.md | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md b/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md new file mode 100644 index 000000000..31daaea1e --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md @@ -0,0 +1,189 @@ +# Paths phase 4: packages via `/packages` — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Package management is whatever is bound at `/packages`. The ModuleWatcher loads every package in that union, looks each one up as a pinned `/@/` URL whose list covers the whole package, and imports it from there, checked against the list. New accounts get `/` and `/packages` bound. Old accounts are migrated once. + +**Architecture:** +- **Filesystem:** lists of packages go deep, and the walk exposes the union member a package was found in. +- **Bootloader resolver:** + - a redirect to a container carries its whole list; + - reads at `heads` ignore whether the path is canonical. +- **ModuleWatcher:** a `folder` source lists `/packages` and watches it. A composite tracker sends `/packages` packages through `/@/` lookups, and everything else through the legacy tracker. +- **Patchwork:** binds `/packages` on new accounts, migrates old ones, and wires the source. + +**Tech Stack:** TypeScript, automerge-repo, vitest (node, plus the bootloader's service worker harness). e2e runs in CI only. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 4, revised in 274dc28d, and section 5 for lists. + +## Global Constraints + +- The module list is the `/packages` union listing, deduplicated by document id. Two packages with the same name both load. +- Packages are tracked by document id; a rename reloads nothing. +- When a package's own heads move, the watcher looks it up again and imports the new pinned URL. Other packages keep their URLs. +- A link under `/packages` that isn't a package (no `package.json`) is warned about and skipped. +- Lookup: `/@/<acctHash>/packages/<memberIdhash>@<memberHeads>/<encoded name>` → 307 → the pinned URL. The watcher imports with `importPackageFromHttpUrl(pinnedUrl)`. +- A container with a `package.json` file lists every file under it, through nested folders. Others list only their own files. +- A redirect to a container carries the container's whole list; a redirect to a file, the one-file list. +- The module-settings doc stays a source; its automerge entries are deduplicated against `/packages` by document id. +- `createDefaultAccount` binds `/` to the root folder and `/packages` to a new folder. +- Migration happens once at boot, when the account has `moduleSettingsUrl` and no `/packages` binding: + - create a folder; + - link each automerge entry, named by its `package.json` `name`, falling back to the doc title, then to its documentId; + - bind the folder at `/packages`, leaving `/` to fall back to `rootFolderUrl` as before; + - leave the module-settings doc untouched. +- `AccountDoc.moduleSettingsUrl` is `@deprecated`. + +## Decisions this plan makes + +1. **Package lists recurse into nested folders,** but only for a container that has a `package.json`, so fetching a user folder never lists the whole tree. Tails cross folders by name (`sub/x.js`). Each nested file records its own `heads`, so a resolver walks to it by name and reads it at those heads. +2. **Nested folders aren't pinned by heads in a package's list.** A rebuild walks them at their current heads. If a nested folder was restructured, a rebuild of an old list won't match, and that old URL fails as a network error. The live lookup gives a new URL, so imports recover. +3. **Lookups always use the member step.** The tab lists the union, so it knows each member's idhash and heads. The member step finds shadowed packages too, and works the same for unique names. +4. **Composite tracker.** The ModuleWatcher keeps one tracker: + - `findLatest(url)` answers with the `/@/` lookup for a document id the `/packages` index knows, and with the legacy service worker redirect otherwise; + - `watch` is shared. + + Its `version` strings can be `/@/` URLs. `importPackageSafe` already sends anything that isn't an automerge URL to `importPackageFromHttpUrl`. +5. **Discovery off the main thread is skipped** for `/@/` packages: the module-loader worker path takes automerge URLs only. Moving it over is a follow-up. +6. **The `/packages` index runs in the tab.** It loads the account doc (already loaded) and each member folder doc (small), never the package docs. Each package's `package.json` presence is checked through the lookup instead: a lookup whose list has no `package.json` entry is warned about and skipped. + +## Review Focus + +- **A package that appears both in module settings and `/packages`** is imported once, through `/packages`. Pinned in Task 4. +- **A rename inside `/packages`** reloads nothing. Removing the link unloads the package. Pinned in Task 4. +- **The same name in two union members:** both load. Pinned in Task 4. +- **A folder package with `./sub/x.js`** imports through its pinned URL. Pinned in Tasks 1 and 2 (filesystem and harness). +- **Migration runs once.** A second boot changes nothing, and an account with `/packages` already bound isn't migrated. Pinned in Task 5. + +--- + +### Task 1: Deep lists for packages; the redirect to a container carries its list + +**Files:** +- `core/filesystem/src/walk.ts` (`listFiles` gains `deep`) and `src/file-list.ts`. +- `core/bootloader/src/resolver.ts`. +- Tests: `core/filesystem/test/file-list.test.ts` and `core/bootloader/test/resolver.test.ts`. + +**Interfaces:** +- `listFiles(repo, path, options & { deep?: boolean })`. With `deep`, a link child whose document has a provider is listed into, with its name as a tail prefix, instead of being skipped. Branches and account docs are still skipped. Each recursion guards against cycles by documentId. +- `buildFileList` sets `deep` when the container's own listing has a `package.json` entry. That check runs on the shallow pass, then the list is redone deep. +- `pinnedPath(…, { only: "end" })`: + - when the path ends at a container (`end.tail === ""` and `end.atDocument`), it builds the container's full list; + - otherwise it builds the one-file list. +- In the resolver's `resolveWalkPath`, a request with `heads` reads with `readPath(…, { heads })` whether or not the path is canonical. Only a request without heads and without `list` redirects when the path isn't canonical. + +- [ ] **Step 1: Failing tests** + +```ts +// file-list.test.ts +it("lists a package's nested folders deep", ...) + // a folder package {package.json, sub → folder {x.js}} → tails include "package.json", "sub/x.js" +it("lists a user folder shallow", ...) // Projects/foo has sub folder, no package.json → no "sub/…" +it("reads a nested file at its recorded heads", ...) // readPath(`${prefix}/sub/x.js`, {heads}) after x changed → old +// resolver.test.ts +it("redirects a live path to a container with its whole list", ...) // body list keys include every file +it("reads at heads through a folder reached by name", ...) // path pkg@h/sub/x.js with heads → 200 old bytes +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** both suites. +- [ ] **Step 5: Commit** `paths: package lists go deep; a redirect to a folder carries its list` + +### Task 2: The service worker serves a folder package through the harness + +**Files:** `core/bootloader/test/service-worker.test.ts` only, unless the test exposes a bug. + +- [ ] **Step 1: Test.** An account binds `/packages` to a folder holding a folder package: `package.json` → `{"main":"index.js"}`, `index.js` → `export * from "./sub/x.js"`, and `sub/x.js`. Then: + - fetch the live `/@/<a>/packages/pkg` → 307; + - fetch `<location>/package.json` and `<location>/sub/x.js` → both 200, `immutable`; + - store holds the list and both files. +- [ ] **Step 2: Run.** It should pass if Task 1 is right; a failure here is a bug to fix with systematic debugging. +- [ ] **Step 3: Commit** `service worker: test a folder package end to end` + +### Task 3: The `/packages` index and a composite tracker + +**Files:** create `core/filesystem/src/packages-folder.ts`; modify `src/index.ts`; test `core/filesystem/test/packages-folder.test.ts`. + +**Interfaces:** +```ts +export type PackageEntry = { + url: AutomergeUrl; // the package doc, without heads + name: string; + memberStep: string; // docStep(memberIdhash, memberHeads) +}; +/** Watch an account's /packages union: call onChange with its entries (deduplicated by documentId, first binding wins) now and whenever the account doc or a member folder changes. */ +export function watchPackagesFolder(repo: Repo, account: DocHandle<AccountNamespaceDoc>, + onChange: (entries: PackageEntry[]) => void): () => void; +/** The live lookup path for an entry. */ +export function packageLookupPath(acctHash: string, entry: PackageEntry): string; + // `${acctHash}/packages/${entry.memberStep}/${encodeURIComponent(entry.name)}` +/** A tracker that answers findLatest through /@/ for the urls `entries()` knows, and through `fallback` otherwise. */ +export function trackPackagesFolder(acctHash: Promise<string>, entries: () => PackageEntry[], + fallback: PackageTracker, lookUp?: (path: string) => Promise<string>): PackageTracker; + // lookUp default: fetch(`/@/${path}`) → response.url (redirected) → the pinned URL; throws if not redirected +``` + +- [ ] **Step 1: Failing tests:** entries from a two-member union, deduplicated, with the same name in both; onChange after a member folder gains an entry; `packageLookupPath`; the tracker routes a known url to `lookUp` and an unknown one to `fallback`. +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement.** **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `filesystem: watch /packages and look packages up through it` + +### Task 4: The ModuleWatcher loads `/packages` + +**Files:** `core/filesystem/src/module-watcher.ts`; test `core/filesystem/test/module-watcher.test.ts`. + +**Interfaces:** +- `ModuleWatcher.addPackagesFolder(name: string, entries: () => PackageEntry[], subscribe: (onChange: () => void) => () => void): Promise<void>`. +- The module list is the settings docs' modules plus the folder's entry urls, deduplicated by `stripHeads` (document id). An entry present in `/packages` is keyed by its bare url. The tracker (composite, constructed by the caller) decides how it's looked up. + +- [ ] **Step 1: Failing tests** (existing fakes in module-watcher.test.ts): + +```ts +it("loads every package in the folder", ...) +it("loads a package in both settings and /packages once", ...) +it("doesn't reload a renamed package", ...) +it("unloads a package removed from the folder", ...) +it("loads both packages with one name in two members", ...) +``` + +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement.** **Step 4: Run the filesystem suite, expect PASS.** +- [ ] **Step 5: Commit** `filesystem: ModuleWatcher loads the /packages folder` + +### Task 5: Accounts bind `/packages`; migration; wiring + +**Files:** +- `core/patchwork/src/createAccount.ts`. +- Create `core/patchwork/src/migrate-packages.ts`. +- `core/patchwork/src/index.ts` (wire the folder source and the composite tracker). +- `core/plugins/src/account.ts` (`@deprecated` on `moduleSettingsUrl`). +- Test `core/patchwork/test/migrate-packages.test.ts`. + +**Interfaces:** +- `migratePackages(repo, account): Promise<boolean>`: + - true if it migrated; + - a no-op when `/packages` is bound or there's no `moduleSettingsUrl`; + - names come from `package.json` `name` (read with `resolvePath(repo, pkg, ["package.json"])`), then the doc's `title`, then its documentId. +- `createDefaultAccount`: `doc.namespace = { "/": [{ url: rootFolder.url }], "/packages": [{ url: packagesFolder.url }] }`. The packages folder is titled "Packages". `moduleSettingsUrl` is still created, for the Packages UI. + +- [ ] **Step 1: Failing tests:** + - migration links each automerge entry by its `package.json` name and binds `/packages`; + - a second run is a no-op; + - an http entry is left to module settings; + - an account already bound isn't touched; + - `createDefaultAccount` binds both mounts. +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement**, then wire in `index.ts`: + - after `registerAccount`, `await migratePackages`; + - `watchPackagesFolder` feeds `moduleWatcher.addPackagesFolder("packages", …)`; + - the tracker passed to the ModuleWatcher is `trackPackagesFolder(acctHash, () => latestEntries, trackPackagesViaServiceWorker(…))`; + - `acctHash` comes from `registerAccount`'s return. The watcher is built before the account resolves, so the tracker takes promises/getters. +- [ ] **Step 4: Run the patchwork suite and typecheck, expect PASS.** +- [ ] **Step 5: Commit** `patchwork: /packages on new accounts, migration, and wiring` + +### Task 6: Changeset + +- [ ] **One changeset** for the filesystem, bootloader, plugins and patchwork packages, saying: + - `/packages` is package management; + - packages are imported through checked `/@/` URLs; + - module settings is deprecated but still read; + - what moved: `moduleSettingsUrl` is `@deprecated`. +- [ ] Commit. From a616a4a59588fe199cd70726646aec618a5cd407 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:20:53 +0000 Subject: [PATCH 41/81] router: review fixes for path routing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A test let the router's background path check outlive it, and the suite exited 1 on the leaked error. - A path walked on its own no longer opens its document once the hash has moved on. - A path that doesn't resolve (yet) opens nothing and stays in the address bar, instead of being wiped; beside doc=, it's dropped only when it names a different document. - path= survives a tool switch on the same document, as draft= does. - The router tests now fire hashchange the way a browser does, so the open → hashchange → route cycle is covered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/open-document-path.md | 2 +- .changeset/paths-routing.md | 2 +- core/patchwork/src/router.ts | 33 ++++++---- core/patchwork/test/router.test.ts | 97 +++++++++++++++++++++++++----- 4 files changed, 106 insertions(+), 28 deletions(-) diff --git a/.changeset/open-document-path.md b/.changeset/open-document-path.md index 19451f1c0..5f5cbd2a5 100644 --- a/.changeset/open-document-path.md +++ b/.changeset/open-document-path.md @@ -3,4 +3,4 @@ "@inkandswitch/patchwork": patch --- -`patchwork:open-document` (and `openDocument`, `patchwork.open`, `<patchwork-view path>`) can say where a document was opened from: `path` is the live path it was opened at, like a cwd, e.g. `<acctHash>/Projects/foo` with each segment `encodeURIComponent`-encoded. The router writes it to `&path=`. A `<patchwork-view>` hands a new path to its tool without remounting it. +`patchwork:open-document` (and `openDocument`, `patchwork.open`, `<patchwork-view path>`) can say where a document was opened from: `path` is the live path it was opened at, like a cwd, e.g. `<acctHash>/Projects/foo` with each segment `encodeURIComponent`-encoded. A `<patchwork-view>` hands a new path to its tool without remounting it. diff --git a/.changeset/paths-routing.md b/.changeset/paths-routing.md index 0f557825d..88ad4c6a1 100644 --- a/.changeset/paths-routing.md +++ b/.changeset/paths-routing.md @@ -3,4 +3,4 @@ "@inkandswitch/patchwork-e2e": patch --- -The router keeps `&path=` in the hash: the live path a document was opened at (`<acctHash>/Projects/foo`), written whenever `patchwork:open-document` carries one and its segments kept exactly as encoded. A hash with `path=` and no `doc=` opens what the path names. When `doc=` and `path=` disagree, because something moved or was renamed, `doc=` wins and `path=` is dropped with a console warning. Only paths under the site's own account resolve. The e2e suite gains a test that opens a document from `&path=` alone. +The router keeps `&path=` in the hash: the live path a document was opened at (`<acctHash>/Projects/foo`), written whenever `patchwork:open-document` carries one and its segments kept exactly as encoded. A hash with `path=` and no `doc=` opens what the path names; one that doesn't resolve (yet) opens nothing and stays in the address bar. When `path=` names a different document than `doc=`, because something moved or was renamed, `doc=` wins and `path=` is dropped with a console warning. `path=` survives a tool switch on the same document. Paths under the site's own account resolve, and so do share links. The e2e suite gains a test that opens a document from `&path=` alone. diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index 3a357b31e..9af7fed63 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -77,10 +77,16 @@ export function createRouter({ params.delete("path"); window.location.hash = serializeHashParams(params); }; - /** `doc=` says what is open; when `path=` names something else, `doc=` wins. */ + /** + * `doc=` says what is open; when `path=` names something else, `doc=` wins. + * A path that doesn't resolve stays: it may name something not synced yet. + */ const checkPath = async (docUrl: AutomergeUrl, path: string) => { const handle = await resolvePathParam(path); - if (!handle) return dropPath(path, "it names nothing"); + if (!handle) { + console.warn(`patchwork: path=${path} doesn't resolve here (yet)`); + return; + } if (handle.documentId !== parseAutomergeUrl(docUrl).documentId) { dropPath(path, `it names ${handle.url}, not ${docUrl}`); } @@ -163,10 +169,14 @@ export function createRouter({ open(docUrl); if (path) void checkPath(docUrl, path); } else if (path) { - // Just a path: open what it names. + // Just a path: open what it names, unless the hash has moved on while + // it was walked. One that doesn't resolve is left in the address bar, + // to be fixed or retried. const handle = await resolvePathParam(path); + const now = parseHashParams(window.location.hash.slice(1)); + if (now.get("path") !== path || now.get("doc")) return; if (handle) open(handle.url); - else dropPath(path, "it names nothing"); + else console.warn(`patchwork: path=${path} doesn't resolve here (yet); opening nothing`); } }; @@ -187,15 +197,16 @@ export function createRouter({ // navigating to a different document invalidates the selection, while // same-doc navigation (tool switches, heads changes) keeps it. const prevDoc = docParamToUrl(params.get("doc")); - if ( - prevDoc && - parseAutomergeUrl(prevDoc).documentId !== parseAutomergeUrl(url).documentId - ) { - params.delete("draft"); - } + const sameDoc = + !!prevDoc && + parseAutomergeUrl(prevDoc).documentId === parseAutomergeUrl(url).documentId; + if (prevDoc && !sameDoc) params.delete("draft"); + // Where the doc was opened from survives a tool switch or heads change on + // the same doc, as `draft` does. + const keptPath = !path && sameDoc ? params.get("path") ?? undefined : path; params.set("doc", url); for (const [key, value] of [ - ["path", path], + ["path", keptPath], ["tool", toolId], ["title", title], ["type", type], diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts index 8e786529c..261220f66 100644 --- a/core/patchwork/test/router.test.ts +++ b/core/patchwork/test/router.test.ts @@ -25,17 +25,31 @@ class FakeElement extends EventTarget { } } -/** A location whose hash reads back with its "#", as a browser's does. */ -function fakeLocation(initial: string) { +/** + * A window whose location's hash reads back with its "#", and which fires + * hashchange (after the current task) only when the hash really changes, as a + * browser does. + */ +function fakeWindow(initial: string) { let hash = initial; - return { + const target = new EventTarget(); + const location = { get hash() { return hash ? `#${hash}` : ""; }, set hash(value: string) { - hash = value.replace(/^#/, ""); + const next = value.replace(/^#/, ""); + if (next === hash) return; + hash = next; + setTimeout(() => target.dispatchEvent(new Event("hashchange"))); }, }; + const window = { + location, + addEventListener: (type: string, listener: EventListener) => + target.addEventListener(type, listener), + }; + return { window, location }; } async function setup(hash: (a: string, notes: AutomergeUrl) => string) { @@ -56,9 +70,9 @@ async function setup(hash: (a: string, notes: AutomergeUrl) => string) { }); const a = await idhash(account.documentId); - const location = fakeLocation(hash(a, notes.url)); + const { window, location } = fakeWindow(hash(a, notes.url)); vi.stubGlobal("location", location); - vi.stubGlobal("window", { location, addEventListener() {} }); + vi.stubGlobal("window", window); vi.stubGlobal("document", { title: "" }); const rootElement = new FakeElement(); @@ -83,7 +97,14 @@ const hashParam = (location: { hash: string }, key: string) => // A document this repo doesn't have, for a doc= the path disagrees with. const OTHER = "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ"; -afterEach(() => vi.unstubAllGlobals()); +/** Let the router's background work (path walks, hashchanges) settle. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 300)); + +afterEach(async () => { + await settle(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); describe("router: path=", () => { it("writes an opened document's path to the hash", async () => { @@ -95,9 +116,10 @@ describe("router: path=", () => { expect(hashParam(location, "path")).toBe(live); }); - it("drops path from the hash when a document is opened without one", async () => { - const { other, location, rootElement } = await setup( - (_, n) => `doc=${n}&path=${livePath("x", ["y"])}` + it("drops path from the hash when another document is opened without one", async () => { + const { live, notes, other, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) ); rootElement.dispatchEvent( new CustomEvent("patchwork:open-document", { detail: { url: other.url } }) @@ -109,17 +131,62 @@ describe("router: path=", () => { const { live, notes, opened, location } = await setup((a) => `path=${livePath(a, ["Projects", "foo", "notes"])}` ); - await vi.waitFor(() => expect(opened).toHaveLength(1)); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); expect(opened[0]).toMatchObject({ url: notes.url, path: live }); expect(hashParam(location, "doc")).toBe(notes.url); }); - it("warns and drops a path that names nothing, opening nothing", async () => { + it("keeps path when the same document is opened without one, as on a tool switch", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, toolId: "other" } }) + ); + expect(hashParam(location, "path")).toBe(live); + expect(hashParam(location, "tool")).toBe("other"); + }); + + it("warns about a path that names nothing, opens nothing, and leaves the hash", async () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - const { opened, location } = await setup((a) => `path=${livePath(a, ["nope"])}`); - await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined()); + const { a, opened, location } = await setup((a) => `path=${livePath(a, ["nope"])}`); + await vi.waitFor(() => expect(warn).toHaveBeenCalled()); + await settle(); expect(opened).toHaveLength(0); - expect(warn).toHaveBeenCalled(); + expect(hashParam(location, "path")).toBe(livePath(a, ["nope"])); + }); + + it("keeps a path beside doc when the path can't be resolved", async () => { + const { a, location } = await setup((a, n) => `doc=${n}&path=${livePath(a, ["nope"])}`); + await settle(); + expect(hashParam(location, "path")).toBe(livePath(a, ["nope"])); + }); + + it("doesn't open a path's document after the hash has moved on", async () => { + const { notes, opened, location } = await setup( + (a) => `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + // a click lands while the path is still being walked + location.hash = `doc=${OTHER}`; + await settle(); + expect(opened.map((o) => o.url)).not.toContain(notes.url); + }); + + it("settles after opening from a path alone: doc= and path= stay, nothing loops", async () => { + const { live, notes, opened, location } = await setup( + (a) => `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(hashParam(location, "doc")).toBe(notes.url)); + await settle(); + const settledHash = location.hash; + const count = opened.length; + await settle(); + expect(location.hash).toBe(settledHash); + expect(opened.length).toBe(count); + expect(hashParam(location, "path")).toBe(live); + // the open, then the hashchange it caused: no more than that + expect(count).toBeLessThanOrEqual(2); }); it("keeps a path that agrees with doc, opening once", async () => { From 8da1f717442445b08023094cb21d3139839138a6 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:23:19 +0000 Subject: [PATCH 42/81] paths: package lists go deep; a redirect to a folder carries its list A container with a package.json lists every file under it, through nested folders, so a folder package's ./sub/x.js is in its own list. A live redirect to a folder carries the folder's whole list. A read at a list's recorded heads goes through folders reached by name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/bootloader/src/resolver.ts | 21 +++++++-- core/bootloader/test/resolver.test.ts | 33 ++++++++++++++ core/filesystem/src/walk.ts | 63 +++++++++++++++++++++----- core/filesystem/test/file-list.test.ts | 58 +++++++++++++++++++++++- 4 files changed, 158 insertions(+), 17 deletions(-) diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 6a6040ca9..0a130e950 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -222,7 +222,9 @@ export function waitingFind( * - A live path, or a pinned-looking one that isn't canonical: a 307 to its pinned form, whose * list covers just the file asked for; the body is that list, for the service worker to check * and keep. - * - A canonical pinned path: the bytes, read at `heads` if given (what its file list recorded). + * - With `heads`: the file the path names, read at those heads (what its file list recorded), + * whether or not the path is canonical. + * - A canonical pinned path: the bytes. * - One that doesn't resolve: 404. * * Pinned heads that never arrive throw {@link AbortHandoff}. @@ -265,6 +267,19 @@ async function resolveWalkPath( }); } + if (heads) { + // A file a list recorded, read at the heads it recorded. A file in a + // package's nested folder is reached by name, through a link without + // heads, so the path needn't be canonical; the service worker checks the + // bytes against the list either way. + const read = await readPath(repo, path, { ...options, heads: parseHeads(heads) }); + if (!read) return notFound(); + return new Response(toBytes(read.content) as BlobPart, { + status: 200, + headers: { "content-type": read.type }, + }); + } + const end = await walkPath(repo, path, options); if (!end) return notFound(); if (!end.canonical) { @@ -287,9 +302,7 @@ async function resolveWalkPath( }); } - const read = heads - ? await readPath(repo, path, { ...options, heads: parseHeads(heads) }) - : await readNode(repo, end.node); + const read = await readNode(repo, end.node); if (!read) return notFound(); return new Response(toBytes(read.content) as BlobPart, { status: 200, diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index be447f51b..7d4f4794c 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -159,6 +159,39 @@ describe("resolver: /@/ paths", () => { }); }); +describe("resolver: folder packages", () => { + async function folderPackage() { + const f = await fixture(); + const x = f.repo.create({ "@patchwork": { type: "file" }, content: "x", mimeType: "text/plain" }); + const sub = folder(f.repo, { "x.js": x.url }); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"index.js"}'), + sub: sub.url, + }); + const packages = folder(f.repo, { fpkg: pkg.url }); + f.account.change((d: any) => (d.namespace["/packages"] = [{ url: packages.url }])); + return { ...f, x }; + } + + it("redirects a live path to a folder with the folder's whole list", async () => { + const { a, resolver } = await folderPackage(); + const reply = await resolver.answer(pathRequest(livePath(a, ["packages", "fpkg"]))); + expect(inline(reply).status).toBe(307); + const list = parseFileList(inline(reply).body as Uint8Array)!; + expect(Object.keys(list.files)).toEqual(expect.arrayContaining(["package.json", "sub/x.js"])); + }); + + it("reads at heads through a folder reached by name", async () => { + const { repo, a, accounts, resolver, x } = await folderPackage(); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const heads = formatHeads(x.heads()); + x.change((d: any) => (d.content = "changed")); + const reply = await resolver.answer(withRequest(`${prefix}/sub/x.js`, { heads })); + expect(inline(reply).status).toBe(200); + expect(body(reply)).toBe("x"); + }); +}); + describe("resolver: accounts it doesn't know", () => { it("a tab declines, so another tab or the worker is asked", async () => { const { repo, a } = await fixture(); diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 4da3f88a7..928c35069 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -19,6 +19,7 @@ import { type Child, type Lookup, directoryProvider, + folderProvider, type PathProvider, accountProvider, branchesProvider, @@ -351,9 +352,12 @@ export async function pinnedPath( const end = await walkPath(repo, path, options); if (!end) return undefined; if (!end.container) return { path: end.pinned }; + // A path that ends at a container lists the whole container; one that ends + // at a file, just that file. + const atContainer = end.tail === "" && end.atDocument; const built = await buildFileList(repo, end.container.pinned, { ...options, - only: options.only === "end" ? [end.tail] : undefined, + only: options.only === "end" && !atContainer ? [end.tail] : undefined, }); if (!built) return { path: end.pinned }; if (built.missing.length) throw new IncompleteFileListError(built.missing); @@ -485,20 +489,33 @@ const LIST_CONCURRENCY = 8; export async function listFiles( repo: Repo, path: string, - options: WalkOptions & { findFile?: WalkOptions["find"]; only?: string[] } + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: string[]; + /** List into nested folders. Default: when the container has a `package.json`. */ + deep?: boolean; + } ): Promise<{ prefix: string; files: ListedFile[]; missing: string[] } | undefined> { const end = await walkPath(repo, path, options); if (!end?.canonical || !end.container) return undefined; const find = options.findFile ?? options.find ?? ((url: AutomergeUrl) => repo.find(url)); const root = end.container.handle.doc(); - const provider = providerFor(root); - const documentId = end.container.handle.documentId; + const rootProvider = providerFor(root); + const rootId = end.container.handle.documentId; + // A package's relative imports reach into its nested folders by name, so a + // package lists everything under it. Anything else lists its own files. + const deep = + options.deep ?? + (rootProvider?.list(root) ?? []).some((child) => child.name === "package.json"); const found = new Map<string, { keys: number[]; file: ListedFile }>(); const missing: string[] = []; const tailOf = (parts: string[]) => parts.map(encodeURIComponent).join("/"); const wanted = (parts: string[]) => !options.only || options.only.includes(tailOf(parts)); + // A folder is worth entering if anything wanted could be under it. + const wantedBelow = (parts: string[]) => + !options.only || options.only.some((tail) => tail.startsWith(`${tailOf(parts)}/`)); const add = async ( parts: string[], keys: number[], @@ -521,7 +538,14 @@ export async function listFiles( // Values are read in place; links are gathered and read below. const links: { parts: string[]; keys: number[]; link: AutomergeUrl }[] = []; - const visit = async (node: unknown, parts: string[], keys: number[]) => { + const entered = new Set<string>([rootId]); + const visit = async ( + node: unknown, + provider: PathProvider | undefined, + holder: DocumentId, + parts: string[], + keys: number[] + ) => { for (const child of provider?.list(node) ?? []) { // Only a directory's keys span several path segments; a folder entry's name is one. const names = @@ -529,18 +553,21 @@ export async function listFiles( const childParts = [...parts, ...names]; const childKeys = [...keys, names.length]; if ("value" in child) { - await add(childParts, childKeys, child.value, documentId); + await add(childParts, childKeys, child.value, holder); // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { - await visit(child.value, childParts, childKeys); + await visit(child.value, provider, holder, childParts, childKeys); } - } else if ("link" in child && wanted(childParts)) { + } else if ( + "link" in child && + (wanted(childParts) || (deep && wantedBelow(childParts))) + ) { links.push({ parts: childParts, keys: childKeys, link: child.link }); } } }; - await add([], [], root, documentId); - await visit(root, [], []); + await add([], [], root, rootId); + await visit(root, rootProvider, rootId, [], []); let next = 0; const readLinks = async () => { @@ -554,8 +581,20 @@ export async function listFiles( ); const heads = parsed.heads ?? live.heads(); const doc = live.view(heads).doc(); - if (doc === undefined || providerFor(doc)) continue; - await add(parts, keys, doc, parsed.documentId, heads); + if (doc === undefined) continue; + const provider = providerFor(doc); + if (!provider) { + await add(parts, keys, doc, parsed.documentId, heads); + } else if ( + deep && + (provider === folderProvider || provider === directoryProvider) && + !entered.has(parsed.documentId) + ) { + // Into a nested folder, by name: its files are under its name here. + entered.add(parsed.documentId); + await add(parts, keys, doc, parsed.documentId, heads); + await visit(doc, provider, parsed.documentId, parts, keys); + } } catch { missing.push(tailOf(parts)); } diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index ea13684ed..6e222839b 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -4,7 +4,7 @@ import { formatHeads, livePath, parseHeads } from "../src/paths.js"; import { IncompleteFileListError, pinnedPath, readPath } from "../src/walk.js"; import { contentHash, toBytes } from "../src/content-hash.js"; import { buildFileList, canonicalJson, parseFileList } from "../src/file-list.js"; -import { chain, fixture } from "./namespace-fixture.js"; +import { chain, file, fixture, folder } from "./namespace-fixture.js"; const decode = (content: string | Uint8Array) => typeof content === "string" ? content : new TextDecoder().decode(content); @@ -94,3 +94,59 @@ describe("pinnedPath", () => { await expect(pinning).rejects.toMatchObject({ missing: ["package.json"] }); }); }); + +describe("package lists", () => { + /** A folder package under /packages: package.json, index.js, and sub/x.js. */ + async function folderPackage() { + const f = await fixture(); + const x = file(f.repo, "x"); + const sub = folder(f.repo, { "x.js": x.url }); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"index.js"}').url, + "index.js": file(f.repo, "export * from './sub/x.js'").url, + sub: sub.url, + }); + f.mine.change((d: any) => d.docs.push({ name: "fpkg", type: "folder", url: pkg.url })); + const prefix = await chain(f.repo, livePath(f.a, ["packages", "fpkg"]), f.accounts); + return { ...f, x, sub, pkg, prefix: prefix! }; + } + + it("lists a package's nested folders deep", async () => { + const { repo, accounts, prefix } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + expect(Object.keys(built!.list.files)).toEqual( + expect.arrayContaining(["package.json", "index.js", "sub/x.js"]) + ); + }); + + it("lists a user folder shallow", async () => { + const { repo, a, accounts, foo } = await fixture(); + const sub = folder(repo, { inner: file(repo, "inner").url }); + foo.change((d: any) => d.docs.push({ name: "sub", type: "folder", url: sub.url })); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(Object.keys(built!.list.files).some((tail) => tail.startsWith("sub/"))).toBe(false); + }); + + it("reads a nested file at its recorded heads", async () => { + const { repo, accounts, prefix, x } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + const entry = built!.list.files["sub/x.js"]; + x.change((d: any) => (d.content = "changed")); + const then = await readPath(repo, `${prefix}/sub/x.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("x"); + }); + + it("pins a live path to a folder with the folder's whole list", async () => { + const { repo, a, accounts } = await folderPackage(); + const pinned = await pinnedPath(repo, livePath(a, ["packages", "fpkg"]), { + accounts, + only: "end", + }); + const list = parseFileList(pinned!.list!)!; + expect(Object.keys(list.files)).toEqual(expect.arrayContaining(["package.json", "sub/x.js"])); + }); +}); From f7f07e949a606ab3d7099dd8ed5f5fef21cecb23 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:23:35 +0000 Subject: [PATCH 43/81] service worker: test a folder package end to end Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/bootloader/test/service-worker.test.ts | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts index 4f58aa1ae..b9ac74876 100644 --- a/core/bootloader/test/service-worker.test.ts +++ b/core/bootloader/test/service-worker.test.ts @@ -108,6 +108,43 @@ describe("service worker: /@/ paths", () => { expect(await stored.match(`${ORIGIN}/.content/${await contentHash(toBytes("hi"))}`)).toBeDefined(); }); + it("serves a folder package's nested files through its pinned URL", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/javascript" }).url; + const folder = (entries: Record<string, AutomergeUrl>) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }).url; + const pkg = folder({ + "package.json": file('{"main":"index.js"}'), + "index.js": file("export * from './sub/x.js'"), + sub: folder({ "x.js": file("export const x = 1") }), + }); + const acct = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: folder({ fpkg: pkg }) }] }, + }); + const a = await registerAccount(acct.url); + + const pinned = await pin(sw, livePath(a, ["packages", "fpkg"])); + for (const [tail, content] of [ + ["package.json", '{"main":"index.js"}'], + ["sub/x.js", "export const x = 1"], + ]) { + const event = sw.fetch(new Request(`${pinned}/${tail}`), "tab"); + const response = await event.response()!; + expect(await response.text()).toBe(content); + expect(response.headers.get("cache-control")).toContain("immutable"); + await event.settled(); + } + // the list, and the two files read + expect(sw.caches.named.get(`patchwork-content-${a}`)!.entries.size).toBe(3); + }); + it("serves the old bytes for a pinned URL after the file changed", async () => { sw = await startServiceWorker(); const repo = new Repo(); From df1abe12161809850c186252b81acf6488bc132b Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:24:46 +0000 Subject: [PATCH 44/81] filesystem: watch /packages and look packages up through it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/index.ts | 1 + core/filesystem/src/packages-folder.ts | 144 +++++++++++++++++++ core/filesystem/test/packages-folder.test.ts | 92 ++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 core/filesystem/src/packages-folder.ts create mode 100644 core/filesystem/test/packages-folder.test.ts diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index 5c148d184..f647c7c62 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -6,6 +6,7 @@ export * from "./metadata.js"; export * from "./module-watcher.js"; export * from "./package-tracker.js"; export * from "./packages.js"; +export * from "./packages-folder.js"; export * from "./paths.js"; export * from "./providers.js"; export * from "./resolve.js"; diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts new file mode 100644 index 000000000..bd1729205 --- /dev/null +++ b/core/filesystem/src/packages-folder.ts @@ -0,0 +1,144 @@ +// Package management is whatever is bound at `/packages`: the account's union +// of package folders. This lists it for the ModuleWatcher, and looks each +// package up as a pinned `/@/` URL whose file list covers the whole package. +import { + parseAutomergeUrl, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocHandle, + type Repo, +} from "@automerge/automerge-repo/slim"; +import type { PackageTracker } from "./package-tracker.js"; +import { docStep, idhash } from "./paths.js"; +import { accountBindings, providerFor } from "./providers.js"; +import type { AccountNamespaceDoc } from "./types.js"; +import { documentBaseOrigin } from "./urls.js"; + +export type PackageEntry = { + /** The package doc, without heads. */ + url: AutomergeUrl; + name: string; + /** The union member it's in, as a doc step: how a lookup finds it, shadowed or not. */ + memberStep: string; +}; + +const bare = (url: AutomergeUrl) => + stringifyAutomergeUrl({ documentId: parseAutomergeUrl(url).documentId }); + +/** + * Watch an account's `/packages` union. `onChange` gets its packages, one per + * document (the first binding that has it wins), now and whenever the account + * doc or a member folder changes. Only the account and the member folders are + * loaded, never the packages. + */ +export function watchPackagesFolder( + repo: Repo, + account: DocHandle<AccountNamespaceDoc>, + onChange: (entries: PackageEntry[]) => void +): () => void { + let stopped = false; + let members: DocHandle<unknown>[] = []; + let last: string | undefined; + let pending: Promise<void> | undefined; + + const list = async () => { + const urls = accountBindings(account.doc()).get("/packages") ?? []; + const handles = ( + await Promise.all(urls.map((url) => repo.find<unknown>(url).catch(() => undefined))) + ).filter((handle): handle is DocHandle<unknown> => !!handle); + if (stopped) return; + for (const member of members) member.off("change", changed); + members = handles; + for (const member of members) member.on("change", changed); + + const entries: PackageEntry[] = []; + const seen = new Set<string>(); + for (const member of members) { + const doc = member.doc(); + const memberStep = docStep(await idhash(member.documentId), member.heads()); + for (const child of providerFor(doc)?.list(doc) ?? []) { + if (!("link" in child)) continue; + const { documentId } = parseAutomergeUrl(child.link); + if (seen.has(documentId)) continue; + seen.add(documentId); + entries.push({ url: bare(child.link), name: child.name, memberStep }); + } + } + const key = JSON.stringify(entries); + if (stopped || key === last) return; + last = key; + onChange(entries); + }; + + // Changes arrive in bursts; list once per burst, and once more if one + // landed while listing. + let again = false; + function changed() { + if (pending) { + again = true; + return; + } + pending = Promise.resolve() + .then(list) + .catch((error) => console.warn("couldn't list /packages", error)) + .finally(() => { + pending = undefined; + if (again && !stopped) { + again = false; + changed(); + } + }); + } + + account.on("change", changed); + changed(); + return () => { + stopped = true; + account.off("change", changed); + for (const member of members) member.off("change", changed); + }; +} + +/** The live lookup path for a package: through the member that holds it. */ +export function packageLookupPath(acctHash: string, entry: PackageEntry): string { + return `${acctHash}/packages/${entry.memberStep}/${encodeURIComponent(entry.name)}`; +} + +/** + * Follow a live `/@/` path's redirect to its pinned URL. The service worker + * keeps the file list the redirect carries. + */ +async function lookUpViaServiceWorker(path: string): Promise<string> { + const response = await fetch(`${documentBaseOrigin()}/@/${path}`); + if (!response.ok || !response.redirected) { + throw new Error( + `the service worker didn't redirect /@/${path} to a pinned URL (HTTP ${response.status})` + ); + } + return response.url; +} + +/** + * A tracker for the packages under `/packages`: `findLatest` answers with the + * package's pinned `/@/` URL, which the ModuleWatcher imports as an http + * package. Anything else goes to `fallback`, which also does the watching. + */ +export function trackPackagesFolder( + acctHash: Promise<string>, + entries: () => PackageEntry[], + fallback: PackageTracker, + lookUp: (path: string) => Promise<string> = lookUpViaServiceWorker +): PackageTracker { + return { + async findLatest(url) { + const { documentId } = parseAutomergeUrl(url); + const entry = entries().find( + (candidate) => parseAutomergeUrl(candidate.url).documentId === documentId + ); + if (!entry) return fallback.findLatest(url); + // Not an automerge URL: the watcher imports it with importPackageFromHttpUrl. + return (await lookUp(packageLookupPath(await acctHash, entry))) as AutomergeUrl; + }, + watch: (url, onChange) => fallback.watch(url, onChange), + }; +} diff --git a/core/filesystem/test/packages-folder.test.ts b/core/filesystem/test/packages-folder.test.ts new file mode 100644 index 000000000..8b7bba181 --- /dev/null +++ b/core/filesystem/test/packages-folder.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect, vi } from "vitest"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; +import { docStep, idhash } from "../src/paths.js"; +import { + packageLookupPath, + trackPackagesFolder, + watchPackagesFolder, + type PackageEntry, +} from "../src/packages-folder.js"; +import type { PackageTracker } from "../src/package-tracker.js"; + +const folder = (repo: Repo, entries: Array<[string, AutomergeUrl]>) => + repo.create({ + title: "packages", + docs: entries.map(([name, url]) => ({ name, type: "x", url })), + }); + +async function fixture() { + const repo = new Repo(); + const pkg = () => repo.create({ "@patchwork": { type: "directory" } }).url; + const [p1, p2, p3] = [pkg(), pkg(), pkg()]; + const mine = folder(repo, [["foo", p1], ["bar", p2]]); + const team = folder(repo, [["foo", p3], ["bar", p2]]); + const account = repo.create<any>({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: mine.url }, { url: team.url }] }, + }); + return { repo, p1, p2, p3, mine, team, account }; +} + +describe("watchPackagesFolder", () => { + it("lists every package in the union, once per document, first binding first", async () => { + const { repo, p1, p2, p3, mine, team, account } = await fixture(); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBeGreaterThan(0)); + const entries = seen.at(-1)!; + expect(entries.map((e) => [e.name, e.url])).toEqual([ + ["foo", p1], + ["bar", p2], + ["foo", p3], + ]); + expect(entries[0].memberStep).toBe(docStep(await idhash(mine.documentId), mine.heads())); + expect(entries[2].memberStep).toBe(docStep(await idhash(team.documentId), team.heads())); + stop(); + }); + + it("tells again when a member folder gains a package", async () => { + const { repo, mine, account } = await fixture(); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBe(1)); + const added = repo.create({ "@patchwork": { type: "directory" } }).url; + mine.change((d: any) => d.docs.push({ name: "new", type: "x", url: added })); + await vi.waitFor(() => expect(seen.at(-1)!.map((e) => e.url)).toContain(added)); + stop(); + }); + + it("tells nothing for an account with no /packages", async () => { + const repo = new Repo(); + const account = repo.create<any>({ "@patchwork": { type: "account" } }); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen).toEqual([[]])); + stop(); + }); +}); + +describe("packageLookupPath", () => { + it("goes through the member that holds the package", () => { + const entry = { url: "automerge:x" as AutomergeUrl, name: "my pkg", memberStep: "m@h" }; + expect(packageLookupPath("acct", entry)).toBe("acct/packages/m@h/my%20pkg"); + }); +}); + +describe("trackPackagesFolder", () => { + it("looks a /packages package up through /@/, and anything else through the fallback", async () => { + const entry = { url: "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl, name: "foo", memberStep: "m@h" }; + const fallback: PackageTracker = { + findLatest: vi.fn(async (url) => `${url}#heads` as AutomergeUrl), + watch: vi.fn(() => () => {}), + }; + const lookUp = vi.fn(async (path: string) => `https://p.test/@/${path}~L`); + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, lookUp); + + expect(await tracker.findLatest(entry.url)).toBe("https://p.test/@/acct/packages/m@h/foo~L"); + const other = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + expect(await tracker.findLatest(other)).toBe(`${other}#heads`); + tracker.watch(entry.url, () => {}); + expect(fallback.watch).toHaveBeenCalled(); + }); +}); From 4e5849c0d1b670e69243eb6c28a00c8888bee698 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:27:23 +0000 Subject: [PATCH 45/81] filesystem: ModuleWatcher loads the /packages folder Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/module-watcher.ts | 43 +++++++- core/filesystem/test/module-watcher.test.ts | 103 ++++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 3e46aeb5c..16ff372e7 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -11,6 +11,7 @@ import { importPackageFromHttpUrl, } from "./packages.js"; import { type HasPatchworkMetadata } from "./metadata.js"; +import type { PackageEntry } from "./packages-folder.js"; import { DEFAULT_BRANCH, trackPackagesInRepo, @@ -132,6 +133,11 @@ export class ModuleWatcher { #retryTimers = new Map<string, ReturnType<typeof setTimeout>>(); #lastAnnounced = new Map<string, string>(); #activeModules = new Set<string>(); + /** Package folders (`/packages`), by source name: what they list, and how to stop hearing them. */ + #folders = new Map< + string, + { entries: () => PackageEntry[]; unsubscribe: () => void } + >(); #disposed = false; onLoad: (name: string, mod: any) => void; @@ -410,6 +416,8 @@ export class ModuleWatcher { ); if (this.#watchedModules.get(importName) !== entry) return undefined; if (lookup !== entry.lookups) return undefined; + // A /packages package comes back as its pinned /@/ URL, imported over http. + if (!isValidAutomergeUrl(latest)) return latest; const packageUrl = stripHeads(latest); if (packageUrl !== entry.packageUrl) { warnBranchesUnsupported(importName as AutomergeUrl); @@ -495,6 +503,8 @@ export class ModuleWatcher { dispose() { this.#disposed = true; + for (const { unsubscribe } of this.#folders.values()) unsubscribe(); + this.#folders.clear(); for (const handle of Object.values(this.handles ?? {})) { handle.removeListener("change", this.onChange); } @@ -509,11 +519,42 @@ export class ModuleWatcher { this.#activeModules.clear(); } + /** + * Load every package in a `/packages` folder, and keep loading what it lists: + * `subscribe` calls back whenever `entries` may have changed. A package is + * keyed by its document, so a rename reloads nothing, and one that module + * settings also lists loads once, through here. Looking packages up is the + * tracker's job (see trackPackagesFolder). + */ + async addPackagesFolder( + name: string, + entries: () => PackageEntry[], + subscribe: (onChange: () => void) => () => void + ): Promise<void> { + await this.doneLoading; + if (this.#disposed) return; + this.#folders.get(name)?.unsubscribe(); + this.#folders.set(name, { entries, unsubscribe: subscribe(this.onChange) }); + await this.load(); + } + private async load() { if (!this.handles) throw new Error("No handles"); const urls = new Set<string>(); + const inFolders = new Set<string>(); + for (const { entries } of this.#folders.values()) { + for (const entry of entries()) { + urls.add(entry.url); + inFolders.add(parseAutomergeUrl(entry.url).documentId); + } + } for (const { doc } of this.settingsDocs()) { - for (const m of doc?.modules ?? []) urls.add(m); + for (const m of doc?.modules ?? []) { + if (isValidAutomergeUrl(m) && inFolders.has(parseAutomergeUrl(m).documentId)) { + continue; + } + urls.add(m); + } } const previousActive = this.#activeModules; this.#activeModules = urls; diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 7f97a832c..07655989f 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -770,3 +770,106 @@ describe("importPackageFromHttpUrl error handling", () => { expect(err.cause).toBe(parseError); }); }); + +describe("ModuleWatcher /packages folder", () => { + const pause = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + /** Packages as a /packages folder lists them, and a tracker that looks them up as http modules. */ + function folderSource(repo: InstanceType<typeof AutomergeRepo>) { + const docs = [0, 1].map(() => repo.create<any>({ "@patchwork": { type: "directory" } })); + let entries = docs.map((doc, i) => ({ + url: doc.url, + name: `pkg${i}`, + memberStep: "m@h", + })); + let changed: () => void = () => {}; + const lookups: string[] = []; + const tracker: PackageTracker = { + async findLatest(url) { + lookups.push(url); + return dataModule(`export const plugins = []; // ${url}`) as any; + }, + watch: () => () => {}, + }; + return { + docs, + tracker, + lookups, + entries: () => entries, + subscribe: (onChange: () => void) => { + changed = onChange; + return () => {}; + }, + set(next: typeof entries) { + entries = next; + changed(); + }, + }; + } + + it("loads every package in the folder", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const loaded: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, source.tracker); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(loaded).toHaveLength(2); + expect(source.lookups.sort()).toEqual(source.docs.map((d) => d.url).sort()); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("loads a package in both module settings and /packages once", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const atHeads = source.docs[0].view(source.docs[0].heads()).url; + const settings = repo.create<any>({ + "@patchwork": { type: "patchwork:module-settings" }, + modules: [atHeads], + }); + const loaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + undefined, + undefined, + source.tracker + ); + // As patchwork wires them: /packages first, then the user's module settings. + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + await watcher.addUrl("user", settings.url); + await pause(50); + expect(loaded).toHaveLength(2); + expect(source.lookups).not.toContain(atHeads); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("doesn't reload a renamed package, and unloads a removed one", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const loaded: string[] = []; + const unloaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + (name) => unloaded.push(name), + undefined, + source.tracker + ); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(loaded).toHaveLength(2); + + source.set(source.entries().map((e, i) => (i === 0 ? { ...e, name: "renamed" } : e))); + await pause(50); + expect(loaded).toHaveLength(2); + + source.set([source.entries()[0]]); + await pause(50); + expect(unloaded).toEqual([source.docs[1].url]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); +}); From ff61fc9b08f342caf06ad045488838216ec3270c Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:29:28 +0000 Subject: [PATCH 46/81] patchwork: /packages on new accounts, migration, and wiring New accounts bind / to the root folder and /packages to a new Packages folder. An account with module settings and no /packages gets one, linking each automerge package by its package.json name. Boot lists /packages and hands it to the ModuleWatcher before module settings, and looks its packages up as pinned /@/ URLs. moduleSettingsUrl is deprecated, and still read: the Packages UI installs into it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/patchwork/src/createAccount.ts | 14 +++- core/patchwork/src/index.ts | 68 ++++++++++++++- core/patchwork/src/migrate-packages.ts | 83 +++++++++++++++++++ core/patchwork/test/migrate-packages.test.ts | 87 ++++++++++++++++++++ core/plugins/src/account.ts | 5 ++ 5 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 core/patchwork/src/migrate-packages.ts create mode 100644 core/patchwork/test/migrate-packages.test.ts diff --git a/core/patchwork/src/createAccount.ts b/core/patchwork/src/createAccount.ts index be3f3d72a..c0383b339 100644 --- a/core/patchwork/src/createAccount.ts +++ b/core/patchwork/src/createAccount.ts @@ -5,7 +5,7 @@ import type { } from "@inkandswitch/patchwork-plugins"; import type { HasPatchworkMetadata } from "@inkandswitch/patchwork-filesystem"; -async function createSubdoc<D>( +export async function createSubdoc<D>( repo: Repo, type: string, init: (doc: D) => void @@ -22,11 +22,15 @@ export const createDefaultAccount: AccountCreator<AccountDoc> = async ( accountHandle, repo ) => { - const [rootFolder, moduleSettings, contact] = await Promise.all([ + const [rootFolder, packagesFolder, moduleSettings, contact] = await Promise.all([ createSubdoc<{ title: string; docs: unknown[] }>(repo, "folder", (doc) => { doc.title = "New Folder"; doc.docs = []; }), + createSubdoc<{ title: string; docs: unknown[] }>(repo, "folder", (doc) => { + doc.title = "Packages"; + doc.docs = []; + }), createSubdoc<{ modules: string[] }>( repo, "patchwork:module-settings", @@ -41,6 +45,12 @@ export const createDefaultAccount: AccountCreator<AccountDoc> = async ( accountHandle.change((doc) => { doc.rootFolderUrl = rootFolder.url; + doc.namespace = { + "/": [{ url: rootFolder.url }], + // Package management: whatever is bound here. + "/packages": [{ url: packagesFolder.url }], + }; + // Still made: the Packages UI installs into it. doc.moduleSettingsUrl = moduleSettings.url; doc.contactUrl = contact.url; }); diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index 6047f16fb..9f8874549 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -31,7 +31,11 @@ import type { AutomergeRepoKeyhiveBase as AutomergeRepoKeyhive } from "@automerg import { ModuleWatcher, + trackPackagesFolder, trackPackagesViaServiceWorker, + watchPackagesFolder, + type AccountNamespaceDoc, + type PackageEntry, } from "@inkandswitch/patchwork-filesystem"; import { importAutomergePackageViaWorker } from "@inkandswitch/patchwork-bootloader/module-loader"; import { registerPatchworkViewElement } from "@inkandswitch/patchwork-elements"; @@ -63,6 +67,7 @@ import type { } from "./types.js"; import { createRepo, initWasm } from "./repo.js"; import { watchDocument } from "./watch-document.js"; +import { migratePackages } from "./migrate-packages.js"; import { createRouter, type Router } from "./router.js"; import { createDefaultAccount } from "./createAccount.js"; @@ -180,6 +185,11 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { // Started with the site bundle alone so resolveAccountHandle has something to // await on — the `account` datatype lives there. The user's own // module-settings URL is added lazily once it appears on the account doc. + // Packages under /packages are looked up as pinned /@/ URLs, once the + // account is known; anything else (module settings, the site's manifest) + // through the service worker's legacy redirect. + const account = Promise.withResolvers<string>(); + let packageEntries: PackageEntry[] = []; const moduleWatcher = new ModuleWatcher( repo, nameSources(moduleSources), @@ -193,8 +203,12 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { // Look packages up through the automerge worker and watch them through // this tab's Subduction subscriptions, so their docs never load into this // tab's Repo. - trackPackagesViaServiceWorker((url, onChange) => - watchDocument(repo, url, onChange) + trackPackagesFolder( + account.promise, + () => packageEntries, + trackPackagesViaServiceWorker((url, onChange) => + watchDocument(repo, url, onChange) + ) ) ); @@ -204,7 +218,13 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { createAccount: options.createAccount ?? createDefaultAccount, })) as DocHandle<AccountDoc>; // So /@/<acctHash>/… paths under it resolve, here and in the automerge worker. - await registerAccount(accountDocHandle.url); + account.resolve(await registerAccount(accountDocHandle.url)); + await migratePackages(repo, accountDocHandle).catch((error) => + console.warn("couldn't move module settings into /packages", error) + ); + await wirePackagesFolder(repo, accountDocHandle, moduleWatcher, (entries) => { + packageEntries = entries; + }); wireModuleSettings(accountDocHandle, moduleWatcher); @@ -355,6 +375,48 @@ function onModuleLoaded(name: string, mod: any): void { registerPlugins(mod.plugins, name); } +// How long boot waits for /packages' first listing before going on without it. +const FIRST_LISTING_MS = 5_000; + +/** + * Feed the account's /packages folder to the ModuleWatcher. Waits for its + * first listing, so a package that module settings also lists is loaded once, + * through /packages, and adds the folder before module settings is wired. + */ +async function wirePackagesFolder( + repo: Repo, + accountDocHandle: DocHandle<AccountDoc>, + moduleWatcher: ModuleWatcher, + setEntries: (entries: PackageEntry[]) => void +): Promise<void> { + let current: PackageEntry[] = []; + const listeners = new Set<() => void>(); + const first = Promise.withResolvers<void>(); + watchPackagesFolder(repo, accountDocHandle as unknown as DocHandle<AccountNamespaceDoc>, (entries) => { + current = entries; + setEntries(entries); + first.resolve(); + for (const listener of listeners) listener(); + }); + await Promise.race([ + first.promise, + new Promise((resolve) => setTimeout(resolve, FIRST_LISTING_MS)), + ]); + // Not awaited: it loads every package. It registers the folder before + // wireModuleSettings' addUrl does its own load, since both wait on the same + // doneLoading and this one asked first. + moduleWatcher + .addPackagesFolder( + "packages", + () => current, + (onChange) => { + listeners.add(onChange); + return () => listeners.delete(onChange); + } + ) + .catch((error) => console.warn("couldn't load /packages", error)); +} + /** * Feed the account's module settings document to the ModuleWatcher. */ diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts new file mode 100644 index 000000000..9f2c01a4a --- /dev/null +++ b/core/patchwork/src/migrate-packages.ts @@ -0,0 +1,83 @@ +// Once per account: an account from before `/packages` gets a packages folder +// holding a link to each automerge package its module-settings doc lists, +// bound at `/packages`. The module-settings doc is left as it is: the Packages +// UI still installs into it, and builds before this one still read it. +import { + isValidAutomergeUrl, + parseAutomergeUrl, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocHandle, + type Repo, +} from "@automerge/automerge-repo/slim"; +import { + getType, + resolvePath, + type HasPatchworkMetadata, +} from "@inkandswitch/patchwork-filesystem"; +import type { AccountDoc } from "@inkandswitch/patchwork-plugins"; +import { createSubdoc } from "./createAccount.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +/** A package's name: its package.json's, else the doc's title, else its id. */ +async function packageName(repo: Repo, url: AutomergeUrl) { + const handle = await repo.find<Partial<HasPatchworkMetadata> & { title?: string }>(url); + try { + const manifest = await resolvePath(repo, handle, ["package.json"]); + const name = manifest && JSON.parse(decode(manifest.content))?.name; + if (typeof name === "string" && name) return { handle, name }; + } catch { + // no package.json, or not JSON: fall back below + } + return { handle, name: handle.doc()?.title || handle.documentId }; +} + +/** + * Bind `/packages` for an account that has module settings and no packages + * folder. True if it did. Http entries stay where they are, in module + * settings, which is still read. + */ +export async function migratePackages( + repo: Repo, + account: DocHandle<AccountDoc> +): Promise<boolean> { + const doc = account.doc(); + if (doc?.namespace?.["/packages"]?.length || !doc?.moduleSettingsUrl) return false; + const settings = await repo.find<{ modules?: string[] }>(doc.moduleSettingsUrl); + const seen = new Set<string>(); + const urls = (settings.doc()?.modules ?? []).flatMap((entry) => { + if (!isValidAutomergeUrl(entry)) return []; + const { documentId } = parseAutomergeUrl(entry); + if (seen.has(documentId)) return []; + seen.add(documentId); + return [stringifyAutomergeUrl({ documentId })]; + }); + const links = await Promise.all( + urls.map(async (url) => { + try { + const { handle, name } = await packageName(repo, url); + return [{ name, type: getType(handle.doc() ?? {}) ?? "folder", url }]; + } catch (error) { + console.warn(`couldn't move package ${url} into /packages`, error); + return []; + } + }) + ); + // Another tab may have migrated while this one read the packages. + if (account.doc()?.namespace?.["/packages"]?.length) return false; + const folder = await createSubdoc<{ title: string; docs: unknown[] }>( + repo, + "folder", + (folder) => { + folder.title = "Packages"; + folder.docs = links.flat(); + } + ); + account.change((doc) => { + doc.namespace ??= {}; + doc.namespace["/packages"] = [{ url: folder.url }]; + }); + return true; +} diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts new file mode 100644 index 000000000..41f08f954 --- /dev/null +++ b/core/patchwork/test/migrate-packages.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { migratePackages } from "../src/migrate-packages.js"; +import { createDefaultAccount } from "../src/createAccount.js"; + +async function account(repo: Repo, modules: string[]) { + const settings = repo.create({ "@patchwork": { type: "patchwork:module-settings" }, modules }); + const acct = repo.create<any>({ + "@patchwork": { type: "account" }, + moduleSettingsUrl: settings.url, + }); + return acct as DocHandle<any>; +} + +function packages(repo: Repo) { + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "application/json" }).url; + const alpha = repo.create({ + "@patchwork": { type: "directory" }, + "package.json": file('{"name":"alpha","main":"index.js"}'), + }); + const bee = repo.create({ "@patchwork": { type: "folder" }, title: "Bee", docs: [] }); + return { alpha, bee }; +} + +const linked = async (repo: Repo, acct: DocHandle<any>) => { + const binding = acct.doc().namespace?.["/packages"]; + if (!binding) return undefined; + const folder = await repo.find<any>(binding[0].url as AutomergeUrl); + return folder.doc().docs.map((d: any) => [d.name, d.url]); +}; + +describe("migratePackages", () => { + it("links each automerge entry by its package.json name, and binds /packages", async () => { + const repo = new Repo(); + const { alpha, bee } = packages(repo); + const acct = await account(repo, [ + alpha.view(alpha.heads()).url, + bee.url, + "https://cdn.example.test/tool/index.js", + ]); + expect(await migratePackages(repo, acct)).toBe(true); + expect(await linked(repo, acct)).toEqual([ + ["alpha", alpha.url], + ["Bee", bee.url], + ]); + }); + + it("does nothing the second time", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + await migratePackages(repo, acct); + const bound = acct.doc().namespace["/packages"][0].url; + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"][0].url).toBe(bound); + }); + + it("leaves an account that already binds /packages alone", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const existing = repo.create({ title: "mine", docs: [] }).url; + acct.change((d: any) => (d.namespace = { "/packages": [{ url: existing }] })); + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"]).toEqual([{ url: existing }]); + }); + + it("does nothing for an account without module settings", async () => { + const repo = new Repo(); + const acct = repo.create<any>({ "@patchwork": { type: "account" } }); + expect(await migratePackages(repo, acct as DocHandle<any>)).toBe(false); + }); +}); + +describe("createDefaultAccount", () => { + it("binds / to the root folder and /packages to a new one", async () => { + const repo = new Repo(); + const acct = repo.create<any>({ "@patchwork": { type: "account" } }); + await createDefaultAccount(acct as DocHandle<any>, repo as any); + const doc = acct.doc(); + expect(doc.namespace["/"]).toEqual([{ url: doc.rootFolderUrl }]); + const packagesFolder = await repo.find<any>(doc.namespace["/packages"][0].url); + expect(packagesFolder.doc()).toMatchObject({ title: "Packages", docs: [] }); + expect(doc.moduleSettingsUrl).toBeDefined(); + }); +}); diff --git a/core/plugins/src/account.ts b/core/plugins/src/account.ts index 41719592b..ef356437c 100644 --- a/core/plugins/src/account.ts +++ b/core/plugins/src/account.ts @@ -28,6 +28,11 @@ export type AccountDoc = { */ namespace?: Namespace; rootFolderUrl?: AutomergeUrl; + /** + * @deprecated Package management is whatever is bound at `/packages` in + * {@link AccountDoc.namespace}. Still read, since the Packages UI installs + * into it. + */ moduleSettingsUrl?: AutomergeUrl; contactUrl?: AutomergeUrl; }; From 3619c3082c9aa1fd0839581dedb5f70418abb31f Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:30:39 +0000 Subject: [PATCH 47/81] changeset: packages via /packages Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/paths-packages.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/paths-packages.md diff --git a/.changeset/paths-packages.md b/.changeset/paths-packages.md new file mode 100644 index 000000000..5056b76f5 --- /dev/null +++ b/.changeset/paths-packages.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork-plugins": patch +"@inkandswitch/patchwork": patch +--- + +Package management is whatever is bound at `/packages` in the account's namespace. New accounts bind `/` to their root folder and `/packages` to a new Packages folder; an older account gets a Packages folder at boot, linking each automerge package its module-settings doc lists, by its `package.json` name. The ModuleWatcher loads every package in the `/packages` union (`addPackagesFolder`, `watchPackagesFolder`), keyed by document so a rename reloads nothing, and looks each one up as a pinned `/@/` URL (`trackPackagesFolder`) that it imports and the service worker checks against the package's file list. A package's list now covers everything under it, nested folders included, and a redirect to a folder carries the folder's whole list. `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding; it's still read, deduplicated against `/packages`, since the Packages UI installs into it. From 698e7fd4c37ad1bd59a1a556c13f892553d9cfd1 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:45:35 +0000 Subject: [PATCH 48/81] packages: stop reloading every package when /packages changes A package's pinned URL embeds the account's and its folder's heads, so any change to /packages gave every package a new URL and re-imported it. The tracker now keeps a package's URL while its own step is unchanged, and the watcher doesn't look a found package up again until its own watch says it moved. Also: a second package with one name in one folder is skipped with a warning instead of importing the first's code; removing a package unloads it under the url its plugins were registered with; a failed lookup is retried with backoff; and listing reads nested folders' files several at a time instead of one by one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/module-watcher.ts | 39 +++++- core/filesystem/src/packages-folder.ts | 30 ++++- core/filesystem/src/walk.ts | 86 +++++++------ core/filesystem/test/file-list.test.ts | 28 +++++ core/filesystem/test/module-watcher.test.ts | 123 ++++++++++++++++++- core/filesystem/test/packages-folder.test.ts | 37 ++++++ 6 files changed, 299 insertions(+), 44 deletions(-) diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 16ff372e7..e8437a74e 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -131,6 +131,8 @@ export class ModuleWatcher { // eventually succeeding and rolling the registry back to that old version. #announceGenerations = new Map<string, number>(); #retryTimers = new Map<string, ReturnType<typeof setTimeout>>(); + /** Lookups that failed, waiting to try again. */ + #lookupRetries = new Map<string, ReturnType<typeof setTimeout>>(); #lastAnnounced = new Map<string, string>(); #activeModules = new Set<string>(); /** Package folders (`/packages`), by source name: what they list, and how to stop hearing them. */ @@ -244,14 +246,39 @@ export class ModuleWatcher { ); } - private async processModuleEntry(importName: string) { + private async processModuleEntry(importName: string, attempt = 0) { if (isValidAutomergeUrl(importName)) { // Watch before looking up, so a change landing in between still reloads. // Keyed by the settings entry, a branches doc's included, so removing // it from settings unloads this module. + const watched = this.#watchedModules.get(importName); + // Already found: its watch looks it up again when it moves. Looking up + // every package whenever its source's list changes would reload them. + if (watched?.version !== undefined) return; const entry = this.watchPackage(importName, stripHeads(importName)); if (!entry) return; - const latest = await this.findLatestVersion(importName, entry); + let latest: AutomergeUrl | undefined; + try { + latest = await this.findLatestVersion(importName, entry); + } catch (error) { + // Nothing else brings it back until the package moves: back off and + // look again. + const delay = RETRY_DELAYS_MS[attempt]; + console.warn( + `couldn't look up ${importName}${delay === undefined ? "" : `; trying again in ${delay}ms`}`, + error + ); + if (delay !== undefined) { + const timer = setTimeout(() => { + this.#lookupRetries.delete(importName); + if (this.#activeModules.has(importName)) { + void this.processModuleEntry(importName, attempt + 1); + } + }, delay); + this.#lookupRetries.set(importName, timer); + } + return; + } // A newer lookup took over, and announces what it finds. if (!latest) return; entry.version = latest; @@ -497,8 +524,12 @@ export class ModuleWatcher { private unloadModule(entry: string) { this.detachDocWatcher(entry); this.cancelAnnounce(entry); + clearTimeout(this.#lookupRetries.get(entry)); + this.#lookupRetries.delete(entry); + // Its plugins were registered under the url it was imported from. + const announced = this.#lastAnnounced.get(entry); this.#lastAnnounced.delete(entry); - this.onUnload?.(entry); + this.onUnload?.(announced ?? entry); } dispose() { @@ -516,6 +547,8 @@ export class ModuleWatcher { } for (const timer of this.#retryTimers.values()) clearTimeout(timer); this.#retryTimers.clear(); + for (const timer of this.#lookupRetries.values()) clearTimeout(timer); + this.#lookupRetries.clear(); this.#activeModules.clear(); } diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts index bd1729205..77c407852 100644 --- a/core/filesystem/src/packages-folder.ts +++ b/core/filesystem/src/packages-folder.ts @@ -54,10 +54,22 @@ export function watchPackagesFolder( const entries: PackageEntry[] = []; const seen = new Set<string>(); for (const member of members) { - const doc = member.doc(); - const memberStep = docStep(await idhash(member.documentId), member.heads()); + // The doc and the heads it was read at, together. + const heads = member.heads(); + const doc = member.view(heads).doc(); + const memberStep = docStep(await idhash(member.documentId), heads); + const names = new Set<string>(); for (const child of providerFor(doc)?.list(doc) ?? []) { if (!("link" in child)) continue; + // Only the first of two links with one name in a folder is reachable + // by that name, so the second would import the first's code. + if (names.has(child.name)) { + console.warn( + `/packages: skipping a second package named ${JSON.stringify(child.name)} in one folder; rename one` + ); + continue; + } + names.add(child.name); const { documentId } = parseAutomergeUrl(child.link); if (seen.has(documentId)) continue; seen.add(documentId); @@ -129,6 +141,12 @@ export function trackPackagesFolder( fallback: PackageTracker, lookUp: (path: string) => Promise<string> = lookUpViaServiceWorker ): PackageTracker { + // A lookup pins the whole chain, so the URL changes whenever the account or + // the member folder does. A pinned URL stays valid forever, so keep the one + // already imported while the package's own step (its heads and list) is + // the same: a change elsewhere in /packages mustn't reload every package. + const known = new Map<string, string>(); + const ownStep = (url: string) => new URL(url).pathname.split("/").at(-1); return { async findLatest(url) { const { documentId } = parseAutomergeUrl(url); @@ -136,8 +154,14 @@ export function trackPackagesFolder( (candidate) => parseAutomergeUrl(candidate.url).documentId === documentId ); if (!entry) return fallback.findLatest(url); + const latest = await lookUp(packageLookupPath(await acctHash, entry)); + const previous = known.get(documentId); + if (previous && ownStep(previous) === ownStep(latest)) { + return previous as AutomergeUrl; + } + known.set(documentId, latest); // Not an automerge URL: the watcher imports it with importPackageFromHttpUrl. - return (await lookUp(packageLookupPath(await acctHash, entry))) as AutomergeUrl; + return latest as AutomergeUrl; }, watch: (url, onChange) => fallback.watch(url, onChange), }; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 928c35069..f39629a2e 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -537,7 +537,7 @@ export async function listFiles( }; // Values are read in place; links are gathered and read below. - const links: { parts: string[]; keys: number[]; link: AutomergeUrl }[] = []; + type Link = { parts: string[]; keys: number[]; link: AutomergeUrl }; const entered = new Set<string>([rootId]); const visit = async ( node: unknown, @@ -545,7 +545,8 @@ export async function listFiles( holder: DocumentId, parts: string[], keys: number[] - ) => { + ): Promise<Link[]> => { + const links: Link[] = []; for (const child of provider?.list(node) ?? []) { // Only a directory's keys span several path segments; a folder entry's name is one. const names = @@ -556,7 +557,7 @@ export async function listFiles( await add(childParts, childKeys, child.value, holder); // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { - await visit(child.value, provider, holder, childParts, childKeys); + links.push(...(await visit(child.value, provider, holder, childParts, childKeys))); } } else if ( "link" in child && @@ -565,44 +566,57 @@ export async function listFiles( links.push({ parts: childParts, keys: childKeys, link: child.link }); } } + return links; }; - await add([], [], root, rootId); - await visit(root, rootProvider, rootId, [], []); - let next = 0; - const readLinks = async () => { - while (next < links.length) { - const { parts, keys, link } = links[next++]; - try { - const parsed = parseAutomergeUrl(link); - const live = await find( - stringifyAutomergeUrl({ documentId: parsed.documentId }), - parsed.heads - ); - const heads = parsed.heads ?? live.heads(); - const doc = live.view(heads).doc(); - if (doc === undefined) continue; - const provider = providerFor(doc); - if (!provider) { - await add(parts, keys, doc, parsed.documentId, heads); - } else if ( - deep && - (provider === folderProvider || provider === directoryProvider) && - !entered.has(parsed.documentId) - ) { - // Into a nested folder, by name: its files are under its name here. - entered.add(parsed.documentId); - await add(parts, keys, doc, parsed.documentId, heads); - await visit(doc, provider, parsed.documentId, parts, keys); - } - } catch { - missing.push(tailOf(parts)); + // At most LIST_CONCURRENCY file docs read at once, nested folders included. + let active = 0; + const queue: Array<() => void> = []; + const acquire = async () => { + if (active >= LIST_CONCURRENCY) await new Promise<void>((resolve) => queue.push(resolve)); + active++; + }; + const release = () => { + active--; + queue.shift()?.(); + }; + + const read = async ({ parts, keys, link }: Link): Promise<void> => { + let below: Link[] = []; + await acquire(); + try { + const parsed = parseAutomergeUrl(link); + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + parsed.heads + ); + const heads = parsed.heads ?? live.heads(); + const doc = live.view(heads).doc(); + if (doc === undefined) return; + const provider = providerFor(doc); + if (!provider) { + await add(parts, keys, doc, parsed.documentId, heads); + } else if ( + deep && + (provider === folderProvider || provider === directoryProvider) && + !entered.has(parsed.documentId) + ) { + // Into a nested folder, by name: its files are under its name here. + entered.add(parsed.documentId); + await add(parts, keys, doc, parsed.documentId, heads); + below = await visit(doc, provider, parsed.documentId, parts, keys); } + } catch { + missing.push(tailOf(parts)); + } finally { + release(); } + // Read what's below after letting go of the slot, so nesting can't deadlock. + await Promise.all(below.map(read)); }; - await Promise.all( - Array.from({ length: Math.min(LIST_CONCURRENCY, links.length) }, readLinks) - ); + + await add([], [], root, rootId); + await Promise.all((await visit(root, rootProvider, rootId, [], [])).map(read)); return { prefix: end.container.pinned, diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index 6e222839b..0ee078398 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -150,3 +150,31 @@ describe("package lists", () => { expect(Object.keys(list.files)).toEqual(expect.arrayContaining(["package.json", "sub/x.js"])); }); }); + +describe("listFiles concurrency", () => { + it("reads a nested folder's files several at a time", async () => { + const f = await fixture(); + const files = Object.fromEntries( + Array.from({ length: 20 }, (_, i) => [`f${i}.js`, file(f.repo, `${i}`).url]) + ); + const dist = folder(f.repo, files); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"dist/f0.js"}').url, + dist: dist.url, + }); + f.mine.change((d: any) => d.docs.push({ name: "big", type: "folder", url: pkg.url })); + const prefix = await chain(f.repo, livePath(f.a, ["packages", "big"]), f.accounts); + let active = 0; + let peak = 0; + const findFile = async (url: any) => { + active++; + peak = Math.max(peak, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active--; + return f.repo.find(url); + }; + const built = await buildFileList(f.repo, prefix!, { accounts: f.accounts, findFile }); + expect(Object.keys(built!.list.files).filter((t) => t.startsWith("dist/"))).toHaveLength(20); + expect(peak).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 07655989f..23de278dd 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -318,7 +318,8 @@ describe("ModuleWatcher module lifecycle", () => { d.modules = []; }); await pause(100); - expect(unloaded).toEqual([folder.url]); + // Under the url it was imported from, which its plugins are registered under. + expect(unloaded).toEqual([importCalls[0]]); folder.change((d: any) => { d.rev = 1; @@ -868,8 +869,126 @@ describe("ModuleWatcher /packages folder", () => { source.set([source.entries()[0]]); await pause(50); - expect(unloaded).toEqual([source.docs[1].url]); + expect(unloaded).toHaveLength(1); + expect(loaded).toContain(unloaded[0]); watcher.dispose(); await repo.shutdown().catch(() => {}); }); }); + +describe("ModuleWatcher /packages, through the real lookup chain", () => { + const pause = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + async function packagesAccount() { + const { idhash } = await import("../src/paths.js"); + const { pinnedPath } = await import("../src/walk.js"); + const { trackPackagesFolder, watchPackagesFolder } = await import("../src/packages-folder.js"); + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const pkg = () => { + const json = repo.create({ "@patchwork": { type: "file" }, content: "{}", mimeType: "application/json" }); + return repo.create<any>({ "@patchwork": { type: "directory" }, "package.json": json.url }); + }; + const [p1, p2, p3] = [pkg(), pkg(), pkg()]; + const member = repo.create<any>({ + title: "packages", + docs: [ + { name: "one", type: "directory", url: p1.url }, + { name: "two", type: "directory", url: p2.url }, + ], + }); + const account = repo.create<any>({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: member.url }] }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + // The pinned path a lookup would redirect to, as a module that says which. + const lookUp = async (path: string) => { + const pinned = await pinnedPath(repo, path, { accounts, only: "end" }); + return `data:text/javascript,export const plugins = []//${pinned!.path}`; + }; + let entries: any[] = []; + const listeners = new Set<() => void>(); + const first = Promise.withResolvers<void>(); + watchPackagesFolder(repo, account, (next) => { + entries = next; + first.resolve(); + for (const listener of listeners) listener(); + }); + await first.promise; + const fallback: PackageTracker = { findLatest: async (u) => u, watch: () => () => {} }; + const tracker = trackPackagesFolder(Promise.resolve(a), () => entries, fallback, lookUp); + const loaded: string[] = []; + const unloaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + (name) => unloaded.push(name), + undefined, + tracker + ); + await watcher.addPackagesFolder("packages", () => entries, (onChange) => { + listeners.add(onChange); + return () => listeners.delete(onChange); + }); + return { repo, member, p3, watcher, loaded, unloaded }; + } + + it("adding a package to the folder loads only that package", async () => { + const { repo, member, p3, watcher, loaded } = await packagesAccount(); + expect(loaded).toHaveLength(2); + member.change((d: any) => d.docs.push({ name: "three", type: "directory", url: p3.url })); + await pause(300); + expect(loaded).toHaveLength(3); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("renaming a package reloads nothing", async () => { + const { repo, member, watcher, loaded } = await packagesAccount(); + member.change((d: any) => (d.docs[0].name = "uno")); + await pause(300); + expect(loaded).toHaveLength(2); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("removing a package unloads it under the name it was loaded with", async () => { + const { repo, member, watcher, loaded, unloaded } = await packagesAccount(); + member.change((d: any) => d.docs.splice(1, 1)); + await pause(300); + expect(unloaded).toHaveLength(1); + expect(loaded).toContain(unloaded[0]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); +}); + +describe("ModuleWatcher lookup retries", () => { + it("retries a lookup that failed", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create<any>({ "@patchwork": { type: "directory" } }); + let calls = 0; + const tracker: PackageTracker = { + async findLatest() { + if (calls++ === 0) throw new Error("not yet"); + return dataModule("export const plugins = []") as any; + }, + watch: () => () => {}, + }; + const loaded: string[] = []; + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, tracker); + await watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + expect(loaded).toHaveLength(0); + await vi.waitFor(() => expect(loaded).toHaveLength(1), { timeout: 3000 }); + watcher.dispose(); + errors.mockRestore(); + await repo.shutdown().catch(() => {}); + }); +}); diff --git a/core/filesystem/test/packages-folder.test.ts b/core/filesystem/test/packages-folder.test.ts index 8b7bba181..d0256ff6f 100644 --- a/core/filesystem/test/packages-folder.test.ts +++ b/core/filesystem/test/packages-folder.test.ts @@ -90,3 +90,40 @@ describe("trackPackagesFolder", () => { expect(fallback.watch).toHaveBeenCalled(); }); }); + +describe("watchPackagesFolder: duplicate names", () => { + it("skips, and warns about, a second package with one name in one member", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const repo = new Repo(); + const pkg = () => repo.create({ "@patchwork": { type: "directory" } }).url; + const [a, b] = [pkg(), pkg()]; + const mine = folder(repo, [["same", a], ["same", b]]); + const account = repo.create<any>({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: mine.url }] }, + }); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBe(1)); + expect(seen[0].map((e) => e.url)).toEqual([a]); + expect(warn).toHaveBeenCalled(); + stop(); + warn.mockRestore(); + }); +}); + +describe("trackPackagesFolder: stable URLs", () => { + it("keeps a package's URL while its own step is unchanged", async () => { + const entry = { url: "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl, name: "foo", memberStep: "m@h1" }; + const fallback: PackageTracker = { findLatest: vi.fn(), watch: vi.fn(() => () => {}) }; + let root = "acct@r1"; + const lookUp = vi.fn(async () => `https://p.test/@/${root}/packages/m@h1/pkg@p1~L1`); + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, lookUp); + const first = await tracker.findLatest(entry.url); + // the account and the member moved; the package didn't + root = "acct@r2"; + expect(await tracker.findLatest(entry.url)).toBe(first); + lookUp.mockResolvedValue("https://p.test/@/acct@r2/packages/m@h1/pkg@p2~L2"); + expect(await tracker.findLatest(entry.url)).toBe("https://p.test/@/acct@r2/packages/m@h1/pkg@p2~L2"); + }); +}); From 8667e5c80e0b86a4b923abfec3742d1c8e55c85b Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 15:49:35 +0000 Subject: [PATCH 49/81] packages: safer migration; tabs leave /packages to the automerge worker Migration waits for the account to catch up with the sync server, since another device may have migrated already; links a package it can't read under its document id instead of dropping it; leaves entries pinned to heads in module settings rather than unpinning them; names same-named packages apart; and treats an emptied /packages binding as deliberate. Tabs decline /packages requests so the automerge worker answers them and package docs stay out of the tab, and a redirect's file list reads each file without waiting on peers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/bootloader/src/resolver.ts | 22 +++- core/bootloader/src/tab-resolver.ts | 1 + core/bootloader/test/resolver.test.ts | 10 ++ core/bootloader/test/service-worker.test.ts | 13 ++- core/filesystem/src/walk.ts | 5 +- core/patchwork/src/index.ts | 13 ++- core/patchwork/src/migrate-packages.ts | 104 +++++++++++++++---- core/patchwork/test/migrate-packages.test.ts | 51 ++++++++- 8 files changed, 189 insertions(+), 30 deletions(-) diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 0a130e950..8e4e9fa6d 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -285,7 +285,12 @@ async function resolveWalkPath( if (!end.canonical) { let pinned: Awaited<ReturnType<typeof pinnedPath>>; try { - pinned = await pinnedPath(repo, path, { ...options, only: "end" }); + pinned = await pinnedPath(repo, path, { + ...options, + only: "end", + // Each file as this node has it, without waiting on peers for every one. + findFile: waitingFind(repo, signal, missing, { catchUp: false }), + }); } catch (error) { // The file isn't readable here yet: not a 404, which would be memoized. if (error instanceof IncompleteFileListError) throw new AbortHandoff(error.message); @@ -334,8 +339,22 @@ export function createResolver( * answers 404 instead. */ declineUnknownAccounts?: boolean; + /** + * Decline paths through `/packages`, leaving them to the automerge + * worker. A package lookup reads every file in the package, and a tab + * shouldn't hold them all. + */ + declinePackages?: boolean; } = {} ): Resolver { + const throughPackages = (path: string) => { + try { + const [first] = parsePath(path).segments; + return first?.kind === "name" && first.name === "packages"; + } catch { + return false; + } + }; const knowsAccountOf = (path: string) => { try { const { root } = parsePath(path); @@ -413,6 +432,7 @@ export function createResolver( if (options.declineUnknownAccounts && !knowsAccountOf(request.path)) { return undefined; } + if (options.declinePackages && throughPackages(request.path)) return undefined; const repo = await getRepo(); const limit = within(signal); log(`resolving handoff ${id} for /@/${request.path}`); diff --git a/core/bootloader/src/tab-resolver.ts b/core/bootloader/src/tab-resolver.ts index 3f3510206..e1755d72a 100644 --- a/core/bootloader/src/tab-resolver.ts +++ b/core/bootloader/src/tab-resolver.ts @@ -30,6 +30,7 @@ const accounts = new Map<string, AutomergeUrl>(); export function resolveHandoffsWith(repo: Repo): void { resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { declineUnknownAccounts: true, + declinePackages: true, }); } diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index 7d4f4794c..54f124462 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -201,6 +201,16 @@ describe("resolver: accounts it doesn't know", () => { expect(await tab.answer(pathRequest(livePath(a, ["Projects"])))).toBeUndefined(); }); + it("a tab leaves /packages to the automerge worker", async () => { + const { repo, a, accounts } = await fixture(); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + declineUnknownAccounts: true, + declinePackages: true, + }); + expect(await tab.answer(pathRequest(livePath(a, ["packages", "x"])))).toBeUndefined(); + expect(await tab.answer(pathRequest(livePath(a, ["Projects"])))).toBeDefined(); + }); + it("the worker, asked last, answers 404", async () => { const { repo, a } = await fixture(); const worker = createResolver(() => Promise.resolve(repo), undefined, () => new Map()); diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts index b9ac74876..df83fa8b1 100644 --- a/core/bootloader/test/service-worker.test.ts +++ b/core/bootloader/test/service-worker.test.ts @@ -1,4 +1,6 @@ -import { afterEach, describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect, onTestFinished } from "vitest"; +import { createResolver } from "../src/resolver.js"; +import { HANDOFF_CHANNEL } from "../src/types.js"; import { Repo, stringifyAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; import { contentHash, livePath, toBytes } from "@inkandswitch/patchwork-filesystem"; import { @@ -129,6 +131,15 @@ describe("service worker: /@/ paths", () => { namespace: { "/packages": [{ url: folder({ fpkg: pkg }) }] }, }); const a = await registerAccount(acct.url); + // Tabs leave /packages to the automerge worker, which answers broadcasts. + const worker = createResolver(() => Promise.resolve(repo), undefined, () => new Map([[a, acct.url]])); + const channel = new BroadcastChannel(HANDOFF_CHANNEL); + channel.onmessage = async (event) => { + if (event.data?.type !== "request") return; + const reply = await worker.answer(event.data); + if (reply) channel.postMessage(reply); + }; + onTestFinished(() => channel.close()); const pinned = await pin(sw, livePath(a, ["packages", "fpkg"])); for (const [tail, content] of [ diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index f39629a2e..b8bb9fd4d 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -479,8 +479,9 @@ const LIST_CONCURRENCY = 8; /** * Every file under the last container a canonical pinned path enters (see * {@link WalkEnd.container}), keyed by tail: "" for the container itself, encoded names joined - * with "/" below it. Containers below it aren't listed; they have their own doc steps. `path` - * may name the container or anything in it. Undefined unless `path` is canonical. + * with "/" below it. Folders below it are listed into only when it's a package (see `deep`); + * otherwise they have their own doc steps. `path` may name the container or anything in it. + * Undefined unless `path` is canonical. * * File docs are read with `findFile` (default: `find`), several at once. One that can't be read * is left out and its tail named in `missing`, so a caller can tell a partial list from a whole diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index 9f8874549..887634586 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -67,7 +67,7 @@ import type { } from "./types.js"; import { createRepo, initWasm } from "./repo.js"; import { watchDocument } from "./watch-document.js"; -import { migratePackages } from "./migrate-packages.js"; +import { caughtUpWithServer, migratePackages } from "./migrate-packages.js"; import { createRouter, type Router } from "./router.js"; import { createDefaultAccount } from "./createAccount.js"; @@ -200,9 +200,9 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { options.importModulesInWorker === false ? undefined : importAutomergePackageViaWorker, - // Look packages up through the automerge worker and watch them through - // this tab's Subduction subscriptions, so their docs never load into this - // tab's Repo. + // Look packages up through the automerge worker (tabs leave /packages + // lookups to it) and watch them through this tab's Subduction + // subscriptions, so their docs never load into this tab's Repo. trackPackagesFolder( account.promise, () => packageEntries, @@ -219,7 +219,10 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { })) as DocHandle<AccountDoc>; // So /@/<acctHash>/… paths under it resolve, here and in the automerge worker. account.resolve(await registerAccount(accountDocHandle.url)); - await migratePackages(repo, accountDocHandle).catch((error) => + await migratePackages(repo, accountDocHandle, { + // Another device may already have migrated this account. + caughtUp: () => caughtUpWithServer(repo, accountDocHandle), + }).catch((error) => console.warn("couldn't move module settings into /packages", error) ); await wirePackagesFolder(repo, accountDocHandle, moduleWatcher, (entries) => { diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts index 9f2c01a4a..a89f03bfa 100644 --- a/core/patchwork/src/migrate-packages.ts +++ b/core/patchwork/src/migrate-packages.ts @@ -9,6 +9,7 @@ import { type AutomergeUrl, type DocHandle, type Repo, + type StorageId, } from "@automerge/automerge-repo/slim"; import { getType, @@ -21,58 +22,121 @@ import { createSubdoc } from "./createAccount.js"; const decode = (content: string | Uint8Array) => typeof content === "string" ? content : new TextDecoder().decode(content); -/** A package's name: its package.json's, else the doc's title, else its id. */ -async function packageName(repo: Repo, url: AutomergeUrl) { +const within = <T>(work: Promise<T>, ms: number) => + Promise.race([ + work, + new Promise<never>((_, reject) => + setTimeout(() => reject(new Error(`no answer in ${ms}ms`)), ms) + ), + ]); + +/** A package's name and type: its package.json's name, else the doc's title. */ +async function describePackage(repo: Repo, url: AutomergeUrl) { const handle = await repo.find<Partial<HasPatchworkMetadata> & { title?: string }>(url); + const type = getType(handle.doc() ?? {}) ?? "folder"; try { const manifest = await resolvePath(repo, handle, ["package.json"]); const name = manifest && JSON.parse(decode(manifest.content))?.name; - if (typeof name === "string" && name) return { handle, name }; + if (typeof name === "string" && name) return { name, type }; } catch { // no package.json, or not JSON: fall back below } - return { handle, name: handle.doc()?.title || handle.documentId }; + return { name: handle.doc()?.title || handle.documentId, type }; +} + +/** + * Whether `handle` has caught up with every sync server this repo is + * connected to, waiting up to `timeoutMs`. False when there's no server to + * ask: offline, the account may be behind another device's. + */ +export async function caughtUpWithServer( + repo: Repo, + handle: DocHandle<unknown>, + timeoutMs = 10_000 +): Promise<boolean> { + if (!repo.isSubductionConnected()) return false; + const peers = (await repo.connectedSubductionPeerIds()) as StorageId[]; + if (!peers.length) return false; + const done = () => peers.every((peer) => handle.isCaughtUpWith(peer) === true); + if (done()) return true; + return new Promise((resolve) => { + const finish = (caughtUp: boolean) => { + clearTimeout(timer); + handle.off("remote-heads", check); + handle.off("heads-changed", check); + resolve(caughtUp); + }; + const check = () => { + if (done()) finish(true); + }; + const timer = setTimeout(() => finish(done()), timeoutMs); + handle.on("remote-heads", check); + handle.on("heads-changed", check); + }); } /** - * Bind `/packages` for an account that has module settings and no packages - * folder. True if it did. Http entries stay where they are, in module - * settings, which is still read. + * Bind `/packages` for an account that has module settings and has never + * had a `/packages` binding (an empty one was emptied on purpose). True if it + * did. + * + * - `caughtUp` is asked first: another device may have migrated already, and + * two migrations would each set `/packages`, one losing. If it says no, the + * migration waits for a later boot. + * - A package it can't read is linked under its document id. + * - Entries pinned to heads, and http entries, stay where they are, in module + * settings, which is still read; a `/packages` link would float to latest. + * - Two packages with one name get distinct names, since only the first of a + * name in a folder is reachable by it. */ export async function migratePackages( repo: Repo, - account: DocHandle<AccountDoc> + account: DocHandle<AccountDoc>, + options: { caughtUp?: () => Promise<boolean>; readTimeoutMs?: number } = {} ): Promise<boolean> { - const doc = account.doc(); - if (doc?.namespace?.["/packages"]?.length || !doc?.moduleSettingsUrl) return false; - const settings = await repo.find<{ modules?: string[] }>(doc.moduleSettingsUrl); + const unmigrated = () => { + const doc = account.doc(); + return !!doc?.moduleSettingsUrl && !(doc.namespace && "/packages" in doc.namespace); + }; + if (!unmigrated()) return false; + if (options.caughtUp && !(await options.caughtUp())) return false; + if (!unmigrated()) return false; + + const settings = await repo.find<{ modules?: string[] }>(account.doc().moduleSettingsUrl!); const seen = new Set<string>(); const urls = (settings.doc()?.modules ?? []).flatMap((entry) => { if (!isValidAutomergeUrl(entry)) return []; - const { documentId } = parseAutomergeUrl(entry); - if (seen.has(documentId)) return []; + const { documentId, heads } = parseAutomergeUrl(entry); + if (heads || seen.has(documentId)) return []; seen.add(documentId); return [stringifyAutomergeUrl({ documentId })]; }); - const links = await Promise.all( + + const described = await Promise.all( urls.map(async (url) => { try { - const { handle, name } = await packageName(repo, url); - return [{ name, type: getType(handle.doc() ?? {}) ?? "folder", url }]; + return { url, ...(await within(describePackage(repo, url), options.readTimeoutMs ?? 10_000)) }; } catch (error) { - console.warn(`couldn't move package ${url} into /packages`, error); - return []; + console.warn(`linking package ${url} into /packages by its id: couldn't read it`, error); + return { url, name: parseAutomergeUrl(url).documentId as string, type: "folder" }; } }) ); + const taken = new Map<string, number>(); + const links = described.map(({ url, name, type }) => { + const count = (taken.get(name) ?? 0) + 1; + taken.set(name, count); + return { name: count === 1 ? name : `${name} (${count})`, type, url }; + }); + // Another tab may have migrated while this one read the packages. - if (account.doc()?.namespace?.["/packages"]?.length) return false; + if (!unmigrated()) return false; const folder = await createSubdoc<{ title: string; docs: unknown[] }>( repo, "folder", (folder) => { folder.title = "Packages"; - folder.docs = links.flat(); + folder.docs = links; } ); account.change((doc) => { diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts index 41f08f954..543c5e7ca 100644 --- a/core/patchwork/test/migrate-packages.test.ts +++ b/core/patchwork/test/migrate-packages.test.ts @@ -35,7 +35,7 @@ describe("migratePackages", () => { const repo = new Repo(); const { alpha, bee } = packages(repo); const acct = await account(repo, [ - alpha.view(alpha.heads()).url, + alpha.url, bee.url, "https://cdn.example.test/tool/index.js", ]); @@ -73,6 +73,55 @@ describe("migratePackages", () => { }); }); +describe("migratePackages: what it can't simply link", () => { + it("names two packages with one package.json name apart", async () => { + const repo = new Repo(); + const twin = () => { + const json = repo.create({ "@patchwork": { type: "file" }, content: '{"name":"alpha"}', mimeType: "application/json" }); + return repo.create({ "@patchwork": { type: "directory" }, "package.json": json.url }); + }; + const [a, b] = [twin(), twin()]; + const acct = await account(repo, [a.url, b.url]); + await migratePackages(repo, acct); + expect(await linked(repo, acct)).toEqual([ + ["alpha", a.url], + ["alpha (2)", b.url], + ]); + }); + + it("links a package it can't read under its document id", async () => { + const repo = new Repo(); + const missing = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + const acct = await account(repo, [missing]); + await migratePackages(repo, acct, { readTimeoutMs: 200 }); + expect(await linked(repo, acct)).toEqual([["4NMNnkMhL8jXrdJ9jamS58PAVdXu", missing]]); + }); + + it("leaves an entry pinned to heads in module settings", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.view(alpha.heads()).url]); + await migratePackages(repo, acct); + expect(await linked(repo, acct)).toEqual([]); + }); + + it("waits for the account to catch up, and leaves it for next boot if it can't", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + expect(await migratePackages(repo, acct, { caughtUp: async () => false })).toBe(false); + expect(acct.doc().namespace).toBeUndefined(); + }); + + it("doesn't migrate again after /packages was emptied on purpose", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + acct.change((d: any) => (d.namespace = { "/packages": [] })); + expect(await migratePackages(repo, acct)).toBe(false); + }); +}); + describe("createDefaultAccount", () => { it("binds / to the root folder and /packages to a new one", async () => { const repo = new Repo(); From 19a1871501380c947b9161cd3e31dac332f339c0 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 17:27:35 +0000 Subject: [PATCH 50/81] plan: paths in the sidebar (packages folder, status, open at a path) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../plans/2026-10-28-paths-sidebar.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-28-paths-sidebar.md diff --git a/docs/superpowers/plans/2026-10-28-paths-sidebar.md b/docs/superpowers/plans/2026-10-28-paths-sidebar.md new file mode 100644 index 000000000..b61d1da42 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-sidebar.md @@ -0,0 +1,64 @@ +# Paths in the sidebar: packages as a real folder, status, open at a path — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** In the shell, `/packages` shows up as a real `packages` folder in the sidebar tree. Each package row shows whether it loaded. Opening anything from the sidebar opens it at its live path: `&path=` is set, and the tool gets `<patchwork-view path>`. The account bar's Packages link goes once `/packages` is bound. + +**Architecture:** This spans two repos. +- **system** (`paths-design`): + - the ModuleWatcher reports each package's status; + - accounts get a root-folder link to their `/packages` folder. +- **base** (`paths`), each tool standalone, with no build-time links between them: + - `providers`: `selected-view` carries `path`; + - `threepane`: passes it to the main view, and hides the Packages link; + - `sideboard`: rows open with their live path and show package status, and the separate Packages section is removed. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 3 (*Opening at a path*: "the sidebar dispatches `{ url, path }` on click, using the names it walked") and section 4. + +## Decisions + +1. **The `packages` folder is real.** It's a link named `packages` in the root folder, to the first folder bound at `/packages`. The mount shadows a same-named entry in `/`, so `<acct>/packages/x` means the same thing either way. It's added only if the root folder has no entry by that name. Removing or renaming the link changes only the view. +2. **Status belongs to the document, not the folder.** + - `ModuleWatcher.packageStatus(urlOrDocumentId)` returns `{ state: "loading" | "loaded" | "failed", error?, plugins?: string[] }`, or undefined for anything that isn't a package. + - The watcher dispatches a `"status"` event when one changes. It's an `EventTarget`. + - Rows read it late-bound through `globalThis.patchwork?.packages`, and look as they do today when it's missing. +3. **A row's live path is built from the names walked from the root folder.** + - Each `DocumentList` passes `names` down. + - The acctHash is idhash(account documentId), copied into sideboard (SHA-256, first 12 bytes, base58, about 15 lines), since tools can't import across repos at build time. + - The account comes from `globalThis.patchwork?.account`. A panel whose folder isn't that account's `rootFolderUrl` opens without a path. +4. **`selected-view` carries `path`.** threepane passes it to `<patchwork-view path>` on the main view. On boot, the provider reads `path=` raw from the hash, since `URLSearchParams` decodes `%2F`. + +## Tasks + +### system + +- [ ] **S1: Package status on the ModuleWatcher.** Test (`module-watcher.test.ts`): + - a package goes `loading` → `loaded` with its plugin ids; + - a failing one goes to `failed` with the error; + - `"status"` fires; + - an http module has no status. + + Implement: a `#status` map keyed by documentId, updated in `announce`. `importPackageSafe` returns the error instead of swallowing it. +- [ ] **S2: The root-folder `packages` link.** Test (`migrate-packages.test.ts`): + - `createDefaultAccount` adds `{ name: "packages", type: "folder", url: <packages folder> }` to the root folder; + - migration adds it when there's no `packages` entry, and leaves an existing one alone. +- [ ] **S3: Changeset**, then build, test (by exit code), and commit. + +### base (`paths` branch) + +- [ ] **B1: providers.** `SelectedView.path`, set from `detail.path` and from the raw `path=` in the hash on boot. Test it. +- [ ] **B2: threepane.** + - `MainDocumentView` sets `path` on the main `<patchwork-view>`. + - The account bar's Packages button shows only when `/packages` isn't bound; the namespace is read from the account doc the frame already holds. + - Test the bar. +- [ ] **B3: sideboard paths.** + - `DocumentList` threads `names`. + - Rows and folders open with `path` when the panel is the account's root. + - `idhash` gets a known-vector test. + - Test that a nested row's open event carries `<acctHash>/a/b`. +- [ ] **B4: sideboard status.** + - A row whose document has a package status shows a dot (`data-package-state`) with a tooltip. + - It re-renders on `"status"`. + - Test it with a fake `globalThis.patchwork.packages`. +- [ ] **B5: Remove the Packages section** from `sideboard.tsx`, and `account-bar.tsx`'s use of it. threepane inlines its own bar, and `chee/account-bar` stays as it was. Keep the panel's menu-scoping fix, which still matters for two list widgets. +- [ ] **B6: Build, test and lint** each touched tool, then commit and push `paths`. From 81229b26fd999c3141da0b0d32185fa42412b0b1 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 17:29:42 +0000 Subject: [PATCH 51/81] filesystem: the ModuleWatcher reports each package's status packageStatus(url or documentId) says whether a package is loading, loaded (with the plugins it brought) or failed (with why), and a "status" event fires when one changes, so a sidebar can show it beside the package. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/module-watcher.ts | 71 ++++++++++++++++++--- core/filesystem/test/module-watcher.test.ts | 53 +++++++++++++++ 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index e8437a74e..42a25de22 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -101,6 +101,9 @@ type WatchedModule = { const stripHeads = (url: AutomergeUrl) => stringifyAutomergeUrl({ documentId: parseAutomergeUrl(url).documentId }); +const describeError = (error: unknown) => + error instanceof Error ? error.message : String(error); + function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { console.warn( `module ${branchesDocUrl} is a branches doc. Branches docs are no longer supported as modules; falling back to the "${DEFAULT_BRANCH}" branch. Please let us know you hit this: post in #patchwork-testers or email chee@inkandswitch.com` @@ -110,6 +113,12 @@ function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { // todo this can be a function that takes a plugin system and returns a change // handler +/** How a package is doing: what a sidebar shows beside it. */ +export type PackageStatus = + | { state: "loading" } + | { state: "loaded"; plugins: string[] } + | { state: "failed"; error: string }; + /** * Settings sources are passed in keyed by name (e.g. `{ system, user }`). Each * source URL may be either an Automerge module-settings doc (`automerge:...`, @@ -117,8 +126,11 @@ function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { * (fetched once at construction). The two kinds can be freely mixed, and the * module URLs *within* either kind can themselves point at Automerge package * docs (folders or directories) or plain HTTP(S) bundles. + * + * Dispatches a `"status"` event whenever a package's {@link PackageStatus} + * changes; read it with {@link ModuleWatcher.packageStatus}. */ -export class ModuleWatcher { +export class ModuleWatcher extends EventTarget { repo: Repo; urls: Record<string, string>; handles: Record<string, DocHandle<ModuleSettingsDoc>> | undefined; @@ -141,6 +153,8 @@ export class ModuleWatcher { { entries: () => PackageEntry[]; unsubscribe: () => void } >(); #disposed = false; + /** Package status, by the package doc's documentId. */ + #status = new Map<string, PackageStatus>(); onLoad: (name: string, mod: any) => void; onUnload?: (name: string) => void; @@ -169,6 +183,7 @@ export class ModuleWatcher { importPackageFromFolderDocUrl(url as AutomergeUrl), tracker: PackageTracker = trackPackagesInRepo(repo) ) { + super(); this.repo = repo; this.urls = { ...urls }; this.onLoad = callback; @@ -268,6 +283,7 @@ export class ModuleWatcher { `couldn't look up ${importName}${delay === undefined ? "" : `; trying again in ${delay}ms`}`, error ); + this.#setStatus(importName, { state: "failed", error: describeError(error) }); if (delay !== undefined) { const timer = setTimeout(() => { this.#lookupRetries.delete(importName); @@ -291,23 +307,44 @@ export class ModuleWatcher { await this.announce(importName); } - private async importPackageSafe(importName: string): Promise<any> { + /** + * How the package doc `urlOrDocumentId` is doing, or undefined if it isn't + * a package this watcher loads (an http module has none). + */ + packageStatus(urlOrDocumentId: string): PackageStatus | undefined { + const documentId = isValidAutomergeUrl(urlOrDocumentId) + ? parseAutomergeUrl(urlOrDocumentId).documentId + : urlOrDocumentId; + return this.#status.get(documentId); + } + + #setStatus(moduleKey: string, status: PackageStatus) { + if (!isValidAutomergeUrl(moduleKey)) return; + this.#status.set(parseAutomergeUrl(moduleKey).documentId, status); + this.dispatchEvent(new Event("status")); + } + + private async importPackageSafe( + importName: string + ): Promise<{ mod: any } | { error: unknown }> { try { if (isValidAutomergeUrl(importName)) { // Pin to heads so descriptor discovery and any later main-thread load // resolve against the exact same version of the package doc. - return await this.importAutomergePackage( - await this.tracker.findLatest(importName) - ); + return { + mod: await this.importAutomergePackage( + await this.tracker.findLatest(importName) + ), + }; } - return await importPackageFromHttpUrl(importName); + return { mod: await importPackageFromHttpUrl(importName) }; } catch (error) { console.error( `%c Failed to import ${importName}`, "color: #000, background: #ffbcef", error ); - return undefined; + return { error }; } } @@ -347,15 +384,27 @@ export class ModuleWatcher { this.#announceGenerations.get(moduleKey) === generation; const attempt = async (attemptIndex: number): Promise<void> => { - const mod = await this.importPackageSafe(importUrl); + if (attemptIndex === 0) this.#setStatus(moduleKey, { state: "loading" }); + const result = await this.importPackageSafe(importUrl); // A newer announce for this module started while we were importing or // backing off — its result wins; announcing ours now could regress the // registry to an older version. if (!current()) return; - if (mod) { + if ("mod" in result && result.mod) { + const mod = result.mod; this.#lastAnnounced.set(moduleKey, versionKey); + this.#setStatus(moduleKey, { + state: "loaded", + plugins: Array.isArray(mod.plugins) + ? mod.plugins.map((plugin: { id?: string }) => plugin?.id).filter(Boolean) + : [], + }); return this.onLoad(importUrl, mod); } + this.#setStatus(moduleKey, { + state: "failed", + error: describeError("error" in result ? result.error : "it exported nothing"), + }); const delay = RETRY_DELAYS_MS[attemptIndex]; if (delay === undefined) return; const timer = setTimeout(() => { @@ -529,6 +578,10 @@ export class ModuleWatcher { // Its plugins were registered under the url it was imported from. const announced = this.#lastAnnounced.get(entry); this.#lastAnnounced.delete(entry); + if (isValidAutomergeUrl(entry)) { + this.#status.delete(parseAutomergeUrl(entry).documentId); + this.dispatchEvent(new Event("status")); + } this.onUnload?.(announced ?? entry); } diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 23de278dd..0198d5269 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -992,3 +992,56 @@ describe("ModuleWatcher lookup retries", () => { await repo.shutdown().catch(() => {}); }); }); + +describe("ModuleWatcher package status", () => { + function setup(module: string) { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create<any>({ "@patchwork": { type: "directory" } }); + const tracker: PackageTracker = { + findLatest: async () => dataModule(module) as any, + watch: () => () => {}, + }; + const watcher = new ModuleWatcher(repo, {}, () => {}, undefined, undefined, tracker); + const events: string[] = []; + watcher.addEventListener("status", () => + events.push(watcher.packageStatus(doc.url)?.state ?? "none") + ); + const add = () => + watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + return { repo, doc, watcher, events, add }; + } + + it("says a package loaded, and which plugins it brought", async () => { + const { repo, doc, watcher, events, add } = setup( + "export const plugins = [{ type: 'patchwork:tool', id: 'demo' }]" + ); + await add(); + expect(watcher.packageStatus(doc.url)).toEqual({ state: "loaded", plugins: ["demo"] }); + expect(watcher.packageStatus(doc.documentId)?.state).toBe("loaded"); + expect(events).toContain("loading"); + expect(events.at(-1)).toBe("loaded"); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("says a package failed, and why", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { repo, doc, watcher, add } = setup("throw new Error('kaboom')"); + await add(); + expect(watcher.packageStatus(doc.url)).toMatchObject({ state: "failed" }); + expect(watcher.packageStatus(doc.url)?.error).toContain("kaboom"); + watcher.dispose(); + errors.mockRestore(); + await repo.shutdown().catch(() => {}); + }); + + it("knows nothing of a document that isn't a package", async () => { + const { repo, watcher } = setup("export const plugins = []"); + expect(watcher.packageStatus("automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu")).toBeUndefined(); + await repo.shutdown().catch(() => {}); + }); +}); From 9a36e52921294d37c192e30f24ea2e1fb9d26dd9 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 17:30:52 +0000 Subject: [PATCH 52/81] patchwork: show /packages as a packages folder in the root folder createDefaultAccount and the migration link the packages folder into the root folder as packages, unless it already has an entry by that name, so the sidebar shows it as a real folder. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/paths-packages.md | 2 +- core/patchwork/src/createAccount.ts | 23 ++++++++++++++- core/patchwork/src/migrate-packages.ts | 5 +++- core/patchwork/test/migrate-packages.test.ts | 30 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/.changeset/paths-packages.md b/.changeset/paths-packages.md index 5056b76f5..082817a55 100644 --- a/.changeset/paths-packages.md +++ b/.changeset/paths-packages.md @@ -5,4 +5,4 @@ "@inkandswitch/patchwork": patch --- -Package management is whatever is bound at `/packages` in the account's namespace. New accounts bind `/` to their root folder and `/packages` to a new Packages folder; an older account gets a Packages folder at boot, linking each automerge package its module-settings doc lists, by its `package.json` name. The ModuleWatcher loads every package in the `/packages` union (`addPackagesFolder`, `watchPackagesFolder`), keyed by document so a rename reloads nothing, and looks each one up as a pinned `/@/` URL (`trackPackagesFolder`) that it imports and the service worker checks against the package's file list. A package's list now covers everything under it, nested folders included, and a redirect to a folder carries the folder's whole list. `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding; it's still read, deduplicated against `/packages`, since the Packages UI installs into it. +Package management is whatever is bound at `/packages` in the account's namespace. New accounts bind `/` to their root folder and `/packages` to a new Packages folder; an older account gets a Packages folder at boot, linking each automerge package its module-settings doc lists, by its `package.json` name. The ModuleWatcher loads every package in the `/packages` union (`addPackagesFolder`, `watchPackagesFolder`), keyed by document so a rename reloads nothing, and looks each one up as a pinned `/@/` URL (`trackPackagesFolder`) that it imports and the service worker checks against the package's file list. A package's list now covers everything under it, nested folders included, and a redirect to a folder carries the folder's whole list. New accounts, and migrated ones, also get a `packages` link in their root folder to that folder, so `/packages` shows in the tree like any other folder. The ModuleWatcher reports each package's status (`packageStatus(url)`: loading, loaded with its plugins, or failed with why) and fires a `"status"` event when one changes; it is now an `EventTarget`. `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding; it's still read, deduplicated against `/packages`, since the Packages UI installs into it. diff --git a/core/patchwork/src/createAccount.ts b/core/patchwork/src/createAccount.ts index c0383b339..a3679d678 100644 --- a/core/patchwork/src/createAccount.ts +++ b/core/patchwork/src/createAccount.ts @@ -1,4 +1,4 @@ -import type { Repo } from "@automerge/automerge-repo/slim"; +import type { AutomergeUrl, Repo } from "@automerge/automerge-repo/slim"; import type { AccountCreator, AccountDoc, @@ -18,6 +18,26 @@ export async function createSubdoc<D>( return handle; } +/** + * Show `/packages` in the account's tree: a link named `packages` in its root + * folder, to the packages folder, unless the root folder already has an entry + * by that name. The `/packages` mount shadows a same-named entry in `/`, so the + * link means the same thing; renaming or removing it changes only the view. + */ +export async function linkPackagesIntoRoot( + repo: Repo, + rootFolderUrl: AutomergeUrl | undefined, + packagesFolderUrl: AutomergeUrl +): Promise<void> { + if (!rootFolderUrl) return; + const root = await repo.find<{ docs?: Array<{ name: string }> }>(rootFolderUrl); + if (!Array.isArray(root.doc()?.docs)) return; + if (root.doc()!.docs!.some((entry) => String(entry?.name) === "packages")) return; + root.change((doc) => { + doc.docs!.push({ name: "packages", type: "folder", url: packagesFolderUrl } as never); + }); +} + export const createDefaultAccount: AccountCreator<AccountDoc> = async ( accountHandle, repo @@ -54,4 +74,5 @@ export const createDefaultAccount: AccountCreator<AccountDoc> = async ( doc.moduleSettingsUrl = moduleSettings.url; doc.contactUrl = contact.url; }); + await linkPackagesIntoRoot(repo, rootFolder.url, packagesFolder.url); }; diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts index a89f03bfa..2d3930d6f 100644 --- a/core/patchwork/src/migrate-packages.ts +++ b/core/patchwork/src/migrate-packages.ts @@ -17,7 +17,7 @@ import { type HasPatchworkMetadata, } from "@inkandswitch/patchwork-filesystem"; import type { AccountDoc } from "@inkandswitch/patchwork-plugins"; -import { createSubdoc } from "./createAccount.js"; +import { createSubdoc, linkPackagesIntoRoot } from "./createAccount.js"; const decode = (content: string | Uint8Array) => typeof content === "string" ? content : new TextDecoder().decode(content); @@ -143,5 +143,8 @@ export async function migratePackages( doc.namespace ??= {}; doc.namespace["/packages"] = [{ url: folder.url }]; }); + await linkPackagesIntoRoot(repo, account.doc()?.rootFolderUrl, folder.url).catch( + (error) => console.warn("couldn't show /packages in the root folder", error) + ); return true; } diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts index 543c5e7ca..79d5c2ef6 100644 --- a/core/patchwork/test/migrate-packages.test.ts +++ b/core/patchwork/test/migrate-packages.test.ts @@ -122,6 +122,32 @@ describe("migratePackages: what it can't simply link", () => { }); }); +describe("migratePackages: the packages folder in the tree", () => { + it("links the new folder into the root folder as packages", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const root = repo.create<any>({ title: "root", docs: [] }); + acct.change((d: any) => (d.rootFolderUrl = root.url)); + await migratePackages(repo, acct); + expect(root.doc().docs).toEqual([ + { name: "packages", type: "folder", url: acct.doc().namespace["/packages"][0].url }, + ]); + }); + + it("leaves a root folder that already has something named packages alone", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const mine = repo.create({ title: "mine", docs: [] }).url; + const root = repo.create<any>({ title: "root", docs: [{ name: "packages", type: "folder", url: mine }] }); + acct.change((d: any) => (d.rootFolderUrl = root.url)); + await migratePackages(repo, acct); + expect(root.doc().docs).toHaveLength(1); + expect(root.doc().docs[0].url).toBe(mine); + }); +}); + describe("createDefaultAccount", () => { it("binds / to the root folder and /packages to a new one", async () => { const repo = new Repo(); @@ -131,6 +157,10 @@ describe("createDefaultAccount", () => { expect(doc.namespace["/"]).toEqual([{ url: doc.rootFolderUrl }]); const packagesFolder = await repo.find<any>(doc.namespace["/packages"][0].url); expect(packagesFolder.doc()).toMatchObject({ title: "Packages", docs: [] }); + const root = await repo.find<any>(doc.rootFolderUrl); + expect(root.doc().docs).toEqual([ + { name: "packages", type: "folder", url: packagesFolder.url }, + ]); expect(doc.moduleSettingsUrl).toBeDefined(); }); }); From f4d3d70488a2629ec93682eb2d132ac2f776cb87 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 17:57:57 +0000 Subject: [PATCH 53/81] spec: propose package-supplied path providers (section 6, not approved) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...28-paths-namespace-content-cache-design.md | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 402c11454..404ad2af0 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -389,6 +389,75 @@ store, and both are checked against the URL the watcher pinned. - **Legacy `/automerge%3A…` URLs** stay keyed by URL and unchecked. Their replies still ride `HANDOFF_CHANNEL`, which any context can answer first. +## 6. Package-supplied path providers (proposed, not approved) + +The sidebar can open up a doc that isn't a folder when a built-in provider +walks it: a directory doc, a branches doc, the account. This section proposes +letting a datatype say what its children are too: a tldraw doc's embedded +docs, say, or the docs a markdown doc links to. It lifts the non-goal +"Path providers supplied by packages". + +### The plugin + +A `patchwork:path-provider` plugin is a `PathProvider`: `matches`, `lookup` +and `list`, exported from `patchwork-filesystem` already. A datatype package +registers one beside its datatype. + +- **Built-ins come first and can't be overridden.** Account, directory, folder + and branches are checked first; package providers after, in registration + order. A package can't take over how folders resolve. +- **It's a pure function of the document.** `list` and `lookup` read only the + node they're given, at the heads they're given. That's what lets a file + list built through a provider be rebuilt to the same bytes. + +### Where it runs: tabs only + +The automerge worker never loads package code, so it never runs package +providers. A tab does: its sidebar, its walks, and its resolver. + +- **The service worker asks tabs first.** Live → pinned redirects and reads at + heads through a package provider are answered by a tab. A page made the + request, so there is one. +- **`/packages` is the worker's.** Tabs decline it (section 4), so package + providers don't apply under `/packages`. Packages are folders and + directories anyway. +- **A pinned URL needs a tab to rebuild its list.** If a pinned URL goes + through a package provider and its list is lost, only a tab with that + provider loaded can rebuild it. Without one, it fails as a network error. + +### Trust + +Nothing between a URL and its bytes trusts a provider (section 5). A file list +built through one is checked against the URL; bytes are checked against the +list. + +What a provider *can* do is the same as any package code in a tab: +- name the wrong children in the sidebar; +- resolve a live path to a document it shouldn't; +- by being pinned, choose the version a URL names. + +That's the "whoever builds a URL chooses its version" gap, applied to +providers. It adds no access a package doesn't already have. + +### Versions + +A new version of the providing package can change what a live path resolves +to. Pinned paths don't change. A list rebuilt through a changed provider won't +match its hash, so the URL fails rather than serving something else. + +### Open questions + +1. **Editing.** Should a provider be able to say how to add, rename or remove + a child, so those rows aren't read-only? Or is that the datatype's own + tool's job? +2. **Values.** Should children that are values inside the doc (not links) be + openable, as a sub-document view with its own path? Today only link + children show. +3. **Rows without a type.** The sidebar can only tell a row expands from its + link `type`. A provider for a type that isn't `directory` or `branches` + needs its type registered as expandable. Should the plugin declare + `supportedDatatypes` for that, like tools do? + ## Remaining gaps These replace the deleted `package-load-gate.md` note. From 976dd5428141f0e18e4a95ff053cd16945356e18 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 18:03:48 +0000 Subject: [PATCH 54/81] =?UTF-8?q?spec:=20section=206=20approved=20?= =?UTF-8?q?=E2=80=94=20providers=20declare=20datatypes,=20may=20offer=20ed?= =?UTF-8?q?iting=20and=20sub-documents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...28-paths-namespace-content-cache-design.md | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 404ad2af0..f50cf7de8 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -26,7 +26,7 @@ authenticated channel) and gets reworked to this one. - Running package code on a separate origin. This design is meant not to preclude it (the store and file lists work the same there), but doesn't do it. - Mount points below the top level of the namespace. -- Path providers supplied by packages. +- Path providers supplied by packages, until section 6 is built. - Garbage collection of the content store. ## Threat model @@ -389,7 +389,9 @@ store, and both are checked against the URL the watcher pinned. - **Legacy `/automerge%3A…` URLs** stay keyed by URL and unchecked. Their replies still ride `HANDOFF_CHANNEL`, which any context can answer first. -## 6. Package-supplied path providers (proposed, not approved) +## 6. Package-supplied path providers + +Approved in conversation 2026-10-28. Not implemented. The sidebar can open up a doc that isn't a folder when a built-in provider walks it: a directory doc, a branches doc, the account. This section proposes @@ -409,6 +411,9 @@ registers one beside its datatype. - **It's a pure function of the document.** `list` and `lookup` read only the node they're given, at the heads they're given. That's what lets a file list built through a provider be rebuilt to the same bytes. +- **It says which datatypes it walks** with `supportedDatatypes`, as a tool + does. That's how the sidebar knows a row of that type can open up, from its + link's `type`, without loading the doc. ### Where it runs: tabs only @@ -445,18 +450,33 @@ A new version of the providing package can change what a live path resolves to. Pinned paths don't change. A list rebuilt through a changed provider won't match its hash, so the URL fails rather than serving something else. -### Open questions - -1. **Editing.** Should a provider be able to say how to add, rename or remove - a child, so those rows aren't read-only? Or is that the datatype's own - tool's job? -2. **Values.** Should children that are values inside the doc (not links) be - openable, as a sub-document view with its own path? Today only link - children show. -3. **Rows without a type.** The sidebar can only tell a row expands from its - link `type`. A provider for a type that isn't `directory` or `branches` - needs its type registered as expandable. Should the plugin declare - `supportedDatatypes` for that, like tools do? +### What a provider offers + +The sidebar shows what a provider offers and no more. Two things are optional. + +**Editing.** A provider may implement any of these: +- `add(handle, node, child, index?)`; +- `rename(handle, node, name, to)`; +- `remove(handle, node, name)`. + +Each takes the doc's handle and changes the doc. The sidebar turns on Rename, +Remove, drops and new-doc only where the provider offers them. A child of a +provider that offers none is read-only, as directory and branches children are +now. + +The built-in folder provider would offer all three. That would make the +sidebar's folder editing one case of this rather than its own code path, +though there's no need to move it yet. + +**Values as sub-documents.** A child that is a value inside the doc, not a +link, can be offered as a sub-document: +- `{ name, sub: [...segments] }` is opened as automerge-repo's sub-handle, + `handle.sub(...segments)`, at the URL `automerge:<docId>/<path>`; +- the tool gets that handle, so it edits just that part of the doc; +- the live path names it as an in-doc step (rule 3): a plain name after the + doc's own step, pinned by the doc's heads. + +Values a provider doesn't offer as sub-documents stay hidden, as now. ## Remaining gaps From 0c03d47b132f7927c659853f537501b3646f1d17 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 20:45:42 +0000 Subject: [PATCH 55/81] filesystem: sub-document children, provider editing methods, extra providers in walks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/providers.ts | 42 +++- core/filesystem/src/resolve.ts | 6 +- core/filesystem/src/walk.ts | 101 ++++++++-- .../filesystem/test/package-providers.test.ts | 130 ++++++++++++ .../2026-10-28-paths-path-provider-plugins.md | 187 ++++++++++++++++++ 5 files changed, 441 insertions(+), 25 deletions(-) create mode 100644 core/filesystem/test/package-providers.test.ts create mode 100644 docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts index 0716aa4fe..36c2cfbdf 100644 --- a/core/filesystem/src/providers.ts +++ b/core/filesystem/src/providers.ts @@ -1,5 +1,6 @@ import { type AutomergeUrl, + type DocHandle, isImmutableString, isValidAutomergeUrl, } from "@automerge/automerge-repo/slim"; @@ -11,14 +12,20 @@ import type { FolderDoc, } from "./types.js"; +/** A step into a document: an object key or an array index. */ +export type SubSegment = string | number; + /** * What a provider finds under a name: a link to another document, a value inside the same - * document (a nested directory, a file's bytes), or an ordered union of documents (a mount). + * document (a nested directory, a file's bytes), an ordered union of documents (a mount), or a + * value offered as a sub-document, opened as `handle.sub(...sub)`. `sub` is relative to the node + * the provider was given. */ export type Child = | { name: string; link: AutomergeUrl } | { name: string; value: unknown } - | { name: string; union: AutomergeUrl[] }; + | { name: string; union: AutomergeUrl[] } + | { name: string; sub: SubSegment[] }; /** * A child and how many of the leading names it used. A step that consumes no name (the `/` @@ -29,13 +36,42 @@ export type Lookup = { child: Child; consumed: number }; /** * How to walk one kind of document. `lookup` and `list` take the document or a value nested in * it, and never leave it: following a link is the walker's job. `lookup` is called with at least - * one name. + * one name. They read only the node they're given, so a walk at the same heads gives the same + * answer. + * + * The editing methods are optional; the sidebar offers only those a provider has. Each changes + * the document through `handle`; `node` is what `list` was given. */ export interface PathProvider { readonly kind: string; matches(doc: unknown): boolean; lookup(node: unknown, names: string[]): Lookup | undefined; list(node: unknown): Child[]; + add?( + handle: DocHandle<unknown>, + node: unknown, + child: { name: string; link: AutomergeUrl }, + index?: number + ): void | Promise<void>; + rename?(handle: DocHandle<unknown>, node: unknown, name: string, to: string): void | Promise<void>; + remove?(handle: DocHandle<unknown>, node: unknown, name: string): void | Promise<void>; +} + +/** + * Providers asked after the built-ins, which can't be overridden. A tab passes the ones + * packages register; the automerge worker passes none. + */ +export type ExtraProviders = (doc: unknown) => Promise<PathProvider | undefined>; + +/** The value at `segments` below `node`, or undefined. */ +export function valueAt(node: unknown, segments: SubSegment[]): unknown { + let at = node; + for (const segment of segments) { + if (!at || typeof at !== "object" || at instanceof Uint8Array) return undefined; + if (!Object.hasOwn(at, segment)) return undefined; + at = (at as Record<SubSegment, unknown>)[segment]; + } + return at; } /** diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 0306176b0..4fe9f1aed 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -3,7 +3,7 @@ import { type Repo, isImmutableString, } from "@automerge/automerge-repo/slim"; -import { type PathProvider, asLink, providerFor } from "./providers.js"; +import { type PathProvider, asLink, providerFor, valueAt } from "./providers.js"; export type Resolved = { content: string | Uint8Array; @@ -54,6 +54,10 @@ async function resolveIn( const { child, consumed } = found; const rest = parts.slice(consumed); if ("value" in child) return resolveIn(repo, provider, child.value, rest, walk); + if ("sub" in child) { + const value = valueAt(node, child.sub); + return value === undefined ? undefined : resolveIn(repo, provider, value, rest, walk); + } // A union (an account's mount) is read through its first binding. const link = "link" in child ? child.link : child.union[0]; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index b8bb9fd4d..1afa5cf7b 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -17,7 +17,9 @@ import { } from "./paths.js"; import { type Child, + type ExtraProviders, type Lookup, + type SubSegment, directoryProvider, folderProvider, type PathProvider, @@ -25,6 +27,7 @@ import { branchesProvider, defaultBranch, providerFor, + valueAt, } from "./providers.js"; import { buildFileList } from "./file-list.js"; import { readNode } from "./resolve.js"; @@ -40,8 +43,41 @@ export type WalkOptions = { * the walk. */ find?: (url: AutomergeUrl, heads?: UrlHeads) => Promise<DocHandle<unknown>>; + /** Providers for documents no built-in walks. A tab passes the ones packages register. */ + providers?: ExtraProviders; }; +/** The built-in provider for `doc`, else one of `extra`'s; one that fails is none. */ +async function providerOf( + doc: unknown, + extra?: ExtraProviders +): Promise<PathProvider | undefined> { + const builtIn = providerFor(doc); + if (builtIn || !extra) return builtIn; + try { + return await extra(doc); + } catch { + return undefined; + } +} + +// A package's provider is package code: one that throws has no children here. +function lookUp(provider: PathProvider, node: unknown, names: string[]): Lookup | undefined { + try { + return provider.lookup(node, names); + } catch { + return undefined; + } +} + +function listOf(provider: PathProvider | undefined, node: unknown): Child[] { + try { + return provider?.list(node) ?? []; + } catch { + return []; + } +} + // idhashes never change, and every walk hashes the same few documents again. const idhashes = new Map<string, Promise<string>>(); function memoIdhash(documentId: string): Promise<string> { @@ -76,6 +112,8 @@ export type WalkEnd = { container?: { handle: DocHandle<unknown>; pinned: string; listHash?: string }; /** The encoded names after the container, joined with "/": "" at the container itself. */ tail: string; + /** Where the walk ended inside the document, when that is a sub-document: `handle.sub(...sub)`. */ + sub?: SubSegment[]; }; /** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ @@ -86,6 +124,8 @@ type At = node: unknown; provider?: PathProvider; inside: boolean; + /** The segments from the document to `node`, while every step inside was a sub-document. */ + sub?: SubSegment[]; } | { kind: "union"; members: AutomergeUrl[] }; @@ -128,7 +168,8 @@ export async function walkPath( const handle = live.view(at ?? live.heads()); const node = handle.doc(); if (node === undefined) return undefined; - return { kind: "doc", handle, node, provider: providerFor(node), inside: false }; + const provider = await providerOf(node, options.providers); + return { kind: "doc", handle, node, provider, inside: false }; } catch { return undefined; } @@ -214,7 +255,10 @@ export async function walkPath( const names = namesFrom(i); for (const url of at.members) { const candidate = await enter(url); - if (!names.length || candidate?.provider?.lookup(candidate.node, names)) { + if ( + !names.length || + (candidate?.provider && lookUp(candidate.provider, candidate.node, names)) + ) { member = url; next = candidate; break; @@ -244,7 +288,7 @@ export async function walkPath( } if (segment.kind === "doc") { - const links = (at.provider?.list(at.node) ?? []).flatMap((child) => + const links = listOf(at.provider, at.node).flatMap((child) => "link" in child ? [child.link] : "union" in child ? child.union : [] ); const link = await matchLink(links, segment); @@ -262,7 +306,7 @@ export async function walkPath( // A document without a provider has no children. if (!at.provider) return undefined; const names = namesFrom(i); - const found: Lookup | undefined = at.provider.lookup(at.node, names); + const found: Lookup | undefined = lookUp(at.provider, at.node, names); if (!found) return undefined; const { child, consumed }: Lookup = found; const written = names.slice(0, consumed).map(encodeURIComponent); @@ -270,7 +314,13 @@ export async function walkPath( if ("value" in child) { out.push(...written); - at = { ...at, node: child.value, inside: true }; + at = { ...at, node: child.value, inside: true, sub: undefined }; + } else if ("sub" in child) { + const node = valueAt(at.node, child.sub); + if (node === undefined) return undefined; + out.push(...written); + const sub: SubSegment[] | undefined = at.inside && !at.sub ? undefined : [...(at.sub ?? []), ...child.sub]; + at = { ...at, node, inside: true, sub }; } else if ("union" in child) { out.push(...written); at = { kind: "union", members: child.union }; @@ -301,6 +351,7 @@ export async function walkPath( listHash: lists.get(container.index), }, tail: container ? out.slice(container.index + 1).join("/") : "", + ...(at.sub ? { sub: at.sub } : {}), }; } @@ -308,8 +359,9 @@ export async function walkPath( * The document a live or pinned path names. For a canonical pinned path (see * {@link WalkEnd.canonical}) that is a view at the heads it was read at; a file reached by name * after the last pinned document is read at its current heads, or the heads its link carries. - * Anything else gets the live handle. Undefined if the path doesn't resolve, or ends at a value - * inside a document. + * Anything else gets the live handle. A path that ends at a sub-document gets that handle's + * sub-handle. Undefined if the path doesn't resolve, or ends at any other value inside a + * document. */ export async function resolveLivePath( repo: Repo, @@ -317,9 +369,11 @@ export async function resolveLivePath( options: WalkOptions ): Promise<DocHandle<unknown> | undefined> { const end = await walkPath(repo, path, options); - if (!end?.atDocument) return undefined; - if (end.canonical) return end.handle; - return repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); + if (!end || (!end.atDocument && !end.sub)) return undefined; + const handle = end.canonical + ? end.handle + : await repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); + return end.sub ? handle.sub(...end.sub) : handle; } /** @@ -378,13 +432,14 @@ export async function pinPath( /** Every binding's children, concatenated in binding order. */ export async function listUnion( repo: Repo, - urls: AutomergeUrl[] + urls: AutomergeUrl[], + providers?: ExtraProviders ): Promise<Child[]> { const lists = await Promise.all( urls.map(async (url) => { try { const doc = (await repo.find(url)).doc(); - return providerFor(doc)?.list(doc) ?? []; + return listOf(await providerOf(doc, providers), doc); } catch { return []; } @@ -399,19 +454,20 @@ export async function listUnion( */ export async function listChildren( repo: Repo, - handle: DocHandle<unknown> + handle: DocHandle<unknown>, + providers?: ExtraProviders ): Promise<Child[]> { const doc = handle.doc(); - const provider = providerFor(doc); + const provider = await providerOf(doc, providers); if (!provider) return []; - const children = provider.list(doc); + const children = listOf(provider, doc); if (provider !== accountProvider) return children; const mounts = children.filter((child) => child.name !== ""); const root = children.find((child) => child.name === ""); const shadowed = new Set(mounts.map((child) => child.name)); const rootChildren = - root && "union" in root ? await listUnion(repo, root.union) : []; + root && "union" in root ? await listUnion(repo, root.union, providers) : []; return [ ...rootChildren.filter((child) => !shadowed.has(child.name)), ...mounts, @@ -502,13 +558,13 @@ export async function listFiles( const find = options.findFile ?? options.find ?? ((url: AutomergeUrl) => repo.find(url)); const root = end.container.handle.doc(); - const rootProvider = providerFor(root); + const rootProvider = await providerOf(root, options.providers); const rootId = end.container.handle.documentId; // A package's relative imports reach into its nested folders by name, so a // package lists everything under it. Anything else lists its own files. const deep = options.deep ?? - (rootProvider?.list(root) ?? []).some((child) => child.name === "package.json"); + listOf(rootProvider, root).some((child) => child.name === "package.json"); const found = new Map<string, { keys: number[]; file: ListedFile }>(); const missing: string[] = []; @@ -548,13 +604,16 @@ export async function listFiles( keys: number[] ): Promise<Link[]> => { const links: Link[] = []; - for (const child of provider?.list(node) ?? []) { + for (const child of listOf(provider, node)) { // Only a directory's keys span several path segments; a folder entry's name is one. const names = provider === directoryProvider ? child.name.split("/") : [child.name]; const childParts = [...parts, ...names]; const childKeys = [...keys, names.length]; - if ("value" in child) { + if ("sub" in child) { + const value = valueAt(node, child.sub); + if (value !== undefined) await add(childParts, childKeys, value, holder); + } else if ("value" in child) { await add(childParts, childKeys, child.value, holder); // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { @@ -594,7 +653,7 @@ export async function listFiles( const heads = parsed.heads ?? live.heads(); const doc = live.view(heads).doc(); if (doc === undefined) return; - const provider = providerFor(doc); + const provider = await providerOf(doc, options.providers); if (!provider) { await add(parts, keys, doc, parsed.documentId, heads); } else if ( diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts new file mode 100644 index 000000000..30b939f65 --- /dev/null +++ b/core/filesystem/test/package-providers.test.ts @@ -0,0 +1,130 @@ +// Providers a tab passes in (`WalkOptions.providers`), after the built-ins, and +// children offered as sub-documents. +import { describe, it, expect } from "vitest"; +import type { AutomergeUrl, DocHandle } from "@automerge/automerge-repo"; +import { livePath } from "../src/paths.js"; +import { getType } from "../src/metadata.js"; +import type { Child, PathProvider } from "../src/providers.js"; +import { listChildren, listFiles, resolveLivePath, walkPath } from "../src/walk.js"; +import { file, fixture } from "./namespace-fixture.js"; + +type BoardDoc = { + "@patchwork": { type: "board" }; + links: { name: string; url: AutomergeUrl }[]; + notes?: { content: string; mimeType: string }; +}; + +const listBoard = (node: unknown): Child[] => { + const board = node as BoardDoc; + const children: Child[] = board.links.map(({ name, url }) => ({ name, link: url })); + if (board.notes) children.push({ name: "notes", sub: ["notes"] }); + return children; +}; + +const board: PathProvider = { + kind: "board", + matches: (doc) => getType(doc as BoardDoc) === "board", + list: listBoard, + lookup(node, names) { + const child = listBoard(node).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, +}; + +const providers = async (doc: unknown) => (board.matches(doc) ? board : undefined); + +async function boardFixture() { + const base = await fixture(); + const child = file(base.repo, "child"); + const b = base.repo.create<BoardDoc>({ + "@patchwork": { type: "board" }, + links: [{ name: "child", url: child.url }], + notes: { content: "n", mimeType: "text/plain" }, + }); + base.root.change((d: any) => d.docs.push({ name: "b", type: "board", url: b.url })); + return { ...base, b, child }; +} + +describe("extra providers", () => { + it("walks through a provider it's given, and not without it", async () => { + const { repo, a, accounts, child } = await boardFixture(); + const path = livePath(a, ["b", "child"]); + expect((await resolveLivePath(repo, path, { accounts, providers }))?.url).toBe(child.url); + expect(await resolveLivePath(repo, path, { accounts })).toBeUndefined(); + }); + + it("asks the built-ins first", async () => { + const { repo, a, accounts, notes } = await fixture(); + const greedy: PathProvider = { kind: "greedy", matches: () => true, list: () => [], lookup: () => undefined }; + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes"]), { + accounts, + providers: async () => greedy, + }); + expect(handle?.url).toBe(notes.url); + }); + + it("treats a provider that throws as having no children", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const broken: PathProvider = { + ...board, + list: () => { + throw new Error("list"); + }, + lookup: () => { + throw new Error("lookup"); + }, + }; + const options = { accounts, providers: async () => broken }; + expect(await walkPath(repo, livePath(a, ["b", "child"]), options)).toBeUndefined(); + expect(await listChildren(repo, b as DocHandle<unknown>, options.providers)).toEqual([]); + }); + + it("lists a provider's children when given it", async () => { + const { repo, b } = await boardFixture(); + const names = (await listChildren(repo, b as DocHandle<unknown>, providers)).map((c) => c.name); + expect(names).toEqual(["child", "notes"]); + }); +}); + +describe("sub-documents", () => { + it("ends at a sub-document as a name after the doc's step", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const end = await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); + expect(end?.atDocument).toBe(false); + expect(end?.sub).toEqual(["notes"]); + expect(end?.handle.documentId).toBe(b.documentId); + expect(end?.pinned.endsWith("/notes")).toBe(true); + }); + + it("resolves to the sub-handle", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const handle = await resolveLivePath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); + expect(handle?.url).toBe(b.sub("notes").url); + expect(handle?.url).toMatch(new RegExp(`^automerge:${b.documentId}/notes`)); + expect((handle?.doc() as any)?.content).toBe("n"); + }); + + it("names nothing once the value is gone", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const sub: PathProvider = { + ...board, + list: (node) => [...board.list(node), { name: "gone", sub: ["missing"] }], + lookup: (node, names) => + names[0] === "gone" + ? { child: { name: "gone", sub: ["missing"] }, consumed: 1 } + : board.lookup(node, names), + }; + expect( + await walkPath(repo, livePath(a, ["b", "gone"]), { accounts, providers: async () => sub }) + ).toBeUndefined(); + b.change((d) => delete d.notes); + expect(await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers })).toBeUndefined(); + }); + + it("lists a sub-document in the container's file list", async () => { + const { repo, a, accounts } = await boardFixture(); + const end = await walkPath(repo, livePath(a, ["b"]), { accounts, providers }); + const listed = await listFiles(repo, end!.pinned, { accounts, providers }); + expect(listed?.files.map((f) => f.tail).sort()).toEqual(["", "child", "notes"]); + }); +}); diff --git a/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md b/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md new file mode 100644 index 000000000..1e4d320e0 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md @@ -0,0 +1,187 @@ +# Package-supplied path providers — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** A datatype package can register a `patchwork:path-provider` plugin that says what a document of its type contains. Tabs walk paths through it. The sidebar opens those documents up, offers whatever editing the provider offers, and opens sub-document children as automerge sub-handles. + +**Architecture:** This spans two repos. +- **system** (`paths-design`): + - `patchwork-filesystem` gains the optional editing methods and the `sub` child on `PathProvider`. Walks take extra providers as an option, checked after the built-ins. + - `patchwork-plugins` owns the plugin type, and `pathProviderFor(doc)`: built-ins first, then plugins. filesystem can't import plugins, since plugins depends on filesystem. + - The tab resolver and the router pass plugin providers into their walks. The automerge worker passes none. +- **base** (`paths`): the sidebar finds providers through the host's `patchwork-plugins` (an importmap external), feature-detected. On an older host it behaves as it does now. + +**Tech stack:** TypeScript, automerge-repo 2.6 sub-handles (`handle.sub(...segments)`; `repo.find` of `automerge:<id>/<path>` returns the sub-handle), vitest, Solid + happy-dom in base. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 6. + +## Global Constraints + +- Built-ins (account, directory, folder, branches) are checked first and can't be overridden. Package providers come after, in registration order. +- A provider is a pure function of the node it's given: `list` and `lookup` read nothing else. +- Package providers run in tabs only. Nothing on the automerge worker's path imports `patchwork-plugins` provider code or passes `providers`. +- `/packages` stays the worker's. Tabs already decline it, so no change is needed there. +- The sidebar enables only the editing a provider offers. A child of a provider that offers none is read-only. +- A changeset for every published package touched (filesystem, plugins, bootloader, patchwork). `patch` level. + +## Decisions + +1. **`supportedDatatypes: string[]`**, with no `"*"`. A provider that claims every type would be asked about every document the sidebar shows. `"*"` is ignored, with a warning. +2. **Sub-document segments are relative to the node passed to `lookup`/`list`.** A walk accumulates them across consecutive `sub` steps. A plain `value` step ends the chain: what's below it isn't openable as a sub-document. +3. **Sidebar drops into a provider container link, never move.** A document dropped on a row whose provider offers `add` calls `add(handle, doc, { name, link })`. Its source is left alone, as in copy mode. File drops and new-document drops aren't accepted there; this first cut doesn't try to map folder semantics onto arbitrary providers. +4. **The folder provider gets no editing methods yet.** The spec says it *would* offer all three, and that there's no need to move folder editing yet. +5. **A loaded provider is checked for shape.** `matches`, `lookup` and `list` must be functions; one that isn't is skipped, with a warning. So is one that fails to load. + +## Review Focus + +1. **A plugin that registers after the sidebar has rendered.** A row of its type should become expandable without a reload. Test: B1 `canExpand` follows the registry's `"changed"` event. +2. **A provider that throws in `list` or `lookup`.** The walk or listing treats that document as having no children; it doesn't fail the page. Test: S1 `walkPath` with a throwing provider returns undefined, and B2 renders no children. +3. **A sub child whose segments don't resolve** (the value was deleted). The walk returns undefined, and the sidebar hides the row. Test: S1. +4. **Automerge worker isolation.** The worker's resolver must never consult plugins. Test: S3 asserts that a resolver created without `providers` can't walk through a plugin-typed doc that a tab resolver can. +5. **Built-in precedence.** A plugin whose `supportedDatatypes` includes `"folder"`, or whose `matches` returns true for a folder doc, never walks one. Test: S2. + +--- + +## Tasks + +### system + +### Task S1: filesystem — `sub` children, editing methods, extra providers in walks + +**Files:** modify `core/filesystem/src/providers.ts`, `core/filesystem/src/walk.ts`; test `core/filesystem/test/walk.test.ts` (or the existing walk test file; same fixtures). + +**Interfaces — produces:** +```ts +// providers.ts +export type Segment = string | number; +export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] } + | { name: string; sub: Segment[] }; +export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; + list(node: unknown): Child[]; + add?(handle: DocHandle<unknown>, node: unknown, child: { name: string; link: AutomergeUrl }, index?: number): void | Promise<void>; + rename?(handle: DocHandle<unknown>, node: unknown, name: string, to: string): void | Promise<void>; + remove?(handle: DocHandle<unknown>, node: unknown, name: string): void | Promise<void>; +} +/** Extra providers, asked after the built-ins (a tab passes the plugin ones). */ +export type ExtraProviders = (doc: unknown) => Promise<PathProvider | undefined>; +export function valueAt(node: unknown, segments: Segment[]): unknown; +// walk.ts +WalkOptions.providers?: ExtraProviders; +WalkEnd.sub?: Segment[]; // set when the walk ends at a sub-document +``` + +- [ ] **Step 1: Write failing tests** with a fake provider `{ kind: "board", matches: d => getType(d) === "board", list: d => [...d.links as {name, link}, { name: "notes", sub: ["notes"] }], lookup: (d, [n]) => … }` over a doc of `@patchwork.type: "board"`: + - `walkPath(repo, "<acct>/b/child", { accounts, providers })` reaches the linked doc; without `providers` it returns undefined. + - `walkPath(..., "<acct>/b/notes")` ends with `atDocument: false` and `sub: ["notes"]`, and its pinned path ends `<boardStep>/notes`. + - `resolveLivePath` of that path returns a handle whose `url` is `stringifyAutomergeUrl({ documentId: board, segments: ["notes"] })`. + - Deleting `notes` from the doc makes the same walk return undefined. + - A provider whose `list`/`lookup` throws: the walk returns undefined, and it doesn't throw. + - `listFiles` on a board container lists `notes` (read from `valueAt`) next to its linked files. `listChildren` returns the board's children when `providers` is passed. +- [ ] **Step 2: Run them** with `pnpm vitest run core/filesystem`. Expected: the new tests fail (no `providers` option; `sub` unknown). +- [ ] **Step 3: Implement.** + - In `walk.ts`, `const providerOf = async (doc) => providerFor(doc) ?? (await safe(options.providers?.(doc)))`. Use it everywhere `providerFor` is called today (`enter`, `listFiles` root and `read`). `listUnion` and `listChildren` take an optional `providers` argument. + - A `sub` child is walked like `value`, with `node = valueAt(node, sub)`. Undefined means the walk returns undefined. It extends `at.sub`; a `value` step clears it. + - A provider's `lookup`/`list` that throws is caught: the walk ends undefined, and the listing gives `[]`. + - `resolveLivePath`: when `!atDocument && end.sub`, return the handle (canonical view, or live) `.sub(...end.sub)`. +- [ ] **Step 4: Run the tests.** Expected: the whole filesystem suite passes (exit 0). +- [ ] **Step 5: Commit** with message `filesystem: sub-document children, provider editing methods, extra providers in walks`. + +### Task S2: plugins — the `patchwork:path-provider` plugin type + +**Files:** create `core/plugins/src/path-providers.ts`; modify `core/plugins/src/registry/types.ts` (add it to `RegistryTypeMap`) and `core/plugins/src/index.ts` (export it); test `core/plugins/test/path-providers.test.ts`. + +**Interfaces — produces:** +```ts +export type PathProviderDescription = PluginDescription & { + type: "patchwork:path-provider"; + supportedDatatypes: string[]; +}; +export type PathProviderPlugin = LoadablePlugin<PathProviderDescription, PathProvider>; +/** Built-ins first, then the first plugin, in registration order, that supports the doc's type, loads, has the right shape, and `matches` the doc. */ +export function pathProviderFor(doc: unknown): Promise<PathProvider | undefined>; +/** Only the plugin step: what walks get as `providers`. */ +export const packagePathProviders: ExtraProviders; +/** Every type some path-provider plugin supports (never "*"). */ +export function pathProviderTypes(): Set<string>; +``` + +- [ ] **Step 1: Write failing tests:** + - For a registered `board` provider, `pathProviderFor(boardDoc)` returns it. + - For a folder doc, `pathProviderFor` returns `folderProvider`, even when a plugin lists `"folder"` and its `matches` returns true. + - Two plugins for one type: the first registered wins. If the first fails to load, or its module lacks `list`, the second is used, and a warning is logged. + - `"*"` in `supportedDatatypes` isn't honored, and is warned about. + - `pathProviderTypes()` gives `{"board"}`. +- [ ] **Step 2: Run** `pnpm vitest run core/plugins`. Expected: FAIL (the module doesn't exist). +- [ ] **Step 3: Implement** on `getRegistry<PathProviderDescription>("patchwork:path-provider")`, with `.filter` and `.load(id)`. +- [ ] **Step 4: Run the tests.** Expected: the plugins suite passes (exit 0). +- [ ] **Step 5: Commit** with message `plugins: patchwork:path-provider plugins`. + +### Task S3: tabs walk through plugin providers; the worker doesn't + +**Files:** modify `core/bootloader/src/resolver.ts` (an option `providers?: ExtraProviders` on `createResolver`, threaded into `resolveWalkPath`'s walk options), `core/bootloader/src/tab-resolver.ts` (passes `packagePathProviders`), `core/patchwork/src/router.ts` (`resolveLivePath(repo, path, { accounts, providers: packagePathProviders })`); tests in `core/bootloader/test/resolver.test.ts` and `core/patchwork/test/router.test.ts`. + +**Consumes:** S1 `WalkOptions.providers`; S2 `packagePathProviders`. + +- [ ] **Step 1: Write failing tests:** + - A resolver created with `providers` answers a `/@/` live path through a board doc with a 307. One created without it (the worker's) answers 404. + - The router opens `&path=<acct>/b/child` at the linked doc, and `&path=<acct>/b/notes` at `automerge:<board>/notes`, with a board path-provider plugin registered. +- [ ] **Step 2: Run** `pnpm vitest run core/bootloader core/patchwork`. Expected: FAIL. +- [ ] **Step 3: Implement** the threading. Confirm that `automerge-protocol-handler.ts` (the worker) doesn't pass `providers`. +- [ ] **Step 4: Run the tests.** Expected: the bootloader and patchwork suites pass (exit 0). +- [ ] **Step 5: Changeset, spec, commit.** + - Changeset `.changeset/path-provider-plugins.md` for filesystem, plugins, bootloader and patchwork, saying what was added: `Child`'s `sub`, the editing methods, `WalkOptions.providers`, and `pathProviderFor`/`packagePathProviders`/`pathProviderTypes`. + - Spec section 6: "Approved … Not implemented" becomes "Implemented"; drop the non-goal's "until section 6 is built" clause. + - Build `core/filesystem`, `core/plugins`, `core/bootloader` and `core/patchwork` (tsc only, no shell build). Commit. + +### base (`paths` branch) + +### Task B1: the sidebar finds providers through the host + +**Files:** modify `sideboard/src/sideboard/container.ts`; test `sideboard/src/sideboard/container.test.ts`. + +**Interfaces — produces:** +```ts +/** Host's plugins.pathProviderFor, else filesystem.providerFor, else nothing. */ +export function providerOf(doc: unknown, host?: HostPaths): Promise<PathProvider | undefined>; +/** Reactive: true for directory/branches, or a type a path-provider plugin supports. */ +export function useExpandable(host?: HostPaths): (type: string) => boolean; +/** Link and sub children; values without `sub` stay hidden. */ +export function containerChildren(doc: unknown, provider: PathProvider | undefined): Array<{ name: string; link: AutomergeUrl } | { name: string; sub: (string | number)[] }>; +``` +`HostPaths` is the feature-detected subset of the host's `patchwork-plugins` and `patchwork-filesystem` modules. Tests inject it, as `live-path.ts` does. + +- [ ] **Step 1: Write failing tests:** + - `providerOf` prefers `pathProviderFor`, falls back to `providerFor`, and gives undefined when neither exists. + - `useExpandable` turns `"board"` true after a fake registry fires `"changed"`. + - `containerChildren` returns `sub` children and drops bare values. +- [ ] **Step 2: Run** `pnpm vitest run src/sideboard/container.test.ts` in `sideboard/`. Expected: FAIL. +- [ ] **Step 3: Implement.** `canExpand` stays exported (now a non-reactive check) for the existing callers. +- [ ] **Step 4: Run.** Expected: PASS. +- [ ] **Step 5: Commit.** + +### Task B2: provider children, sub-documents and editing in `ContainerItem` + +**Files:** modify `sideboard/src/sideboard/document-list/container-item.tsx`, `item.tsx` (an optional `onProviderDrop(items: DndPayloadItem[])`: when set on a read-only row, it accepts document drags as copy and calls it), `item-menu.tsx`/`state.ts` (`MenuTarget` gets `canRename?`/`canRemove?`, which default to `!readOnly`), `document-list.tsx` (uses `useExpandable`); test `sideboard/src/sideboard/document-list/container-item.test.tsx`. + +**Consumes:** B1. + +- [ ] **Step 1: Write failing tests** with a fake board provider: + - Expanding a board row lists its link child and its `notes` sub child. + - Opening `notes` dispatches `{ url: "automerge:<board>/notes", path: "<acct>/…/b/notes" }`. + - With `rename` and `remove` offered, the child's menu target has `canRename`/`canRemove`, and invoking them calls `provider.rename(handle, doc, "child", "new")` and `provider.remove(handle, doc, "child")`. + - With none offered, the child is read-only, as now. + - Dropping a document payload on the board row calls `provider.add(handle, doc, { name, link })` when `add` is offered, and does nothing when it isn't. + - A provider whose `list` throws renders no children. +- [ ] **Step 2: Run** `pnpm vitest run src/sideboard/document-list/container-item.test.tsx`. Expected: FAIL. +- [ ] **Step 3: Implement.** + - `ContainerChildren` resolves `providerOf(doc())` (a resource keyed on the doc) and renders `ChildRow` for links and `SubRow` for subs. + - A `SubRow` opens `stringifyAutomergeUrl({ documentId, segments })`. Its type is the value's `@patchwork.type`, or empty. + - Rename and remove go to the provider, with the container's handle and current doc. +- [ ] **Step 4: Run all sideboard tests, then tsc and lint.** Expected: everything passes. tsc shows only the 2–3 errors that were already there (`create-new.tsx`, `document-list-panel.tsx`). +- [ ] **Step 5: Commit, and push `paths`** (PR #94 is open on it). From b3dbd063d33f44b2b9fafc681abb861f17ec22e4 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Wed, 28 Oct 2026 20:46:53 +0000 Subject: [PATCH 56/81] plugins: patchwork:path-provider plugins Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/plugins/package.json | 2 +- core/plugins/src/index.ts | 1 + core/plugins/src/path-providers.ts | 84 +++++++++++++++++++ core/plugins/src/registry/types.ts | 2 + core/plugins/test/path-providers.test.ts | 100 +++++++++++++++++++++++ core/plugins/vitest.config.ts | 8 ++ vitest.config.ts | 1 + 7 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 core/plugins/src/path-providers.ts create mode 100644 core/plugins/test/path-providers.test.ts create mode 100644 core/plugins/vitest.config.ts diff --git a/core/plugins/package.json b/core/plugins/package.json index a374e17c0..80ee2f246 100644 --- a/core/plugins/package.json +++ b/core/plugins/package.json @@ -13,7 +13,7 @@ "scripts": { "build": "tsc", "dev": "tsc -w --preserveWatchOutput", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run" }, "keywords": [], "license": "ISC", diff --git a/core/plugins/src/index.ts b/core/plugins/src/index.ts index ce36107fc..a714f5153 100644 --- a/core/plugins/src/index.ts +++ b/core/plugins/src/index.ts @@ -4,3 +4,4 @@ export * from "./datatypes.js"; export * from "./registry/guards.js"; export * from "./account.js"; export * from "./keyhive.js"; +export * from "./path-providers.js"; diff --git a/core/plugins/src/path-providers.ts b/core/plugins/src/path-providers.ts new file mode 100644 index 000000000..002755784 --- /dev/null +++ b/core/plugins/src/path-providers.ts @@ -0,0 +1,84 @@ +// A datatype package says what a document of its type contains by registering +// a `patchwork:path-provider` plugin. Tabs walk paths through them; the +// automerge worker never loads package code, so it walks built-ins only. +import { + getType, + providerFor, + type ExtraProviders, + type HasPatchworkMetadata, + type PathProvider, +} from "@inkandswitch/patchwork-filesystem"; +import { getRegistry } from "./registry/index.js"; +import type { LoadablePlugin, PluginDescription } from "./registry/types.js"; + +export const PATH_PROVIDER = "patchwork:path-provider"; + +export type PathProviderDescription = PluginDescription & { + type: typeof PATH_PROVIDER; + /** The datatypes it walks. `"*"` isn't honored: built-ins aside, a provider says what it walks. */ + supportedDatatypes: string[]; +}; + +export type PathProviderPlugin = LoadablePlugin<PathProviderDescription, PathProvider>; + +const registry = () => getRegistry<PathProviderDescription>(PATH_PROVIDER); + +const warned = new Set<string>(); +const warnOnce = (key: string, ...message: unknown[]) => { + if (warned.has(key)) return; + warned.add(key); + console.warn(...message); +}; + +const datatypesOf = (plugin: PathProviderDescription): string[] => { + const types = Array.isArray(plugin.supportedDatatypes) ? plugin.supportedDatatypes : []; + if (types.includes("*")) { + warnOnce(`${plugin.id} *`, `path provider ${plugin.id}: "*" in supportedDatatypes is ignored`); + } + return types.filter((type) => typeof type === "string" && type !== "*"); +}; + +const isProvider = (module: unknown): module is PathProvider => + !!module && + typeof module === "object" && + ["matches", "lookup", "list"].every( + (method) => typeof (module as Record<string, unknown>)[method] === "function" + ); + +/** + * The plugin provider for `doc`: the first, in registration order, that supports its type, + * loads, is shaped like a provider, and `matches` it. What a tab passes walks as `providers`. + */ +export const packagePathProviders: ExtraProviders = async (doc) => { + const type = getType((doc ?? {}) as HasPatchworkMetadata); + if (!type) return undefined; + for (const plugin of registry().filter((p) => datatypesOf(p).includes(type))) { + let module: unknown; + try { + module = (await registry().load(plugin.id))?.module; + } catch (error) { + warnOnce(`${plugin.id} load`, `path provider ${plugin.id} didn't load`, error); + continue; + } + if (!isProvider(module)) { + warnOnce(`${plugin.id} shape`, `path provider ${plugin.id} lacks matches, lookup or list`); + continue; + } + try { + if (module.matches(doc)) return module; + } catch { + // a provider whose matches throws walks nothing + } + } + return undefined; +}; + +/** Built-ins first, which a plugin can't override, then {@link packagePathProviders}. */ +export async function pathProviderFor(doc: unknown): Promise<PathProvider | undefined> { + return providerFor(doc) ?? (await packagePathProviders(doc)); +} + +/** Every type some path-provider plugin supports, without loading any. */ +export function pathProviderTypes(): Set<string> { + return new Set(registry().all().flatMap((plugin) => datatypesOf(plugin))); +} diff --git a/core/plugins/src/registry/types.ts b/core/plugins/src/registry/types.ts index 93f130472..3e35ee2b5 100644 --- a/core/plugins/src/registry/types.ts +++ b/core/plugins/src/registry/types.ts @@ -1,5 +1,6 @@ import { DatatypeDescription } from "../datatypes.js"; import { ToolDescription } from "../tools.js"; +import type { PathProviderDescription } from "../path-providers.js"; export interface PluginRegistryEvents<D extends PluginDescription, I = any> { registered: (plugin: Plugin<D, I>) => void; @@ -16,6 +17,7 @@ export interface PluginRegistryEvents<D extends PluginDescription, I = any> { export type RegistryTypeMap = { "patchwork:tool": ToolDescription; "patchwork:datatype": DatatypeDescription; + "patchwork:path-provider": PathProviderDescription; }; /** diff --git a/core/plugins/test/path-providers.test.ts b/core/plugins/test/path-providers.test.ts new file mode 100644 index 000000000..0734602a6 --- /dev/null +++ b/core/plugins/test/path-providers.test.ts @@ -0,0 +1,100 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { folderProvider, type PathProvider } from "@inkandswitch/patchwork-filesystem"; +import { getRegistry, registerPlugins, unregisterPlugins } from "../src/registry/index.js"; +import { + packagePathProviders, + pathProviderFor, + pathProviderTypes, + type PathProviderDescription, + type PathProviderPlugin, +} from "../src/path-providers.js"; + +const boardDoc = { "@patchwork": { type: "board" }, links: [] }; +const folderDoc = { title: "f", docs: [] }; + +const provider = (kind: string, extra: Partial<PathProvider> = {}): PathProvider => ({ + kind, + matches: () => true, + lookup: () => undefined, + list: () => [], + ...extra, +}); + +const plugin = ( + id: string, + supportedDatatypes: string[], + load: () => Promise<unknown> +): PathProviderPlugin => + ({ + id, + name: id, + type: "patchwork:path-provider", + supportedDatatypes, + load, + }) as PathProviderPlugin; + +const URL_ = "https://example.test/pkg.js"; + +afterEach(() => { + unregisterPlugins(URL_); + vi.restoreAllMocks(); +}); + +describe("pathProviderFor", () => { + it("finds a plugin for the doc's type", async () => { + const board = provider("board"); + registerPlugins([plugin("board", ["board"], async () => board)], URL_); + expect(await pathProviderFor(boardDoc)).toBe(board); + expect(await packagePathProviders(boardDoc)).toBe(board); + }); + + it("asks the built-ins first, whatever a plugin claims", async () => { + registerPlugins([plugin("greedy", ["folder"], async () => provider("greedy"))], URL_); + expect(await pathProviderFor(folderDoc)).toBe(folderProvider); + }); + + it("only gives plugins documents of a type they support", async () => { + registerPlugins([plugin("board", ["board"], async () => provider("board"))], URL_); + expect(await pathProviderFor({ "@patchwork": { type: "other" } })).toBeUndefined(); + expect(await packagePathProviders(folderDoc)).toBeUndefined(); + }); + + it("takes the first registered, skipping one that fails to load or isn't a provider", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const second = provider("second"); + registerPlugins( + [ + plugin("broken", ["board"], () => Promise.reject(new Error("nope"))), + plugin("shapeless", ["board"], async () => ({ kind: "shapeless" })), + plugin("not-matching", ["board"], async () => provider("x", { matches: () => false })), + plugin("second", ["board"], async () => second), + plugin("third", ["board"], async () => provider("third")), + ], + URL_ + ); + expect(await pathProviderFor(boardDoc)).toBe(second); + expect(warn).toHaveBeenCalled(); + }); + + it("doesn't honor \"*\"", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + registerPlugins([plugin("star", ["*"], async () => provider("star"))], URL_); + expect(await pathProviderFor(boardDoc)).toBeUndefined(); + expect(pathProviderTypes().has("*")).toBe(false); + expect(warn).toHaveBeenCalled(); + }); +}); + +describe("pathProviderTypes", () => { + it("is every type some plugin supports", () => { + registerPlugins( + [ + plugin("a", ["board", "canvas"], async () => provider("a")), + plugin("b", ["board"], async () => provider("b")), + ], + URL_ + ); + expect(pathProviderTypes()).toEqual(new Set(["board", "canvas"])); + expect(getRegistry<PathProviderDescription>("patchwork:path-provider").all()).toHaveLength(2); + }); +}); diff --git a/core/plugins/vitest.config.ts b/core/plugins/vitest.config.ts new file mode 100644 index 000000000..ed8bf7739 --- /dev/null +++ b/core/plugins/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["test/**/*.test.ts"], + }, +}); diff --git a/vitest.config.ts b/vitest.config.ts index d7c73226d..53a7fe9dc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,6 +6,7 @@ export default defineConfig({ "core/bootloader", "core/filesystem", "core/patchwork", + "core/plugins", "packages/edge-handles", ], }, From 8b0aa348b96009f67105d8d6f5bbfae5b6044463 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 02:25:12 +0000 Subject: [PATCH 57/81] tabs and the router walk through path-provider plugins; the worker doesn't Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/path-provider-plugins.md | 16 +++++ core/bootloader/src/resolver.ts | 13 +++- core/bootloader/src/tab-resolver.ts | 3 + core/bootloader/test/resolver.test.ts | 57 ++++++++++++++++++ core/patchwork/src/router.ts | 7 ++- core/patchwork/test/router.test.ts | 60 ++++++++++++++++++- ...28-paths-namespace-content-cache-design.md | 9 ++- 7 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 .changeset/path-provider-plugins.md diff --git a/.changeset/path-provider-plugins.md b/.changeset/path-provider-plugins.md new file mode 100644 index 000000000..7f8aa4517 --- /dev/null +++ b/.changeset/path-provider-plugins.md @@ -0,0 +1,16 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +--- + +Packages can now say what a document of their type contains. A `patchwork:path-provider` plugin is a `PathProvider` with `supportedDatatypes`. The built-in providers (account, directory, folder, branches) are always asked first and can't be overridden. After them come plugins, in registration order. + +- `patchwork-filesystem`: + - a `Child` can be `{ name, sub: [...segments] }`, a value inside the document offered as a sub-document. A walk that ends at one reports it as `WalkEnd.sub`, and `resolveLivePath` returns `handle.sub(...segments)`; + - `PathProvider` gains the optional `add`, `rename` and `remove`, for whoever shows a provider's children to edit them; + - walks take `WalkOptions.providers` (an `ExtraProviders`), asked after the built-ins, as do `listChildren` and `listUnion`. A provider that throws is treated as having no children. +- `patchwork-plugins`: the plugin type `PathProviderDescription`/`PathProviderPlugin`, with `pathProviderFor(doc)` (built-ins, then plugins), `packagePathProviders` (plugins only, for `WalkOptions.providers`), and `pathProviderTypes()`. `"*"` in `supportedDatatypes` is ignored. +- `patchwork-bootloader`: `createResolver` takes `providers`. A tab's resolver passes the registered plugins, and the automerge worker's passes none, since it never loads package code. +- `patchwork`: `&path=` resolves through registered path providers too, and a path that ends at a sub-document opens it at its sub-handle's URL. diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 8e4e9fa6d..4b0e3f542 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -28,6 +28,7 @@ import { resolvePath, toBytes, walkPath, + type ExtraProviders, type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; @@ -234,7 +235,8 @@ async function resolveWalkPath( path: string, accounts: ReadonlyMap<string, AutomergeUrl>, signal: AbortSignal, - { heads, list, only }: { heads?: string; list?: boolean; only?: string } = {} + { heads, list, only }: { heads?: string; list?: boolean; only?: string } = {}, + providers?: ExtraProviders ): Promise<Response> { try { parsePath(path); @@ -242,7 +244,7 @@ async function resolveWalkPath( return new Response(String(error), { status: 400 }); } const missing = new Set<string>(); - const options = { accounts, find: waitingFind(repo, signal, missing) }; + const options = { accounts, providers, find: waitingFind(repo, signal, missing) }; const notFound = () => { if (missing.size) { throw new AbortHandoff(`heads not found for ${[...missing].join(", ")}`); @@ -345,6 +347,11 @@ export function createResolver( * shouldn't hold them all. */ declinePackages?: boolean; + /** + * Providers for documents no built-in walks: a tab passes the ones packages register. The + * automerge worker never loads package code, so it passes none. + */ + providers?: ExtraProviders; } = {} ): Resolver { const throughPackages = (path: string) => { @@ -437,7 +444,7 @@ export function createResolver( const limit = within(signal); log(`resolving handoff ${id} for /@/${request.path}`); response = await Promise.race([ - resolveWalkPath(repo, request.path, accounts(), limit, request), + resolveWalkPath(repo, request.path, accounts(), limit, request, options.providers), new Promise<never>((_, reject) => limit.addEventListener("abort", () => reject(new AbortHandoff(`resolving /@/${request.path} timed out`)) diff --git a/core/bootloader/src/tab-resolver.ts b/core/bootloader/src/tab-resolver.ts index e1755d72a..28851d5b9 100644 --- a/core/bootloader/src/tab-resolver.ts +++ b/core/bootloader/src/tab-resolver.ts @@ -8,6 +8,7 @@ import { type Repo, } from "@automerge/automerge-repo/slim"; import { idhash } from "@inkandswitch/patchwork-filesystem"; +import { packagePathProviders } from "@inkandswitch/patchwork-plugins"; import { createResolver, type Resolver } from "./resolver.js"; import type { @@ -31,6 +32,8 @@ export function resolveHandoffsWith(repo: Repo): void { resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { declineUnknownAccounts: true, declinePackages: true, + // Package code runs in tabs, so tabs walk through the providers packages register. + providers: packagePathProviders, }); } diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts index 54f124462..cc7a75d52 100644 --- a/core/bootloader/test/resolver.test.ts +++ b/core/bootloader/test/resolver.test.ts @@ -14,6 +14,7 @@ import { pinnedPath, walkPath, } from "@inkandswitch/patchwork-filesystem"; +import { registerPlugins, unregisterPlugins } from "@inkandswitch/patchwork-plugins"; import { createResolver } from "../src/resolver.js"; import { registerAccount, resolveHandoffsWith, handleResolveRequest } from "../src/tab-resolver.js"; import type { HandoffRequestMessage, HandoffResponseMessage } from "../src/types.js"; @@ -266,3 +267,59 @@ describe("registerAccount", () => { expect(inline(replies.at(-1)).status).toBe(307); }); }); + +describe("package path providers", () => { + const PLUGIN_URL = "https://example.test/board.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [{ name: "child", link: doc.child }], + lookup: (doc: any, names: string[]) => + names[0] === "child" ? { child: { name: "child", link: doc.child }, consumed: 1 } : undefined, + }; + + async function boardFixture() { + const base = await fixture(); + const b = base.repo.create({ "@patchwork": { type: "board" }, child: file(base.repo, "c") }); + const root = await base.repo.find<any>(base.account.doc()!.namespace["/"][0].url); + root.change((d: any) => d.docs.push({ name: "b", type: "board", url: b.url })); + return base; + } + + it("walk through the providers a resolver is given; the worker's has none", async () => { + const { repo, a, accounts } = await boardFixture(); + const path = livePath(a, ["b", "child"]); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + providers: async (doc) => (board.matches(doc) ? board : undefined), + }); + expect(inline(await tab.answer(pathRequest(path))).status).toBe(307); + const worker = createResolver(() => Promise.resolve(repo), undefined, () => accounts); + expect(inline(await worker.answer(pathRequest(path))).status).toBe(404); + }); + + it("a tab's resolver walks through registered path-provider plugins", async () => { + const { repo, a, account } = await boardFixture(); + registerPlugins( + [{ id: "board", name: "board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + try { + resolveHandoffsWith(repo); + await registerAccount(account.url); + const { port1, port2 } = new MessageChannel(); + const replies: unknown[] = []; + port1.onmessage = (event) => replies.push(event.data); + handleResolveRequest( + new MessageEvent("message", { + data: { ...pathRequest(livePath(a, ["b", "child"])), type: "resolve" }, + ports: [port2], + }) + ); + await expect.poll(() => replies.length, { timeout: 5000 }).toBeGreaterThanOrEqual(2); + port1.close(); + expect(inline(replies.at(-1)).status).toBe(307); + } finally { + unregisterPlugins(PLUGIN_URL); + } + }); +}); diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index 9af7fed63..86b2a2c41 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -21,6 +21,7 @@ import { type DatatypeImplementation, type ToolDescription, getRegistry, + packagePathProviders, } from "@inkandswitch/patchwork-plugins"; // Legacy big-patchwork hash shape: `<slug>--<documentId>[?…]`. The slug can @@ -64,7 +65,11 @@ export function createRouter({ ); const resolvePathParam = async (path: string) => { try { - return await resolveLivePath(repo, path, { accounts: await accounts }); + // Package code runs here, so paths go through the providers packages register. + return await resolveLivePath(repo, path, { + accounts: await accounts, + providers: packagePathProviders, + }); } catch { return undefined; // a path that doesn't parse names nothing } diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts index 261220f66..291402340 100644 --- a/core/patchwork/test/router.test.ts +++ b/core/patchwork/test/router.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; import { livePath } from "@inkandswitch/patchwork-filesystem"; +import { registerPlugins, unregisterPlugins } from "@inkandswitch/patchwork-plugins"; vi.mock("@inkandswitch/patchwork-elements", () => ({ openDocument: (element: EventTarget, url: string) => @@ -52,7 +53,10 @@ function fakeWindow(initial: string) { return { window, location }; } -async function setup(hash: (a: string, notes: AutomergeUrl) => string) { +async function setup( + hash: (a: string, notes: AutomergeUrl) => string, + extra: (repo: Repo) => Record<string, AutomergeUrl> = () => ({}) +) { const repo = new Repo(); const file = (content: string) => repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); @@ -63,7 +67,10 @@ async function setup(hash: (a: string, notes: AutomergeUrl) => string) { }); const notes = file("hi"); const other = file("other"); - const root = folder({ Projects: folder({ foo: folder({ notes: notes.url }).url }).url }); + const root = folder({ + Projects: folder({ foo: folder({ notes: notes.url }).url }).url, + ...extra(repo), + }); const account = repo.create({ "@patchwork": { type: "account" }, namespace: { "/": [{ url: root.url }] }, @@ -88,7 +95,7 @@ async function setup(hash: (a: string, notes: AutomergeUrl) => string) { siteTitle: "test", }); const live = livePath(a, ["Projects", "foo", "notes"]); - return { a, live, notes, other, location, rootElement, opened, router }; + return { a, repo, live, notes, other, location, rootElement, opened, router }; } const hashParam = (location: { hash: string }, key: string) => @@ -211,3 +218,50 @@ describe("router: path=", () => { expect(warn).toHaveBeenCalled(); }); }); + +describe("router: paths through package providers", () => { + const PLUGIN_URL = "https://example.test/board.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [ + { name: "child", link: doc.child }, + { name: "notes", sub: ["notes"] }, + ], + lookup(doc: any, names: string[]) { + const child = board.list(doc).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, + }; + let b: DocHandle<any> | undefined; + const withBoard = (repo: Repo) => { + const child = repo.create({ "@patchwork": { type: "file" }, content: "c", mimeType: "text/plain" }); + b = repo.create({ + "@patchwork": { type: "board" }, + child: child.url, + notes: { content: "n", mimeType: "text/plain" }, + }); + return { b: b.url }; + }; + + afterEach(() => unregisterPlugins(PLUGIN_URL)); + const register = () => + registerPlugins( + [{ id: "board", name: "board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + + it("opens a document a package's provider names", async () => { + register(); + const { opened } = await setup((a) => `path=${livePath(a, ["b", "child"])}`, withBoard); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(b!.doc().child); + }); + + it("opens a sub-document at its sub-handle's url", async () => { + register(); + const { opened } = await setup((a) => `path=${livePath(a, ["b", "notes"])}`, withBoard); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(b!.sub("notes").url); + }); +}); diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index f50cf7de8..2d54d2dce 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -26,7 +26,6 @@ authenticated channel) and gets reworked to this one. - Running package code on a separate origin. This design is meant not to preclude it (the store and file lists work the same there), but doesn't do it. - Mount points below the top level of the namespace. -- Path providers supplied by packages, until section 6 is built. - Garbage collection of the content store. ## Threat model @@ -391,7 +390,13 @@ store, and both are checked against the URL the watcher pinned. ## 6. Package-supplied path providers -Approved in conversation 2026-10-28. Not implemented. +Approved in conversation 2026-10-28. Implemented in `patchwork-filesystem` +(`sub` children, the editing methods, `WalkOptions.providers`), +`patchwork-plugins` (`pathProviderFor`, `packagePathProviders`, +`pathProviderTypes`), the tab resolver and the router, and the base +sidebar. The built-in folder provider doesn't offer editing yet. In the +sidebar, a document dropped on a provider's row is added (linked) and left +where it was; files and new documents can't be dropped there. The sidebar can open up a doc that isn't a folder when a built-in provider walks it: a directory doc, a branches doc, the account. This section proposes From dacd2abeb95e531646d5df1325adf18494892906 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 02:39:49 +0000 Subject: [PATCH 58/81] filesystem: package providers only for docs a walk goes through; no spinning, nested subs listed, sub links refused - A lookup inside a document that consumes no name, or more names than there are, is no child, so a buggy provider can't spin the walk. - Whether a document a path ends at is a container is the built-ins' call, and so is what listFiles counts as a file: a path pins and lists the same with or without a package's provider, and installing one doesn't change what an existing pinned URL means. - listFiles lists what's inside the values a package's provider hands out, as far as a walk can reach. - A link with path segments (a sub-document) isn't walked or listed as the whole document. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/walk.ts | 56 ++++++++++---- .../filesystem/test/package-providers.test.ts | 73 ++++++++++++++++++- 2 files changed, 113 insertions(+), 16 deletions(-) diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 1afa5cf7b..866f3ff27 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -154,12 +154,19 @@ export async function walkPath( const stepOf = async (handle: DocHandle<unknown>) => docStep(await hashOf(handle.url), handle.heads()); + // Package providers apply only to a document the walk goes on through. Whether a document the + // path ends at is a container (a doc step) or a file (a name) is the built-ins' call, so a + // path pins the same with or without a package's provider, and a plugin installed later + // doesn't change what an existing pinned URL means. async function enter( url: AutomergeUrl, - heads?: UrlHeads + heads: UrlHeads | undefined, + through: boolean ): Promise<AtDoc | undefined> { try { const parsed = parseAutomergeUrl(url); + // A link to a sub-document: walking it as the whole document would name the wrong thing. + if (parsed.segments?.length) return undefined; const at = heads ?? parsed.heads; const live = await find( stringifyAutomergeUrl({ documentId: parsed.documentId }), @@ -168,7 +175,7 @@ export async function walkPath( const handle = live.view(at ?? live.heads()); const node = handle.doc(); if (node === undefined) return undefined; - const provider = await providerOf(node, options.providers); + const provider = through ? await providerOf(node, options.providers) : providerFor(node); return { kind: "doc", handle, node, provider, inside: false }; } catch { return undefined; @@ -215,12 +222,16 @@ export async function walkPath( let at: At | undefined; if (root.kind === "account") { const url = options.accounts.get(root.acctHash); - at = url && (await enter(url, root.heads)); + at = url && (await enter(url, root.heads, segments.length > 0)); if (!at) return undefined; out.push(docStep(root.acctHash, at.handle.heads())); if (root.listHash) lists.set(0, root.listHash); } else { - at = await enter(stringifyAutomergeUrl({ documentId: root.documentId }), root.heads); + at = await enter( + stringifyAutomergeUrl({ documentId: root.documentId }), + root.heads, + segments.length > 0 + ); if (!at) return undefined; out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); if (root.listHash) lists.set(0, root.listHash); @@ -248,13 +259,13 @@ export async function walkPath( listHash = segment.listHash; i++; if (!guard(member, segments.length - i)) return undefined; - next = await enter(member, segment.heads); + next = await enter(member, segment.heads, i < segments.length); } } if (!member) { const names = namesFrom(i); for (const url of at.members) { - const candidate = await enter(url); + const candidate = await enter(url, undefined, names.length > 0); if ( !names.length || (candidate?.provider && lookUp(candidate.provider, candidate.node, names)) @@ -279,7 +290,7 @@ export async function walkPath( if (at.provider !== branchesProvider || at.inside) break; const link = defaultBranch(at.node); if (!link || !guard(link, 0)) return undefined; - const next = await enter(link); + const next = await enter(link, undefined, false); if (!next) return undefined; out.push(await stepOf(next.handle)); live = true; @@ -293,7 +304,7 @@ export async function walkPath( ); const link = await matchLink(links, segment); if (link) { - const next = await enter(link, segment.heads); + const next = await enter(link, segment.heads, i + 1 < segments.length); if (!next) return undefined; out.push(await stepOf(next.handle)); if (segment.listHash) lists.set(out.length - 1, segment.listHash); @@ -309,6 +320,11 @@ export async function walkPath( const found: Lookup | undefined = lookUp(at.provider, at.node, names); if (!found) return undefined; const { child, consumed }: Lookup = found; + // A step inside the document must use a name, or the walk would never move on; and none + // can use more names than there are. + if (consumed > names.length || (consumed < 1 && ("value" in child || "sub" in child))) { + return undefined; + } const written = names.slice(0, consumed).map(encodeURIComponent); i += consumed; @@ -326,7 +342,7 @@ export async function walkPath( at = { kind: "union", members: child.union }; } else { if (!consumed && !guard(child.link, segments.length - i)) return undefined; - const next = await enter(child.link); + const next = await enter(child.link, undefined, i < segments.length); if (!next) return undefined; if (next.provider || !consumed) { out.push(await stepOf(next.handle)); @@ -610,11 +626,18 @@ export async function listFiles( provider === directoryProvider ? child.name.split("/") : [child.name]; const childParts = [...parts, ...names]; const childKeys = [...keys, names.length]; - if ("sub" in child) { - const value = valueAt(node, child.sub); - if (value !== undefined) await add(childParts, childKeys, value, holder); - } else if ("value" in child) { - await add(childParts, childKeys, child.value, holder); + const builtIn = provider === directoryProvider || provider === folderProvider; + if ("sub" in child || "value" in child) { + const value = "sub" in child ? valueAt(node, child.sub) : child.value; + if (value === undefined) continue; + await add(childParts, childKeys, value, holder); + // A package's provider walks on into the values it hands out, so list what's in them. + if (!builtIn && value && typeof value === "object" && !(value instanceof Uint8Array)) { + links.push(...(await visit(value, provider, holder, childParts, childKeys))); + continue; + } + } + if ("value" in child) { // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { links.push(...(await visit(child.value, provider, holder, childParts, childKeys))); @@ -646,6 +669,8 @@ export async function listFiles( await acquire(); try { const parsed = parseAutomergeUrl(link); + // A link to a sub-document isn't a file a walk can reach (see walkPath). + if (parsed.segments?.length) return; const live = await find( stringifyAutomergeUrl({ documentId: parsed.documentId }), parsed.heads @@ -653,7 +678,8 @@ export async function listFiles( const heads = parsed.heads ?? live.heads(); const doc = live.view(heads).doc(); if (doc === undefined) return; - const provider = await providerOf(doc, options.providers); + // Built-ins decide what's a file, as the walk does for a path that ends here. + const provider = providerFor(doc); if (!provider) { await add(parts, keys, doc, parsed.documentId, heads); } else if ( diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts index 30b939f65..47483163d 100644 --- a/core/filesystem/test/package-providers.test.ts +++ b/core/filesystem/test/package-providers.test.ts @@ -123,8 +123,79 @@ describe("sub-documents", () => { it("lists a sub-document in the container's file list", async () => { const { repo, a, accounts } = await boardFixture(); - const end = await walkPath(repo, livePath(a, ["b"]), { accounts, providers }); + // A path through the board: one that ends at it treats it as a file (see below). + const end = await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); const listed = await listFiles(repo, end!.pinned, { accounts, providers }); expect(listed?.files.map((f) => f.tail).sort()).toEqual(["", "child", "notes"]); }); }); + +describe("review fixes", () => { + it("treats a value or sub child that consumes no name as no child, rather than spinning", async () => { + const { repo, a, accounts } = await boardFixture(); + for (const child of [{ name: "x", sub: ["notes"] }, { name: "x", value: 1 }]) { + const spin: PathProvider = { ...board, lookup: () => ({ child, consumed: 0 }) }; + expect( + await walkPath(repo, livePath(a, ["b", "x"]), { accounts, providers: async () => spin }) + ).toBeUndefined(); + } + const greedy: PathProvider = { + ...board, + lookup: (node, names) => ({ child: board.lookup(node, names)!.child, consumed: names.length + 1 }), + }; + expect( + await walkPath(repo, livePath(a, ["b", "child"]), { accounts, providers: async () => greedy }) + ).toBeUndefined(); + }, 5000); + + it("lists nested sub-documents a walk can reach", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const nested: PathProvider = { + ...board, + list: (node: any) => + node?.links ? board.list(node) : node?.inner ? [{ name: "inner", sub: ["inner"] }] : [], + lookup(node, names) { + const child = nested.list(node).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, + }; + b.change((d: any) => (d.notes.inner = { content: "i", mimeType: "text/plain" })); + const options = { accounts, providers: async (doc: unknown) => (board.matches(doc) ? nested : undefined) }; + const end = await walkPath(repo, livePath(a, ["b", "notes", "inner"]), options); + expect(end?.sub).toEqual(["notes", "inner"]); + const listed = await listFiles(repo, end!.pinned, options); + expect(listed?.files.map((f) => f.tail)).toContain("notes/inner"); + }); + + it("pins and lists a doc the path ends at the same way, plugin or not", async () => { + const { repo, a, accounts, b } = await boardFixture(); + // A board that reads as a file too, so a listing has something to show for it. + b.change((d: any) => { + d.content = "board"; + d.mimeType = "text/plain"; + }); + const path = livePath(a, ["b"]); + const without = await walkPath(repo, path, { accounts }); + const withIt = await walkPath(repo, path, { accounts, providers }); + expect(withIt?.pinned).toBe(without?.pinned); + expect(withIt?.canonical).toBe(without?.canonical); + + // The root folder's list, from a path that ends at the board. + const tails = async (options: object) => + (await listFiles(repo, without!.pinned, { accounts, ...options }))?.files.map((f) => f.tail).sort(); + expect(await tails({ providers })).toEqual(await tails({})); + expect(await tails({ providers })).toContain("b"); + }); + + it("doesn't walk a link to a sub-document as if it were the whole document", async () => { + const { repo, a, accounts, root, b } = await boardFixture(); + const subUrl = b.sub("notes").url; + root.change((d: any) => d.docs.push({ name: "subby", type: "x", url: subUrl })); + expect(await walkPath(repo, livePath(a, ["subby"]), { accounts })).toBeUndefined(); + const listed = await listFiles(repo, (await walkPath(repo, livePath(a, ["b"]), { accounts }))!.pinned, { + accounts, + }); + expect(listed?.files.map((f) => f.tail)).not.toContain("subby"); + expect(listed?.missing).not.toContain("subby"); + }); +}); From ee74993179a195c3aad2542dcc331262b5e41e13 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 02:39:58 +0000 Subject: [PATCH 59/81] spec, changeset: package providers apply only to docs a walk goes through Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/path-provider-plugins.md | 2 ++ .../2026-10-28-paths-namespace-content-cache-design.md | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.changeset/path-provider-plugins.md b/.changeset/path-provider-plugins.md index 7f8aa4517..a28b92989 100644 --- a/.changeset/path-provider-plugins.md +++ b/.changeset/path-provider-plugins.md @@ -11,6 +11,8 @@ Packages can now say what a document of their type contains. A `patchwork:path-p - a `Child` can be `{ name, sub: [...segments] }`, a value inside the document offered as a sub-document. A walk that ends at one reports it as `WalkEnd.sub`, and `resolveLivePath` returns `handle.sub(...segments)`; - `PathProvider` gains the optional `add`, `rename` and `remove`, for whoever shows a provider's children to edit them; - walks take `WalkOptions.providers` (an `ExtraProviders`), asked after the built-ins, as do `listChildren` and `listUnion`. A provider that throws is treated as having no children. + - extra providers apply only to a document a walk goes on through. Whether a document a path ends at is a container, and what a file list counts as a file, are the built-ins' call, so installing a provider doesn't change what an existing pinned URL means; + - a link with path segments (to a sub-document) isn't walked or listed as the whole document. - `patchwork-plugins`: the plugin type `PathProviderDescription`/`PathProviderPlugin`, with `pathProviderFor(doc)` (built-ins, then plugins), `packagePathProviders` (plugins only, for `WalkOptions.providers`), and `pathProviderTypes()`. `"*"` in `supportedDatatypes` is ignored. - `patchwork-bootloader`: `createResolver` takes `providers`. A tab's resolver passes the registered plugins, and the automerge worker's passes none, since it never loads package code. - `patchwork`: `&path=` resolves through registered path providers too, and a path that ends at a sub-document opens it at its sub-handle's URL. diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 2d54d2dce..8fa535aa9 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -452,7 +452,11 @@ providers. It adds no access a package doesn't already have. ### Versions A new version of the providing package can change what a live path resolves -to. Pinned paths don't change. A list rebuilt through a changed provider won't +to. Pinned paths don't change. A package's provider applies only to a document a walk +goes on *through*. Whether a document a path ends at is a container or a +file, and what a file list counts as a file, are the built-ins' call. So a +path pins to the same URL with or without the provider, and a provider +installed later doesn't change a URL pinned before it. A list rebuilt through a changed provider won't match its hash, so the URL fails rather than serving something else. ### What a provider offers From b45d51338db720dacd613cf66ffed9a89d818ab2 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 04:15:23 +0000 Subject: [PATCH 60/81] paths: one tail per provider name, non-array lists, sub-aware path checks, worker isolation test - A name a provider spreads over several segments is written as the file list writes it, so each spelling pins to a tail its list covers. - A provider whose list isn't an array has no children. - The router drops a path= naming a different part of doc='s document. - resolve.ts no longer carries a sub branch nothing can reach. - A static test holds that the automerge worker never gets package providers. - Spec sections 2 and 6 say where package providers run and where drops land. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/path-provider-plugins.md | 3 +- core/bootloader/test/worker-isolation.test.ts | 23 ++++++++++++ core/filesystem/src/resolve.ts | 8 ++-- core/filesystem/src/walk.ts | 22 ++++++++--- .../filesystem/test/package-providers.test.ts | 33 +++++++++++++++++ core/patchwork/src/router.ts | 4 +- core/patchwork/test/router.test.ts | 37 +++++++++++++++++++ ...28-paths-namespace-content-cache-design.md | 15 ++++---- 8 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 core/bootloader/test/worker-isolation.test.ts diff --git a/.changeset/path-provider-plugins.md b/.changeset/path-provider-plugins.md index a28b92989..f761d247d 100644 --- a/.changeset/path-provider-plugins.md +++ b/.changeset/path-provider-plugins.md @@ -13,6 +13,7 @@ Packages can now say what a document of their type contains. A `patchwork:path-p - walks take `WalkOptions.providers` (an `ExtraProviders`), asked after the built-ins, as do `listChildren` and `listUnion`. A provider that throws is treated as having no children. - extra providers apply only to a document a walk goes on through. Whether a document a path ends at is a container, and what a file list counts as a file, are the built-ins' call, so installing a provider doesn't change what an existing pinned URL means; - a link with path segments (to a sub-document) isn't walked or listed as the whole document. + - a name a provider spreads over several path segments is written as the file list writes it, so every spelling of it pins to a tail the list covers. - `patchwork-plugins`: the plugin type `PathProviderDescription`/`PathProviderPlugin`, with `pathProviderFor(doc)` (built-ins, then plugins), `packagePathProviders` (plugins only, for `WalkOptions.providers`), and `pathProviderTypes()`. `"*"` in `supportedDatatypes` is ignored. - `patchwork-bootloader`: `createResolver` takes `providers`. A tab's resolver passes the registered plugins, and the automerge worker's passes none, since it never loads package code. -- `patchwork`: `&path=` resolves through registered path providers too, and a path that ends at a sub-document opens it at its sub-handle's URL. +- `patchwork`: `&path=` resolves through registered path providers too, and a path that ends at a sub-document opens it at its sub-handle's URL. A `path=` that names a different part of the document `doc=` names is dropped, as one naming a different document already was. diff --git a/core/bootloader/test/worker-isolation.test.ts b/core/bootloader/test/worker-isolation.test.ts new file mode 100644 index 000000000..bfbf22d22 --- /dev/null +++ b/core/bootloader/test/worker-isolation.test.ts @@ -0,0 +1,23 @@ +// The automerge worker never loads package code, so it never walks through a +// package's path provider (spec section 6). Held here statically: nothing the +// worker's resolver is built from imports patchwork-plugins, and the worker +// creates its resolver without `providers`. +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const source = (file: string) => + readFileSync(new URL(`../src/${file}`, import.meta.url), "utf8"); + +describe("automerge worker isolation", () => { + it("doesn't import patchwork-plugins in the worker or its resolver", () => { + for (const file of ["automerge-protocol-handler.ts", "resolver.ts"]) { + expect(source(file), file).not.toMatch(/from\s+["']@inkandswitch\/patchwork-plugins["']/); + } + }); + + it("creates the worker's resolver without providers", () => { + const calls = source("automerge-protocol-handler.ts").match(/createResolver\([^;]*\)/g) ?? []; + expect(calls.length).toBeGreaterThan(0); + for (const call of calls) expect(call).not.toMatch(/providers/); + }); +}); diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 4fe9f1aed..71721f145 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -3,7 +3,7 @@ import { type Repo, isImmutableString, } from "@automerge/automerge-repo/slim"; -import { type PathProvider, asLink, providerFor, valueAt } from "./providers.js"; +import { type PathProvider, asLink, providerFor } from "./providers.js"; export type Resolved = { content: string | Uint8Array; @@ -54,10 +54,8 @@ async function resolveIn( const { child, consumed } = found; const rest = parts.slice(consumed); if ("value" in child) return resolveIn(repo, provider, child.value, rest, walk); - if ("sub" in child) { - const value = valueAt(node, child.sub); - return value === undefined ? undefined : resolveIn(repo, provider, value, rest, walk); - } + // Only built-ins walk here, and none hands out sub-documents. + if ("sub" in child) return undefined; // A union (an account's mount) is read through its first binding. const link = "link" in child ? child.link : child.union[0]; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 866f3ff27..86dc4fdb5 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -72,12 +72,21 @@ function lookUp(provider: PathProvider, node: unknown, names: string[]): Lookup function listOf(provider: PathProvider | undefined, node: unknown): Child[] { try { - return provider?.list(node) ?? []; + const children = provider?.list(node); + return Array.isArray(children) ? children : []; } catch { return []; } } +/** + * The path segments a child's name is written as: one, except a directory key, which spans + * several. The walk and file lists both write it this way, so a name reached by several + * spellings ("a%2Fb", or "a/b" consumed together) pins to the one tail its list has. + */ +const nameSegments = (provider: PathProvider | undefined, name: string) => + provider === directoryProvider ? name.split("/") : [name]; + // idhashes never change, and every walk hashes the same few documents again. const idhashes = new Map<string, Promise<string>>(); function memoIdhash(documentId: string): Promise<string> { @@ -325,7 +334,12 @@ export async function walkPath( if (consumed > names.length || (consumed < 1 && ("value" in child || "sub" in child))) { return undefined; } - const written = names.slice(0, consumed).map(encodeURIComponent); + const written = + consumed === 0 + ? [] + : typeof child.name === "string" && child.name + ? nameSegments(at.provider, child.name).map(encodeURIComponent) + : names.slice(0, consumed).map(encodeURIComponent); i += consumed; if ("value" in child) { @@ -621,9 +635,7 @@ export async function listFiles( ): Promise<Link[]> => { const links: Link[] = []; for (const child of listOf(provider, node)) { - // Only a directory's keys span several path segments; a folder entry's name is one. - const names = - provider === directoryProvider ? child.name.split("/") : [child.name]; + const names = nameSegments(provider, child.name); const childParts = [...parts, ...names]; const childKeys = [...keys, names.length]; const builtIn = provider === directoryProvider || provider === folderProvider; diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts index 47483163d..c582db9f7 100644 --- a/core/filesystem/test/package-providers.test.ts +++ b/core/filesystem/test/package-providers.test.ts @@ -199,3 +199,36 @@ describe("review fixes", () => { expect(listed?.missing).not.toContain("subby"); }); }); + +describe("deferred review fixes", () => { + it("writes a name a provider spreads over several segments as the list does", async () => { + const { repo, a, accounts, b, child } = await boardFixture(); + const slashy: PathProvider = { + ...board, + list: () => [{ name: "a/b", link: child.url }], + lookup: (_node, names) => + names[0] === "a/b" + ? { child: { name: "a/b", link: child.url }, consumed: 1 } + : names[0] === "a" && names[1] === "b" + ? { child: { name: "a/b", link: child.url }, consumed: 2 } + : undefined, + }; + const options = { accounts, providers: async (doc: unknown) => (board.matches(doc) ? slashy : undefined) }; + const spread = await walkPath(repo, `${livePath(a, ["b"])}/a/b`, options); + const encoded = await walkPath(repo, livePath(a, ["b", "a/b"]), options); + expect(spread?.tail).toBe("a%2Fb"); + expect(encoded?.tail).toBe("a%2Fb"); + const listed = await listFiles(repo, spread!.pinned, options); + expect(listed?.files.map((f) => f.tail)).toContain(spread!.tail); + expect(b).toBeDefined(); + }); + + it("treats a list that isn't an array as no children", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const odd: PathProvider = { ...board, list: () => ({}) as unknown as Child[] }; + const providers = async (doc: unknown) => (board.matches(doc) ? odd : undefined); + expect(await listChildren(repo, b as DocHandle<unknown>, providers)).toEqual([]); + const end = await walkPath(repo, livePath(a, ["b", "child"]), { accounts, providers }); + await expect(listFiles(repo, end!.pinned, { accounts, providers })).resolves.toBeDefined(); + }); +}); diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index 86b2a2c41..d898f8231 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -92,7 +92,9 @@ export function createRouter({ console.warn(`patchwork: path=${path} doesn't resolve here (yet)`); return; } - if (handle.documentId !== parseAutomergeUrl(docUrl).documentId) { + // The document, and the part of it (`automerge:<id>/<path>`), without heads. + const named = (url: string) => url.split("#")[0]; + if (named(handle.url) !== named(docUrl)) { dropPath(path, `it names ${handle.url}, not ${docUrl}`); } }; diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts index 291402340..affe3b590 100644 --- a/core/patchwork/test/router.test.ts +++ b/core/patchwork/test/router.test.ts @@ -265,3 +265,40 @@ describe("router: paths through package providers", () => { expect(opened[0].url).toBe(b!.sub("notes").url); }); }); + +describe("router: paths to part of a document", () => { + const PLUGIN_URL = "https://example.test/board2.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: () => [{ name: "notes", sub: ["notes"] }], + lookup: (_doc: any, names: string[]) => + names[0] === "notes" ? { child: { name: "notes", sub: ["notes"] }, consumed: 1 } : undefined, + }; + let b: DocHandle<any> | undefined; + const withBoard = (repo: Repo) => { + b = repo.create({ "@patchwork": { type: "board" }, notes: { content: "n" } }); + return { b: b.url }; + }; + afterEach(() => unregisterPlugins(PLUGIN_URL)); + + it("drops a path that names part of doc when doc is the whole document", async () => { + registerPlugins( + [{ id: "board2", name: "board2", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let boardUrl = ""; + const { location } = await setup((a) => { + return `path=${livePath(a, ["b", "notes"])}`; + }, (repo) => { + const entries = withBoard(repo); + boardUrl = entries.b; + return entries; + }); + location.hash = `doc=${boardUrl}&path=${hashParam(location, "path")}`; + await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined(), { timeout: 3000 }); + expect(hashParam(location, "doc")).toBe(boardUrl); + warn.mockRestore(); + }); +}); diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 8fa535aa9..ba9c8507f 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -139,11 +139,10 @@ Built-in providers, checked in order, first match wins: stands for its `default` branch. This replaces the special case in `core/bootloader/src/resolver.ts`. -Providers are built in only, for now. Nothing trusts a provider's answers -any more, since every byte is checked against the URL. But a provider runs -inside every resolver, the automerge worker's included, so a package -supplying one would run package code there. The provider interface is -exported, shaped so package providers can be added later. +These built-ins run in every resolver, the automerge worker's included. +Nothing trusts a provider's answers, since every byte is checked against the +URL. Packages can supply providers too (section 6), but they run only in +tabs, after the built-ins, and only for documents a walk goes through. ### Namespace @@ -395,8 +394,10 @@ Approved in conversation 2026-10-28. Implemented in `patchwork-filesystem` `patchwork-plugins` (`pathProviderFor`, `packagePathProviders`, `pathProviderTypes`), the tab resolver and the router, and the base sidebar. The built-in folder provider doesn't offer editing yet. In the -sidebar, a document dropped on a provider's row is added (linked) and left -where it was; files and new documents can't be dropped there. +sidebar, a document dropped among an opened provider's children is added +(linked) and left where it was. The row itself still takes drops as a folder +entry. Files, new documents and sub-documents can't be dropped among a +provider's children. The sidebar can open up a doc that isn't a folder when a built-in provider walks it: a directory doc, a branches doc, the account. This section proposes From e6765125cfae4303f4abe61e010e6153c2680697 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:40:22 +0000 Subject: [PATCH 61/81] router: checks don't outlive their navigation; paths are per sub-document; path-only hashes retry when providers register - A doc=/path= check that resolves after the hash moved on leaves the new hash alone: dropPath now needs doc= unchanged too. - Opening another part of the same document without a path drops the path; only the same part (a tool or heads change) keeps it. - A path-only hash that doesn't resolve is walked again whenever path providers change, so a cold boot opens a path through a package's datatype once that package registers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/patchwork/src/router.ts | 66 ++++++++++++++++++++++++----- core/patchwork/test/router.test.ts | 67 ++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 11 deletions(-) diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index d898f8231..c33b9bb5f 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -74,10 +74,17 @@ export function createRouter({ return undefined; // a path that doesn't parse names nothing } }; - /** Take `path=` out of the hash, if it's still the one that was checked. */ - const dropPath = (path: string, reason: string) => { + // The document, and the part of it (`automerge:<id>/<path>`), without heads. + const named = (url: string) => url.split("#")[0]; + /** + * Take `path=` out of the hash, if it and `doc=` are still the ones that were + * checked: a check that resolves after a navigation is about the old one. + */ + const dropPath = (path: string, docUrl: AutomergeUrl, reason: string) => { const params = parseHashParams(window.location.hash.slice(1)); if (params.get("path") !== path) return; + const doc = docParamToUrl(params.get("doc")); + if (!doc || named(doc) !== named(docUrl)) return; console.warn(`patchwork: dropping path=${path} from the URL: ${reason}`); params.delete("path"); window.location.hash = serializeHashParams(params); @@ -92,10 +99,8 @@ export function createRouter({ console.warn(`patchwork: path=${path} doesn't resolve here (yet)`); return; } - // The document, and the part of it (`automerge:<id>/<path>`), without heads. - const named = (url: string) => url.split("#")[0]; if (named(handle.url) !== named(docUrl)) { - dropPath(path, `it names ${handle.url}, not ${docUrl}`); + dropPath(path, docUrl, `it names ${handle.url}, not ${docUrl}`); } }; @@ -179,14 +184,51 @@ export function createRouter({ // Just a path: open what it names, unless the hash has moved on while // it was walked. One that doesn't resolve is left in the address bar, // to be fixed or retried. - const handle = await resolvePathParam(path); - const now = parseHashParams(window.location.hash.slice(1)); - if (now.get("path") !== path || now.get("doc")) return; - if (handle) open(handle.url); - else console.warn(`patchwork: path=${path} doesn't resolve here (yet); opening nothing`); + await openPath(path, open, true); } }; + // A path-only hash that didn't resolve may go through a datatype whose + // package hasn't registered its path provider yet (on a cold boot, say). + // It's walked again whenever path providers change, until it resolves or + // the hash moves on. + let stopRetrying: (() => void) | undefined; + const pathOnly = (path: string) => { + const now = parseHashParams(window.location.hash.slice(1)); + return now.get("path") === path && !now.get("doc"); + }; + async function openPath( + path: string, + open: (url: AutomergeUrl) => void, + first: boolean + ) { + const handle = await resolvePathParam(path); + if (!pathOnly(path)) return; + if (handle) { + stopRetrying?.(); + open(handle.url); + return; + } + if (!first) return; + console.warn(`patchwork: path=${path} doesn't resolve here (yet); opening nothing`); + stopRetrying?.(); + const stop = () => { + off(); + if (stopRetrying === stop) stopRetrying = undefined; + }; + // Runs inside whoever registered a plugin, so it mustn't throw at them. + const off = getRegistry("patchwork:path-provider").on("changed", () => { + try { + if (pathOnly(path)) void openPath(path, open, false).catch(stop); + else stop(); + } catch { + stop(); + } + }); + stopRetrying = stop; + stopRetrying = off; + } + rootElement.addEventListener("patchwork:open-document", async (event) => { const { url, toolId, type, title, path } = event.detail as { url: AutomergeUrl; @@ -210,7 +252,9 @@ export function createRouter({ if (prevDoc && !sameDoc) params.delete("draft"); // Where the doc was opened from survives a tool switch or heads change on // the same doc, as `draft` does. - const keptPath = !path && sameDoc ? params.get("path") ?? undefined : path; + // Only on the same part of it, though: another sub-document is elsewhere. + const samePart = !!prevDoc && named(prevDoc) === named(url); + const keptPath = !path && samePart ? params.get("path") ?? undefined : path; params.set("doc", url); for (const [key, value] of [ ["path", keptPath], diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts index affe3b590..ed6cd4750 100644 --- a/core/patchwork/test/router.test.ts +++ b/core/patchwork/test/router.test.ts @@ -302,3 +302,70 @@ describe("router: paths to part of a document", () => { warn.mockRestore(); }); }); + +describe("router: copilot review", () => { + const PLUGIN_URL = "https://example.test/late-board.js"; + afterEach(() => unregisterPlugins(PLUGIN_URL)); + + it("doesn't let a check for an earlier doc= drop the path from a newer one", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { live, notes, location } = await setup((a) => + `doc=${OTHER}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + // Before the first check resolves, navigate to the doc the path names. + location.hash = `doc=${notes.url}&path=${live}`; + await settle(); + expect(hashParam(location, "doc")).toBe(notes.url); + expect(hashParam(location, "path")).toBe(live); + warn.mockRestore(); + }); + + it("drops the path when another part of the same document is opened without one", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: `${notes.url}/content` } }) + ); + expect(hashParam(location, "path")).toBeUndefined(); + // A tool switch on the same part keeps it. + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, toolId: "other" } }) + ); + expect(hashParam(location, "path")).toBe(live); + }); + + it("opens a path-only hash once the provider it needs registers", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let childUrl = ""; + const { opened } = await setup( + (a) => `path=${livePath(a, ["b", "child"])}`, + (repo) => { + const child = repo.create({ "@patchwork": { type: "file" }, content: "c" }); + childUrl = child.url; + const b = repo.create({ "@patchwork": { type: "board" }, child: child.url }); + return { b: b.url }; + } + ); + await settle(); + expect(opened).toHaveLength(0); + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [{ name: "child", link: doc.child }], + lookup: (doc: any, names: string[]) => + names[0] === "child" ? { child: { name: "child", link: doc.child }, consumed: 1 } : undefined, + }; + registerPlugins( + [{ id: "late-board", name: "late-board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(childUrl); + warn.mockRestore(); + }); +}); From 4d5a6e4ecbbcaed71307434082e8df1bbc663a45 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:46:56 +0000 Subject: [PATCH 62/81] file lists: a __proto__ tail is listed; prototype names aren't entries Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/bootloader/src/pinned.ts | 3 ++- core/bootloader/test/pinned.test.ts | 9 +++++++++ core/filesystem/src/file-list.ts | 3 ++- core/filesystem/test/file-list.test.ts | 10 ++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index 716ed4944..6e66bda7e 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -203,7 +203,8 @@ export async function servePinned( if (!list) { throw new PinnedAbort(`no file list ${listed.listHash} to check /@/${path} against`); } - const entry = list.files[listed.tail]; + // Own entries only: a tail like "constructor" isn't in a list that doesn't name it. + const entry = Object.hasOwn(list.files, listed.tail) ? list.files[listed.tail] : undefined; if (!entry) { return new Response(`/@/${path} isn't in its file list`, { status: 404, diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index 61ec0af4a..f4a821d13 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -192,6 +192,15 @@ describe("servePinned", () => { expect(deps.resolve).not.toHaveBeenCalled(); }); + it("answers 404 for a tail named like an Object.prototype member", async () => { + const { deps, listHash } = await setup(); + for (const tail of ["constructor", "toString", "__proto__"]) { + const response = await servePinned(normalizePinned(`acct@h/pkg@h~${listHash}/${tail}`)!, deps); + expect(response.status, tail).toBe(404); + } + expect(deps.resolve).not.toHaveBeenCalled(); + }); + it("never stores bytes the list says not to", async () => { const { deps, sha256, normalized, settle } = await setup({ store: false }); expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); diff --git a/core/filesystem/src/file-list.ts b/core/filesystem/src/file-list.ts index 5f061e2be..aa6bdf4f8 100644 --- a/core/filesystem/src/file-list.ts +++ b/core/filesystem/src/file-list.ts @@ -85,7 +85,8 @@ export async function buildFileList( const throughPackages = first?.kind === "name" && first.name === "packages"; const isProtected = options.isProtected ?? isKeyhiveDocumentId; - const files: Record<string, FileEntry> = {}; + // Tails are names people chose: "__proto__" must be a key like any other. + const files: Record<string, FileEntry> = Object.create(null); for (const file of listed.files) { files[file.tail] = { sha256: await contentHash(toBytes(file.content)), diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index 0ee078398..401665d6e 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -47,6 +47,16 @@ describe("buildFileList", () => { expect(text).not.toContain(parseAutomergeUrl(fileUrl as any).documentId); }); + it("lists a file named __proto__ like any other", async () => { + const { repo, a, accounts, foo } = await fixture(); + foo.change((d: any) => d.docs.push({ name: "__proto__", type: "x", url: file(repo, "p").url })); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(Object.hasOwn(built!.list.files, "__proto__")).toBe(true); + expect(new TextDecoder().decode(built!.bytes)).toContain('"__proto__":'); + expect(Object.hasOwn(parseFileList(built!.bytes)!.files, "__proto__")).toBe(true); + }); + it("lists only what it's asked to", async () => { const { repo, a, accounts } = await fixture(); const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); From 664e2b7a1e347879af9bba667efaedb97cacf7a6 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:47:45 +0000 Subject: [PATCH 63/81] migration: unique package names, and a /packages binding that arrives mid-migration wins Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/patchwork/src/migrate-packages.ts | 19 ++++++-- core/patchwork/test/migrate-packages.test.ts | 51 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts index 2d3930d6f..103cd1c33 100644 --- a/core/patchwork/src/migrate-packages.ts +++ b/core/patchwork/src/migrate-packages.ts @@ -122,11 +122,13 @@ export async function migratePackages( } }) ); - const taken = new Map<string, number>(); + // "alpha", "alpha (2)", "alpha": the third is "alpha (3)", not a second "alpha (2)". + const used = new Set<string>(); const links = described.map(({ url, name, type }) => { - const count = (taken.get(name) ?? 0) + 1; - taken.set(name, count); - return { name: count === 1 ? name : `${name} (${count})`, type, url }; + let unique = name; + for (let n = 2; used.has(unique); n++) unique = `${name} (${n})`; + used.add(unique); + return { name: unique, type, url }; }); // Another tab may have migrated while this one read the packages. @@ -139,10 +141,19 @@ export async function migratePackages( folder.docs = links; } ); + // A binding that arrived while the folder was made (another device's + // migration, synced in) wins: this folder is new and holds only links, so + // dropping it loses nothing. Two migrations that never see each other's + // binding before they merge still conflict, and Automerge keeps one; both + // folders hold the same links, read from the same module settings. + let bound = false; account.change((doc) => { + if (doc.namespace && "/packages" in doc.namespace) return; doc.namespace ??= {}; doc.namespace["/packages"] = [{ url: folder.url }]; + bound = true; }); + if (!bound) return false; await linkPackagesIntoRoot(repo, account.doc()?.rootFolderUrl, folder.url).catch( (error) => console.warn("couldn't show /packages in the root folder", error) ); diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts index 79d5c2ef6..1e11ccd9e 100644 --- a/core/patchwork/test/migrate-packages.test.ts +++ b/core/patchwork/test/migrate-packages.test.ts @@ -164,3 +164,54 @@ describe("createDefaultAccount", () => { expect(doc.moduleSettingsUrl).toBeDefined(); }); }); + +describe("migratePackages: copilot review", () => { + const named = (repo: Repo, name: string) => { + const manifest = repo.create({ + "@patchwork": { type: "file" }, + content: JSON.stringify({ name }), + mimeType: "application/json", + }).url; + return repo.create({ "@patchwork": { type: "directory" }, "package.json": manifest }); + }; + + it("never gives two packages one name, whatever they're called", async () => { + const repo = new Repo(); + const one = named(repo, "alpha"); + const two = named(repo, "alpha (2)"); + const three = named(repo, "alpha"); + const acct = await account(repo, [one.url, two.url, three.url]); + await migratePackages(repo, acct); + const names = (await linked(repo, acct))!.map(([name]: string[]) => name); + expect(new Set(names).size).toBe(3); + expect(names.slice(0, 2)).toEqual(["alpha", "alpha (2)"]); + }); + + it("leaves a /packages binding that arrives while its folder is made", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const theirs = repo.create({ title: "theirs", docs: [] }); + // Another device's migration syncs in just as this one creates its folder. + const create = repo.create.bind(repo); + (repo as any).create = (...args: any[]) => { + if (!acct.doc().namespace) { + acct.change((d: any) => (d.namespace = { "/packages": [{ url: theirs.url }] })); + } + return create(...args); + }; + for (const name of ["create2"] as const) { + const original = (repo as any)[name]?.bind(repo); + if (original) { + (repo as any)[name] = (...args: any[]) => { + if (!acct.doc().namespace) { + acct.change((d: any) => (d.namespace = { "/packages": [{ url: theirs.url }] })); + } + return original(...args); + }; + } + } + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"]).toEqual([{ url: theirs.url }]); + }); +}); From 77adaf46f39e2fd5c195498fe74b882dd8a709d6 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:50:43 +0000 Subject: [PATCH 64/81] packages: a /packages package that's a branches doc reloads when its branch's code changes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/packages-folder.ts | 39 +++++++++++++++++++- core/filesystem/test/packages-folder.test.ts | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts index 77c407852..989b0524f 100644 --- a/core/filesystem/src/packages-folder.ts +++ b/core/filesystem/src/packages-folder.ts @@ -2,6 +2,7 @@ // of package folders. This lists it for the ModuleWatcher, and looks each // package up as a pinned `/@/` URL whose file list covers the whole package. import { + isValidAutomergeUrl, parseAutomergeUrl, stringifyAutomergeUrl, type AutomergeUrl, @@ -163,6 +164,42 @@ export function trackPackagesFolder( // Not an automerge URL: the watcher imports it with importPackageFromHttpUrl. return latest as AutomergeUrl; }, - watch: (url, onChange) => fallback.watch(url, onChange), + watch(url, onChange) { + const documentId = parseAutomergeUrl(url).documentId; + const isEntry = () => + entries().some((entry) => parseAutomergeUrl(entry.url).documentId === documentId); + // A package that's a branches doc stands for its default branch, whose + // code changes without the branches doc changing: watch that too, and + // follow it when the branch is pointed somewhere else. + let stopped = false; + let branch: AutomergeUrl | undefined; + let unwatchBranch = () => {}; + const follow = async () => { + if (!isEntry()) return; + let target: AutomergeUrl | undefined; + try { + const latest = await fallback.findLatest(url); + if (isValidAutomergeUrl(latest)) target = bare(latest); + } catch { + return; // keep the branch already watched; the next change asks again + } + if (stopped) return; + if (target && parseAutomergeUrl(target).documentId === documentId) target = undefined; + if (target === branch) return; + unwatchBranch(); + branch = target; + unwatchBranch = target ? fallback.watch(target, onChange) : () => {}; + }; + const unwatch = fallback.watch(url, (seen) => { + void follow(); + onChange(seen); + }); + void follow(); + return () => { + stopped = true; + unwatch(); + unwatchBranch(); + }; + }, }; } diff --git a/core/filesystem/test/packages-folder.test.ts b/core/filesystem/test/packages-folder.test.ts index d0256ff6f..15042bf22 100644 --- a/core/filesystem/test/packages-folder.test.ts +++ b/core/filesystem/test/packages-folder.test.ts @@ -127,3 +127,41 @@ describe("trackPackagesFolder: stable URLs", () => { expect(await tracker.findLatest(entry.url)).toBe("https://p.test/@/acct@r2/packages/m@h1/pkg@p2~L2"); }); }); + +describe("trackPackagesFolder: a branches package", () => { + it("watches the branch it stands for, and follows the branch when it moves", async () => { + const BRANCHES = "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl; + const MAIN = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + const NEXT = "automerge:3tmVsDzywTYecevUsD21YABfXKon" as AutomergeUrl; + const entry = { url: BRANCHES, name: "foo", memberStep: "m@h" }; + let current = MAIN; + const watchers = new Map<string, Set<(seen?: string) => void>>(); + const fire = (url: string) => watchers.get(url)?.forEach((cb) => cb()); + const fallback: PackageTracker = { + findLatest: vi.fn(async (url) => (url === BRANCHES ? current : url)), + watch: vi.fn((url, onChange) => { + if (!watchers.has(url)) watchers.set(url, new Set()); + watchers.get(url)!.add(onChange); + return () => watchers.get(url)!.delete(onChange); + }), + }; + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, vi.fn()); + const changes = vi.fn(); + const stop = tracker.watch(BRANCHES, changes); + + // New code on the default branch, with the branch pointer unchanged. + await vi.waitFor(() => expect(watchers.get(MAIN)?.size).toBe(1)); + fire(MAIN); + expect(changes).toHaveBeenCalledTimes(1); + + // The branch moves: the watch follows it. + current = NEXT; + fire(BRANCHES); + await vi.waitFor(() => expect(watchers.get(NEXT)?.size).toBe(1)); + expect(watchers.get(MAIN)?.size).toBe(0); + + stop(); + expect(watchers.get(NEXT)?.size).toBe(0); + expect(watchers.get(BRANCHES)?.size).toBe(0); + }); +}); From 10f3f2a1e0927029ebd98f3e40dfdf44af73a9fc Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:54:17 +0000 Subject: [PATCH 65/81] paths: pinning through a package's container lists it; values in nested docs read at their heads - A doc named by a doc step is walked with package providers too: only a container is written as one, so a pinned prefix through a package's container rebuilds that container's list, not its parent's. A path that ends at a doc by name still decides container or file by the built-ins. - A value held by a doc below the container (a nested directory in a deep package) is listed with that doc's heads, and read back at them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/walk.ts | 53 +++++++++++++------ core/filesystem/test/file-list.test.ts | 19 +++++++ .../filesystem/test/package-providers.test.ts | 18 ++++++- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index 86dc4fdb5..c3337e737 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -163,10 +163,11 @@ export async function walkPath( const stepOf = async (handle: DocHandle<unknown>) => docStep(await hashOf(handle.url), handle.heads()); - // Package providers apply only to a document the walk goes on through. Whether a document the - // path ends at is a container (a doc step) or a file (a name) is the built-ins' call, so a - // path pins the same with or without a package's provider, and a plugin installed later - // doesn't change what an existing pinned URL means. + // Package providers apply to a document the walk goes on through, and to one named by a doc + // step (only a container is ever written as one, so a pinned prefix stays its container). A + // document a path ends at *by name* is a container or a file by the built-ins' call, so a path + // pins the same with or without a package's provider, and a plugin installed later doesn't + // change what an existing pinned URL means. async function enter( url: AutomergeUrl, heads: UrlHeads | undefined, @@ -231,7 +232,7 @@ export async function walkPath( let at: At | undefined; if (root.kind === "account") { const url = options.accounts.get(root.acctHash); - at = url && (await enter(url, root.heads, segments.length > 0)); + at = url && (await enter(url, root.heads, true)); if (!at) return undefined; out.push(docStep(root.acctHash, at.handle.heads())); if (root.listHash) lists.set(0, root.listHash); @@ -239,7 +240,7 @@ export async function walkPath( at = await enter( stringifyAutomergeUrl({ documentId: root.documentId }), root.heads, - segments.length > 0 + true ); if (!at) return undefined; out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); @@ -268,7 +269,7 @@ export async function walkPath( listHash = segment.listHash; i++; if (!guard(member, segments.length - i)) return undefined; - next = await enter(member, segment.heads, i < segments.length); + next = await enter(member, segment.heads, true); } } if (!member) { @@ -313,7 +314,7 @@ export async function walkPath( ); const link = await matchLink(links, segment); if (link) { - const next = await enter(link, segment.heads, i + 1 < segments.length); + const next = await enter(link, segment.heads, true); if (!next) return undefined; out.push(await stepOf(next.handle)); if (segment.listHash) lists.set(out.length - 1, segment.listHash); @@ -519,11 +520,26 @@ export async function readPath( const end = await walkPath(repo, path, options); if (!end) return undefined; let node = end.node; - if (options.heads && end.atDocument) { + if (options.heads) { + const heads = options.heads; + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const holder = end.handle.documentId; try { - const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); - const bare = stringifyAutomergeUrl({ documentId: end.handle.documentId }); - node = (await find(bare, options.heads)).view(options.heads).doc(); + if (end.atDocument) { + const bare = stringifyAutomergeUrl({ documentId: holder }); + node = (await find(bare, heads)).view(heads).doc(); + } else { + // A value inside a document: walk again with that document read at the heads. + const atHeads = await walkPath(repo, path, { + ...options, + find: async (url, at) => + parseAutomergeUrl(url).documentId === holder + ? (await find(url, heads)).view(heads) + : find(url, at), + }); + if (!atHeads) return undefined; + node = atHeads.node; + } } catch { return undefined; } @@ -626,12 +642,15 @@ export async function listFiles( // Values are read in place; links are gathered and read below. type Link = { parts: string[]; keys: number[]; link: AutomergeUrl }; const entered = new Set<string>([rootId]); + // `heads`: what the holder was read at, when it's a doc below the container (reached by a + // link, so not pinned by the container's step): its values are read back at those heads. const visit = async ( node: unknown, provider: PathProvider | undefined, holder: DocumentId, parts: string[], - keys: number[] + keys: number[], + heads?: UrlHeads ): Promise<Link[]> => { const links: Link[] = []; for (const child of listOf(provider, node)) { @@ -642,17 +661,17 @@ export async function listFiles( if ("sub" in child || "value" in child) { const value = "sub" in child ? valueAt(node, child.sub) : child.value; if (value === undefined) continue; - await add(childParts, childKeys, value, holder); + await add(childParts, childKeys, value, holder, heads); // A package's provider walks on into the values it hands out, so list what's in them. if (!builtIn && value && typeof value === "object" && !(value instanceof Uint8Array)) { - links.push(...(await visit(value, provider, holder, childParts, childKeys))); + links.push(...(await visit(value, provider, holder, childParts, childKeys, heads))); continue; } } if ("value" in child) { // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { - links.push(...(await visit(child.value, provider, holder, childParts, childKeys))); + links.push(...(await visit(child.value, provider, holder, childParts, childKeys, heads))); } } else if ( "link" in child && @@ -702,7 +721,7 @@ export async function listFiles( // Into a nested folder, by name: its files are under its name here. entered.add(parsed.documentId); await add(parts, keys, doc, parsed.documentId, heads); - below = await visit(doc, provider, parsed.documentId, parts, keys); + below = await visit(doc, provider, parsed.documentId, parts, keys, heads); } } catch { missing.push(tailOf(parts)); diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index 401665d6e..c1bf2ed0a 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -150,6 +150,25 @@ describe("package lists", () => { expect(decode(then!.content)).toBe("x"); }); + it("records the heads of a nested doc that holds values, and reads them there", async () => { + const { repo, a, accounts, pkg } = await folderPackage(); + const dir = repo.create<any>({ + "@patchwork": { type: "directory" }, + "a.js": { content: "old", mimeType: "text/javascript" }, + }); + pkg.change((d: any) => d.docs.push({ name: "dir", type: "directory", url: dir.url })); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dir/a.js"]; + expect(entry?.heads).toBeDefined(); + dir.change((d: any) => (d["a.js"].content = "new")); + const then = await readPath(repo, `${built!.prefix}/dir/a.js`, { + accounts, + heads: parseHeads(entry!.heads!), + }); + expect(decode(then!.content)).toBe("old"); + }); + it("pins a live path to a folder with the folder's whole list", async () => { const { repo, a, accounts } = await folderPackage(); const pinned = await pinnedPath(repo, livePath(a, ["packages", "fpkg"]), { diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts index c582db9f7..a1b69b2af 100644 --- a/core/filesystem/test/package-providers.test.ts +++ b/core/filesystem/test/package-providers.test.ts @@ -5,7 +5,8 @@ import type { AutomergeUrl, DocHandle } from "@automerge/automerge-repo"; import { livePath } from "../src/paths.js"; import { getType } from "../src/metadata.js"; import type { Child, PathProvider } from "../src/providers.js"; -import { listChildren, listFiles, resolveLivePath, walkPath } from "../src/walk.js"; +import { listChildren, listFiles, pinnedPath, resolveLivePath, walkPath } from "../src/walk.js"; +import { buildFileList, parseFileList } from "../src/file-list.js"; import { file, fixture } from "./namespace-fixture.js"; type BoardDoc = { @@ -232,3 +233,18 @@ describe("deferred review fixes", () => { await expect(listFiles(repo, end!.pinned, { accounts, providers })).resolves.toBeDefined(); }); }); + +describe("pinning through a package's container", () => { + it("pins a path through it to a URL whose list covers the file", async () => { + const { repo, a, accounts } = await boardFixture(); + for (const name of ["child", "notes"]) { + const pinned = await pinnedPath(repo, livePath(a, ["b", name]), { accounts, providers }); + const [prefix, tail] = [pinned!.path.slice(0, pinned!.path.lastIndexOf("/")), name]; + const list = parseFileList(pinned!.list!)!; + expect(Object.keys(list.files), name).toContain(tail); + // The list rebuilds from the container's own prefix, as the service worker asks for it. + const rebuilt = await buildFileList(repo, prefix.replace(/~[^/]*$/, ""), { accounts, providers }); + expect(rebuilt?.listHash).toBe(prefix.match(/~([^/]*)$/)![1]); + } + }); +}); From 91ce8d44ed3c3287f5e30072a85bda5865a74264 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 10:59:00 +0000 Subject: [PATCH 66/81] packages: entries migration moved to /packages load only through it, so removing one there removes it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/module-watcher.ts | 14 +++++++++++-- core/filesystem/test/module-watcher.test.ts | 22 ++++++++++++++++++++ core/patchwork/src/migrate-packages.ts | 14 +++++++++++-- core/patchwork/test/migrate-packages.test.ts | 13 ++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 42a25de22..8a250c971 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -23,6 +23,13 @@ const log = debug("patchwork:modules"); export type ModuleSettingsDoc = { modules: AutomergeUrl[]; + /** + * Documents whose unpinned entries `/packages` took over (see + * migratePackages): those entries load through `/packages` or not at all, + * so removing a package there removes it. Builds that predate `/packages` + * ignore this and keep loading them. + */ + movedToPackages?: string[]; } & HasPatchworkMetadata & { "@patchwork": { type: "patchwork:module-settings" }; }; @@ -635,9 +642,12 @@ export class ModuleWatcher extends EventTarget { } } for (const { doc } of this.settingsDocs()) { + const moved = new Set(Array.isArray(doc?.movedToPackages) ? doc.movedToPackages : []); for (const m of doc?.modules ?? []) { - if (isValidAutomergeUrl(m) && inFolders.has(parseAutomergeUrl(m).documentId)) { - continue; + if (isValidAutomergeUrl(m)) { + const { documentId, heads } = parseAutomergeUrl(m); + if (inFolders.has(documentId)) continue; + if (!heads && moved.has(documentId)) continue; } urls.add(m); } diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 0198d5269..0f75e98a4 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -847,6 +847,28 @@ describe("ModuleWatcher /packages folder", () => { await repo.shutdown().catch(() => {}); }); + it("doesn't load a module-settings entry migration moved to /packages, once it's removed there", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const [moved, kept] = source.docs; + const settings = repo.create<any>({ + "@patchwork": { type: "patchwork:module-settings" }, + modules: [moved.url, kept.url], + movedToPackages: [moved.documentId], + }); + const loaded: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, source.tracker); + // The moved package was removed from /packages; the other was never in it. + source.set([]); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + await watcher.addUrl("user", settings.url); + await pause(50); + expect(source.lookups).not.toContain(moved.url); + expect(source.lookups).toContain(kept.url); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + it("doesn't reload a renamed package, and unloads a removed one", async () => { const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); const source = folderSource(repo); diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts index 103cd1c33..332673e09 100644 --- a/core/patchwork/src/migrate-packages.ts +++ b/core/patchwork/src/migrate-packages.ts @@ -1,7 +1,9 @@ // Once per account: an account from before `/packages` gets a packages folder // holding a link to each automerge package its module-settings doc lists, -// bound at `/packages`. The module-settings doc is left as it is: the Packages -// UI still installs into it, and builds before this one still read it. +// bound at `/packages`. The module-settings doc keeps its entries: the +// Packages UI still installs into it, and builds before this one still read +// it. It records which it moved (`movedToPackages`), which this build then +// loads only through `/packages`. import { isValidAutomergeUrl, parseAutomergeUrl, @@ -154,6 +156,14 @@ export async function migratePackages( bound = true; }); if (!bound) return false; + // What /packages now holds: the watcher leaves those entries to it, so + // removing a package there removes it. The entries themselves stay, for + // builds that predate /packages. + settings.change((doc) => { + const moved = new Set([...((doc as { movedToPackages?: string[] }).movedToPackages ?? [])]); + for (const url of urls) moved.add(parseAutomergeUrl(url).documentId); + (doc as { movedToPackages?: string[] }).movedToPackages = [...moved]; + }); await linkPackagesIntoRoot(repo, account.doc()?.rootFolderUrl, folder.url).catch( (error) => console.warn("couldn't show /packages in the root folder", error) ); diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts index 1e11ccd9e..e58ce2d99 100644 --- a/core/patchwork/test/migrate-packages.test.ts +++ b/core/patchwork/test/migrate-packages.test.ts @@ -46,6 +46,19 @@ describe("migratePackages", () => { ]); }); + it("records in module settings which entries it moved", async () => { + const repo = new Repo(); + const { alpha, bee } = packages(repo); + const pinned = alpha.view(alpha.heads()).url; + const acct = await account(repo, [alpha.url, bee.url, pinned, "https://cdn.example.test/x.js"]); + await migratePackages(repo, acct); + const settings = await repo.find<any>(acct.doc().moduleSettingsUrl); + expect([...settings.doc().movedToPackages].sort()).toEqual( + [alpha.documentId, bee.documentId].sort() + ); + expect(settings.doc().modules).toHaveLength(4); + }); + it("does nothing the second time", async () => { const repo = new Repo(); const { alpha } = packages(repo); From 40f4358273f23612dba6d9b8aed13dd605830453 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 11:02:12 +0000 Subject: [PATCH 67/81] path providers: a typed doc that's also folder-shaped goes to its type's provider The folder built-in matches anything with a docs array, so a tldraw canvas (which keeps its scripts there) never reached its own provider. A provider declared for such a doc's type now walks it; an untyped or folder-typed folder is still the built-in's. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/path-provider-plugins.md | 2 +- core/filesystem/src/providers.ts | 12 +++++ core/filesystem/src/walk.ts | 7 +-- .../filesystem/test/package-providers.test.ts | 44 +++++++++++++++++++ core/plugins/src/path-providers.ts | 10 ++++- core/plugins/test/path-providers.test.ts | 10 +++++ ...28-paths-namespace-content-cache-design.md | 7 +++ 7 files changed, 86 insertions(+), 6 deletions(-) diff --git a/.changeset/path-provider-plugins.md b/.changeset/path-provider-plugins.md index f761d247d..b67858584 100644 --- a/.changeset/path-provider-plugins.md +++ b/.changeset/path-provider-plugins.md @@ -5,7 +5,7 @@ "@inkandswitch/patchwork": patch --- -Packages can now say what a document of their type contains. A `patchwork:path-provider` plugin is a `PathProvider` with `supportedDatatypes`. The built-in providers (account, directory, folder, branches) are always asked first and can't be overridden. After them come plugins, in registration order. +Packages can now say what a document of their type contains. A `patchwork:path-provider` plugin is a `PathProvider` with `supportedDatatypes`. The built-in providers (account, directory, folder, branches) are always asked first and can't be overridden. After them come plugins, in registration order. The one exception is a typed document that also has a folder's `docs` array (a tldraw canvas): a plugin declared for its type walks it, and the folder provider walks it otherwise (`yieldsToPackage`). - `patchwork-filesystem`: - a `Child` can be `{ name, sub: [...segments] }`, a value inside the document offered as a sub-document. A walk that ends at one reports it as `WalkEnd.sub`, and `resolveLivePath` returns `handle.sub(...segments)`; diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts index 36c2cfbdf..eb6e83488 100644 --- a/core/filesystem/src/providers.ts +++ b/core/filesystem/src/providers.ts @@ -262,3 +262,15 @@ export const PROVIDERS: readonly PathProvider[] = [ export function providerFor(doc: unknown): PathProvider | undefined { return PROVIDERS.find((provider) => provider.matches(doc)); } + +/** + * Whether a package's provider for `doc`'s type may walk it in place of `builtIn`. Built-ins + * can't be overridden, with one exception: the folder provider matches anything with a `docs` + * array, and a typed document (a tldraw canvas keeps its scripts that way) is its own type + * first. A folder-typed or untyped folder is always the built-in's. + */ +export function yieldsToPackage(builtIn: PathProvider | undefined, doc: unknown): boolean { + if (builtIn !== folderProvider) return false; + const type = getType(doc as AccountNamespaceDoc); + return !!type && type !== "folder"; +} diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index c3337e737..a87f4b689 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -28,6 +28,7 @@ import { defaultBranch, providerFor, valueAt, + yieldsToPackage, } from "./providers.js"; import { buildFileList } from "./file-list.js"; import { readNode } from "./resolve.js"; @@ -53,11 +54,11 @@ async function providerOf( extra?: ExtraProviders ): Promise<PathProvider | undefined> { const builtIn = providerFor(doc); - if (builtIn || !extra) return builtIn; + if (!extra || (builtIn && !yieldsToPackage(builtIn, doc))) return builtIn; try { - return await extra(doc); + return (await extra(doc)) ?? builtIn; } catch { - return undefined; + return builtIn; } } diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts index a1b69b2af..6e2e4119e 100644 --- a/core/filesystem/test/package-providers.test.ts +++ b/core/filesystem/test/package-providers.test.ts @@ -248,3 +248,47 @@ describe("pinning through a package's container", () => { } }); }); + +describe("a typed doc that's also folder-shaped", () => { + it("is walked by its type's provider when given one, and as a folder otherwise", async () => { + const { repo, a, accounts, root, child } = await boardFixture(); + // A canvas keeps a folder-compatible `docs` (its scripts) beside its own content. + const canvas = repo.create<any>({ + "@patchwork": { type: "canvas" }, + docs: [{ name: "script", type: "file", url: child.url }], + shapes: { s1: { content: "shape", mimeType: "text/plain" } }, + }); + root.change((d: any) => d.docs.push({ name: "c", type: "canvas", url: canvas.url })); + const canvasProvider: PathProvider = { + kind: "canvas", + matches: (doc: any) => doc?.["@patchwork"]?.type === "canvas", + list: (doc: any) => [ + ...doc.docs.map((e: any) => ({ name: e.name, link: e.url })), + ...Object.keys(doc.shapes).map((id) => ({ name: id, sub: ["shapes", id] })), + ], + lookup(doc, names) { + const found = canvasProvider.list(doc).find((c) => c.name === names[0]); + return found && { child: found, consumed: 1 }; + }, + }; + const providers = async (doc: unknown) => (canvasProvider.matches(doc) ? canvasProvider : undefined); + const shape = await walkPath(repo, livePath(a, ["c", "s1"]), { accounts, providers }); + expect(shape?.sub).toEqual(["shapes", "s1"]); + // Its folder entries resolve the same either way, so pinned paths agree. + const viaPlugin = await walkPath(repo, livePath(a, ["c", "script"]), { accounts, providers }); + const viaFolder = await walkPath(repo, livePath(a, ["c", "script"]), { accounts }); + expect(viaPlugin?.pinned).toBe(viaFolder?.pinned); + // Without the plugin, it's a folder: no shapes. + expect(await walkPath(repo, livePath(a, ["c", "s1"]), { accounts })).toBeUndefined(); + }); + + it("never hands a folder-typed or untyped folder to a package", async () => { + const { repo, a, accounts, notes } = await fixture(); + const greedy: PathProvider = { kind: "greedy", matches: () => true, list: () => [], lookup: () => undefined }; + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes"]), { + accounts, + providers: async () => greedy, + }); + expect(handle?.url).toBe(notes.url); + }); +}); diff --git a/core/plugins/src/path-providers.ts b/core/plugins/src/path-providers.ts index 002755784..0449fecc1 100644 --- a/core/plugins/src/path-providers.ts +++ b/core/plugins/src/path-providers.ts @@ -4,6 +4,7 @@ import { getType, providerFor, + yieldsToPackage, type ExtraProviders, type HasPatchworkMetadata, type PathProvider, @@ -73,9 +74,14 @@ export const packagePathProviders: ExtraProviders = async (doc) => { return undefined; }; -/** Built-ins first, which a plugin can't override, then {@link packagePathProviders}. */ +/** + * Built-ins first, which a plugin can't override, then {@link packagePathProviders}. The one + * exception is a typed document that's also folder-shaped (see `yieldsToPackage`). + */ export async function pathProviderFor(doc: unknown): Promise<PathProvider | undefined> { - return providerFor(doc) ?? (await packagePathProviders(doc)); + const builtIn = providerFor(doc); + if (builtIn && !yieldsToPackage(builtIn, doc)) return builtIn; + return (await packagePathProviders(doc)) ?? builtIn; } /** Every type some path-provider plugin supports, without loading any. */ diff --git a/core/plugins/test/path-providers.test.ts b/core/plugins/test/path-providers.test.ts index 0734602a6..8e8c1e6a5 100644 --- a/core/plugins/test/path-providers.test.ts +++ b/core/plugins/test/path-providers.test.ts @@ -98,3 +98,13 @@ describe("pathProviderTypes", () => { expect(getRegistry<PathProviderDescription>("patchwork:path-provider").all()).toHaveLength(2); }); }); + +describe("pathProviderFor: a typed doc that's also folder-shaped", () => { + it("prefers the plugin for its declared type over the folder built-in", async () => { + const canvas = provider("canvas"); + registerPlugins([plugin("canvas", ["canvas"], async () => canvas)], URL_); + expect(await pathProviderFor({ "@patchwork": { type: "canvas" }, docs: [] })).toBe(canvas); + expect(await pathProviderFor({ "@patchwork": { type: "folder" }, docs: [] })).toBe(folderProvider); + expect(await pathProviderFor({ "@patchwork": { type: "other" }, docs: [] })).toBe(folderProvider); + }); +}); diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index ba9c8507f..869054a52 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -414,6 +414,13 @@ registers one beside its datatype. - **Built-ins come first and can't be overridden.** Account, directory, folder and branches are checked first; package providers after, in registration order. A package can't take over how folders resolve. + - One exception: the folder built-in matches anything with a `docs` array, + so a typed document that keeps one (a tldraw canvas holds its scripts + there) would never reach its own type's provider. Such a document is + walked by a provider declared for its type, if there is one, and as a + folder otherwise. An untyped or `folder`-typed folder is always the + built-in's. A provider for such a type should list the `docs` entries as + the folder provider does, so paths into them pin the same either way. - **It's a pure function of the document.** `list` and `lookup` read only the node they're given, at the heads they're given. That's what lets a file list built through a provider be rebuilt to the same bytes. From 44f9c0b70bf4c7d5d74606335103d3556f47c276 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 11:05:21 +0000 Subject: [PATCH 68/81] packages: a /packages member that can't be found doesn't hold up the others Members are found independently. The first listing waits up to 3s for all of them, then lists what it has; a slow member is listed when it arrives, and one that couldn't be found is asked for again 10s later. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/packages-folder.ts | 57 ++++++++++++++++++-- core/filesystem/test/packages-folder.test.ts | 22 ++++++++ 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts index 989b0524f..d19b1b340 100644 --- a/core/filesystem/src/packages-folder.ts +++ b/core/filesystem/src/packages-folder.ts @@ -35,18 +35,59 @@ const bare = (url: AutomergeUrl) => export function watchPackagesFolder( repo: Repo, account: DocHandle<AccountNamespaceDoc>, - onChange: (entries: PackageEntry[]) => void + onChange: (entries: PackageEntry[]) => void, + /** How long the first listing waits for every member before going without the slow ones. */ + { patienceMs = FIRST_LISTING_PATIENCE_MS }: { patienceMs?: number } = {} ): () => void { let stopped = false; + let patient = true; + const patience = setTimeout(() => { + patient = false; + if (!stopped) changed(); + }, patienceMs); let members: DocHandle<unknown>[] = []; let last: string | undefined; let pending: Promise<void> | undefined; + // Each member is found on its own: one that isn't available yet (or ever) + // mustn't hold up the others. It's listed once it arrives, and one that + // couldn't be found is asked for again a while later. + const found = new Map<AutomergeUrl, DocHandle<unknown>>(); + const finding = new Set<AutomergeUrl>(); + const retries = new Set<ReturnType<typeof setTimeout>>(); + const find = (url: AutomergeUrl) => { + if (found.has(url) || finding.has(url)) return; + finding.add(url); + repo + .find<unknown>(url) + .then( + (handle) => { + found.set(url, handle); + }, + () => { + const retry = setTimeout(() => { + retries.delete(retry); + if (!stopped) changed(); + }, MEMBER_RETRY_MS); + retries.add(retry); + } + ) + .finally(() => { + finding.delete(url); + if (!stopped) changed(); + }); + }; + const list = async () => { const urls = accountBindings(account.doc()).get("/packages") ?? []; - const handles = ( - await Promise.all(urls.map((url) => repo.find<unknown>(url).catch(() => undefined))) - ).filter((handle): handle is DocHandle<unknown> => !!handle); + for (const url of urls) find(url); + // The first listing waits a while for every member, so packages don't + // load in dribs; after that, a member is listed whenever it arrives. + if (last === undefined && patient && urls.some((url) => finding.has(url))) return; + const handles = urls.flatMap((url) => { + const handle = found.get(url); + return handle ? [handle] : []; + }); if (stopped) return; for (const member of members) member.off("change", changed); members = handles; @@ -109,9 +150,17 @@ export function watchPackagesFolder( stopped = true; account.off("change", changed); for (const member of members) member.off("change", changed); + for (const retry of retries) clearTimeout(retry); + clearTimeout(patience); }; } +/** How long the first listing of `/packages` waits for every member folder. */ +const FIRST_LISTING_PATIENCE_MS = 3_000; + +/** How long before a `/packages` member that couldn't be found is asked for again. */ +const MEMBER_RETRY_MS = 10_000; + /** The live lookup path for a package: through the member that holds it. */ export function packageLookupPath(acctHash: string, entry: PackageEntry): string { return `${acctHash}/packages/${entry.memberStep}/${encodeURIComponent(entry.name)}`; diff --git a/core/filesystem/test/packages-folder.test.ts b/core/filesystem/test/packages-folder.test.ts index 15042bf22..0117f0328 100644 --- a/core/filesystem/test/packages-folder.test.ts +++ b/core/filesystem/test/packages-folder.test.ts @@ -165,3 +165,25 @@ describe("trackPackagesFolder: a branches package", () => { expect(watchers.get(BRANCHES)?.size).toBe(0); }); }); + +describe("watchPackagesFolder: a member that can't be found yet", () => { + it("lists the members it has, and the slow one once it arrives", async () => { + const { repo, p1, p2, p3, mine, team, account } = await fixture(); + let release: (() => void) | undefined; + const find = repo.find.bind(repo); + (repo as any).find = (url: AutomergeUrl, ...rest: any[]) => + url === mine.url + ? new Promise((resolve) => (release = () => resolve(find(url, ...rest)))) + : find(url, ...rest); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries), { + patienceMs: 100, + }); + await vi.waitFor(() => expect(seen.length).toBeGreaterThan(0)); + expect(seen.at(-1)!.map((e) => e.url).sort()).toEqual([p3, p2].sort()); + release!(); + await vi.waitFor(() => expect(seen.at(-1)!.map((e) => e.url)).toEqual([p1, p2, p3])); + stop(); + expect(team).toBeDefined(); + }); +}); From 2e233b26bdaa79069e776c41c53fad1cffc23ec9 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 11:06:54 +0000 Subject: [PATCH 69/81] boot: wait at most a second for the move into /packages Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/patchwork/src/index.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index 887634586..0810fddc3 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -219,12 +219,19 @@ async function doSetup(options: PatchworkOptions): Promise<Patchwork> { })) as DocHandle<AccountDoc>; // So /@/<acctHash>/… paths under it resolve, here and in the automerge worker. account.resolve(await registerAccount(accountDocHandle.url)); - await migratePackages(repo, accountDocHandle, { + const migration = migratePackages(repo, accountDocHandle, { // Another device may already have migrated this account. caughtUp: () => caughtUpWithServer(repo, accountDocHandle), }).catch((error) => console.warn("couldn't move module settings into /packages", error) ); + // Waiting to hear from the server can take up to 10s, and offline it never + // says yes, so boot doesn't wait long for it. A migration that finishes + // later binds /packages, and the watcher moves each package over once. + await Promise.race([ + migration, + new Promise((resolve) => setTimeout(resolve, MIGRATION_WAIT_MS)), + ]); await wirePackagesFolder(repo, accountDocHandle, moduleWatcher, (entries) => { packageEntries = entries; }); @@ -380,6 +387,8 @@ function onModuleLoaded(name: string, mod: any): void { // How long boot waits for /packages' first listing before going on without it. const FIRST_LISTING_MS = 5_000; +// How long boot waits for the move from module settings into /packages. +const MIGRATION_WAIT_MS = 1_000; /** * Feed the account's /packages folder to the ModuleWatcher. Waits for its From d629d8a660ceec16536021f492709319d7112d87 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 11:09:05 +0000 Subject: [PATCH 70/81] service worker: a /@/ URL for an account with nothing stored creates no cache The content store is opened lazily: reads and deletes of a cache that doesn't exist find nothing, and only a verified write creates it. Before, every made-up account name in a /@/ link left a cache behind that nothing ever deleted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/bootloader/src/content-store.ts | 22 +++++++++++++++ core/bootloader/src/service-worker.ts | 7 ++--- core/bootloader/test/content-store.test.ts | 28 ++++++++++++++++++- .../bootloader/test/service-worker-harness.ts | 3 ++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/core/bootloader/src/content-store.ts b/core/bootloader/src/content-store.ts index 6c832d82f..826ce9de3 100644 --- a/core/bootloader/src/content-store.ts +++ b/core/bootloader/src/content-store.ts @@ -32,6 +32,28 @@ export function cacheStore(cache: Cache, origin: string): ContentStore { }; } +/** + * {@link cacheStore} in the cache `name`, opened only when something is stored: a read or a + * delete of a cache that doesn't exist finds nothing, and creates nothing. Any `/@/` URL names an + * account, so opening on every request would leave a cache behind for each made-up one. + */ +export function lazyCacheStore(storage: CacheStorage, name: string, origin: string): ContentStore { + let opened: Promise<ContentStore> | undefined; + const open = () => (opened ??= storage.open(name).then((cache) => cacheStore(cache, origin))); + const existing = async () => (opened || (await storage.has(name)) ? open() : undefined); + return { + async get(hash) { + return (await existing())?.get(hash); + }, + async put(hash, bytes) { + await (await open()).put(hash, bytes); + }, + async delete(hash) { + await (await existing())?.delete(hash); + }, + }; +} + /** * Forget everything stored for an account. There's no sign-out flow; a site calls this from its * own. File lists the service worker holds in memory may still name these hashes, which then diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 27d838c8b..5d5730a02 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -17,7 +17,7 @@ import { } from "@inkandswitch/patchwork-filesystem"; import { CONTENT_CACHE_PREFIX, - cacheStore, + lazyCacheStore, contentCacheName, } from "./content-store.js"; import { @@ -570,10 +570,7 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise<Response const store = normalized.acctHash && !normalized.share - ? cacheStore( - await caches.open(contentCacheName(normalized.acctHash)), - self.location.origin - ) + ? lazyCacheStore(caches, contentCacheName(normalized.acctHash), self.location.origin) : undefined; const deps: PinnedDeps = { lists: fileLists, diff --git a/core/bootloader/test/content-store.test.ts b/core/bootloader/test/content-store.test.ts index 2c626244c..644d95f90 100644 --- a/core/bootloader/test/content-store.test.ts +++ b/core/bootloader/test/content-store.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { cacheStore } from "../src/content-store.js"; +import { cacheStore, lazyCacheStore } from "../src/content-store.js"; import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; /** Enough of Cache for the store: put, match and delete by url. */ @@ -44,3 +44,29 @@ describe("cacheStore", () => { expect(await store.get(hash)).toBeUndefined(); }); }); + +describe("lazyCacheStore", () => { + /** Enough of CacheStorage: has, and open (which creates). */ + class FakeStorage { + caches = new Map<string, FakeCache>(); + async has(name: string) { + return this.caches.has(name); + } + async open(name: string) { + if (!this.caches.has(name)) this.caches.set(name, new FakeCache()); + return this.caches.get(name)! as unknown as Cache; + } + } + + it("creates no cache until something is stored", async () => { + const storage = new FakeStorage(); + const store = lazyCacheStore(storage as unknown as CacheStorage, "patchwork-content-x", "https://p.test"); + expect(await store.get("h")).toBeUndefined(); + await store.delete("h"); + expect(storage.caches.size).toBe(0); + const bytes = toBytes("x"); + await store.put("h", bytes); + expect([...storage.caches.keys()]).toEqual(["patchwork-content-x"]); + expect(await store.get("h")).toEqual(bytes); + }); +}); diff --git a/core/bootloader/test/service-worker-harness.ts b/core/bootloader/test/service-worker-harness.ts index 6429637a8..d5909f3ef 100644 --- a/core/bootloader/test/service-worker-harness.ts +++ b/core/bootloader/test/service-worker-harness.ts @@ -30,6 +30,9 @@ export class FakeCaches { if (!cache) this.named.set(name, (cache = new FakeCache())); return cache as unknown as Cache; } + async has(name: string) { + return this.named.has(name); + } async delete(name: string) { return this.named.delete(name); } From fd233f64f21c03eb05522ea821d1e76c5eb2863d Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Thu, 29 Oct 2026 11:10:10 +0000 Subject: [PATCH 71/81] changeset: review fixes to paths, packages and the content store Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/paths-review-fixes.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .changeset/paths-review-fixes.md diff --git a/.changeset/paths-review-fixes.md b/.changeset/paths-review-fixes.md new file mode 100644 index 000000000..eda870e15 --- /dev/null +++ b/.changeset/paths-review-fixes.md @@ -0,0 +1,22 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +--- + +Fixes to paths, packages and the content store from review. + +- File lists: a file named `__proto__` is listed like any other. A tail named like an `Object.prototype` member (`constructor`, `toString`) that the list doesn't name answers 404. +- Pinning through a package's container (a path provider's doc) lists that container, so the URL serves. A doc named by a doc step is walked with package providers; one a path ends at by name is still a container or a file by the built-ins. +- A value held by a doc below a package's root (a nested directory doc) is listed with that doc's heads, and read back at them. +- `/packages`: + - a package that's a branches doc reloads when its default branch's code changes, and follows the branch when it moves; + - a member folder that can't be found doesn't hold up the others: the first listing waits up to 3s, then lists what it has, and the rest arrive as they're found; + - migration records the entries it moved in the module-settings doc (`movedToPackages`). This build loads those only through `/packages`, so removing a package there removes it. Builds that predate `/packages` still read `modules` as before; + - migration never gives two packages one name, and a `/packages` binding that syncs in mid-migration wins; + - boot waits at most a second for migration. +- Routing: + - a `doc=`/`path=` check that finishes after the hash moved on leaves the new hash alone; + - opening another part of the same document drops the path; + - a path-only hash that doesn't resolve is walked again when path providers register, so a cold boot opens a path through a package's datatype. +- The service worker opens an account's content store only to write to it, so a `/@/` link naming an unknown account leaves no cache behind. From dec4afc64126645b9b7c91a321117ed7e3f01dba Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 02:25:19 +0000 Subject: [PATCH 72/81] bootloader: no URLs in console format strings; error responses without stacks From CodeQL on #467. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/codeql-sw-logging.md | 5 +++++ core/bootloader/src/automerge-protocol-handler.ts | 2 +- core/bootloader/src/resolver.ts | 4 +++- core/bootloader/src/service-worker.ts | 15 ++++++++------- 4 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 .changeset/codeql-sw-logging.md diff --git a/.changeset/codeql-sw-logging.md b/.changeset/codeql-sw-logging.md new file mode 100644 index 000000000..8027be357 --- /dev/null +++ b/.changeset/codeql-sw-logging.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-bootloader": patch +--- + +The service worker and resolvers no longer put request URLs or paths into a console format string, where a `%c` or `%s` in a URL was interpreted. Error responses carry the error's message, not its stack; the stack goes to the console. diff --git a/core/bootloader/src/automerge-protocol-handler.ts b/core/bootloader/src/automerge-protocol-handler.ts index f11c25df2..e938f29e7 100644 --- a/core/bootloader/src/automerge-protocol-handler.ts +++ b/core/bootloader/src/automerge-protocol-handler.ts @@ -64,7 +64,7 @@ async function registerAccount({ url }: RegisterAccountMessage) { const { documentId } = parseAutomergeUrl(url as AutomergeUrl); accounts.set(await idhash(documentId), url as AutomergeUrl); } catch (error) { - console.warn(`ignoring a register-account for ${url}`, error); + console.warn("ignoring a register-account for %s", url, error); } } diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 4b0e3f542..553eeb64e 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -241,7 +241,9 @@ async function resolveWalkPath( try { parsePath(path); } catch (error) { - return new Response(String(error), { status: 400 }); + return new Response(error instanceof Error ? error.message : "malformed path", { + status: 400, + }); } const missing = new Set<string>(); const options = { accounts, providers, find: waitingFind(repo, signal, missing) }; diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 5d5730a02..4ab353f9e 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -504,7 +504,7 @@ async function serveHandoff( const copy = response.clone(); fetchEvent.waitUntil( cache.put(fetchEvent.request, copy).catch((error) => { - console.warn(`error caching ${handoffURL} in ${cachename}`, error); + console.warn("error caching %s in %s", handoffURL, cachename, error); }) ); } @@ -584,7 +584,7 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise<Response work.catch((error) => { // Always loud, as for the page cache: a QuotaExceededError here // is the first sign the origin is under storage pressure. - console.warn(`error keeping /@/${path}`, error); + console.warn("error keeping /@/%s", path, error); }) ), }; @@ -599,11 +599,12 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise<Response lifecycle("warn", `failing /@/${path}: ${(error as Error).message}`); throw error; } - console.error(`service worker error serving /@/${path}`, error); - return new Response( - error instanceof Error ? `${error.message}\n\n${error.stack}` : String(error), - { status: 556, headers: { "content-type": "text/plain" } } - ); + console.error("service worker error serving /@/%s", path, error); + // The stack is in the console above; the page gets the message only. + return new Response(error instanceof Error ? error.message : "service worker error", { + status: 556, + headers: { "content-type": "text/plain" }, + }); } } From a9ee6391517883495190248ccba1a9d23bc2bf41 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 04:05:05 +0000 Subject: [PATCH 73/81] plan: the Packages tool installs into /packages Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...30-packages-tool-installs-into-packages.md | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md diff --git a/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md b/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md new file mode 100644 index 000000000..872773289 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md @@ -0,0 +1,97 @@ +# The Packages tool installs into /packages — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** On a host that loads `/packages`, the Packages tool installs a package by linking it into the account's own packages folder, and uninstalls one by removing that link. Module settings stops growing. Once it holds nothing `/packages` doesn't cover, the Packages button retires itself; the rule for that is already built. + +**Architecture:** All in base, in `packages/` (the tool). The tool keeps its handle, the module-settings doc. It finds the account through `globalThis.patchwork.account`, as the sidebar does, and the repo through its element. It writes to the folder bound first at `/packages`. The host's ModuleWatcher already loads, unloads and dedupes whatever that folder lists, so nothing changes in system except the spec. + +**Tech stack:** Solid, vitest with happy-dom, base's standalone-package rules (no build-time imports between tools; `@inkandswitch/patchwork-*` are host externals, feature-detected). + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 4 ("module settings still read; the Packages UI writes it" becomes "the Packages UI writes `/packages`"). + +## Decisions + +1. **Where an install goes.** + - An unpinned automerge URL goes into **your** packages folder: the first binding at `/packages`. + - Pinned URLs (with heads) and http URLs still go into module settings. A folder link floats to the latest version, and a folder can't hold an http module. + - On a host that doesn't load `/packages`, or an account without it bound, everything goes into module settings, as now. +2. **What a link looks like.** `{ name, type, url }`: + - `url` is the bare automerge URL. + - `type` is the package doc's `@patchwork.type`, or `folder`. + - `name` is its package.json `name`, else its title, else its document id. The same allocator as migration keeps names unique (`alpha`, `alpha (2)`, `alpha (3)`), since only the first of a name in a folder is reachable. +3. **"Installed" means in your packages folder, or listed in module settings.** + - A package linked from another `/packages` member (a team folder) is shown as installed, from that folder's title, with no Remove: removing the link there would uninstall it for everyone who shares the folder. + - Module-settings entries recorded in `movedToPackages` count as installed only if the folder still links them, matching what the host loads. +4. **Uninstall:** + - removes the package's link from your folder (every link to that document, in case of duplicates); + - removes an unmoved, unpinned module-settings entry for the same document; + - pinned and http entries are removed from module settings as now; + - `deactivateModule` still runs, so the plugins go away immediately. +5. **No move button for leftovers.** Bare entries an older host adds to module settings later still load, through module settings, and stay removable there. Moving them over can come later if it's wanted. + +## Review focus + +1. **Two tabs installing the same package at once.** Both push a link; the second is a duplicate, and only the first of a name is reachable. The install checks for the document inside the `change` callback, so concurrent installs in one tab can't double up. Across devices it's a merge; the host dedupes by document. Test: installing an already-linked document is a no-op. +2. **A package doc that can't be read yet** (unsynced). It's linked under its document id rather than failing the install. Test. +3. **Your folder not loaded yet when the tool opens.** The install waits for it rather than falling back to module settings, which would put it in the wrong place. Test with a slow `find`. +4. **An older host.** No `watchPackagesFolder` on the host's filesystem: every install goes to module settings. Test. +5. **A link from someone else's folder.** It's never removed by Uninstall. Test. + +--- + +## Tasks + +### Task 1: `packages/src/packages-folder.ts` — where installs go, and the folder edits + +**Files:** create `packages/src/packages-folder.ts` and `packages/src/packages-folder.test.ts`. + +**Produces:** +```ts +export type AccountLike = { doc(): { namespace?: Record<string, { url?: unknown }[]> } | undefined }; +/** Your packages folder (the first /packages binding), on a host that loads /packages; else undefined. */ +export function installFolderUrl(account: AccountLike | undefined, hostLoadsPackages: boolean): AutomergeUrl | undefined; +/** Whether `url` goes into the packages folder: an unpinned automerge url. */ +export function installsIntoFolder(url: string): boolean; +/** A name not in `taken`: `name`, `name (2)`, … */ +export function uniqueName(name: string, taken: Iterable<string>): string; +/** Link `url` into the folder under a unique name; a no-op if it already links that document. */ +export function linkPackage(folder: DocHandle<FolderDoc>, link: { url: AutomergeUrl; name: string; type: string }): void; +/** Remove every link to `documentId` from the folder. True if any went. */ +export function unlinkPackage(folder: DocHandle<FolderDoc>, documentId: string): boolean; +``` + +- [ ] Tests first, covering: + - `installFolderUrl`: the first binding; undefined without `/packages` or on an older host; + - `installsIntoFolder`: true for a bare automerge url; false with heads, and false for http; + - `uniqueName("alpha", ["alpha", "alpha (2)"])` → `"alpha (3)"`; + - `linkPackage`: adds `{name, type, url}`; a second call for the same document adds nothing; + - `unlinkPackage`: removes duplicates too, and returns false when there's nothing to remove. +- [ ] Run `npx vitest run src/packages-folder.test.ts` in `packages/`. It should fail first, then pass once implemented. +- [ ] Commit. + +### Task 2: the tool installs and uninstalls through it + +**Files:** modify `packages/src/packages.tsx` (install, uninstall), `packages/src/use-package-data.ts` (installed keys, and where each package is from), `packages/src/origin.ts` (an origin hint for "in your packages folder" vs "from <folder>"); tests in `packages/src/packages.test.tsx`. + +**Consumes:** Task 1. + +- [ ] Tests first, with a fake account and fake folder handles on a fake repo: + - Installing a bare automerge URL links it into your folder, named from package.json, and leaves module settings alone. + - Installing a pinned or http URL writes module settings. + - On an older host (the host check returns false), everything goes to module settings. + - Uninstalling a package in your folder removes its link and calls `deactivateModule`. + - A package linked only from a second binding shows as installed, with no Remove button. + - A package whose doc can't be read is linked under its document id. + - Installing while your folder is still loading waits for it. +- [ ] Implement: + - `use-package-data` reads the account's `/packages` members (repo `find`, plus their `change` events) and merges their document ids into `installedKeys`, tagging each with its member. + - `install`/`uninstall` follow the decisions above. + - The host check is `typeof filesystem.watchPackagesFolder === "function"`. +- [ ] Run every `packages/` test, `tsc -p` and the package's build. They should pass, with no new tsc errors. +- [ ] Commit. + +### Task 3: spec and push + +- [ ] System spec section 4: the Packages UI installs into `/packages`; module settings keeps pinned and http entries and what older hosts add. Commit to `paths-design`. +- [ ] Push base `paths`. From 77cdf379b6e2d9d3702f668195ed5f276de49d31 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 04:12:34 +0000 Subject: [PATCH 74/81] spec: the Packages tool installs into /packages Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...28-paths-namespace-content-cache-design.md | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 869054a52..9455f3cb5 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -260,10 +260,17 @@ Builds on the `open-document-path` branch (48f4ab98): - `PackageTracker.findLatest` keeps asking the service worker, in the new grammar. - The site's `system` HTTP manifest stays a separate source. -- So does the module-settings doc, for now. The Packages UI (an installed - tool, not in this repo) still installs into it, so the watcher loads both - it and `/packages`, deduplicated by document id. A package in both is - imported through `/packages`. +- So does the module-settings doc, for now. The watcher loads both it and + `/packages`, deduplicated by document id; a package in both is imported + through `/packages`. Entries migration moved (`movedToPackages`) load only + through `/packages`, so removing a package there removes it. +- The Packages tool (base `packages/`), on a host that loads `/packages`, + installs an automerge package by linking it into your packages folder (the + first `/packages` binding) and uninstalls by removing the link. Pinned and + http modules still go into module settings, since a folder link follows + the latest version and a folder can't hold an http module. A package + linked from a folder you share can't be uninstalled from the tool: + removing it there would uninstall it for everyone who shares it. - A link under `/packages` that isn't a package (no `package.json`) is warned about and skipped. @@ -274,10 +281,11 @@ Builds on the `open-document-path` branch (48f4ab98): - Migration, once at boot: if the account has `moduleSettingsUrl` and no `/packages` binding, create a folder, add a link for each automerge entry (named by `package.json` `name`, falling back to the doc title), and bind - it at `/packages`. The module-settings doc is left untouched. -- Packages installed through the old Packages UI after the migration still - land in the module-settings doc, which this build keeps reading, but not - in `/packages`. Builds older than this one never see `/packages`. + it at `/packages`. The module-settings doc keeps its entries, and records + which it moved in `movedToPackages`. +- Packages installed by an older host after the migration still land in the + module-settings doc, which this build keeps reading, but not in + `/packages`. Builds older than this one never see `/packages`. - `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding. From 1abe1b0af6f83db58db461dc28c782bb705a3f6c Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 04:19:28 +0000 Subject: [PATCH 75/81] ModuleWatcher: packageFor maps a loaded /@/ import url back to its package doc Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/paths-review-fixes.md | 1 + core/filesystem/src/module-watcher.ts | 12 ++++++++++++ core/filesystem/test/module-watcher.test.ts | 14 ++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/.changeset/paths-review-fixes.md b/.changeset/paths-review-fixes.md index eda870e15..5f42081cf 100644 --- a/.changeset/paths-review-fixes.md +++ b/.changeset/paths-review-fixes.md @@ -20,3 +20,4 @@ Fixes to paths, packages and the content store from review. - opening another part of the same document drops the path; - a path-only hash that doesn't resolve is walked again when path providers register, so a cold boot opens a path through a package's datatype. - The service worker opens an account's content store only to write to it, so a `/@/` link naming an unknown account leaves no cache behind. +- `ModuleWatcher.packageFor(importUrl)` says which package doc an import url was loaded for. A `/packages` package's plugins are registered under its pinned `/@/` url; this maps it back, so a tool like Packages can tell what's installed. diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 8a250c971..973fc429f 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -325,6 +325,18 @@ export class ModuleWatcher extends EventTarget { return this.#status.get(documentId); } + /** + * The package an import url was loaded for, if this watcher loaded it. A + * `/packages` package is imported from its pinned `/@/` url, which is what + * its plugins are registered under; this maps that back to the package doc. + */ + packageFor(importUrl: string): AutomergeUrl | undefined { + for (const [moduleKey, version] of this.#lastAnnounced) { + if (version === importUrl && isValidAutomergeUrl(moduleKey)) return moduleKey; + } + return undefined; + } + #setStatus(moduleKey: string, status: PackageStatus) { if (!isValidAutomergeUrl(moduleKey)) return; this.#status.set(parseAutomergeUrl(moduleKey).documentId, status); diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 0f75e98a4..ce3bc4c45 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -820,6 +820,20 @@ describe("ModuleWatcher /packages folder", () => { await repo.shutdown().catch(() => {}); }); + it("says which package a loaded import url belongs to", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const announced: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => announced.push(name), undefined, undefined, source.tracker); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(announced).toHaveLength(2); + const packages = announced.map((url) => watcher.packageFor(url)).sort(); + expect(packages).toEqual(source.docs.map((d) => d.url).sort()); + expect(watcher.packageFor("https://elsewhere.test/x.js")).toBeUndefined(); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + it("loads a package in both module settings and /packages once", async () => { const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); const source = folderSource(repo); From 53a53932a289887c08786f6639104e2a1b3a1573 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 04:21:59 +0000 Subject: [PATCH 76/81] spec: where the Packages tool installs, and how it tells what's installed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- ...6-10-28-paths-namespace-content-cache-design.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index 9455f3cb5..eb12c97f9 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -264,13 +264,17 @@ Builds on the `open-document-path` branch (48f4ab98): `/packages`, deduplicated by document id; a package in both is imported through `/packages`. Entries migration moved (`movedToPackages`) load only through `/packages`, so removing a package there removes it. -- The Packages tool (base `packages/`), on a host that loads `/packages`, - installs an automerge package by linking it into your packages folder (the - first `/packages` binding) and uninstalls by removing the link. Pinned and +- The Packages tool (base `packages/`), on a host that loads `/packages` and + viewing the account's own module-settings doc, installs an automerge + package by linking it into your packages folder (the first `/packages` + binding) and uninstalls by removing the link. It tells which package a + plugin came from with `ModuleWatcher.packageFor`, since a `/packages` + package's plugins are registered under its pinned `/@/` url. Pinned and http modules still go into module settings, since a folder link follows the latest version and a folder can't hold an http module. A package - linked from a folder you share can't be uninstalled from the tool: - removing it there would uninstall it for everyone who shares it. + linked from a folder you share (even if yours links it too) can't be + uninstalled from the tool: removing it there would uninstall it for + everyone who shares it, and leaving it there keeps it loaded. - A link under `/packages` that isn't a package (no `package.json`) is warned about and skipped. From db4e1823ffb996435a4d3ef8f7109a56b8d68ec1 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 14:47:56 +0000 Subject: [PATCH 77/81] elements: part of a document opens in its document's tool A sub-document url (a tldraw shape from the sidebar, say) whose value has no @patchwork type picked no tool, and patchwork-view said the document was missing its metadata. The tool is now chosen from the document it's part of, and still gets the sub-handle. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../subdoc-opens-in-its-documents-tool.md | 5 +++ core/elements/src/legacy-impl.ts | 35 ++++++++++++++++-- core/elements/test/patchwork-view.test.ts | 37 +++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 .changeset/subdoc-opens-in-its-documents-tool.md diff --git a/.changeset/subdoc-opens-in-its-documents-tool.md b/.changeset/subdoc-opens-in-its-documents-tool.md new file mode 100644 index 000000000..da40c4494 --- /dev/null +++ b/.changeset/subdoc-opens-in-its-documents-tool.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-elements": patch +--- + +`<patchwork-view>` opens part of a document (`automerge:<id>/<path>`, a sub-handle) that has no `@patchwork` type of its own in the tool for the document it's part of, and hands that tool the sub-handle. Before, such a part, like a tldraw shape opened from the sidebar, failed with "missing @patchwork metadata". diff --git a/core/elements/src/legacy-impl.ts b/core/elements/src/legacy-impl.ts index a8f7030cd..12c9e2c7a 100644 --- a/core/elements/src/legacy-impl.ts +++ b/core/elements/src/legacy-impl.ts @@ -1,5 +1,6 @@ import { parseAutomergeUrl, + stringifyAutomergeUrl, type AutomergeUrl, type DocHandle, type DocHandleChangePayload, @@ -237,6 +238,16 @@ export class LegacyImpl { #docUrl: AutomergeUrl | null = null; #toolId: string | null = null; #handle: DocHandle<HasPatchworkMetadata> | null = null; + /** + * The doc whose type picks the tool: the handle's own, or, for part of a + * document (`automerge:<id>/<path>`) with no type of its own, the document + * it's part of. The tool still gets the part. + */ + #typeHandle: DocHandle<HasPatchworkMetadata> | null = null; + // Called only once there's a handle, as `this.#handle.doc()` was. + #typeDoc(): HasPatchworkMetadata { + return (this.#typeHandle ?? this.#handle)!.doc(); + } #tool: LoadedTool | null = null; #state: State = State.none; #requestedToolImports = new Set<string>(); @@ -380,7 +391,7 @@ export class LegacyImpl { // only change the outcome when no fallback is known yet or the new // tool supports this doc's type. if (this.#handle && this.#couldChangeFallback(addedTool)) { - this.#fallbackId = getFallbackTool(this.#handle.doc())?.id; + this.#fallbackId = getFallbackTool(this.#typeDoc())?.id; } const isFallbackTool = toolId == this.#fallbackId; if (isChosenTool || isFallbackTool) { @@ -443,7 +454,22 @@ export class LegacyImpl { if (epoch !== this.#initEpoch) return; + // Part of a document with no type of its own opens in its document's tool. + let typeHandle: DocHandle<HasPatchworkMetadata> | null = null; + const { documentId, heads, segments } = parseAutomergeUrl(this.#docUrl! as AutomergeUrl); + if (segments?.length && !getType(handle.doc())) { + try { + typeHandle = await repo.find<HasPatchworkMetadata>( + stringifyAutomergeUrl({ documentId, heads }) + ); + } catch { + // its document can't be had: pick from the part alone + } + if (epoch !== this.#initEpoch) return; + } + this.#handle = handle; + this.#typeHandle = typeHandle; this.#handle.on("change", this.#onDocChange); this.#teardowns.add(() => this.#handle!.off("change", this.#onDocChange)); @@ -458,7 +484,7 @@ export class LegacyImpl { const supported = added.supportedDatatypes; if (!supported) return false; if (supported === "*" || supported.includes("*")) return true; - const type = this.#handle ? getType(this.#handle.doc()) : undefined; + const type = this.#handle ? getType(this.#typeDoc()) : undefined; return type != null && supported.includes(type); } @@ -500,6 +526,7 @@ export class LegacyImpl { this.#keyhiveRetrySetup = false; this.#unableNoAccess = false; this.#handle = null; + this.#typeHandle = null; this.#tool = null; this.#requestedToolImports.clear(); this.#content.textContent = ""; @@ -606,7 +633,7 @@ export class LegacyImpl { this.#resetDisplay(); - const doc = this.#handle.doc(); + const doc = this.#typeDoc(); const fallbackTool = getFallbackTool(doc); this.#fallbackId = fallbackTool?.id; const fallingBack = !this.#toolId; @@ -909,7 +936,7 @@ export class LegacyImpl { */ #suggestedImport(): { url: string; loading: boolean } | null { if (!this.#docUrl || !this.#handle) return null; - const url = getSuggestedImportUrl(this.#handle.doc()); + const url = getSuggestedImportUrl(this.#typeDoc()); if (!url) return null; return { url, loading: this.#requestedToolImports.has(url) }; } diff --git a/core/elements/test/patchwork-view.test.ts b/core/elements/test/patchwork-view.test.ts index 6c246608c..749485d70 100644 --- a/core/elements/test/patchwork-view.test.ts +++ b/core/elements/test/patchwork-view.test.ts @@ -170,6 +170,43 @@ describe("patchwork-view (legacy mode)", () => { expect(toast?.querySelector("button.pw-error__load")).toBeTruthy(); }); + it("opens part of a document with no type of its own in its document's tool, handing it the part", async () => { + const id = `test-canvas-tool-${++seq}`; + const handed: string[] = []; + const render = (handle: { url: string }) => { + handed.push(handle.url); + return () => {}; + }; + toolRegistry.register( + { + id, + type: "patchwork:tool", + name: id, + supportedDatatypes: ["test-canvas"], + load: async () => render, + module: render, + } as never, + `test://${id}` + ); + registeredTools.push(id); + + const canvas = repo.create({ + "@patchwork": { type: "test-canvas" }, + store: { "shape:a": { id: "shape:a", typeName: "shape" } }, + }); + const part = canvas.sub("store", "shape:a").url; + const provider = document.createElement("repo-provider"); + const view = document.createElement("patchwork-view-legacy"); + view.setAttribute("doc-url", part); + provider.append(view); + document.body.append(provider); + await settle(); + await settle(); + + expect(view.textContent).not.toContain("missing @patchwork metadata"); + expect(handed).toEqual([part]); + }); + it("still mounts the wildcard stopgap when the doc suggests nothing", async () => { const raw = registerWildcardTool(); From 656cd3b19faffff6e64271fd529f084595d79c15 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 23:06:56 +0000 Subject: [PATCH 78/81] ModuleWatcher: discover /packages packages in the module-loader worker The pinned /@/ URL a /packages lookup answers with went straight to importPackageFromHttpUrl on the main thread. It now goes through the importAutomergePackage hook like any package doc's version, so the worker reads its descriptors and load() re-imports the same pinned URL. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/packages-discovered-off-thread.md | 8 ++++ core/bootloader/src/module-loader-worker.ts | 13 ++++--- core/bootloader/src/module-loader.ts | 13 +++---- core/filesystem/src/module-watcher.ts | 21 ++++++---- core/filesystem/src/packages.ts | 30 ++++++++------ core/filesystem/test/module-watcher.test.ts | 41 ++++++++++++++++++++ 6 files changed, 94 insertions(+), 32 deletions(-) create mode 100644 .changeset/packages-discovered-off-thread.md diff --git a/.changeset/packages-discovered-off-thread.md b/.changeset/packages-discovered-off-thread.md new file mode 100644 index 000000000..b64407e27 --- /dev/null +++ b/.changeset/packages-discovered-off-thread.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +--- + +`/packages` packages are discovered off the main thread, like module-settings packages. The pinned `/@/` URL a `/packages` lookup answers with now goes through the ModuleWatcher's `importAutomergePackage` hook, so the browser's module-loader worker reads its plugin descriptors, and a plugin's `load()` re-imports the same pinned URL on the main thread. Before, those packages' entry points were imported on the main thread to register them. + +`importPluginFromPackage(url, type, id)` takes a folder doc pinned to heads or a pinned `/@/` URL. `importPluginFromFolderDocUrl` is kept as a deprecated alias for it. diff --git a/core/bootloader/src/module-loader-worker.ts b/core/bootloader/src/module-loader-worker.ts index f8607805c..114af505c 100644 --- a/core/bootloader/src/module-loader-worker.ts +++ b/core/bootloader/src/module-loader-worker.ts @@ -6,18 +6,19 @@ // entry point off the main thread purely to read its exported `plugins` array, // strips the non-cloneable `load()` / `import` machinery, and posts the plain // descriptors back. The main thread re-imports the package (at the same heads) -// only when a plugin is actually loaded — see `importPluginFromFolderDocUrl`. +// only when a plugin is actually loaded — see `importPluginFromPackage`. // // Created with type:"module"; its dynamic `import()` of `/<automergeUrl>/…` -// entry points is served by the service worker that controls this worker. +// entry points, and of the pinned `/@/` URLs `/packages` packages load from, +// is served by the service worker that controls this worker. -import { importPackageFromFolderDocUrl } from "@inkandswitch/patchwork-filesystem"; -import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { importPackage } from "@inkandswitch/patchwork-filesystem"; type DiscoverRequest = { type: "discover"; id: number; - url: AutomergeUrl; + /** A folder doc pinned to heads, or a pinned `/@/` URL. */ + url: string; }; // Keep only the structured-cloneable description fields. `load` is a closure @@ -54,7 +55,7 @@ self.addEventListener("message", (event: MessageEvent) => { if (!isDiscoverRequest(data)) return; const { id, url } = data; - importPackageFromFolderDocUrl(url) + importPackage(url) .then((mod) => { const plugins: any[] = Array.isArray(mod?.plugins) ? mod.plugins : []; const descriptors = plugins.map(toDescriptor); diff --git a/core/bootloader/src/module-loader.ts b/core/bootloader/src/module-loader.ts index ae717ce52..166257e9d 100644 --- a/core/bootloader/src/module-loader.ts +++ b/core/bootloader/src/module-loader.ts @@ -5,10 +5,10 @@ // import the package entry point and report which plugins it exports, then // returns the same `{ plugins }` shape the watcher already feeds to // `registerPlugins` — except each plugin's `load()` re-imports the package -// (pinned to the same heads) on this thread and runs the real plugin loader. +// (pinned to the same heads, or the same pinned `/@/` URL) on this thread and +// runs the real plugin loader. -import { importPluginFromFolderDocUrl } from "@inkandswitch/patchwork-filesystem"; -import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { importPluginFromPackage } from "@inkandswitch/patchwork-filesystem"; import { MODULE_LOADER_WORKER } from "./emitted.js"; type Descriptor = Record<string, unknown> & { id?: string; type?: string }; @@ -51,7 +51,7 @@ function getWorker(): Worker { return worker; } -function discoverDescriptors(urlAtHeads: AutomergeUrl): Promise<Descriptor[]> { +function discoverDescriptors(urlAtHeads: string): Promise<Descriptor[]> { const id = nextRequestId++; return new Promise<Descriptor[]>((resolve, reject) => { pending.set(id, { resolve, reject }); @@ -60,9 +60,8 @@ function discoverDescriptors(urlAtHeads: AutomergeUrl): Promise<Descriptor[]> { } export async function importAutomergePackageViaWorker( - urlAtHeads: string + url: string ): Promise<{ plugins: Descriptor[] }> { - const url = urlAtHeads as AutomergeUrl; const descriptors = await discoverDescriptors(url); const plugins = descriptors.map((descriptor) => { const { id, type } = descriptor; @@ -71,7 +70,7 @@ export async function importAutomergePackageViaWorker( if (typeof id !== "string" || typeof type !== "string") return descriptor; return { ...descriptor, - load: () => importPluginFromFolderDocUrl(url, type, id), + load: () => importPluginFromPackage(url, type, id), }; }); return { plugins }; diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 973fc429f..e60aa63d0 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -7,7 +7,7 @@ import { stringifyAutomergeUrl, } from "@automerge/automerge-repo/slim"; import { - importPackageFromFolderDocUrl, + importPackage, importPackageFromHttpUrl, } from "./packages.js"; import { type HasPatchworkMetadata } from "./metadata.js"; @@ -166,9 +166,10 @@ export class ModuleWatcher extends EventTarget { onLoad: (name: string, mod: any) => void; onUnload?: (name: string) => void; /** - * How an Automerge package doc (pinned to heads) is turned into the - * `{ plugins }` shape `onLoad` consumes. Defaults to importing the package - * entry point directly on this thread. The browser bootloader overrides this + * How an Automerge package doc (pinned to heads, or the pinned `/@/` URL a + * `/packages` lookup answers with) is turned into the `{ plugins }` shape + * `onLoad` consumes. Defaults to importing the package entry point directly + * on this thread. The browser bootloader overrides this * to run the entry point in a worker for descriptor discovery and rebuild the * `{ plugins }` shape with main-thread `load()` functions. */ @@ -187,7 +188,7 @@ export class ModuleWatcher extends EventTarget { callback: (name: string, mod: any) => void, onUnload?: (name: string) => void, importAutomergePackage: (urlAtHeads: string) => Promise<any> = (url) => - importPackageFromFolderDocUrl(url as AutomergeUrl), + importPackage(url), tracker: PackageTracker = trackPackagesInRepo(repo) ) { super(); @@ -344,7 +345,8 @@ export class ModuleWatcher extends EventTarget { } private async importPackageSafe( - importName: string + importName: string, + moduleKey: string ): Promise<{ mod: any } | { error: unknown }> { try { if (isValidAutomergeUrl(importName)) { @@ -356,6 +358,11 @@ export class ModuleWatcher extends EventTarget { ), }; } + // The tracker's answer for a package doc: a /packages package's pinned + // /@/ URL. It's still that doc's package, so it's imported the same way. + if (isValidAutomergeUrl(moduleKey)) { + return { mod: await this.importAutomergePackage(importName) }; + } return { mod: await importPackageFromHttpUrl(importName) }; } catch (error) { console.error( @@ -404,7 +411,7 @@ export class ModuleWatcher extends EventTarget { const attempt = async (attemptIndex: number): Promise<void> => { if (attemptIndex === 0) this.#setStatus(moduleKey, { state: "loading" }); - const result = await this.importPackageSafe(importUrl); + const result = await this.importPackageSafe(importUrl, moduleKey); // A newer announce for this module started while we were importing or // backing off — its result wins; announcing ours now could regress the // registry to an older version. diff --git a/core/filesystem/src/packages.ts b/core/filesystem/src/packages.ts index f79f5511c..01a5ac5ac 100644 --- a/core/filesystem/src/packages.ts +++ b/core/filesystem/src/packages.ts @@ -165,9 +165,10 @@ async function httpEntryPointUrl( } /** - * Import the entry point of a package (at `folderDocUrl`, which should be - * pinned to heads) and return the loaded implementation of a single one of the - * plugins it exports, selected by its `pluginType` and `pluginId`. + * Import the entry point of a package (at `packageUrl`: a folder doc pinned + * to heads, or a pinned `/@/` URL) and return the loaded implementation of a + * single one of the plugins it exports, selected by its `pluginType` and + * `pluginId`. * * A plugin `id` is only unique *within* a plugin type (there is a separate * registry per type), so a package can export e.g. a `patchwork:datatype` and @@ -181,25 +182,21 @@ async function httpEntryPointUrl( * with the live descriptor, so the registered description and the loaded * implementation come from the same pinned version. */ -export async function importPluginFromFolderDocUrl( - folderDocUrl: AutomergeUrl, +export async function importPluginFromPackage( + packageUrl: string, pluginType: string, pluginId: string, subpath: string = ".", conditions: string[] = defaultImportConditions ) { - const mod = await importPackageFromFolderDocUrl( - folderDocUrl, - subpath, - conditions - ); + const mod = await importPackage(packageUrl, subpath, conditions); const plugins: any[] = Array.isArray(mod?.plugins) ? mod.plugins : []; const plugin = plugins.find( (p) => p?.type === pluginType && p?.id === pluginId ); if (!plugin) { throw new Error( - `No plugin "${pluginType}:${pluginId}" exported by the package at ${folderDocUrl}` + `No plugin "${pluginType}:${pluginId}" exported by the package at ${packageUrl}` ); } if (typeof plugin.load === "function") { @@ -209,10 +206,19 @@ export async function importPluginFromFolderDocUrl( return import(/* @vite-ignore */ plugin.import); } throw new Error( - `Plugin "${pluginType}:${pluginId}" at ${folderDocUrl} has no load() function or import URL` + `Plugin "${pluginType}:${pluginId}" at ${packageUrl} has no load() function or import URL` ); } +/** @deprecated Use {@link importPluginFromPackage}, which also takes a pinned `/@/` URL. */ +export const importPluginFromFolderDocUrl: ( + folderDocUrl: AutomergeUrl, + pluginType: string, + pluginId: string, + subpath?: string, + conditions?: string[] +) => Promise<any> = importPluginFromPackage; + async function packageJsonContentsFromFolderDocUrl( folderDocUrl: AutomergeUrl ): Promise<Record<string, any>> { diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index ce3bc4c45..10171ef91 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -820,6 +820,47 @@ describe("ModuleWatcher /packages folder", () => { await repo.shutdown().catch(() => {}); }); + it("imports a package's pinned url, first and on reload, through the package import", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create<any>({ "@patchwork": { type: "directory" } }); + let version = 0; + let moved: (seen?: string) => void = () => {}; + const tracker: PackageTracker = { + findLatest: async () => `https://patchwork.test/@/acct/packages/p@v${version}~list` as any, + watch: (_url, onChange) => { + moved = onChange; + return () => {}; + }, + }; + const imported: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + () => {}, + undefined, + async (url) => { + imported.push(url); + return { plugins: [] }; + }, + tracker + ); + await watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + expect(imported).toEqual(["https://patchwork.test/@/acct/packages/p@v0~list"]); + version = 1; + moved(); + await pause(400); + expect(imported).toEqual([ + "https://patchwork.test/@/acct/packages/p@v0~list", + "https://patchwork.test/@/acct/packages/p@v1~list", + ]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + it("says which package a loaded import url belongs to", async () => { const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); const source = folderSource(repo); From 701ada8bbfd1b3d3fdf10fe4bf8d7d3d8af84c4b Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 23:12:47 +0000 Subject: [PATCH 79/81] file lists: record nested folders' heads, so renamed files stay readable A file in a package's nested folder is reached through folder docs linked without heads. After a rename, walking back to it at their current heads found nothing, so its pinned URL failed when the bytes weren't stored. Lists now record those folders' heads as via; the service worker passes them to the resolver, and readPath reads each folder at them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/nested-renames-keep-pinned-urls.md | 8 +++ core/bootloader/src/pinned.ts | 13 ++-- core/bootloader/src/resolver.ts | 17 ++++- core/bootloader/src/service-worker.ts | 5 +- core/bootloader/src/types.ts | 2 + core/bootloader/test/pinned.test.ts | 23 ++++-- core/filesystem/src/file-list.ts | 11 ++- core/filesystem/src/walk.ts | 72 ++++++++++++++----- core/filesystem/test/file-list.test.ts | 43 +++++++++++ ...28-paths-namespace-content-cache-design.md | 10 ++- 10 files changed, 170 insertions(+), 34 deletions(-) create mode 100644 .changeset/nested-renames-keep-pinned-urls.md diff --git a/.changeset/nested-renames-keep-pinned-urls.md b/.changeset/nested-renames-keep-pinned-urls.md new file mode 100644 index 000000000..931dbc40c --- /dev/null +++ b/.changeset/nested-renames-keep-pinned-urls.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +--- + +A pinned URL to a file in a package's nested folder keeps working after the file, or a folder above it, is renamed. A file list now records `via` for such a file: the heads of each nested folder doc on the way to it. The service worker passes those on with `heads` when it asks a resolver for the file, and `readPath` reads any document it reaches by a link without heads at whichever of them it has. Before, the walk read nested folders as they are now, so a renamed file answered nothing when its bytes weren't already stored. + +Lists with no nested folders leave `via` out and hash as before. A package with nested folders gets a new list, and so a new pinned URL, once. diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts index 6e66bda7e..42f7eefd5 100644 --- a/core/bootloader/src/pinned.ts +++ b/core/bootloader/src/pinned.ts @@ -103,12 +103,14 @@ export type PinnedDeps = { */ buildList(prefix: string, only?: string): Promise<Uint8Array | undefined>; /** - * Ask a tab or the automerge worker for the file at `path`, read at `heads` if given: its - * bytes, a Response for any other answer (a redirect, a 404), or undefined. + * Ask a tab or the automerge worker for the file at `path`, read at `heads` if given, with + * the nested folders on the way read at `via`: its bytes, a Response for any other answer (a + * redirect, a 404), or undefined. */ resolve( path: string, - heads?: string + heads?: string, + via?: string[] ): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; warn(message: string): void; /** Keep the service worker alive for work that outlives the response. */ @@ -225,7 +227,7 @@ export async function servePinned( } } - const answer = await deps.resolve(path, entry.heads); + const answer = await deps.resolve(path, entry.heads, entry.via); if (!answer || answer instanceof Response) { throw new PinnedAbort( `/@/${path} is in its file list, but a resolver answered ${answer?.status ?? "nothing"}` @@ -265,7 +267,7 @@ export function sharedRequests<A extends unknown[], T>( export function handoffRequest( request: Request, cachename: string, - to: URL | { path: string; heads?: string; list?: boolean; only?: string } + to: URL | { path: string; heads?: string; via?: string[]; list?: boolean; only?: string } ): HandoffRequestMessage { const target = to instanceof URL @@ -274,6 +276,7 @@ export function handoffRequest( handoffURL: `patchwork-path:${to.path}`, path: to.path, ...(to.heads ? { heads: to.heads } : {}), + ...(to.via?.length ? { via: to.via } : {}), ...(to.list ? { list: true } : {}), ...(to.only !== undefined ? { only: to.only } : {}), }; diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 553eeb64e..476c38ca0 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -224,7 +224,7 @@ export function waitingFind( * list covers just the file asked for; the body is that list, for the service worker to check * and keep. * - With `heads`: the file the path names, read at those heads (what its file list recorded), - * whether or not the path is canonical. + * whether or not the path is canonical, with the nested folders on the way read at `via`. * - A canonical pinned path: the bytes. * - One that doesn't resolve: 404. * @@ -235,7 +235,12 @@ async function resolveWalkPath( path: string, accounts: ReadonlyMap<string, AutomergeUrl>, signal: AbortSignal, - { heads, list, only }: { heads?: string; list?: boolean; only?: string } = {}, + { + heads, + via, + list, + only, + }: { heads?: string; via?: string[]; list?: boolean; only?: string } = {}, providers?: ExtraProviders ): Promise<Response> { try { @@ -276,7 +281,13 @@ async function resolveWalkPath( // package's nested folder is reached by name, through a link without // heads, so the path needn't be canonical; the service worker checks the // bytes against the list either way. - const read = await readPath(repo, path, { ...options, heads: parseHeads(heads) }); + const read = await readPath(repo, path, { + ...options, + heads: parseHeads(heads), + via: (Array.isArray(via) ? via : []) + .map((step) => parseHeads(String(step))) + .filter((step): step is UrlHeads => !!step), + }); if (!read) return notFound(); return new Response(toBytes(read.content) as BlobPart, { status: 200, diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 4ab353f9e..b204f41da 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -353,7 +353,7 @@ function askTab( */ async function handoff( fetchEvent: FetchEvent, - to: URL | { path: string; heads?: string; list?: boolean } + to: URL | { path: string; heads?: string; via?: string[]; list?: boolean; only?: string } ): Promise<HandoffReplyMessage> { const message = handoffRequest(fetchEvent.request, cachename, to); const deadline = Date.now() + HANDOFF_TIMEOUT_MS; @@ -577,7 +577,7 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise<Response store, hash: contentHash, buildList: (prefix, only) => buildList(prefix, only, ask), - resolve: (target, heads) => ask({ path: target, heads }), + resolve: (target, heads, via) => ask({ path: target, heads, via }), warn: (message) => lifecycle("warn", message), keep: (work) => fetchEvent.waitUntil( @@ -612,6 +612,7 @@ async function servePath(fetchEvent: FetchEvent, path: string): Promise<Response type Ask = (to: { path: string; heads?: string; + via?: string[]; list?: boolean; only?: string; }) => Promise<{ body: Uint8Array; contentType: string } | Response>; diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 2990ed290..59bfa8656 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -54,6 +54,8 @@ export interface HandoffRequest { path?: string; /** Read the file `path` names at these heads, as its file list recorded. */ heads?: string; + /** With `heads`: the heads its list recorded for the nested folders on the way. */ + via?: string[]; /** Answer the file list of the container `path` names, not a file. */ list?: boolean; /** With `list`: list only this tail, as the one-file list a redirect hands out. */ diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts index f4a821d13..527f52d94 100644 --- a/core/bootloader/test/pinned.test.ts +++ b/core/bootloader/test/pinned.test.ts @@ -74,12 +74,18 @@ class MemoryStore implements ContentStore { const REAL = toBytes("export default 42"); const FORGED = toBytes("export default 666"); -async function setup(options: { store?: boolean } = {}) { +async function setup(options: { store?: boolean; via?: string[] } = {}) { const sha256 = await contentHash(REAL); const list: FileList = { v: 1, files: { - "index.js": { sha256, type: "text/javascript", heads: "fh", store: options.store ?? true }, + "index.js": { + sha256, + type: "text/javascript", + heads: "fh", + ...(options.via ? { via: options.via } : {}), + store: options.store ?? true, + }, }, }; const listBytes = new TextEncoder().encode(canonicalJson(list)); @@ -93,7 +99,7 @@ async function setup(options: { store?: boolean } = {}) { buildList: vi.fn( async (_prefix: string, _only?: string) => listBytes as Uint8Array | undefined ), - resolve: vi.fn(async (_path: string, _heads?: string) => ({ + resolve: vi.fn(async (_path: string, _heads?: string, _via?: string[]) => ({ body: REAL, contentType: "text/javascript", }) as { body: Uint8Array; contentType: string } | Response | undefined), @@ -161,12 +167,18 @@ describe("servePinned", () => { await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); }); + it("rereads a file in a nested folder with the heads its list recorded on the way", async () => { + const { deps, normalized } = await setup({ via: ["sub"] }); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh", ["sub"]); + }); + it("deletes a forged file, rereads it at the list's heads, and serves", async () => { const { deps, sha256, normalized, settle } = await setup(); await deps.store.put(sha256, FORGED); expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); await settle(); - expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh"); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh", undefined); expect(deps.warn).toHaveBeenCalled(); expect(deps.store.bytes.get(sha256)).toEqual(REAL); }); @@ -282,6 +294,9 @@ describe("handoffRequest", () => { const req = new Request("https://p.test/@/x"); expect(handoffRequest(req, "patchwork", { path: "a@h/x", heads: "fh" }).request.heads).toBe("fh"); expect(handoffRequest(req, "patchwork", { path: "a@h", list: true }).request.list).toBe(true); + const nested = { path: "a@h/sub/x", heads: "fh", via: ["sh"] }; + expect(handoffRequest(req, "patchwork", nested).request.via).toEqual(["sh"]); + expect(handoffRequest(req, "patchwork", { path: "a@h/x", heads: "fh" }).request).not.toHaveProperty("via"); }); it("builds a legacy handoff with no path", () => { diff --git a/core/filesystem/src/file-list.ts b/core/filesystem/src/file-list.ts index aa6bdf4f8..33a88ded5 100644 --- a/core/filesystem/src/file-list.ts +++ b/core/filesystem/src/file-list.ts @@ -15,6 +15,12 @@ export type FileEntry = { type: string; /** The heads the file's document was read at; absent for a value inside the container. */ heads?: string; + /** + * The heads of each nested folder doc on the way to the file, outermost first, for a file in + * a package's nested folder: a walk back to it reads them at these, so renaming the file + * after it was pinned doesn't lose it. Absent otherwise, so other lists hash as before. + */ + via?: string[]; /** Whether the bytes may be kept in the content store. */ store: boolean; }; @@ -40,7 +46,9 @@ const isEntry = (value: unknown): value is FileEntry => { typeof entry.sha256 === "string" && typeof entry.type === "string" && typeof entry.store === "boolean" && - (entry.heads === undefined || typeof entry.heads === "string") + (entry.heads === undefined || typeof entry.heads === "string") && + (entry.via === undefined || + (Array.isArray(entry.via) && entry.via.every((heads) => typeof heads === "string"))) ); }; @@ -92,6 +100,7 @@ export async function buildFileList( sha256: await contentHash(toBytes(file.content)), type: file.type, ...(file.heads ? { heads: formatHeads(file.heads) } : {}), + ...(file.via?.length ? { via: file.via.map(formatHeads) } : {}), store: root.kind === "account" && (throughPackages || !isProtected(file.documentId)), }; } diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts index a87f4b689..12110287e 100644 --- a/core/filesystem/src/walk.ts +++ b/core/filesystem/src/walk.ts @@ -4,10 +4,12 @@ import { type DocumentId, type Repo, type UrlHeads, + decodeHeads, isImmutableString, parseAutomergeUrl, stringifyAutomergeUrl, } from "@automerge/automerge-repo/slim"; +import { hasHeads } from "@automerge/automerge/slim"; import { type DocStep, docStep, @@ -511,14 +513,30 @@ export type FileContent = { content: string | Uint8Array; type: string }; /** * What a path names, read: its pinned form, whether the input was canonical, and the bytes. * With `heads`, the document the path ends at is read at those heads instead: how a file is - * read at the heads its file list recorded, after it has changed. + * read at the heads its file list recorded, after it has changed. With `via`, the heads its + * list recorded for the nested folders on the way: a document the walk reaches by a link + * without heads is read at whichever of these it has, so a file renamed since is still found. */ export async function readPath( repo: Repo, path: string, - options: WalkOptions & { heads?: UrlHeads } + options: WalkOptions & { heads?: UrlHeads; via?: UrlHeads[] } ): Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined> { - const end = await walkPath(repo, path, options); + const recorded = [...(options.via ?? []), ...(options.heads ? [options.heads] : [])]; + const find0 = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const end = await walkPath(repo, path, { + ...options, + find: recorded.length + ? async (url, at) => { + const handle = await find0(url, at); + if (at) return handle; + // A document's own change hashes: only it, or a copy of it, has them. + const doc = handle.doc(); + const mine = doc && recorded.find((heads) => hasHeads(doc, decodeHeads(heads))); + return mine ? handle.view(mine) : handle; + } + : options.find, + }); if (!end) return undefined; let node = end.node; if (options.heads) { @@ -551,12 +569,14 @@ export async function readPath( /** * A file under a pinned document, the document that holds its bytes, and the heads that - * document was read at if it was reached by a link. + * document was read at if it was reached by a link. `via`: the heads each nested folder doc + * on the way to it was read at, outermost first, when it's in a package's nested folder. */ export type ListedFile = FileContent & { tail: string; documentId: DocumentId; heads?: UrlHeads; + via?: UrlHeads[]; }; const isNested = (value: unknown): value is Record<string, unknown> => @@ -625,7 +645,8 @@ export async function listFiles( keys: number[], node: unknown, holder: DocumentId, - heads?: UrlHeads + heads?: UrlHeads, + via: UrlHeads[] = [] ) => { if (!wanted(parts)) return; const tail = tailOf(parts); @@ -635,23 +656,33 @@ export async function listFiles( if (read) { found.set(tail, { keys, - file: { ...read, tail, documentId: holder, ...(heads ? { heads } : {}) }, + file: { + ...read, + tail, + documentId: holder, + ...(heads ? { heads } : {}), + ...(via.length ? { via } : {}), + }, }); } }; - // Values are read in place; links are gathered and read below. - type Link = { parts: string[]; keys: number[]; link: AutomergeUrl }; + // Values are read in place; links are gathered and read below. `via`: the heads of the + // nested folder docs on the way to the link, outermost first. + type Link = { parts: string[]; keys: number[]; link: AutomergeUrl; via: UrlHeads[] }; const entered = new Set<string>([rootId]); // `heads`: what the holder was read at, when it's a doc below the container (reached by a // link, so not pinned by the container's step): its values are read back at those heads. + // `via`: the heads of the nested folder docs above the holder, so a walk back to a file can + // read each at what it was, after a file in one is renamed. const visit = async ( node: unknown, provider: PathProvider | undefined, holder: DocumentId, parts: string[], keys: number[], - heads?: UrlHeads + heads?: UrlHeads, + via: UrlHeads[] = [] ): Promise<Link[]> => { const links: Link[] = []; for (const child of listOf(provider, node)) { @@ -662,23 +693,30 @@ export async function listFiles( if ("sub" in child || "value" in child) { const value = "sub" in child ? valueAt(node, child.sub) : child.value; if (value === undefined) continue; - await add(childParts, childKeys, value, holder, heads); + await add(childParts, childKeys, value, holder, heads, via); // A package's provider walks on into the values it hands out, so list what's in them. if (!builtIn && value && typeof value === "object" && !(value instanceof Uint8Array)) { - links.push(...(await visit(value, provider, holder, childParts, childKeys, heads))); + links.push(...(await visit(value, provider, holder, childParts, childKeys, heads, via))); continue; } } if ("value" in child) { // Nested values are addressable by path only inside a directory doc. if (provider === directoryProvider && isNested(child.value)) { - links.push(...(await visit(child.value, provider, holder, childParts, childKeys, heads))); + links.push( + ...(await visit(child.value, provider, holder, childParts, childKeys, heads, via)) + ); } } else if ( "link" in child && (wanted(childParts) || (deep && wantedBelow(childParts))) ) { - links.push({ parts: childParts, keys: childKeys, link: child.link }); + links.push({ + parts: childParts, + keys: childKeys, + link: child.link, + via: heads ? [...via, heads] : via, + }); } } return links; @@ -696,7 +734,7 @@ export async function listFiles( queue.shift()?.(); }; - const read = async ({ parts, keys, link }: Link): Promise<void> => { + const read = async ({ parts, keys, link, via }: Link): Promise<void> => { let below: Link[] = []; await acquire(); try { @@ -713,7 +751,7 @@ export async function listFiles( // Built-ins decide what's a file, as the walk does for a path that ends here. const provider = providerFor(doc); if (!provider) { - await add(parts, keys, doc, parsed.documentId, heads); + await add(parts, keys, doc, parsed.documentId, heads, via); } else if ( deep && (provider === folderProvider || provider === directoryProvider) && @@ -721,8 +759,8 @@ export async function listFiles( ) { // Into a nested folder, by name: its files are under its name here. entered.add(parsed.documentId); - await add(parts, keys, doc, parsed.documentId, heads); - below = await visit(doc, provider, parsed.documentId, parts, keys, heads); + await add(parts, keys, doc, parsed.documentId, heads, via); + below = await visit(doc, provider, parsed.documentId, parts, keys, heads, via); } } catch { missing.push(tailOf(parts)); diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts index c1bf2ed0a..a2998856e 100644 --- a/core/filesystem/test/file-list.test.ts +++ b/core/filesystem/test/file-list.test.ts @@ -169,6 +169,49 @@ describe("package lists", () => { expect(decode(then!.content)).toBe("old"); }); + it("still reads a nested file renamed since it was pinned", async () => { + const { repo, accounts, prefix, sub } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + const entry = built!.list.files["sub/x.js"]; + expect(entry.via).toEqual([formatHeads(sub.heads())]); + sub.change((d: any) => (d.docs[0].name = "y.js")); + const then = await readPath(repo, `${built!.prefix}/sub/x.js`, { + accounts, + heads: parseHeads(entry.heads!), + via: entry.via!.map((heads) => parseHeads(heads)!), + }); + expect(decode(then!.content)).toBe("x"); + }); + + it("still reads a value renamed in a nested directory doc since it was pinned", async () => { + const { repo, a, accounts, pkg } = await folderPackage(); + const dir = repo.create<any>({ + "@patchwork": { type: "directory" }, + "a.js": { content: "old", mimeType: "text/javascript" }, + }); + pkg.change((d: any) => d.docs.push({ name: "dir", type: "directory", url: dir.url })); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dir/a.js"]; + expect(entry.via).toBeUndefined(); + dir.change((d: any) => { + d["b.js"] = { content: "old", mimeType: "text/javascript" }; + delete d["a.js"]; + }); + const then = await readPath(repo, `${built!.prefix}/dir/a.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("old"); + }); + + it("leaves via out of a list with no nested folders, so it hashes as before", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(new TextDecoder().decode(built!.bytes)).not.toContain('"via"'); + }); + it("pins a live path to a folder with the folder's whole list", async () => { const { repo, a, accounts } = await folderPackage(); const pinned = await pinnedPath(repo, livePath(a, ["packages", "fpkg"]), { diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md index eb12c97f9..389974387 100644 --- a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -326,11 +326,17 @@ was pinned. It is canonical JSON: keys sorted, no whitespace. - `type` is its content type. - `heads` is the heads its document was read at. It's absent for a value inside the container, which the container's own heads pin. + - `via`, for a file in a package's nested folder, is the heads of each + nested folder doc on the way to it, outermost first. Those folders are + reached by links without heads, so a walk back to the file reads each + at the heads it has from `via`: renaming the file, or a folder above + it, doesn't lose it. Absent everywhere else, so lists without nested + folders hash as they did before it. - `store` says whether the bytes may be kept (see [What gets stored](#what-gets-stored)). - **No document ids, by design.** A resolver that needs a file's bytes walks - to it by name from the container at its pinned heads, and reads it at the - recorded heads. Automerge keeps history, so an old version stays readable + to it by name from the container at its pinned heads, through nested + folders at their `via` heads, and reads it at the recorded heads. Automerge keeps history, so an old version stays readable wherever the document is. - **The listHash is the SHA-256 of these bytes.** The list is content like any other: stored under its hash if its container's content may be stored, From b7aa61a888fcadc9c3a2d9d207aa0d71d3ff22e9 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 23:35:45 +0000 Subject: [PATCH 80/81] bootloader tests: give the harness's fake tabs time to accept A fake tab runs on the service worker's event loop, so its accepted can queue behind its own resolving. Under load that passed the 250ms accept timeout, the tab counted as silent, and the request waited out a 35s broadcast nothing answers: the 30s test timeouts. Four parallel runs of the suite failed 12 of 12 times before; 12 of 12 pass after. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .changeset/handoff-accept-timeout-module.md | 5 +++++ core/bootloader/src/handoff-timing.ts | 9 +++++++++ core/bootloader/src/service-worker.ts | 6 +----- core/bootloader/test/service-worker-harness.ts | 5 +++++ 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 .changeset/handoff-accept-timeout-module.md create mode 100644 core/bootloader/src/handoff-timing.ts diff --git a/.changeset/handoff-accept-timeout-module.md b/.changeset/handoff-accept-timeout-module.md new file mode 100644 index 000000000..ac1e35ee2 --- /dev/null +++ b/.changeset/handoff-accept-timeout-module.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-bootloader": patch +--- + +The service worker's tab accept timeout (250ms) moves into its own module, `handoff-timing.ts`, so the node test harness can widen it. Its in-process fake tabs share the service worker's event loop, so on a loaded machine their "accepted" could miss the window and the test would wait out a 35-second broadcast. Behaviour in browsers is unchanged. diff --git a/core/bootloader/src/handoff-timing.ts b/core/bootloader/src/handoff-timing.ts new file mode 100644 index 000000000..497026f78 --- /dev/null +++ b/core/bootloader/src/handoff-timing.ts @@ -0,0 +1,9 @@ +// How long the service worker gives a tab to say it'll resolve a request before +// asking the next one. A tab that hasn't been handed its Repo yet says it can't +// at once, so this is for one that can't answer at all: a frozen background +// tab, or one from a build before tabs resolved. +// +// Its own module so the node test harness can widen it: there, a fake tab runs +// on the service worker's own event loop, and its "accepted" can wait behind +// its own resolving on a loaded machine. +export const ACCEPT_TIMEOUT_MS = 250; diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index b204f41da..4365a642c 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -33,6 +33,7 @@ import { // installs a replacement service worker when they do, and the controllerchange // that follows is how open tabs learn a deploy happened (see setup.ts). import { buildId } from "./build-id.js"; +import { ACCEPT_TIMEOUT_MS } from "./handoff-timing.js"; const DEFAULT_CACHE_NAME = "patchwork"; @@ -253,11 +254,6 @@ async function startWorkerIfMissing(id: string) { /** Signals that respondWith should reject; see {@link HandoffAbortMessage}. */ class HandoffAborted extends Error {} -// How long a tab has to say it'll resolve a request before the service -// worker asks the next one. A tab that hasn't been handed its Repo yet says it -// can't at once, so this is for one that can't answer at all: a frozen -// background tab, or one from a build before tabs resolved. -const ACCEPT_TIMEOUT_MS = 250; // A tab posts its reply and then lets go of its lock, but the lock can be // granted here before the reply is delivered. const REPLY_AFTER_LOCK_MS = 100; diff --git a/core/bootloader/test/service-worker-harness.ts b/core/bootloader/test/service-worker-harness.ts index d5909f3ef..b90fea5d0 100644 --- a/core/bootloader/test/service-worker-harness.ts +++ b/core/bootloader/test/service-worker-harness.ts @@ -131,6 +131,11 @@ export async function startServiceWorker(): Promise<ServiceWorkerHarness> { vi.stubGlobal("caches", caches); vi.stubGlobal("BroadcastChannel", TrackedBroadcastChannel); vi.resetModules(); + // A fake tab shares this event loop, so its "accepted" can queue behind its + // own resolving; under load that passes 250ms, the tab counts as silent, and + // the request waits out a broadcast nothing here answers. Tests that want a + // tab declined have it say so. + vi.doMock("../src/handoff-timing.js", () => ({ ACCEPT_TIMEOUT_MS: 10_000 })); await import("../src/service-worker.js"); // The module's own channels are open; the test's are the real class again. vi.stubGlobal("BroadcastChannel", RealBroadcastChannel); From a1b1853f7c669c3a4a052e9a135cba394cba7a18 Mon Sep 17 00:00:00 2001 From: chee <yay@chee.party> Date: Fri, 30 Oct 2026 23:41:43 +0000 Subject: [PATCH 81/81] packages-folder: say how the watcher imports a /packages URL now Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- core/filesystem/src/packages-folder.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts index d19b1b340..05fc571bd 100644 --- a/core/filesystem/src/packages-folder.ts +++ b/core/filesystem/src/packages-folder.ts @@ -182,8 +182,9 @@ async function lookUpViaServiceWorker(path: string): Promise<string> { /** * A tracker for the packages under `/packages`: `findLatest` answers with the - * package's pinned `/@/` URL, which the ModuleWatcher imports as an http - * package. Anything else goes to `fallback`, which also does the watching. + * package's pinned `/@/` URL, which the ModuleWatcher imports through its + * package import (in a browser, discovered in the module-loader worker). + * Anything else goes to `fallback`, which also does the watching. */ export function trackPackagesFolder( acctHash: Promise<string>, @@ -210,7 +211,7 @@ export function trackPackagesFolder( return previous as AutomergeUrl; } known.set(documentId, latest); - // Not an automerge URL: the watcher imports it with importPackageFromHttpUrl. + // Not an automerge URL: the watcher hands it to its package import as it is. return latest as AutomergeUrl; }, watch(url, onChange) {