diff --git a/.changeset/codeql-sw-logging.md b/.changeset/codeql-sw-logging.md new file mode 100644 index 000000000..8027be357 --- /dev/null +++ b/.changeset/codeql-sw-logging.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-bootloader": patch +--- + +The service worker and resolvers no longer put request URLs or paths into a console format string, where a `%c` or `%s` in a URL was interpreted. Error responses carry the error's message, not its stack; the stack goes to the console. diff --git a/.changeset/handoff-accept-timeout-module.md b/.changeset/handoff-accept-timeout-module.md new file mode 100644 index 000000000..ac1e35ee2 --- /dev/null +++ b/.changeset/handoff-accept-timeout-module.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-bootloader": patch +--- + +The service worker's tab accept timeout (250ms) moves into its own module, `handoff-timing.ts`, so the node test harness can widen it. Its in-process fake tabs share the service worker's event loop, so on a loaded machine their "accepted" could miss the window and the test would wait out a 35-second broadcast. Behaviour in browsers is unchanged. diff --git a/.changeset/nested-renames-keep-pinned-urls.md b/.changeset/nested-renames-keep-pinned-urls.md new file mode 100644 index 000000000..931dbc40c --- /dev/null +++ b/.changeset/nested-renames-keep-pinned-urls.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +--- + +A pinned URL to a file in a package's nested folder keeps working after the file, or a folder above it, is renamed. A file list now records `via` for such a file: the heads of each nested folder doc on the way to it. The service worker passes those on with `heads` when it asks a resolver for the file, and `readPath` reads any document it reaches by a link without heads at whichever of them it has. Before, the walk read nested folders as they are now, so a renamed file answered nothing when its bytes weren't already stored. + +Lists with no nested folders leave `via` out and hash as before. A package with nested folders gets a new list, and so a new pinned URL, once. diff --git a/.changeset/open-document-path.md b/.changeset/open-document-path.md new file mode 100644 index 000000000..5f5cbd2a5 --- /dev/null +++ b/.changeset/open-document-path.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork-elements": patch +"@inkandswitch/patchwork": patch +--- + +`patchwork:open-document` (and `openDocument`, `patchwork.open`, ``) can say where a document was opened from: `path` is the live path it was opened at, like a cwd, e.g. `/Projects/foo` with each segment `encodeURIComponent`-encoded. A `` hands a new path to its tool without remounting it. diff --git a/.changeset/packages-discovered-off-thread.md b/.changeset/packages-discovered-off-thread.md new file mode 100644 index 000000000..b64407e27 --- /dev/null +++ b/.changeset/packages-discovered-off-thread.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +--- + +`/packages` packages are discovered off the main thread, like module-settings packages. The pinned `/@/` URL a `/packages` lookup answers with now goes through the ModuleWatcher's `importAutomergePackage` hook, so the browser's module-loader worker reads its plugin descriptors, and a plugin's `load()` re-imports the same pinned URL on the main thread. Before, those packages' entry points were imported on the main thread to register them. + +`importPluginFromPackage(url, type, id)` takes a folder doc pinned to heads or a pinned `/@/` URL. `importPluginFromFolderDocUrl` is kept as a deprecated alias for it. diff --git a/.changeset/path-provider-plugins.md b/.changeset/path-provider-plugins.md new file mode 100644 index 000000000..b67858584 --- /dev/null +++ b/.changeset/path-provider-plugins.md @@ -0,0 +1,19 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +--- + +Packages can now say what a document of their type contains. A `patchwork:path-provider` plugin is a `PathProvider` with `supportedDatatypes`. The built-in providers (account, directory, folder, branches) are always asked first and can't be overridden. After them come plugins, in registration order. The one exception is a typed document that also has a folder's `docs` array (a tldraw canvas): a plugin declared for its type walks it, and the folder provider walks it otherwise (`yieldsToPackage`). + +- `patchwork-filesystem`: + - a `Child` can be `{ name, sub: [...segments] }`, a value inside the document offered as a sub-document. A walk that ends at one reports it as `WalkEnd.sub`, and `resolveLivePath` returns `handle.sub(...segments)`; + - `PathProvider` gains the optional `add`, `rename` and `remove`, for whoever shows a provider's children to edit them; + - walks take `WalkOptions.providers` (an `ExtraProviders`), asked after the built-ins, as do `listChildren` and `listUnion`. A provider that throws is treated as having no children. + - extra providers apply only to a document a walk goes on through. Whether a document a path ends at is a container, and what a file list counts as a file, are the built-ins' call, so installing a provider doesn't change what an existing pinned URL means; + - a link with path segments (to a sub-document) isn't walked or listed as the whole document. + - a name a provider spreads over several path segments is written as the file list writes it, so every spelling of it pins to a tail the list covers. +- `patchwork-plugins`: the plugin type `PathProviderDescription`/`PathProviderPlugin`, with `pathProviderFor(doc)` (built-ins, then plugins), `packagePathProviders` (plugins only, for `WalkOptions.providers`), and `pathProviderTypes()`. `"*"` in `supportedDatatypes` is ignored. +- `patchwork-bootloader`: `createResolver` takes `providers`. A tab's resolver passes the registered plugins, and the automerge worker's passes none, since it never loads package code. +- `patchwork`: `&path=` resolves through registered path providers too, and a path that ends at a sub-document opens it at its sub-handle's URL. A `path=` that names a different part of the document `doc=` names is dropped, as one naming a different document already was. diff --git a/.changeset/paths-content-store.md b/.changeset/paths-content-store.md new file mode 100644 index 000000000..7801c9c95 --- /dev/null +++ b/.changeset/paths-content-store.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The service worker serves `/@/` paths. A live path (`/@//Projects/foo/x.js`) redirects to its pinned form. A pinned one names its folder's file list by hash (`…/@~/x.js`): the list records each file's content hash and the heads it was read at, so the service worker checks the list against the URL and the bytes against the list, and serves a pinned URL's old bytes even after the file changes. Lists and bytes are kept in a content-addressed store (`patchwork-content-`) that anything on the origin can write and that is checked on every read. Links no longer need heads to pin what they point at, and no document id appears in a URL, a file list or a store key (a stored folder's own JSON still holds its children's URLs). A pinned URL whose list has been lost is rebuilt from its pinned heads, trying the one-file list a redirect hands out before the whole folder's; `pinPath` refuses to pin a list it couldn't read every file for (`IncompleteFileListError`). Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker itself. New in `@inkandswitch/patchwork-bootloader`: `registerAccount` (patchwork calls it at boot) and `clearAccountStore`. New in `@inkandswitch/patchwork-filesystem`: `buildFileList`, `canonicalJson`, `parseFileList`, `contentHash`, `toBytes`, `readPath` (optionally at given heads), `listFiles`, `readNode`, `WalkOptions.find`, `WalkEnd.container` and `WalkEnd.tail`. The e2e suite gains a test that a forged store entry is never served. diff --git a/.changeset/paths-grammar-providers.md b/.changeset/paths-grammar-providers.md new file mode 100644 index 000000000..6f469dfa6 --- /dev/null +++ b/.changeset/paths-grammar-providers.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +--- + +Add paths. `idhash`, `livePath` and `parsePath` read and write live paths (`/Projects/foo`) and pinned ones (`@/@~/…`), where a container's step may name a hash of its file list. `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren` and `listUnion` walk an account's namespace to find a document or pin a path; `pinnedPath` and `pinPath` return the pinned path or URL together with the file list it names. Path providers (`PathProvider`, `accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `providerFor`) replace `FolderStrategy`, which is now a deprecated alias of `PathProvider`. `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. `isKeyhiveDoc` in `@inkandswitch/patchwork-plugins` now goes through filesystem's new `isKeyhiveDocumentId`, so the rule lives in one place. diff --git a/.changeset/paths-packages.md b/.changeset/paths-packages.md new file mode 100644 index 000000000..082817a55 --- /dev/null +++ b/.changeset/paths-packages.md @@ -0,0 +1,8 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork-plugins": patch +"@inkandswitch/patchwork": patch +--- + +Package management is whatever is bound at `/packages` in the account's namespace. New accounts bind `/` to their root folder and `/packages` to a new Packages folder; an older account gets a Packages folder at boot, linking each automerge package its module-settings doc lists, by its `package.json` name. The ModuleWatcher loads every package in the `/packages` union (`addPackagesFolder`, `watchPackagesFolder`), keyed by document so a rename reloads nothing, and looks each one up as a pinned `/@/` URL (`trackPackagesFolder`) that it imports and the service worker checks against the package's file list. A package's list now covers everything under it, nested folders included, and a redirect to a folder carries the folder's whole list. New accounts, and migrated ones, also get a `packages` link in their root folder to that folder, so `/packages` shows in the tree like any other folder. The ModuleWatcher reports each package's status (`packageStatus(url)`: loading, loaded with its plugins, or failed with why) and fires a `"status"` event when one changes; it is now an `EventTarget`. `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` binding; it's still read, deduplicated against `/packages`, since the Packages UI installs into it. diff --git a/.changeset/paths-review-fixes.md b/.changeset/paths-review-fixes.md new file mode 100644 index 000000000..5f42081cf --- /dev/null +++ b/.changeset/paths-review-fixes.md @@ -0,0 +1,23 @@ +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +--- + +Fixes to paths, packages and the content store from review. + +- File lists: a file named `__proto__` is listed like any other. A tail named like an `Object.prototype` member (`constructor`, `toString`) that the list doesn't name answers 404. +- Pinning through a package's container (a path provider's doc) lists that container, so the URL serves. A doc named by a doc step is walked with package providers; one a path ends at by name is still a container or a file by the built-ins. +- A value held by a doc below a package's root (a nested directory doc) is listed with that doc's heads, and read back at them. +- `/packages`: + - a package that's a branches doc reloads when its default branch's code changes, and follows the branch when it moves; + - a member folder that can't be found doesn't hold up the others: the first listing waits up to 3s, then lists what it has, and the rest arrive as they're found; + - migration records the entries it moved in the module-settings doc (`movedToPackages`). This build loads those only through `/packages`, so removing a package there removes it. Builds that predate `/packages` still read `modules` as before; + - migration never gives two packages one name, and a `/packages` binding that syncs in mid-migration wins; + - boot waits at most a second for migration. +- Routing: + - a `doc=`/`path=` check that finishes after the hash moved on leaves the new hash alone; + - opening another part of the same document drops the path; + - a path-only hash that doesn't resolve is walked again when path providers register, so a cold boot opens a path through a package's datatype. +- The service worker opens an account's content store only to write to it, so a `/@/` link naming an unknown account leaves no cache behind. +- `ModuleWatcher.packageFor(importUrl)` says which package doc an import url was loaded for. A `/packages` package's plugins are registered under its pinned `/@/` url; this maps it back, so a tool like Packages can tell what's installed. diff --git a/.changeset/paths-routing.md b/.changeset/paths-routing.md new file mode 100644 index 000000000..88ad4c6a1 --- /dev/null +++ b/.changeset/paths-routing.md @@ -0,0 +1,6 @@ +--- +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The router keeps `&path=` in the hash: the live path a document was opened at (`/Projects/foo`), written whenever `patchwork:open-document` carries one and its segments kept exactly as encoded. A hash with `path=` and no `doc=` opens what the path names; one that doesn't resolve (yet) opens nothing and stays in the address bar. When `path=` names a different document than `doc=`, because something moved or was renamed, `doc=` wins and `path=` is dropped with a console warning. `path=` survives a tool switch on the same document. Paths under the site's own account resolve, and so do share links. The e2e suite gains a test that opens a document from `&path=` alone. diff --git a/.changeset/subdoc-opens-in-its-documents-tool.md b/.changeset/subdoc-opens-in-its-documents-tool.md new file mode 100644 index 000000000..da40c4494 --- /dev/null +++ b/.changeset/subdoc-opens-in-its-documents-tool.md @@ -0,0 +1,5 @@ +--- +"@inkandswitch/patchwork-elements": patch +--- + +`` opens part of a document (`automerge:/`, a sub-handle) that has no `@patchwork` type of its own in the tool for the document it's part of, and hands that tool the sub-handle. Before, such a part, like a tldraw shape opened from the sidebar, failed with "missing @patchwork metadata". diff --git a/.github/workflows/spike-sw-sharedworker.yml b/.github/workflows/spike-sw-sharedworker.yml new file mode 100644 index 000000000..d29dc6807 --- /dev/null +++ b/.github/workflows/spike-sw-sharedworker.yml @@ -0,0 +1,53 @@ +name: spike sw-sharedworker + +# Records how each browser lets a service worker and a module SharedWorker +# message each other, and what it tells the service worker about the sender. +# See packages/e2e/spikes/sw-sharedworker. The JSON lands in the job summary. + +permissions: + contents: read + +on: + workflow_dispatch: + push: + paths: + - packages/e2e/spikes/sw-sharedworker/** + - .github/workflows/spike-sw-sharedworker.yml + +jobs: + spike: + name: spike (${{ matrix.browser }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + run_install: false + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + + - name: install the e2e package + run: pnpm install --frozen-lockfile --filter @inkandswitch/patchwork-e2e + + - name: install ${{ matrix.browser }} + working-directory: packages/e2e + run: pnpm exec playwright install --with-deps ${{ matrix.browser }} + + - name: run the spike + working-directory: packages/e2e + run: pnpm exec playwright test -c spikes/sw-sharedworker/playwright.config.mjs --project=${{ matrix.browser }} + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: spike-sw-sharedworker-${{ matrix.browser }} + path: packages/e2e/spikes/sw-sharedworker/results/ + retention-days: 14 diff --git a/core/bootloader/package.json b/core/bootloader/package.json index 207a095d6..2faa571df 100644 --- a/core/bootloader/package.json +++ b/core/bootloader/package.json @@ -113,6 +113,7 @@ }, "scripts": { "build": "tsc && cp src/global.css dist/global.css", - "dev": "tsc -w --preserveWatchOutput" + "dev": "tsc -w --preserveWatchOutput", + "test": "vitest run" } } diff --git a/core/bootloader/src/automerge-protocol-handler.ts b/core/bootloader/src/automerge-protocol-handler.ts index 9f0a457ec..e938f29e7 100644 --- a/core/bootloader/src/automerge-protocol-handler.ts +++ b/core/bootloader/src/automerge-protocol-handler.ts @@ -17,9 +17,12 @@ import initSubduction from "@automerge/automerge-subduction/slim"; import { Repo, + parseAutomergeUrl, + type AutomergeUrl, type DocumentId, type PeerId, } from "@automerge/automerge-repo/slim"; +import { idhash } from "@inkandswitch/patchwork-filesystem"; import { IndexedDBStorageAdapter } from "@automerge/automerge-repo-storage-indexeddb"; import type { @@ -37,6 +40,7 @@ import { HANDOFF_CHANNEL, type HandoffOnlineMessage, type HandoffRequestMessage, + type RegisterAccountMessage, } from "./types.js"; declare const __SYNC_SERVER__: { @@ -50,11 +54,30 @@ const syncServer = ? __SYNC_SERVER__ : { url: "wss://subduction.sync.inkandswitch.com" }; +// acctHash → account url: the accounts whose `/@/` paths this worker resolves. +// Tabs tell us theirs; the hash is computed here, so no tab can name an +// account by the wrong one. +const accounts = new Map(); + +async function registerAccount({ url }: RegisterAccountMessage) { + try { + const { documentId } = parseAutomergeUrl(url as AutomergeUrl); + accounts.set(await idhash(documentId), url as AutomergeUrl); + } catch (error) { + console.warn("ignoring a register-account for %s", url, error); + } +} + const control = startWorkerControl("automerge-protocol-handler-worker", { onRestart: drainForRestart, // Held only so the tabs and the service worker can tell whether this // instance is still around; nothing here waits for it. lock: AUTOMERGE_WORKER_LOCK, + onMessage(data) { + if (data?.type === "register-account") { + void registerAccount(data as RegisterAccountMessage); + } + }, }); const log = control.log; @@ -169,7 +192,7 @@ async function buildKeyhiveRepo( // them can (one still booting, one from an older build, a site that brings // its own Repo). -const resolver = createResolver(getRepo, log); +const resolver = createResolver(getRepo, log, () => accounts); const handoffChannel = new BroadcastChannel(HANDOFF_CHANNEL); const EVICT_IDLE_MS = 5_000; diff --git a/core/bootloader/src/content-store.ts b/core/bootloader/src/content-store.ts new file mode 100644 index 000000000..826ce9de3 --- /dev/null +++ b/core/bootloader/src/content-store.ts @@ -0,0 +1,64 @@ +// Bytes by their hash, in CacheStorage. Nothing here is trusted: any code on the origin can +// write these caches, so every read is hashed and checked against the hash a pinned URL vouches +// for, through its file list, before it's served (see pinned.ts). Reads and writes go through this one interface so the bytes can be +// encrypted at rest later. + +export interface ContentStore { + get(hash: string): Promise; + put(hash: string, bytes: Uint8Array): Promise; + delete(hash: string): Promise; +} + +export const CONTENT_CACHE_PREFIX = "patchwork-content-"; + +/** One store per account, apart from the `patchwork` cache of site pages and assets. */ +export const contentCacheName = (acctHash: string) => + `${CONTENT_CACHE_PREFIX}${acctHash}`; + +/** A store in `cache`, keyed by `/.content/`: an internal key, never fetched. */ +export function cacheStore(cache: Cache, origin: string): ContentStore { + const key = (hash: string) => `${origin}/.content/${hash}`; + return { + async get(hash) { + const response = await cache.match(key(hash)); + return response ? new Uint8Array(await response.arrayBuffer()) : undefined; + }, + async put(hash, bytes) { + await cache.put(key(hash), new Response(bytes as BlobPart)); + }, + async delete(hash) { + await cache.delete(key(hash)); + }, + }; +} + +/** + * {@link cacheStore} in the cache `name`, opened only when something is stored: a read or a + * delete of a cache that doesn't exist finds nothing, and creates nothing. Any `/@/` URL names an + * account, so opening on every request would leave a cache behind for each made-up one. + */ +export function lazyCacheStore(storage: CacheStorage, name: string, origin: string): ContentStore { + let opened: Promise | undefined; + const open = () => (opened ??= storage.open(name).then((cache) => cacheStore(cache, origin))); + const existing = async () => (opened || (await storage.has(name)) ? open() : undefined); + return { + async get(hash) { + return (await existing())?.get(hash); + }, + async put(hash, bytes) { + await (await open()).put(hash, bytes); + }, + async delete(hash) { + await (await existing())?.delete(hash); + }, + }; +} + +/** + * Forget everything stored for an account. There's no sign-out flow; a site calls this from its + * own. File lists the service worker holds in memory may still name these hashes, which then + * read as misses. + */ +export function clearAccountStore(acctHash: string): Promise { + return caches.delete(contentCacheName(acctHash)); +} diff --git a/core/bootloader/src/handoff-timing.ts b/core/bootloader/src/handoff-timing.ts new file mode 100644 index 000000000..497026f78 --- /dev/null +++ b/core/bootloader/src/handoff-timing.ts @@ -0,0 +1,9 @@ +// How long the service worker gives a tab to say it'll resolve a request before +// asking the next one. A tab that hasn't been handed its Repo yet says it can't +// at once, so this is for one that can't answer at all: a frozen background +// tab, or one from a build before tabs resolved. +// +// Its own module so the node test harness can widen it: there, a fake tab runs +// on the service worker's own event loop, and its "accepted" can wait behind +// its own resolving on a loaded machine. +export const ACCEPT_TIMEOUT_MS = 250; diff --git a/core/bootloader/src/module-loader-worker.ts b/core/bootloader/src/module-loader-worker.ts index f8607805c..114af505c 100644 --- a/core/bootloader/src/module-loader-worker.ts +++ b/core/bootloader/src/module-loader-worker.ts @@ -6,18 +6,19 @@ // entry point off the main thread purely to read its exported `plugins` array, // strips the non-cloneable `load()` / `import` machinery, and posts the plain // descriptors back. The main thread re-imports the package (at the same heads) -// only when a plugin is actually loaded — see `importPluginFromFolderDocUrl`. +// only when a plugin is actually loaded — see `importPluginFromPackage`. // // Created with type:"module"; its dynamic `import()` of `//…` -// entry points is served by the service worker that controls this worker. +// entry points, and of the pinned `/@/` URLs `/packages` packages load from, +// is served by the service worker that controls this worker. -import { importPackageFromFolderDocUrl } from "@inkandswitch/patchwork-filesystem"; -import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { importPackage } from "@inkandswitch/patchwork-filesystem"; type DiscoverRequest = { type: "discover"; id: number; - url: AutomergeUrl; + /** A folder doc pinned to heads, or a pinned `/@/` URL. */ + url: string; }; // Keep only the structured-cloneable description fields. `load` is a closure @@ -54,7 +55,7 @@ self.addEventListener("message", (event: MessageEvent) => { if (!isDiscoverRequest(data)) return; const { id, url } = data; - importPackageFromFolderDocUrl(url) + importPackage(url) .then((mod) => { const plugins: any[] = Array.isArray(mod?.plugins) ? mod.plugins : []; const descriptors = plugins.map(toDescriptor); diff --git a/core/bootloader/src/module-loader.ts b/core/bootloader/src/module-loader.ts index ae717ce52..166257e9d 100644 --- a/core/bootloader/src/module-loader.ts +++ b/core/bootloader/src/module-loader.ts @@ -5,10 +5,10 @@ // import the package entry point and report which plugins it exports, then // returns the same `{ plugins }` shape the watcher already feeds to // `registerPlugins` — except each plugin's `load()` re-imports the package -// (pinned to the same heads) on this thread and runs the real plugin loader. +// (pinned to the same heads, or the same pinned `/@/` URL) on this thread and +// runs the real plugin loader. -import { importPluginFromFolderDocUrl } from "@inkandswitch/patchwork-filesystem"; -import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { importPluginFromPackage } from "@inkandswitch/patchwork-filesystem"; import { MODULE_LOADER_WORKER } from "./emitted.js"; type Descriptor = Record & { id?: string; type?: string }; @@ -51,7 +51,7 @@ function getWorker(): Worker { return worker; } -function discoverDescriptors(urlAtHeads: AutomergeUrl): Promise { +function discoverDescriptors(urlAtHeads: string): Promise { const id = nextRequestId++; return new Promise((resolve, reject) => { pending.set(id, { resolve, reject }); @@ -60,9 +60,8 @@ function discoverDescriptors(urlAtHeads: AutomergeUrl): Promise { } export async function importAutomergePackageViaWorker( - urlAtHeads: string + url: string ): Promise<{ plugins: Descriptor[] }> { - const url = urlAtHeads as AutomergeUrl; const descriptors = await discoverDescriptors(url); const plugins = descriptors.map((descriptor) => { const { id, type } = descriptor; @@ -71,7 +70,7 @@ export async function importAutomergePackageViaWorker( if (typeof id !== "string" || typeof type !== "string") return descriptor; return { ...descriptor, - load: () => importPluginFromFolderDocUrl(url, type, id), + load: () => importPluginFromPackage(url, type, id), }; }); return { plugins }; diff --git a/core/bootloader/src/pinned.ts b/core/bootloader/src/pinned.ts new file mode 100644 index 000000000..42f7eefd5 --- /dev/null +++ b/core/bootloader/src/pinned.ts @@ -0,0 +1,296 @@ +// Serving a pinned `/@/` URL. Its last container step names a file list by hash +// (`@~`), so the URL vouches for the list and the list for each file's +// bytes. Lists, store entries and resolvers' answers are all checked against those hashes; none +// is trusted for who made it (spec, section 5). +import { + parseFileList, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; +import type { ContentStore } from "./content-store.js"; +import type { HandoffRequestMessage } from "./types.js"; + +const BASE58 = "[1-9A-HJ-NP-Za-km-z]+"; +const LIST = "~[A-Za-z0-9_-]+"; +const DOC_STEP = new RegExp(`^${BASE58}@${BASE58}(?:\\.${BASE58})*(?:${LIST})?$`); +const ACCOUNT_ROOT = new RegExp(`^(${BASE58})(@${BASE58}(?:\\.${BASE58})*(?:${LIST})?)?$`); +const SHARE_PREFIX = "automerge:"; + +const reencode = (segment: string) => encodeURIComponent(decodeURIComponent(segment)); +const listHashOf = (step: string) => /~([A-Za-z0-9_-]+)$/.exec(step)?.[1]; +const withoutList = (step: string) => step.replace(/~[A-Za-z0-9_-]+$/, ""); + +/** + * One reading of a pinned path: the container's pinned path (without its own `~listHash`), the + * names after it, and the list hash its step carries, if any. + */ +export type Candidate = { prefix: string; tail: string; listHash?: string }; + +export type NormalizedPath = { + /** + * The readings of this path, most likely first. A segment shaped like a doc step + * (`react@18.3.1`) may be a name, so there's a reading for each place the container could be; + * every segment after it is a name, re-encoded. + */ + candidates: Candidate[]; + acctHash?: string; + /** False for a live path, whose names resolve at current heads. */ + pinned: boolean; + /** A share link, `automerge:@`: nothing under it is stored. */ + share: boolean; +}; + +/** + * The path after `/@/`, normalized so every spelling of one file has the same candidates: + * segments are percent-decoded and re-encoded, and empty ones dropped. Undefined for a malformed + * escape or a root that isn't an account hash or a share link. + */ +export function normalizePinned(path: string): NormalizedPath | undefined { + const [root, ...rest] = path.split("/").filter(Boolean); + if (!root) return undefined; + try { + const decodedRoot = decodeURIComponent(root); + const share = decodedRoot.startsWith(SHARE_PREFIX); + let acctHash: string | undefined; + let pinned = true; + if (!share) { + const match = ACCOUNT_ROOT.exec(root); + if (!match) return undefined; + acctHash = match[1]; + pinned = !!match[2]; + } + const head = share ? decodedRoot : root; + + // The container is the segment at `split`, or the root at -1. + const reading = (split: number): Candidate => { + const container = split < 0 ? head : rest[split]; + const before = + split < 0 + ? [] + : [head, ...rest.slice(0, split).map((s) => (DOC_STEP.test(s) ? s : reencode(s)))]; + const listHash = listHashOf(container); + return { + prefix: [...before, withoutList(container)].join("/"), + tail: rest.slice(split + 1).map(reencode).join("/"), + ...(listHash ? { listHash } : {}), + }; + }; + + // Doc-step-shaped segments after the root, latest first; then the root, when it's a + // container step itself (a share link, or a root that names a list), or when nothing else is. + const splits = rest + .map((segment, i) => (DOC_STEP.test(segment) ? i : -1)) + .filter((i) => i >= 0) + .reverse(); + if (share || listHashOf(head) || !splits.length) splits.push(-1); + return { candidates: splits.map(reading), acctHash, pinned, share }; + } catch { + return undefined; + } +} + +/** Thrown to fail the request as a network error: nothing that can be memoized. */ +export class PinnedAbort extends Error {} + +export type PinnedDeps = { + /** File lists held in memory, by list hash: ones that may not be stored, and a cache. */ + lists: Map; + /** Undefined: nothing is read or written (a share link). */ + store: ContentStore | undefined; + hash(bytes: Uint8Array): Promise; + /** + * Ask a resolver for the list of the container at `prefix`, covering only the tail `only` if + * given, else every file: its bytes, or undefined. + */ + buildList(prefix: string, only?: string): Promise; + /** + * Ask a tab or the automerge worker for the file at `path`, read at `heads` if given, with + * the nested folders on the way read at `via`: its bytes, a Response for any other answer (a + * redirect, a 404), or undefined. + */ + resolve( + path: string, + heads?: string, + via?: string[] + ): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; + warn(message: string): void; + /** Keep the service worker alive for work that outlives the response. */ + keep(work: Promise): void; +}; + +/** + * Keep a list if `bytes` hash to `listHash`: in memory always, and in the store too when every + * file in it may be stored. False if they don't match, or aren't a list. + */ +export async function keepFileList( + bytes: Uint8Array, + listHash: string, + deps: Pick +): Promise { + if ((await deps.hash(bytes)) !== listHash) return false; + const list = parseFileList(bytes); + if (!list) return false; + deps.lists.set(listHash, list); + if (deps.store && Object.values(list.files).every((entry) => entry.store)) { + deps.keep(deps.store.put(listHash, bytes)); + } + return true; +} + +/** + * The list a URL names: from memory, from the store (checked), or rebuilt (checked). A rebuild + * tries the one-file list a redirect hands out first, which costs one file read, then the whole + * container's. + */ +async function loadList( + { prefix, tail, listHash }: { prefix: string; tail: string; listHash: string }, + deps: PinnedDeps +): Promise { + const known = deps.lists.get(listHash); + if (known) return known; + const stored = await deps.store?.get(listHash); + if (stored) { + // Already in the store: kept in memory only. + if (await keepFileList(stored, listHash, { ...deps, store: undefined })) { + return deps.lists.get(listHash); + } + deps.warn(`content store entry ${listHash} isn't the file list it's named for; deleting it`); + await deps.store!.delete(listHash); + } + for (const only of [tail, undefined]) { + const built = await deps.buildList(prefix, only); + if (built && (await keepFileList(built, listHash, deps))) return deps.lists.get(listHash); + } + return undefined; +} + +const respond = (bytes: Uint8Array, contentType: string, immutable: boolean) => + new Response(bytes as BlobPart, { + headers: { + "content-type": contentType, + "cache-control": immutable ? "public, max-age=31536000, immutable" : "no-store", + }, + }); + +const join = (prefix: string, tail: string) => (tail ? `${prefix}/${tail}` : prefix); + +/** + * Serve a pinned URL (see the spec's *Request flow for a pinned URL*): + * + * 1. Load the list its container step names; none to be had fails as a network error. + * 2. Look the file up in it; not there, 404. + * 3. Read the store by the file's hash and check it; a mismatch is deleted. + * 4. Ask a resolver for the file at the heads the list recorded and check that; a match is served + * (and stored, if the list allows), anything else fails as a network error. + * + * A path with no list anywhere is served live from a resolver: `no-store`, unchecked. + */ +export async function servePinned( + normalized: Pick, + deps: PinnedDeps +): Promise { + const listed = normalized.candidates.find( + (candidate): candidate is Candidate & { listHash: string } => !!candidate.listHash + ); + if (!listed) { + const [first] = normalized.candidates; + const path = join(first.prefix, first.tail); + const answer = await deps.resolve(path, undefined); + if (answer instanceof Response) return answer; + if (!answer) return new Response(`nothing at /@/${path}`, { status: 404 }); + return respond(answer.body, answer.contentType, false); + } + + const path = join(listed.prefix, listed.tail); + const list = await loadList(listed, deps); + if (!list) { + throw new PinnedAbort(`no file list ${listed.listHash} to check /@/${path} against`); + } + // Own entries only: a tail like "constructor" isn't in a list that doesn't name it. + const entry = Object.hasOwn(list.files, listed.tail) ? list.files[listed.tail] : undefined; + if (!entry) { + return new Response(`/@/${path} isn't in its file list`, { + status: 404, + headers: { "content-type": "text/plain", "cache-control": "no-store" }, + }); + } + + if (deps.store) { + const stored = await deps.store.get(entry.sha256); + if (stored) { + if ((await deps.hash(stored)) === entry.sha256) { + return respond(stored, entry.type, true); + } + deps.warn( + `content store entry ${entry.sha256} for /@/${path} doesn't hash to its key; deleting it` + ); + await deps.store.delete(entry.sha256); + } + } + + const answer = await deps.resolve(path, entry.heads, entry.via); + if (!answer || answer instanceof Response) { + throw new PinnedAbort( + `/@/${path} is in its file list, but a resolver answered ${answer?.status ?? "nothing"}` + ); + } + if ((await deps.hash(answer.body)) !== entry.sha256) { + throw new PinnedAbort(`/@/${path} resolved to bytes that don't match its file list`); + } + if (entry.store && deps.store) deps.keep(deps.store.put(entry.sha256, answer.body)); + return respond(answer.body, entry.type, true); +} + +/** + * Wrap `fn` so concurrent calls with one key share a request, made with the first call's other + * arguments. A settled request is forgotten. + */ +export function sharedRequests( + fn: (key: string, ...args: A) => Promise +): (key: string, ...args: A) => Promise { + const inFlight = new Map>(); + return (key, ...args) => { + let request = inFlight.get(key); + if (!request) { + request = fn(key, ...args).finally(() => inFlight.delete(key)); + inFlight.set(key, request); + } + return request; + }; +} + +/** + * The handoff the service worker sends tabs and the automerge worker: `to` is a legacy special + * URL (`automerge:…`), or a `/@/` path, with the heads to read its file at, or a request for + * its container's file list. A path's `handoffURL` is `patchwork-path:` and the path, which + * resolvers from before paths parse and then decline. + */ +export function handoffRequest( + request: Request, + cachename: string, + to: URL | { path: string; heads?: string; via?: string[]; list?: boolean; only?: string } +): HandoffRequestMessage { + const target = + to instanceof URL + ? { handoffURL: to.href } + : { + handoffURL: `patchwork-path:${to.path}`, + path: to.path, + ...(to.heads ? { heads: to.heads } : {}), + ...(to.via?.length ? { via: to.via } : {}), + ...(to.list ? { list: true } : {}), + ...(to.only !== undefined ? { only: to.only } : {}), + }; + return { + id: crypto.randomUUID(), + type: "request", + cachename, + request: { + url: request.url, + ...target, + headers: Object.fromEntries(request.headers.entries()), + method: request.method, + destination: request.destination, + referrer: request.referrer, + }, + }; +} diff --git a/core/bootloader/src/resolver.ts b/core/bootloader/src/resolver.ts index 72e0f0395..476c38ca0 100644 --- a/core/bootloader/src/resolver.ts +++ b/core/bootloader/src/resolver.ts @@ -1,20 +1,35 @@ -// Turning a handoff request into a reply, given a Repo: find the automerge -// URL's document, resolve the path in it, and either write the response into -// the service worker's cache or hand it back inline. A tab answers the service -// worker with its own Repo this way (see tab-resolver.ts), and the automerge -// worker answers the broadcasts nobody else could (automerge-protocol-handler). +// Turning a handoff request into a reply, given a Repo: walk a `/@/` path, or +// find a legacy automerge URL's document and resolve the path in it, and hand +// the response back inline. What to keep is the service worker's call, never a +// resolver's. A tab answers the service worker with its own Repo this way (see +// tab-resolver.ts), and the automerge worker answers the broadcasts nobody else +// could (automerge-protocol-handler). import { hasHeads } from "@automerge/automerge/slim"; import { + decodeHeads, isValidAutomergeUrl, parseAutomergeUrl, stringifyAutomergeUrl, + type AutomergeUrl, type DocHandle, type Repo, type StorageId, + type UrlHeads, } from "@automerge/automerge-repo/slim"; import { + buildFileList, getDefaultBranch, + IncompleteFileListError, + parseHeads, + parsePath, + pinnedPath, + readNode, + readPath, resolvePath, + toBytes, + walkPath, + type ExtraProviders, + type WalkOptions, } from "@inkandswitch/patchwork-filesystem"; import type { @@ -24,8 +39,6 @@ import type { const RESOLVE_TIMEOUT_MS = 30_000; -const CACHEABLE_STATUSES = [200, 203, 204]; - const CATCH_UP_MS = 3_000; function waitForHeads( @@ -178,19 +191,203 @@ function impatience(limit: number) { ); } +/** + * The walk's `find`: wait for pinned heads to arrive, or for a live document to catch up with + * the sync server, within `signal`. Heads that never arrive are remembered in `missing`, so a + * walk that fails because of them answers with a network error rather than a 404. + */ +export function waitingFind( + repo: Repo, + signal: AbortSignal, + missing: Set, + { catchUp = true }: { catchUp?: boolean } = {} +): NonNullable { + return async (url: AutomergeUrl, heads?: UrlHeads) => { + const handle = await repo.find(url, { signal }); + if (heads) { + if (!(await waitForHeads(handle, decodeHeads(heads), signal))) { + missing.add(url); + throw new AbortHandoff(`heads ${heads.join(",")} of ${url} not found`); + } + } else if (catchUp) { + await caughtUpWithPeers(repo, handle, signal); + } + return handle; + }; +} + +/** + * A `/@/` path, answered inline: + * + * - `list`: the file list of the container the path names (canonical), as its bytes. + * - A live path, or a pinned-looking one that isn't canonical: a 307 to its pinned form, whose + * list covers just the file asked for; the body is that list, for the service worker to check + * and keep. + * - With `heads`: the file the path names, read at those heads (what its file list recorded), + * whether or not the path is canonical, with the nested folders on the way read at `via`. + * - A canonical pinned path: the bytes. + * - One that doesn't resolve: 404. + * + * Pinned heads that never arrive throw {@link AbortHandoff}. + */ +async function resolveWalkPath( + repo: Repo, + path: string, + accounts: ReadonlyMap, + signal: AbortSignal, + { + heads, + via, + list, + only, + }: { heads?: string; via?: string[]; list?: boolean; only?: string } = {}, + providers?: ExtraProviders +): Promise { + try { + parsePath(path); + } catch (error) { + return new Response(error instanceof Error ? error.message : "malformed path", { + status: 400, + }); + } + const missing = new Set(); + const options = { accounts, providers, find: waitingFind(repo, signal, missing) }; + const notFound = () => { + if (missing.size) { + throw new AbortHandoff(`heads not found for ${[...missing].join(", ")}`); + } + return new Response(`nothing at ${path}`, { + status: 404, + headers: { "cache-control": "no-store" }, + }); + }; + + if (list) { + const built = await buildFileList(repo, path, { + ...options, + only: only === undefined ? undefined : [only], + // Each file as this node has it, without waiting on peers for every one. + findFile: waitingFind(repo, signal, missing, { catchUp: false }), + }); + if (!built) return notFound(); + return new Response(built.bytes as BlobPart, { + status: 200, + headers: { "content-type": "application/json", "cache-control": "no-store" }, + }); + } + + if (heads) { + // A file a list recorded, read at the heads it recorded. A file in a + // package's nested folder is reached by name, through a link without + // heads, so the path needn't be canonical; the service worker checks the + // bytes against the list either way. + const read = await readPath(repo, path, { + ...options, + heads: parseHeads(heads), + via: (Array.isArray(via) ? via : []) + .map((step) => parseHeads(String(step))) + .filter((step): step is UrlHeads => !!step), + }); + if (!read) return notFound(); + return new Response(toBytes(read.content) as BlobPart, { + status: 200, + headers: { "content-type": read.type }, + }); + } + + const end = await walkPath(repo, path, options); + if (!end) return notFound(); + if (!end.canonical) { + let pinned: Awaited>; + try { + pinned = await pinnedPath(repo, path, { + ...options, + only: "end", + // Each file as this node has it, without waiting on peers for every one. + findFile: waitingFind(repo, signal, missing, { catchUp: false }), + }); + } catch (error) { + // The file isn't readable here yet: not a 404, which would be memoized. + if (error instanceof IncompleteFileListError) throw new AbortHandoff(error.message); + throw error; + } + if (!pinned) return notFound(); + return new Response((pinned.list ?? null) as BlobPart | null, { + status: 307, + headers: { + location: `/@/${pinned.path}`, + "content-type": "application/json", + "cache-control": "no-store", + }, + }); + } + + const read = await readNode(repo, end.node); + if (!read) return notFound(); + return new Response(toBytes(read.content) as BlobPart, { + status: 200, + headers: { "content-type": read.type }, + }); +} + export type Resolver = { /** - * Resolve `message`'s request and say how the service worker should answer - * it, having written the response into its cache if it's one to keep. - * Undefined if the special URL isn't an `automerge:` one. + * Resolve `message`'s request and hand the response back inline, for the + * service worker to serve and, if it chooses, keep. Undefined if it's neither + * a `/@/` path nor an `automerge:` URL. `signal` cuts the 30s allowance short. */ - answer(message: HandoffRequestMessage): Promise; + answer( + message: HandoffRequestMessage, + signal?: AbortSignal + ): Promise; }; export function createResolver( getRepo: () => Promise, - log: (...args: unknown[]) => void = () => {} + log: (...args: unknown[]) => void = () => {}, + accounts: () => ReadonlyMap = () => new Map(), + options: { + /** + * Decline (answer undefined) a path under an account this resolver + * doesn't know, so the service worker asks someone else. A tab learns its + * account only once it has loaded; the automerge worker, asked last, + * answers 404 instead. + */ + declineUnknownAccounts?: boolean; + /** + * Decline paths through `/packages`, leaving them to the automerge + * worker. A package lookup reads every file in the package, and a tab + * shouldn't hold them all. + */ + declinePackages?: boolean; + /** + * Providers for documents no built-in walks: a tab passes the ones packages register. The + * automerge worker never loads package code, so it passes none. + */ + providers?: ExtraProviders; + } = {} ): Resolver { + const throughPackages = (path: string) => { + try { + const [first] = parsePath(path).segments; + return first?.kind === "name" && first.name === "packages"; + } catch { + return false; + } + }; + const knowsAccountOf = (path: string) => { + try { + const { root } = parsePath(path); + return root.kind !== "account" || accounts().has(root.acctHash); + } catch { + return true; // answered with a 400 below + } + }; + const within = (signal?: AbortSignal) => + signal + ? AbortSignal.any([signal, AbortSignal.timeout(RESOLVE_TIMEOUT_MS)]) + : AbortSignal.timeout(RESOLVE_TIMEOUT_MS); + const resolveWithin = async (handoff: URL): Promise => { const repo = await getRepo(); const signal = AbortSignal.timeout(RESOLVE_TIMEOUT_MS); @@ -237,21 +434,48 @@ export function createResolver( response: { status, body, headers: { "content-type": "text/plain" } }, }) satisfies HandoffReplyMessage; - return { - async answer({ id, cachename, request }) { - let handoff: URL; - try { - handoff = new URL(request.handoffURL); - } catch { - console.error("couldn't parse handoff url", request); - return inline(id, 400, `couldn't parse a special url out of ${request.url}`); - } - if (handoff.protocol !== "automerge:") return undefined; + const reply = async (id: string, response: Response): Promise => ({ + id, + type: "response", + response: { + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: response.body ? new Uint8Array(await response.arrayBuffer()) : undefined, + }, + }); + return { + async answer({ id, request }, signal) { let response: Response; try { - log(`resolving handoff ${id} for ${handoff}`); - response = await resolveHandoff(handoff); + if (request.path !== undefined) { + if (options.declineUnknownAccounts && !knowsAccountOf(request.path)) { + return undefined; + } + if (options.declinePackages && throughPackages(request.path)) return undefined; + const repo = await getRepo(); + const limit = within(signal); + log(`resolving handoff ${id} for /@/${request.path}`); + response = await Promise.race([ + resolveWalkPath(repo, request.path, accounts(), limit, request, options.providers), + new Promise((_, reject) => + limit.addEventListener("abort", () => + reject(new AbortHandoff(`resolving /@/${request.path} timed out`)) + ) + ), + ]); + } else { + let handoff: URL; + try { + handoff = new URL(request.handoffURL); + } catch { + console.error("couldn't parse handoff url", request); + return inline(id, 400, `couldn't parse a special url out of ${request.url}`); + } + if (handoff.protocol !== "automerge:") return undefined; + log(`resolving handoff ${id} for ${handoff}`); + response = await resolveHandoff(handoff); + } } catch (error) { if (error instanceof AbortHandoff) { return { id, type: "abort", reason: error.message }; @@ -267,41 +491,10 @@ export function createResolver( } try { - const immutable = - CACHEABLE_STATUSES.includes(response.status) && - !/(^|,)\s*no-store(\s*,|$)/.test( - response.headers.get("cache-control") ?? "" - ); - if (!immutable) { - // Errors, redirects and the like go back inline for the service - // worker to serve directly. Mutable responses aren't cached because - // the same URL can resolve to different content later. - log(`responding inline to ${request.url} with ${response.status}`); - return { - id, - type: "response", - response: { - status: response.status, - headers: Object.fromEntries(response.headers.entries()), - body: response.body - ? new Uint8Array(await response.arrayBuffer()) - : undefined, - }, - }; - } - - // Reconstruct the request the service worker is holding so the entry - // matches its cache.match. `destination` isn't constructible but - // doesn't participate in cache matching. - const cacheKey = new Request(request.url, { - method: request.method, - headers: request.headers, - referrer: request.referrer, - }); - const cache = await caches.open(cachename); - await cache.put(cacheKey, response); - log(`cached ${cacheKey.url} in ${cachename}`); - return { id, type: "cached" }; + // Everything goes back inline. The service worker keeps what's + // immutable (see its serveHandoff), or checks it against a file list. + log(`responding inline to ${request.url} with ${response.status}`); + return await reply(id, response); } catch (error) { console.error(`failed to reply for ${request.url}`, error); return inline(id, 558, String(error)); diff --git a/core/bootloader/src/service-worker.ts b/core/bootloader/src/service-worker.ts index 4a4878598..4365a642c 100644 --- a/core/bootloader/src/service-worker.ts +++ b/core/bootloader/src/service-worker.ts @@ -6,13 +6,34 @@ import { type AutomergeWorkerMissingMessage, type HandoffReplyMessage, type HandoffRequestMessage, + type HandoffResponse, type ResolveAcceptedMessage, type ResolveRequestMessage, } from "./types.js"; +import { + contentHash, + toBytes, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; +import { + CONTENT_CACHE_PREFIX, + lazyCacheStore, + contentCacheName, +} from "./content-store.js"; +import { + PinnedAbort, + handoffRequest, + keepFileList, + normalizePinned, + servePinned, + sharedRequests, + type PinnedDeps, +} from "./pinned.js"; // Referenced so this script's bytes change with every build. The browser only // installs a replacement service worker when they do, and the controllerchange // that follows is how open tabs learn a deploy happened (see setup.ts). import { buildId } from "./build-id.js"; +import { ACCEPT_TIMEOUT_MS } from "./handoff-timing.js"; const DEFAULT_CACHE_NAME = "patchwork"; @@ -85,7 +106,11 @@ async function clearOtherCaches() { const names = await caches.keys(); await Promise.all( names - .filter((name) => name !== cachename) + // Content stores are per account and checked on every read; they + // outlive cache renames and deploys. + .filter( + (name) => name !== cachename && !name.startsWith(CONTENT_CACHE_PREFIX) + ) .map((name) => caches.delete(name)) ); } @@ -184,6 +209,11 @@ handoffChannel.addEventListener("message", (event) => { for (const { message } of stranded) handoffChannel.postMessage(message); }); +// File lists this worker holds, by list hash: ones that may not be stored, and +// ones read from the store. Lost when it stops; the store, or a resolver +// rebuilding a list from its pinned heads, brings them back. +const fileLists = new Map(); + const MISSING_WORKER_NUDGE_MS = 5_000; // A worker that's been spawned only asks for the lock once its imports have // evaluated, so on a cold start it looks missing for a while. @@ -224,11 +254,6 @@ async function startWorkerIfMissing(id: string) { /** Signals that respondWith should reject; see {@link HandoffAbortMessage}. */ class HandoffAborted extends Error {} -// How long a tab has to say it'll resolve a request before the service -// worker asks the next one. A tab that hasn't been handed its Repo yet says it -// can't at once, so this is for one that can't answer at all: a frozen -// background tab, or one from a build before tabs resolved. -const ACCEPT_TIMEOUT_MS = 250; // A tab posts its reply and then lets go of its lock, but the lock can be // granted here before the reply is delivered. const REPLY_AFTER_LOCK_MS = 100; @@ -324,22 +349,9 @@ function askTab( */ async function handoff( fetchEvent: FetchEvent, - handoffURL: URL + to: URL | { path: string; heads?: string; via?: string[]; list?: boolean; only?: string } ): Promise { - const request = fetchEvent.request; - const message: HandoffRequestMessage = { - id: crypto.randomUUID(), - type: "request", - cachename, - request: { - url: request.url, - handoffURL: handoffURL.href, - headers: Object.fromEntries(request.headers.entries()), - method: request.method, - destination: request.destination, - referrer: request.referrer, - }, - }; + const message = handoffRequest(fetchEvent.request, cachename, to); const deadline = Date.now() + HANDOFF_TIMEOUT_MS; for (const tab of await resolvingTabs(fetchEvent.clientId)) { const answer = await askTab(tab, message, deadline); @@ -479,10 +491,20 @@ async function serveHandoff( if (reply.type === "response") { log(`serving handed-off response for ${handoffURL}`, reply); - return new Response(reply.response.body ?? null, { - status: reply.response.status ?? 200, - headers: reply.response.headers, - }); + const response = inlineResponse(reply.response); + // Resolvers hand everything back inline; what's immutable is kept here, + // under the request, as resolvers used to keep it. Not cachePage: a + // navigation to a document in a doc must not replace the app shell's + // offline copy at / and /index.html. + if (isImmutable(response)) { + const copy = response.clone(); + fetchEvent.waitUntil( + cache.put(fetchEvent.request, copy).catch((error) => { + console.warn("error caching %s in %s", handoffURL, cachename, error); + }) + ); + } + return response; } const stored = await cache.match(fetchEvent.request); @@ -496,6 +518,139 @@ async function serveHandoff( return stored; } +const inlineResponse = (response: HandoffResponse) => + new Response(response.body ?? null, { + status: response.status ?? 200, + headers: response.headers, + }); + +const IMMUTABLE_STATUSES = [200, 203, 204]; + +function isImmutable(response: Response): boolean { + return ( + IMMUTABLE_STATUSES.includes(response.status) && + !/(^|,)\s*no-store(\s*,|$)/.test(response.headers.get("cache-control") ?? "") + ); +} + +/** + * A `/@/` path. A live one is handed off, and the redirect it answers with is + * served without its body, the file list the pinned URL names, which is kept + * once it checks out. A pinned one is served checked against its own list + * (pinned.ts). + */ +async function servePath(fetchEvent: FetchEvent, path: string): Promise { + const normalized = normalizePinned(path); + if (!normalized) { + return new Response(`not a path: /@/${path}`, { + status: 400, + headers: { "content-type": "text/plain" }, + }); + } + + const ask: Ask = async (to) => { + const reply = await handoff(fetchEvent, to); + if (reply.type === "abort") throw new HandoffAborted(reply.reason); + if (reply.type !== "response") { + return new Response(`unexpected ${reply.type} reply for /@/${to.path}`, { + status: 502, + }); + } + const { status = 200, body, headers } = reply.response; + if (status !== 200) return inlineResponse(reply.response); + return { + body: toBytes(body ?? new Uint8Array()), + contentType: headers?.["content-type"] ?? "application/octet-stream", + }; + }; + + const store = + normalized.acctHash && !normalized.share + ? lazyCacheStore(caches, contentCacheName(normalized.acctHash), self.location.origin) + : undefined; + const deps: PinnedDeps = { + lists: fileLists, + store, + hash: contentHash, + buildList: (prefix, only) => buildList(prefix, only, ask), + resolve: (target, heads, via) => ask({ path: target, heads, via }), + warn: (message) => lifecycle("warn", message), + keep: (work) => + fetchEvent.waitUntil( + work.catch((error) => { + // Always loud, as for the page cache: a QuotaExceededError here + // is the first sign the origin is under storage pressure. + console.warn("error keeping /@/%s", path, error); + }) + ), + }; + + try { + const response = await servePinned(normalized, deps); + if (response.status === 307) return keepRedirectedList(response, deps); + return response; + } catch (error) { + // Deliberate: fail the request as a network error, with no response. + if (error instanceof PinnedAbort || error instanceof HandoffAborted) { + lifecycle("warn", `failing /@/${path}: ${(error as Error).message}`); + throw error; + } + console.error("service worker error serving /@/%s", path, error); + // The stack is in the console above; the page gets the message only. + return new Response(error instanceof Error ? error.message : "service worker error", { + status: 556, + headers: { "content-type": "text/plain" }, + }); + } +} + +/** A handoff's reply, as bytes, or a Response for anything else. */ +type Ask = (to: { + path: string; + heads?: string; + via?: string[]; + list?: boolean; + only?: string; +}) => Promise<{ body: Uint8Array; contentType: string } | Response>; + +// Concurrent requests for files of one package share a rebuild of its list. +const sharedListBuilds = sharedRequests( + async ( + _key: string, + ask: Ask, + prefix: string, + only: string | undefined + ): Promise => { + const answer = await ask({ path: prefix, list: true, only }); + return answer instanceof Response ? undefined : answer.body; + } +); +const buildList = (prefix: string, only: string | undefined, ask: Ask) => + sharedListBuilds(`${prefix}\n${only ?? ""}`, ask, prefix, only); + +/** + * A resolver's redirect from a live path carries, as its body, the file list + * the pinned URL names. Keep it if it hashes to the `~listHash` in `location`, + * and serve the redirect without it. + */ +async function keepRedirectedList( + response: Response, + deps: PinnedDeps +): Promise { + const location = response.headers.get("location") ?? ""; + const listHash = normalizePinned(location.replace(/^\/@\//, ""))?.candidates.find( + (candidate) => candidate.listHash + )?.listHash; + const body = new Uint8Array(await response.arrayBuffer()); + if (listHash && body.length && !(await keepFileList(body, listHash, deps))) { + lifecycle("warn", `the file list with the redirect to ${location} isn't the one it names`); + } + return new Response(null, { + status: response.status, + headers: { location, "cache-control": "no-store" }, + }); +} + async function servePassthrough( fetchEvent: FetchEvent, cache: Cache, @@ -570,6 +725,15 @@ async function respond( self.addEventListener("fetch", (fetchEvent: FetchEvent) => { const request = fetchEvent.request; log("fetch event", request.url); + const url = new URL(request.url); + const ours = url.origin === self.location.origin; + if (request.method !== "GET") return; + + if (ours && url.pathname.startsWith("/@/")) { + fetchEvent.respondWith(servePath(fetchEvent, url.pathname.slice("/@/".length))); + return; + } + fetchEvent.respondWith(respond(fetchEvent, specialURLFor(request))); }); diff --git a/core/bootloader/src/setup.ts b/core/bootloader/src/setup.ts index 0b1eb1335..ac2febdd7 100644 --- a/core/bootloader/src/setup.ts +++ b/core/bootloader/src/setup.ts @@ -1,12 +1,18 @@ -import type { - NewBuildInfo, - SetupServiceWorkerOptions, - SetupServiceWorkerResult, +import { + AUTOMERGE_WORKER_PATH, + type RegisterAccountMessage, + type NewBuildInfo, + type SetupServiceWorkerOptions, + type SetupServiceWorkerResult, } from "./types.js"; import { buildId, isNewerBuild } from "./build-id.js"; import { promptForNewBuild } from "./new-build-prompt.js"; import debug from "debug"; -import { handleResolveRequest } from "./tab-resolver.js"; +import type { AutomergeUrl } from "@automerge/automerge-repo/slim"; +import { + handleResolveRequest, + registerAccount as resolveAccountInTab, +} from "./tab-resolver.js"; import { forwardWorkerConsole, lifecycleLog, @@ -15,6 +21,7 @@ import { export { lifecycleLog }; export { resolveHandoffsWith } from "./tab-resolver.js"; +export { clearAccountStore } from "./content-store.js"; export { connectClassicSync } from "./classic-sync.js"; const serviceWorkerDebugging = debug.enabled("patchwork:serviceworker"); @@ -107,8 +114,23 @@ function serviceWorkerBuild(sw: ServiceWorker): Promise { // port closes, or when the Web Lock it holds for its lifetime is released (see // AUTOMERGE_WORKER_LOCK). -let automergeProtocolHandlerWorkerPath = - "/automerge-protocol-handler-worker.js"; +let automergeProtocolHandlerWorkerPath = AUTOMERGE_WORKER_PATH; + +const registeredAccounts = new Set(); + +/** + * Let `/@//…` paths under this account resolve: in this tab, and in + * the automerge worker, which is told again whenever it's replaced. Returns the + * account's acctHash. + */ +export async function registerAccount(url: AutomergeUrl): Promise { + registeredAccounts.add(url); + automergeProtocolHandlerWorker.post({ + type: "register-account", + url, + } satisfies RegisterAccountMessage); + return resolveAccountInTab(url); +} const automergeProtocolHandlerWorker = sharedWorkerHandle( "patchwork-automerge-protocol-handler", @@ -118,6 +140,15 @@ const automergeProtocolHandlerWorker = sharedWorkerHandle( onMessage(event) { forwardWorkerConsole("automerge-protocol-handler-worker", event.data); }, + // A new instance knows no accounts until it's told again. + onSpawn(worker) { + for (const url of registeredAccounts) { + worker.port.postMessage({ + type: "register-account", + url, + } satisfies RegisterAccountMessage); + } + }, onRestarting(message) { // The worker restarts because a newer tab connected. If that build is // newer than ours too, we're the stale one. diff --git a/core/bootloader/src/tab-resolver.ts b/core/bootloader/src/tab-resolver.ts index c0f6ec0af..28851d5b9 100644 --- a/core/bootloader/src/tab-resolver.ts +++ b/core/bootloader/src/tab-resolver.ts @@ -2,7 +2,13 @@ // service worker asks the tab that made a request first, then any other tab, // and only broadcasts to the automerge worker when none of them can; a tab // that hasn't been handed its Repo yet says so straight away. -import type { Repo } from "@automerge/automerge-repo/slim"; +import { + parseAutomergeUrl, + type AutomergeUrl, + type Repo, +} from "@automerge/automerge-repo/slim"; +import { idhash } from "@inkandswitch/patchwork-filesystem"; +import { packagePathProviders } from "@inkandswitch/patchwork-plugins"; import { createResolver, type Resolver } from "./resolver.js"; import type { @@ -14,13 +20,31 @@ import type { let resolver: Resolver | undefined; +/** acctHash → account url: the accounts whose `/@/` paths this tab resolves. */ +const accounts = new Map(); + /** - * Resolve the service worker's `automerge:` requests with `repo` from now on, - * rather than leaving them to the automerge worker. What they load stays in - * `repo`. + * Resolve the service worker's `automerge:` requests and `/@/` paths with + * `repo` from now on, rather than leaving them to the automerge worker. What + * they load stays in `repo`. */ export function resolveHandoffsWith(repo: Repo): void { - resolver = createResolver(() => Promise.resolve(repo)); + resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + declineUnknownAccounts: true, + declinePackages: true, + // Package code runs in tabs, so tabs walk through the providers packages register. + providers: packagePathProviders, + }); +} + +/** + * Resolve `/@/` paths rooted at this account in this tab. Returns its acctHash, + * computed here from the url, so nothing can name an account by the wrong hash. + */ +export async function registerAccount(url: AutomergeUrl): Promise { + const acctHash = await idhash(parseAutomergeUrl(url).documentId); + accounts.set(acctHash, url); + return acctHash; } /** Answer a ResolveRequestMessage; false if `event` isn't one. */ diff --git a/core/bootloader/src/types.ts b/core/bootloader/src/types.ts index 0ddc6d4ec..59bfa8656 100644 --- a/core/bootloader/src/types.ts +++ b/core/bootloader/src/types.ts @@ -40,8 +40,26 @@ export interface HandoffRequest { * `https://…/automerge%3Aabc/…` form) — the cache key. */ url: string; - /** the decoded special URL, e.g. `automerge:abc/some/path` */ + /** + * the decoded special URL, e.g. `automerge:abc/some/path`. For a `/@/` path + * it's `patchwork-path:` and the path, which older resolvers parse and then + * decline, since it isn't `automerge:`. + */ handoffURL: string; + /** + * For a `/@/` URL, the path after `/@/`: live (answered with a redirect to + * its pinned form, whose body is the file list it names) or pinned (answered + * with its bytes, inline). + */ + path?: string; + /** Read the file `path` names at these heads, as its file list recorded. */ + heads?: string; + /** With `heads`: the heads its list recorded for the nested folders on the way. */ + via?: string[]; + /** Answer the file list of the container `path` names, not a file. */ + list?: boolean; + /** With `list`: list only this tail, as the one-file list a redirect hands out. */ + only?: string; /** * @deprecated A briefly-deployed shape put the special URL in `url` and * the cache key here. Only read, never sent. @@ -193,3 +211,15 @@ export type SetupServiceWorkerResult = { shared?: SharedWorker; kill?: () => void; }; + +/** + * Where the automerge worker's script is served, by default. The vite plugin keeps this name + * from one build to the next, since it's started by URL. + */ +export const AUTOMERGE_WORKER_PATH = "/automerge-protocol-handler-worker.js"; + +/** Tab → automerge worker: resolve paths under this account. */ +export interface RegisterAccountMessage { + type: "register-account"; + url: string; +} diff --git a/core/bootloader/test/content-store.test.ts b/core/bootloader/test/content-store.test.ts new file mode 100644 index 000000000..644d95f90 --- /dev/null +++ b/core/bootloader/test/content-store.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect } from "vitest"; +import { cacheStore, lazyCacheStore } from "../src/content-store.js"; +import { contentHash, toBytes } from "@inkandswitch/patchwork-filesystem"; + +/** Enough of Cache for the store: put, match and delete by url. */ +export class FakeCache { + entries = new Map(); + async put(request: RequestInfo | URL, response: Response) { + this.entries.set(new Request(request).url, response.clone()); + } + async match(request: RequestInfo | URL) { + return this.entries.get(new Request(request).url)?.clone(); + } + async delete(request: RequestInfo | URL) { + return this.entries.delete(new Request(request).url); + } +} + +describe("contentHash", () => { + it("is base64url sha-256 without padding", async () => { + // sha256("") = e3b0c442…b855 + expect(await contentHash(new Uint8Array())).toBe( + "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" + ); + }); + + it("hashes strings as UTF-8", async () => { + expect(await contentHash(toBytes("héllo ✓"))).toBe( + await contentHash(new TextEncoder().encode("héllo ✓")) + ); + }); +}); + +describe("cacheStore", () => { + it("round-trips bytes under /.content/", async () => { + const cache = new FakeCache(); + const store = cacheStore(cache as unknown as Cache, "https://p.test"); + const bytes = toBytes("export default 1"); + const hash = await contentHash(bytes); + await store.put(hash, bytes); + expect([...cache.entries.keys()]).toEqual([`https://p.test/.content/${hash}`]); + expect(await store.get(hash)).toEqual(bytes); + await store.delete(hash); + expect(await store.get(hash)).toBeUndefined(); + }); +}); + +describe("lazyCacheStore", () => { + /** Enough of CacheStorage: has, and open (which creates). */ + class FakeStorage { + caches = new Map(); + async has(name: string) { + return this.caches.has(name); + } + async open(name: string) { + if (!this.caches.has(name)) this.caches.set(name, new FakeCache()); + return this.caches.get(name)! as unknown as Cache; + } + } + + it("creates no cache until something is stored", async () => { + const storage = new FakeStorage(); + const store = lazyCacheStore(storage as unknown as CacheStorage, "patchwork-content-x", "https://p.test"); + expect(await store.get("h")).toBeUndefined(); + await store.delete("h"); + expect(storage.caches.size).toBe(0); + const bytes = toBytes("x"); + await store.put("h", bytes); + expect([...storage.caches.keys()]).toEqual(["patchwork-content-x"]); + expect(await store.get("h")).toEqual(bytes); + }); +}); diff --git a/core/bootloader/test/pinned.test.ts b/core/bootloader/test/pinned.test.ts new file mode 100644 index 000000000..527f52d94 --- /dev/null +++ b/core/bootloader/test/pinned.test.ts @@ -0,0 +1,308 @@ +import { describe, it, expect, vi } from "vitest"; +import { + canonicalJson, + contentHash, + toBytes, + type FileList, +} from "@inkandswitch/patchwork-filesystem"; +import type { ContentStore } from "../src/content-store.js"; +import { + PinnedAbort, + handoffRequest, + keepFileList, + normalizePinned, + servePinned, + sharedRequests, + type PinnedDeps, +} from "../src/pinned.js"; + +const candidatesOf = (path: string) => normalizePinned(path)!.candidates; + +describe("normalizePinned", () => { + it("gives every spelling of a file one candidate", () => { + const [canonical] = candidatesOf("acct@h/x@h~L/a%40b.js"); + expect(canonical).toEqual({ prefix: "acct@h/x@h", tail: "a%40b.js", listHash: "L" }); + for (const spelling of ["acct@h/x@h~L/a@b.js", "/acct@h//x@h~L/a%40b.js/"]) { + expect(candidatesOf(spelling)).toContainEqual(canonical); + } + expect(candidatesOf("acct@h/x@h~L/日本")).toEqual(candidatesOf("acct@h/x@h~L/%E6%97%A5%E6%9C%AC")); + }); + + it("tries the latest doc step first, then each earlier one", () => { + expect(candidatesOf("acct@h/packages/m@h/f@h~L/vendor/react@18.3.1/index.js")).toEqual([ + { prefix: "acct@h/packages/m@h/f@h~L/vendor/react@18.3.1", tail: "index.js" }, + { prefix: "acct@h/packages/m@h/f@h", tail: "vendor/react%4018.3.1/index.js", listHash: "L" }, + { prefix: "acct@h/packages/m@h", tail: "f%40h~L/vendor/react%4018.3.1/index.js" }, + ]); + }); + + it("says a live path is live", () => { + expect(normalizePinned("acct/Projects")).toMatchObject({ pinned: false, acctHash: "acct" }); + }); + + it("reads a share root, and a doc-step-shaped name under it", () => { + const id = "2j9knpCseyhnK8izDmLpGP5WMdZQ"; + const n = normalizePinned(`automerge%3A${id}@h~L/react@18.3.1/x.js`)!; + expect(n).toMatchObject({ pinned: true, share: true, acctHash: undefined }); + expect(n.candidates).toContainEqual({ + prefix: `automerge:${id}@h`, + tail: "react%4018.3.1/x.js", + listHash: "L", + }); + }); + + it("refuses a malformed escape or a bad root", () => { + expect(normalizePinned("acct@h/%zz")).toBeUndefined(); + expect(normalizePinned("")).toBeUndefined(); + expect(normalizePinned("not-base58!/x")).toBeUndefined(); + }); +}); + +class MemoryStore implements ContentStore { + bytes = new Map(); + async get(hash: string) { + return this.bytes.get(hash); + } + async put(hash: string, bytes: Uint8Array) { + this.bytes.set(hash, bytes); + } + async delete(hash: string) { + this.bytes.delete(hash); + } +} + +const REAL = toBytes("export default 42"); +const FORGED = toBytes("export default 666"); + +async function setup(options: { store?: boolean; via?: string[] } = {}) { + const sha256 = await contentHash(REAL); + const list: FileList = { + v: 1, + files: { + "index.js": { + sha256, + type: "text/javascript", + heads: "fh", + ...(options.via ? { via: options.via } : {}), + store: options.store ?? true, + }, + }, + }; + const listBytes = new TextEncoder().encode(canonicalJson(list)); + const listHash = await contentHash(listBytes); + const path = `acct@h/pkg@h~${listHash}/index.js`; + const kept: Promise[] = []; + const deps = { + lists: new Map(), + store: new MemoryStore(), + hash: contentHash, + buildList: vi.fn( + async (_prefix: string, _only?: string) => listBytes as Uint8Array | undefined + ), + resolve: vi.fn(async (_path: string, _heads?: string, _via?: string[]) => ({ + body: REAL, + contentType: "text/javascript", + }) as { body: Uint8Array; contentType: string } | Response | undefined), + warn: vi.fn(), + keep: (work: Promise) => void kept.push(work), + } satisfies PinnedDeps; + return { + deps, + sha256, + list, + listBytes, + listHash, + normalized: normalizePinned(path)!, + settle: () => Promise.all(kept), + }; +} + +const text = (response: Response) => response.text(); + +describe("servePinned", () => { + it("serves a file whose list and bytes are in the store, asking no resolver", async () => { + const { deps, sha256, listBytes, listHash, normalized } = await setup(); + await deps.store.put(listHash, listBytes); + await deps.store.put(sha256, REAL); + const response = await servePinned(normalized, deps); + expect(await text(response)).toBe("export default 42"); + expect(response.headers.get("content-type")).toBe("text/javascript"); + expect(response.headers.get("cache-control")).toContain("immutable"); + expect(deps.resolve).not.toHaveBeenCalled(); + expect(deps.buildList).not.toHaveBeenCalled(); + }); + + it("deletes a forged list, rebuilds it, and serves", async () => { + const { deps, listHash, listBytes, normalized, settle } = await setup(); + await deps.store.put(listHash, toBytes('{"v":1,"files":{}}')); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + await settle(); + expect(deps.warn).toHaveBeenCalled(); + expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h", "index.js"); + expect(deps.store.bytes.get(listHash)).toEqual(listBytes); + }); + + it("rebuilds a one-file list, as a redirect hands out, before a whole one", async () => { + const { deps, listBytes, normalized } = await setup(); + deps.buildList.mockImplementation(async (_prefix: string, only?: string) => + only === "index.js" ? listBytes : toBytes('{"files":{},"v":1}') + ); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.buildList).toHaveBeenCalledTimes(1); + expect(deps.buildList).toHaveBeenCalledWith("acct@h/pkg@h", "index.js"); + }); + + it("falls back to the whole list when the one-file rebuild doesn't match", async () => { + const { deps, listBytes, normalized } = await setup(); + deps.buildList.mockImplementation(async (_prefix: string, only?: string) => + only === undefined ? listBytes : toBytes('{"files":{},"v":1}') + ); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.buildList).toHaveBeenLastCalledWith("acct@h/pkg@h", undefined); + }); + + it("fails when the rebuilt list doesn't match the URL", async () => { + const { deps, normalized } = await setup(); + deps.buildList.mockResolvedValue(toBytes('{"files":{},"v":1}')); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); + }); + + it("rereads a file in a nested folder with the heads its list recorded on the way", async () => { + const { deps, normalized } = await setup({ via: ["sub"] }); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh", ["sub"]); + }); + + it("deletes a forged file, rereads it at the list's heads, and serves", async () => { + const { deps, sha256, normalized, settle } = await setup(); + await deps.store.put(sha256, FORGED); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + await settle(); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", "fh", undefined); + expect(deps.warn).toHaveBeenCalled(); + expect(deps.store.bytes.get(sha256)).toEqual(REAL); + }); + + it("fails on a resolver's bytes that don't match, storing nothing", async () => { + const { deps, sha256, normalized, settle } = await setup(); + deps.resolve.mockResolvedValue({ body: FORGED, contentType: "text/javascript" }); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); + await settle(); + expect(deps.store.bytes.has(sha256)).toBe(false); + }); + + it("fails when a resolver answers anything but the bytes", async () => { + const { deps, normalized } = await setup(); + deps.resolve.mockResolvedValue(new Response("nope", { status: 404 })); + await expect(servePinned(normalized, deps)).rejects.toBeInstanceOf(PinnedAbort); + }); + + it("answers 404 for a tail the list doesn't cover", async () => { + const { deps, listHash } = await setup(); + const response = await servePinned(normalizePinned(`acct@h/pkg@h~${listHash}/other.js`)!, deps); + expect(response.status).toBe(404); + expect(deps.resolve).not.toHaveBeenCalled(); + }); + + it("answers 404 for a tail named like an Object.prototype member", async () => { + const { deps, listHash } = await setup(); + for (const tail of ["constructor", "toString", "__proto__"]) { + const response = await servePinned(normalizePinned(`acct@h/pkg@h~${listHash}/${tail}`)!, deps); + expect(response.status, tail).toBe(404); + } + expect(deps.resolve).not.toHaveBeenCalled(); + }); + + it("never stores bytes the list says not to", async () => { + const { deps, sha256, normalized, settle } = await setup({ store: false }); + expect(await text(await servePinned(normalized, deps))).toBe("export default 42"); + await settle(); + expect(deps.store.bytes.has(sha256)).toBe(false); + }); + + it("reads and writes no store for a share link", async () => { + const { deps, normalized } = await setup(); + const response = await servePinned(normalized, { ...deps, store: undefined }); + expect(await text(response)).toBe("export default 42"); + }); + + it("serves a path with no list live and unchecked", async () => { + const { deps } = await setup(); + const response = await servePinned(normalizePinned("acct@h/pkg@h/index.js")!, deps); + expect(await text(response)).toBe("export default 42"); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(deps.resolve).toHaveBeenCalledWith("acct@h/pkg@h/index.js", undefined); + }); + + it("passes a resolver's redirect through for a path with no list", async () => { + const { deps } = await setup(); + deps.resolve.mockResolvedValue( + new Response(null, { status: 307, headers: { location: "/@/acct@h/pkg@h~L" } }) + ); + const response = await servePinned(normalizePinned("acct@h/pkg@h")!, deps); + expect(response.status).toBe(307); + }); +}); + +describe("keepFileList", () => { + it("keeps a list in memory, and stores it only if every entry may be stored", async () => { + const stored = await setup(); + expect(await keepFileList(stored.listBytes, stored.listHash, stored.deps)).toBe(true); + await stored.settle(); + expect(stored.deps.lists.get(stored.listHash)).toEqual(stored.list); + expect(stored.deps.store.bytes.has(stored.listHash)).toBe(true); + + const memory = await setup({ store: false }); + expect(await keepFileList(memory.listBytes, memory.listHash, memory.deps)).toBe(true); + await memory.settle(); + expect(memory.deps.lists.has(memory.listHash)).toBe(true); + expect(memory.deps.store.bytes.has(memory.listHash)).toBe(false); + }); + + it("refuses a list that doesn't hash to its name", async () => { + const { deps, listBytes } = await setup(); + expect(await keepFileList(listBytes, "not-its-hash", deps)).toBe(false); + expect(deps.lists.size).toBe(0); + }); +}); + +describe("sharedRequests", () => { + it("shares one request among concurrent calls for a key, and forgets it once settled", async () => { + let release!: () => void; + const gate = new Promise((resolve) => (release = resolve)); + const fn = vi.fn(async (key: string) => (await gate, key)); + const shared = sharedRequests(fn); + const all = [shared("a"), shared("a"), shared("a"), shared("b")]; + release(); + expect(await Promise.all(all)).toEqual(["a", "a", "a", "b"]); + expect(fn).toHaveBeenCalledTimes(2); + await shared("a"); + expect(fn).toHaveBeenCalledTimes(3); + }); +}); + +describe("handoffRequest", () => { + it("builds a path handoff that old resolvers decline", () => { + const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "patchwork", { + path: "acct/Projects", + }); + expect(m.request.path).toBe("acct/Projects"); + expect(new URL(m.request.handoffURL).protocol).not.toBe("automerge:"); + }); + + it("carries the heads to read at, or a request for the list", () => { + const req = new Request("https://p.test/@/x"); + expect(handoffRequest(req, "patchwork", { path: "a@h/x", heads: "fh" }).request.heads).toBe("fh"); + expect(handoffRequest(req, "patchwork", { path: "a@h", list: true }).request.list).toBe(true); + const nested = { path: "a@h/sub/x", heads: "fh", via: ["sh"] }; + expect(handoffRequest(req, "patchwork", nested).request.via).toEqual(["sh"]); + expect(handoffRequest(req, "patchwork", { path: "a@h/x", heads: "fh" }).request).not.toHaveProperty("via"); + }); + + it("builds a legacy handoff with no path", () => { + const special = new URL("automerge:abc/x.js"); + const m = handoffRequest(new Request("https://p.test/automerge%3Aabc/x.js"), "patchwork", special); + expect(m.request.handoffURL).toBe(special.href); + expect(m.request.path).toBeUndefined(); + }); +}); diff --git a/core/bootloader/test/resolver.test.ts b/core/bootloader/test/resolver.test.ts new file mode 100644 index 000000000..cc7a75d52 --- /dev/null +++ b/core/bootloader/test/resolver.test.ts @@ -0,0 +1,325 @@ +import { describe, it, expect } from "vitest"; +import { + Repo, + encodeHeads, + stringifyAutomergeUrl, + type AutomergeUrl, +} from "@automerge/automerge-repo"; +import { + contentHash, + formatHeads, + idhash, + livePath, + parseFileList, + pinnedPath, + walkPath, +} from "@inkandswitch/patchwork-filesystem"; +import { registerPlugins, unregisterPlugins } from "@inkandswitch/patchwork-plugins"; +import { createResolver } from "../src/resolver.js"; +import { registerAccount, resolveHandoffsWith, handleResolveRequest } from "../src/tab-resolver.js"; +import type { HandoffRequestMessage, HandoffResponseMessage } from "../src/types.js"; + +// No `caches` in node: a resolver that still wrote the cache would throw. + +const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }).url; +const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + +async function fixture() { + const repo = new Repo(); + const notes = file(repo, "hi"); + const foo = folder(repo, { notes }); + const root = folder(repo, { Projects: folder(repo, { foo: foo.url }).url }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + const resolver = createResolver(() => Promise.resolve(repo), undefined, () => accounts); + return { repo, a, accounts, account, resolver, notes }; +} + +/** A path's chain of doc steps, without file lists: what a resolver is asked to read. */ +const chain = async (repo: Repo, path: string, accounts: ReadonlyMap) => + (await walkPath(repo, path, { accounts }))!.pinned; + +const withRequest = ( + path: string, + extra: { heads?: string; list?: boolean; only?: string } +): HandoffRequestMessage => { + const message = pathRequest(path); + return { ...message, request: { ...message.request, ...extra } }; +}; + +const pathRequest = (path: string): HandoffRequestMessage => ({ + id: "1", + type: "request", + cachename: "patchwork", + request: { + url: `https://p.test/@/${path}`, + handoffURL: `patchwork-path:${path}`, + path, + headers: {}, + method: "GET", + destination: "script", + referrer: "", + }, +}); + +const inline = (reply: unknown) => (reply as HandoffResponseMessage).response; +const body = (reply: unknown) => { + const b = inline(reply).body; + return typeof b === "string" ? b : new TextDecoder().decode(b); +}; + +describe("resolver: /@/ paths", () => { + it("answers a live path with a 307 whose body is the list it names", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const live = livePath(a, ["Projects", "foo", "notes"]); + const reply = await resolver.answer(pathRequest(live)); + expect(reply?.type).toBe("response"); + expect(inline(reply).status).toBe(307); + const pinned = await pinnedPath(repo, live, { accounts, only: "end" }); + expect(inline(reply).headers?.location).toBe(`/@/${pinned!.path}`); + expect(inline(reply).headers?.["cache-control"]).toBe("no-store"); + const list = inline(reply).body as Uint8Array; + expect(pinned!.path).toContain(`~${await contentHash(list)}/notes`); + expect(Object.keys(parseFileList(list)!.files)).toEqual(["notes"]); + }); + + it("answers a container's full list", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const foo = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const reply = await resolver.answer(withRequest(foo, { list: true })); + expect(inline(reply).status).toBe(200); + const pinned = await pinnedPath(repo, foo, { accounts }); + expect(inline(reply).body).toEqual(pinned!.list); + }); + + it("answers a one-file list, the same a redirect hands out", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const live = livePath(a, ["Projects", "foo", "notes"]); + const foo = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const reply = await resolver.answer(withRequest(foo, { list: true, only: "notes" })); + const pinned = await pinnedPath(repo, live, { accounts, only: "end" }); + expect(inline(reply).body).toEqual(pinned!.list); + }); + + it("reads a file at given heads, after it changed", async () => { + const { repo, a, accounts, resolver, notes } = await fixture(); + const path = await chain(repo, livePath(a, ["Projects", "foo", "notes"]), accounts); + const handle = await repo.find<{ content: string }>(notes); + const heads = formatHeads(handle.heads()); + handle.change((d) => (d.content = "changed")); + expect(body(await resolver.answer(pathRequest(path)))).toBe("changed"); + expect(body(await resolver.answer(withRequest(path, { heads })))).toBe("hi"); + }); + + it("answers a pinned path with its bytes inline", async () => { + const { repo, a, accounts, resolver } = await fixture(); + const pinned = await chain(repo, livePath(a, ["Projects", "foo", "notes"]), accounts); + const reply = await resolver.answer(pathRequest(pinned)); + expect(inline(reply).status).toBe(200); + expect(inline(reply).headers?.["content-type"]).toBe("text/plain"); + expect(body(reply)).toBe("hi"); + }); + + it("redirects a pinned-looking path that isn't canonical", async () => { + const { repo, a, account, resolver } = await fixture(); + const heads = [...account.heads()].sort().join("."); + const reply = await resolver.answer(pathRequest(`${a}@${heads}/Projects`)); + expect(inline(reply).status).toBe(307); + void repo; + }); + + it("answers an unknown path with 404", async () => { + const { a, resolver } = await fixture(); + const reply = await resolver.answer(pathRequest(livePath(a, ["nope"]))); + expect(inline(reply).status).toBe(404); + }); + + it("aborts a pinned path whose heads never arrive", async () => { + const { a, resolver } = await fixture(); + const [unknown] = encodeHeads(["00".repeat(32)]); + const reply = await resolver.answer( + pathRequest(`${a}@${unknown}/Projects`), + AbortSignal.timeout(200) + ); + expect(reply?.type).toBe("abort"); + }); + + it("answers 404 for heads that can't be heads", async () => { + const { a, resolver } = await fixture(); + const reply = await resolver.answer(pathRequest(`${a}@3vQB7B6MrGQZaxCuFg4oh/Projects`)); + expect(inline(reply).status).toBe(404); + }); +}); + +describe("resolver: folder packages", () => { + async function folderPackage() { + const f = await fixture(); + const x = f.repo.create({ "@patchwork": { type: "file" }, content: "x", mimeType: "text/plain" }); + const sub = folder(f.repo, { "x.js": x.url }); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"index.js"}'), + sub: sub.url, + }); + const packages = folder(f.repo, { fpkg: pkg.url }); + f.account.change((d: any) => (d.namespace["/packages"] = [{ url: packages.url }])); + return { ...f, x }; + } + + it("redirects a live path to a folder with the folder's whole list", async () => { + const { a, resolver } = await folderPackage(); + const reply = await resolver.answer(pathRequest(livePath(a, ["packages", "fpkg"]))); + expect(inline(reply).status).toBe(307); + const list = parseFileList(inline(reply).body as Uint8Array)!; + expect(Object.keys(list.files)).toEqual(expect.arrayContaining(["package.json", "sub/x.js"])); + }); + + it("reads at heads through a folder reached by name", async () => { + const { repo, a, accounts, resolver, x } = await folderPackage(); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const heads = formatHeads(x.heads()); + x.change((d: any) => (d.content = "changed")); + const reply = await resolver.answer(withRequest(`${prefix}/sub/x.js`, { heads })); + expect(inline(reply).status).toBe(200); + expect(body(reply)).toBe("x"); + }); +}); + +describe("resolver: accounts it doesn't know", () => { + it("a tab declines, so another tab or the worker is asked", async () => { + const { repo, a } = await fixture(); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => new Map(), { + declineUnknownAccounts: true, + }); + expect(await tab.answer(pathRequest(livePath(a, ["Projects"])))).toBeUndefined(); + }); + + it("a tab leaves /packages to the automerge worker", async () => { + const { repo, a, accounts } = await fixture(); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + declineUnknownAccounts: true, + declinePackages: true, + }); + expect(await tab.answer(pathRequest(livePath(a, ["packages", "x"])))).toBeUndefined(); + expect(await tab.answer(pathRequest(livePath(a, ["Projects"])))).toBeDefined(); + }); + + it("the worker, asked last, answers 404", async () => { + const { repo, a } = await fixture(); + const worker = createResolver(() => Promise.resolve(repo), undefined, () => new Map()); + const reply = await worker.answer(pathRequest(livePath(a, ["Projects"]))); + expect(inline(reply).status).toBe(404); + }); +}); + +describe("resolver: legacy automerge URLs", () => { + it("answers inline, leaving caching to the service worker", async () => { + const { repo, resolver } = await fixture(); + const pkg = repo.create({ + "@patchwork": { type: "directory" }, + "index.js": { content: "export default 1", mimeType: "text/javascript" }, + }); + const url = stringifyAutomergeUrl({ documentId: pkg.documentId, heads: pkg.heads() }); + const reply = await resolver.answer({ + ...pathRequest(""), + request: { + url: `https://p.test/${encodeURIComponent(url)}/index.js`, + handoffURL: `${url}/index.js`, + headers: {}, + method: "GET", + destination: "script", + referrer: "", + }, + }); + expect(reply?.type).toBe("response"); + expect(inline(reply).status).toBe(200); + expect(inline(reply).headers?.["cache-control"]).toContain("immutable"); + expect(body(reply)).toBe("export default 1"); + }); +}); + +describe("registerAccount", () => { + it("makes the tab's resolver answer for that account", async () => { + const { repo, a, account } = await fixture(); + resolveHandoffsWith(repo); + expect(await registerAccount(account.url)).toBe(a); + + const { port1, port2 } = new MessageChannel(); + const replies: unknown[] = []; + port1.onmessage = (event) => replies.push(event.data); + const handled = handleResolveRequest( + new MessageEvent("message", { + data: { ...pathRequest(livePath(a, ["Projects"])), type: "resolve" }, + ports: [port2], + }) + ); + expect(handled).toBe(true); + await expect.poll(() => replies.length, { timeout: 5000 }).toBeGreaterThanOrEqual(2); + port1.close(); + expect(inline(replies.at(-1)).status).toBe(307); + }); +}); + +describe("package path providers", () => { + const PLUGIN_URL = "https://example.test/board.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [{ name: "child", link: doc.child }], + lookup: (doc: any, names: string[]) => + names[0] === "child" ? { child: { name: "child", link: doc.child }, consumed: 1 } : undefined, + }; + + async function boardFixture() { + const base = await fixture(); + const b = base.repo.create({ "@patchwork": { type: "board" }, child: file(base.repo, "c") }); + const root = await base.repo.find(base.account.doc()!.namespace["/"][0].url); + root.change((d: any) => d.docs.push({ name: "b", type: "board", url: b.url })); + return base; + } + + it("walk through the providers a resolver is given; the worker's has none", async () => { + const { repo, a, accounts } = await boardFixture(); + const path = livePath(a, ["b", "child"]); + const tab = createResolver(() => Promise.resolve(repo), undefined, () => accounts, { + providers: async (doc) => (board.matches(doc) ? board : undefined), + }); + expect(inline(await tab.answer(pathRequest(path))).status).toBe(307); + const worker = createResolver(() => Promise.resolve(repo), undefined, () => accounts); + expect(inline(await worker.answer(pathRequest(path))).status).toBe(404); + }); + + it("a tab's resolver walks through registered path-provider plugins", async () => { + const { repo, a, account } = await boardFixture(); + registerPlugins( + [{ id: "board", name: "board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + try { + resolveHandoffsWith(repo); + await registerAccount(account.url); + const { port1, port2 } = new MessageChannel(); + const replies: unknown[] = []; + port1.onmessage = (event) => replies.push(event.data); + handleResolveRequest( + new MessageEvent("message", { + data: { ...pathRequest(livePath(a, ["b", "child"])), type: "resolve" }, + ports: [port2], + }) + ); + await expect.poll(() => replies.length, { timeout: 5000 }).toBeGreaterThanOrEqual(2); + port1.close(); + expect(inline(replies.at(-1)).status).toBe(307); + } finally { + unregisterPlugins(PLUGIN_URL); + } + }); +}); diff --git a/core/bootloader/test/service-worker-harness.ts b/core/bootloader/test/service-worker-harness.ts new file mode 100644 index 000000000..b90fea5d0 --- /dev/null +++ b/core/bootloader/test/service-worker-harness.ts @@ -0,0 +1,174 @@ +// Runs the real service worker module in node against a fake global scope: +// `self` (listeners, clients, location), `caches` as in-memory maps, and node's +// own BroadcastChannel and navigator.locks. Tabs and the automerge worker are +// fake clients; give them real resolvers to test the whole round trip. +import { vi } from "vitest"; + +export const ORIGIN = "https://p.test"; + +/** Enough of Cache: put, match and delete, keyed by url. */ +export class FakeCache { + entries = new Map(); + async put(request: RequestInfo | URL, response: Response) { + this.entries.set(new Request(request).url, response.clone()); + } + async match(request: RequestInfo | URL) { + return this.entries.get(new Request(request).url)?.clone(); + } + async delete(request: RequestInfo | URL) { + return this.entries.delete(new Request(request).url); + } + async keys() { + return [...this.entries.keys()].map((url) => new Request(url)); + } +} + +export class FakeCaches { + named = new Map(); + async open(name: string) { + let cache = this.named.get(name); + if (!cache) this.named.set(name, (cache = new FakeCache())); + return cache as unknown as Cache; + } + async has(name: string) { + return this.named.has(name); + } + async delete(name: string) { + return this.named.delete(name); + } + async keys() { + return [...this.named.keys()]; + } + async match(request: RequestInfo | URL) { + for (const cache of this.named.values()) { + const hit = await cache.match(request); + if (hit) return hit; + } + return undefined; + } +} + +export type FakeClient = { + id: string; + type: "window" | "worker" | "sharedworker"; + url: string; + focused?: boolean; + visibilityState?: "visible" | "hidden"; + postMessage(message: unknown, transfer?: Transferable[]): void; +}; + +/** A fetch event the test can await: what `respondWith` was given, and every `waitUntil`. */ +export type FakeFetchEvent = { + request: Request; + clientId: string; + respondWith(response: Response | Promise): void; + waitUntil(work: Promise): void; + /** The response, or undefined if the worker let the request through. */ + response(): Promise | undefined; + /** Every waitUntil, settled. */ + settled(): Promise; +}; + +export type ServiceWorkerHarness = { + caches: FakeCaches; + clients: Map; + /** Dispatch a fetch event to the worker. */ + fetch(request: Request, clientId?: string): FakeFetchEvent; + /** Dispatch a lifecycle event (install, activate) and wait for its waitUntil. */ + lifecycle(type: "install" | "activate"): Promise; + close(): void; +}; + +/** + * A request the platform builds but scripts can't: a navigation. `mode` and + * `destination` are read-only getters on Request.prototype, so shadow them. + */ +export function navigation(url: string): Request { + const request = new Request(url); + Object.defineProperty(request, "mode", { value: "navigate" }); + Object.defineProperty(request, "destination", { value: "document" }); + return request; +} + +/** Install the fake scope and import a fresh copy of the service worker module. */ +export async function startServiceWorker(): Promise { + const listeners = new Map void>>(); + const clients = new Map(); + const caches = new FakeCaches(); + + // Every channel the module opens, so a test's service worker doesn't hear + // the next test's traffic. + const channels: BroadcastChannel[] = []; + const RealBroadcastChannel = globalThis.BroadcastChannel; + class TrackedBroadcastChannel extends RealBroadcastChannel { + constructor(name: string) { + super(name); + channels.push(this); + } + } + + const scope = { + location: new URL(`${ORIGIN}/service-worker.js`), + registration: { scope: `${ORIGIN}/` }, + addEventListener(type: string, listener: (event: any) => void) { + listeners.set(type, [...(listeners.get(type) ?? []), listener]); + }, + skipWaiting: async () => {}, + clients: { + async get(id: string) { + return clients.get(id); + }, + async matchAll(options: { type?: string } = {}) { + return [...clients.values()].filter( + (client) => !options.type || options.type === "all" || client.type === options.type + ); + }, + async claim() {}, + }, + }; + + vi.stubGlobal("self", scope); + vi.stubGlobal("caches", caches); + vi.stubGlobal("BroadcastChannel", TrackedBroadcastChannel); + vi.resetModules(); + // A fake tab shares this event loop, so its "accepted" can queue behind its + // own resolving; under load that passes 250ms, the tab counts as silent, and + // the request waits out a broadcast nothing here answers. Tests that want a + // tab declined have it say so. + vi.doMock("../src/handoff-timing.js", () => ({ ACCEPT_TIMEOUT_MS: 10_000 })); + await import("../src/service-worker.js"); + // The module's own channels are open; the test's are the real class again. + vi.stubGlobal("BroadcastChannel", RealBroadcastChannel); + + const dispatch = (type: string, event: unknown) => { + for (const listener of listeners.get(type) ?? []) listener(event); + }; + + return { + caches, + clients, + fetch(request, clientId = "") { + let responded: Promise | undefined; + const waits: Promise[] = []; + const event: FakeFetchEvent = { + request, + clientId, + respondWith: (response) => void (responded = Promise.resolve(response)), + waitUntil: (work) => void waits.push(work), + response: () => responded, + settled: () => Promise.allSettled(waits), + }; + dispatch("fetch", event); + return event; + }, + async lifecycle(type) { + const waits: Promise[] = []; + dispatch(type, { waitUntil: (work: Promise) => waits.push(work) }); + await Promise.all(waits); + }, + close() { + for (const channel of channels) channel.close(); + vi.unstubAllGlobals(); + }, + }; +} diff --git a/core/bootloader/test/service-worker.test.ts b/core/bootloader/test/service-worker.test.ts new file mode 100644 index 000000000..df83fa8b1 --- /dev/null +++ b/core/bootloader/test/service-worker.test.ts @@ -0,0 +1,242 @@ +import { afterEach, describe, it, expect, onTestFinished } from "vitest"; +import { createResolver } from "../src/resolver.js"; +import { HANDOFF_CHANNEL } from "../src/types.js"; +import { Repo, stringifyAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; +import { contentHash, livePath, toBytes } from "@inkandswitch/patchwork-filesystem"; +import { + handleResolveRequest, + registerAccount, + resolveHandoffsWith, +} from "../src/tab-resolver.js"; +import { + ORIGIN, + navigation, + startServiceWorker, + type ServiceWorkerHarness, +} from "./service-worker-harness.js"; + +let sw: ServiceWorkerHarness | undefined; +afterEach(() => { + sw?.close(); + sw = undefined; +}); + +/** A tab that answers the service worker with `repo`, as tab-resolver.ts does in a page. */ +function addTab(harness: ServiceWorkerHarness, repo: Repo, id = "tab") { + resolveHandoffsWith(repo); + harness.clients.set(id, { + id, + type: "window", + url: `${ORIGIN}/`, + focused: true, + visibilityState: "visible", + postMessage(message, transfer) { + handleResolveRequest( + new MessageEvent("message", { + data: message, + ports: (transfer ?? []) as MessagePort[], + }) + ); + }, + }); +} + +describe("service worker: legacy automerge URLs", () => { + it("keeps an immutable navigation under its own URL, not over the app shell", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const doc = repo.create({ + "@patchwork": { type: "directory" }, + "index.html": { content: "

a doc

", mimeType: "text/html" }, + }); + const url = stringifyAutomergeUrl({ documentId: doc.documentId, heads: doc.heads() }); + const request = navigation(`${ORIGIN}/${encodeURIComponent(url)}/index.html`); + + const event = sw.fetch(request, "tab"); + const response = await event.response()!; + expect(await response.text()).toBe("

a doc

"); + await event.settled(); + + const kept = [...(sw.caches.named.get("patchwork")?.entries.keys() ?? [])]; + expect(kept).toEqual([request.url]); + }); +}); + +/** An account whose / is a folder Projects/foo holding one file, `notes`. */ +async function account(repo: Repo) { + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + const folder = (entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + const notes = file("hi"); + const foo = folder({ notes: notes.url }); + const root = folder({ Projects: folder({ foo: foo.url }).url }); + const acct = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await registerAccount(acct.url); + return { a, notes }; +} + +/** Follow a live /@/ URL's redirect, as fetch() would: the pinned URL it names. */ +async function pin(harness: ServiceWorkerHarness, live: string) { + const response = await harness.fetch(new Request(`${ORIGIN}/@/${live}`), "tab").response()!; + expect(response.status).toBe(307); + expect(response.body).toBeNull(); + return `${ORIGIN}${response.headers.get("location")}`; +} + +describe("service worker: /@/ paths", () => { + it("redirects a live path, then serves the pinned file from its list", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + const event = sw.fetch(new Request(pinned), "tab"); + const response = await event.response()!; + expect(await response.text()).toBe("hi"); + expect(response.headers.get("cache-control")).toContain("immutable"); + await event.settled(); + // the list and the file's bytes, each under its own hash + const stored = sw.caches.named.get(`patchwork-content-${a}`)!; + expect(stored.entries.size).toBe(2); + expect(await stored.match(`${ORIGIN}/.content/${await contentHash(toBytes("hi"))}`)).toBeDefined(); + }); + + it("serves a folder package's nested files through its pinned URL", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/javascript" }).url; + const folder = (entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }).url; + const pkg = folder({ + "package.json": file('{"main":"index.js"}'), + "index.js": file("export * from './sub/x.js'"), + sub: folder({ "x.js": file("export const x = 1") }), + }); + const acct = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: folder({ fpkg: pkg }) }] }, + }); + const a = await registerAccount(acct.url); + // Tabs leave /packages to the automerge worker, which answers broadcasts. + const worker = createResolver(() => Promise.resolve(repo), undefined, () => new Map([[a, acct.url]])); + const channel = new BroadcastChannel(HANDOFF_CHANNEL); + channel.onmessage = async (event) => { + if (event.data?.type !== "request") return; + const reply = await worker.answer(event.data); + if (reply) channel.postMessage(reply); + }; + onTestFinished(() => channel.close()); + + const pinned = await pin(sw, livePath(a, ["packages", "fpkg"])); + for (const [tail, content] of [ + ["package.json", '{"main":"index.js"}'], + ["sub/x.js", "export const x = 1"], + ]) { + const event = sw.fetch(new Request(`${pinned}/${tail}`), "tab"); + const response = await event.response()!; + expect(await response.text()).toBe(content); + expect(response.headers.get("cache-control")).toContain("immutable"); + await event.settled(); + } + // the list, and the two files read + expect(sw.caches.named.get(`patchwork-content-${a}`)!.entries.size).toBe(3); + }); + + it("serves the old bytes for a pinned URL after the file changed", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a, notes } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + notes.change((d: any) => (d.content = "changed")); + // nothing stored: the bytes come from a resolver, read at the heads the list recorded + await sw.caches.delete(`patchwork-content-${a}`); + expect(await (await sw.fetch(new Request(pinned), "tab").response()!).text()).toBe("hi"); + }); + + it("serves a redirect's pinned URL again after the service worker restarts", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + // a fresh service worker: no lists in memory, and nothing stored + sw.close(); + sw = await startServiceWorker(); + addTab(sw, repo); + expect(await (await sw.fetch(new Request(pinned), "tab").response()!).text()).toBe("hi"); + }); + + it("never serves a forged store entry", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + const first = sw.fetch(new Request(pinned), "tab"); + await (await first.response()!).text(); + await first.settled(); + + const key = `${ORIGIN}/.content/${await contentHash(toBytes("hi"))}`; + const store = sw.caches.named.get(`patchwork-content-${a}`)!; + await store.put(key, new Response("forged")); + const again = sw.fetch(new Request(pinned), "tab"); + expect(await (await again.response()!).text()).toBe("hi"); + await again.settled(); + expect(await (await store.match(key))!.text()).toBe("hi"); + }); + + it("fails, rather than serving, when a tab answers with other bytes", async () => { + sw = await startServiceWorker(); + const repo = new Repo(); + addTab(sw, repo); + const { a } = await account(repo); + const pinned = await pin(sw, livePath(a, ["Projects", "foo", "notes"])); + + // a tab that answers everything with bytes of its own + sw.clients.set("tab", { + id: "tab", + type: "window", + url: `${ORIGIN}/`, + focused: true, + visibilityState: "visible", + postMessage(message: any, transfer) { + const [port] = (transfer ?? []) as MessagePort[]; + if (!port) return; // not a request to resolve + port.postMessage({ type: "accepted" }); + port.postMessage({ + id: message.id, + type: "response", + response: { status: 200, body: toBytes("forged"), headers: { "content-type": "text/plain" } }, + }); + }, + }); + await expect(sw.fetch(new Request(pinned), "tab").response()!).rejects.toThrow(); + }); +}); + +describe("service worker: lifecycle", () => { + it("keeps content stores when it clears old caches", async () => { + sw = await startServiceWorker(); + await sw.caches.open("patchwork-content-acct"); + await sw.caches.open("some-old-cache"); + await sw.lifecycle("activate"); + expect((await sw.caches.keys()).sort()).toEqual(["patchwork", "patchwork-content-acct"]); + }); +}); diff --git a/core/bootloader/test/setup.ts b/core/bootloader/test/setup.ts new file mode 100644 index 000000000..c8dc9e97e --- /dev/null +++ b/core/bootloader/test/setup.ts @@ -0,0 +1,10 @@ +// Initialize both Wasm modules before any test runs. +// automerge-repo@subduction.9 always creates a SubductionSource, +// which imports from @automerge/automerge-subduction/slim — the +// Wasm must be initialized first. +// +// Importing the fat entry points auto-calls initSync / UseApi(). +// The vitest.config.ts resolve aliases ensure a single copy is used +// even when automerge-repo is linked locally. +import "@automerge/automerge"; +import "@automerge/automerge-subduction"; diff --git a/core/bootloader/test/worker-isolation.test.ts b/core/bootloader/test/worker-isolation.test.ts new file mode 100644 index 000000000..bfbf22d22 --- /dev/null +++ b/core/bootloader/test/worker-isolation.test.ts @@ -0,0 +1,23 @@ +// The automerge worker never loads package code, so it never walks through a +// package's path provider (spec section 6). Held here statically: nothing the +// worker's resolver is built from imports patchwork-plugins, and the worker +// creates its resolver without `providers`. +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const source = (file: string) => + readFileSync(new URL(`../src/${file}`, import.meta.url), "utf8"); + +describe("automerge worker isolation", () => { + it("doesn't import patchwork-plugins in the worker or its resolver", () => { + for (const file of ["automerge-protocol-handler.ts", "resolver.ts"]) { + expect(source(file), file).not.toMatch(/from\s+["']@inkandswitch\/patchwork-plugins["']/); + } + }); + + it("creates the worker's resolver without providers", () => { + const calls = source("automerge-protocol-handler.ts").match(/createResolver\([^;]*\)/g) ?? []; + expect(calls.length).toBeGreaterThan(0); + for (const call of calls) expect(call).not.toMatch(/providers/); + }); +}); diff --git a/core/bootloader/vitest.config.ts b/core/bootloader/vitest.config.ts new file mode 100644 index 000000000..f4ad63c16 --- /dev/null +++ b/core/bootloader/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["test/**/*.{test,spec}.ts"], + testTimeout: 30_000, + setupFiles: ["./test/setup.ts"], + // automerge-repo's DocumentQuery rejects internally-held promises when a + // pending query is torn down after a test ends; that's upstream noise, not + // a test failure (see patchwork-filesystem's config). This vitest has no + // per-error filter, only the big switch. + dangerouslyIgnoreUnhandledErrors: true, + }, +}); diff --git a/core/elements/src/events.ts b/core/elements/src/events.ts index 7a29f6fda..f9bc99238 100644 --- a/core/elements/src/events.ts +++ b/core/elements/src/events.ts @@ -5,6 +5,13 @@ export interface OpenDocumentEventDetail { toolId?: string; title?: string; type?: string; + /** + * The live path the document was opened at, like a cwd: + * `/Projects/foo`, each segment `encodeURIComponent`-encoded + * (see `livePath` in `@inkandswitch/patchwork-filesystem`). The router + * writes it to `&path=`. + */ + path?: string; } export class OpenDocumentEvent extends CustomEvent { @@ -58,7 +65,8 @@ declare global { export const openDocument = ( element: HTMLElement | ShadowRoot, url: AutomergeUrl, - toolId?: string + toolId?: string, + path?: string ) => { - element.dispatchEvent(new OpenDocumentEvent({ url, toolId })); + element.dispatchEvent(new OpenDocumentEvent({ url, toolId, path })); }; diff --git a/core/elements/src/index.ts b/core/elements/src/index.ts index dff235b4b..b15983098 100644 --- a/core/elements/src/index.ts +++ b/core/elements/src/index.ts @@ -11,6 +11,7 @@ import type {} from "solid-js"; type LegacyPatchworkViewAttrs = { "doc-url"?: string; "tool-id"?: string | null; + path?: string | null; component?: never; url?: never; }; @@ -20,6 +21,7 @@ type PatchworkViewComponentAttrs = { url?: string; "doc-url"?: string; "tool-id"?: string | null; + path?: string | null; }; type PatchworkViewAttrs = diff --git a/core/elements/src/legacy-impl.ts b/core/elements/src/legacy-impl.ts index a8f7030cd..12c9e2c7a 100644 --- a/core/elements/src/legacy-impl.ts +++ b/core/elements/src/legacy-impl.ts @@ -1,5 +1,6 @@ import { parseAutomergeUrl, + stringifyAutomergeUrl, type AutomergeUrl, type DocHandle, type DocHandleChangePayload, @@ -237,6 +238,16 @@ export class LegacyImpl { #docUrl: AutomergeUrl | null = null; #toolId: string | null = null; #handle: DocHandle | null = null; + /** + * The doc whose type picks the tool: the handle's own, or, for part of a + * document (`automerge:/`) with no type of its own, the document + * it's part of. The tool still gets the part. + */ + #typeHandle: DocHandle | null = null; + // Called only once there's a handle, as `this.#handle.doc()` was. + #typeDoc(): HasPatchworkMetadata { + return (this.#typeHandle ?? this.#handle)!.doc(); + } #tool: LoadedTool | null = null; #state: State = State.none; #requestedToolImports = new Set(); @@ -380,7 +391,7 @@ export class LegacyImpl { // only change the outcome when no fallback is known yet or the new // tool supports this doc's type. if (this.#handle && this.#couldChangeFallback(addedTool)) { - this.#fallbackId = getFallbackTool(this.#handle.doc())?.id; + this.#fallbackId = getFallbackTool(this.#typeDoc())?.id; } const isFallbackTool = toolId == this.#fallbackId; if (isChosenTool || isFallbackTool) { @@ -443,7 +454,22 @@ export class LegacyImpl { if (epoch !== this.#initEpoch) return; + // Part of a document with no type of its own opens in its document's tool. + let typeHandle: DocHandle | null = null; + const { documentId, heads, segments } = parseAutomergeUrl(this.#docUrl! as AutomergeUrl); + if (segments?.length && !getType(handle.doc())) { + try { + typeHandle = await repo.find( + stringifyAutomergeUrl({ documentId, heads }) + ); + } catch { + // its document can't be had: pick from the part alone + } + if (epoch !== this.#initEpoch) return; + } + this.#handle = handle; + this.#typeHandle = typeHandle; this.#handle.on("change", this.#onDocChange); this.#teardowns.add(() => this.#handle!.off("change", this.#onDocChange)); @@ -458,7 +484,7 @@ export class LegacyImpl { const supported = added.supportedDatatypes; if (!supported) return false; if (supported === "*" || supported.includes("*")) return true; - const type = this.#handle ? getType(this.#handle.doc()) : undefined; + const type = this.#handle ? getType(this.#typeDoc()) : undefined; return type != null && supported.includes(type); } @@ -500,6 +526,7 @@ export class LegacyImpl { this.#keyhiveRetrySetup = false; this.#unableNoAccess = false; this.#handle = null; + this.#typeHandle = null; this.#tool = null; this.#requestedToolImports.clear(); this.#content.textContent = ""; @@ -606,7 +633,7 @@ export class LegacyImpl { this.#resetDisplay(); - const doc = this.#handle.doc(); + const doc = this.#typeDoc(); const fallbackTool = getFallbackTool(doc); this.#fallbackId = fallbackTool?.id; const fallingBack = !this.#toolId; @@ -909,7 +936,7 @@ export class LegacyImpl { */ #suggestedImport(): { url: string; loading: boolean } | null { if (!this.#docUrl || !this.#handle) return null; - const url = getSuggestedImportUrl(this.#handle.doc()); + const url = getSuggestedImportUrl(this.#typeDoc()); if (!url) return null; return { url, loading: this.#requestedToolImports.has(url) }; } diff --git a/core/elements/src/patchwork-view.ts b/core/elements/src/patchwork-view.ts index 51a66b307..025e4259f 100644 --- a/core/elements/src/patchwork-view.ts +++ b/core/elements/src/patchwork-view.ts @@ -60,6 +60,7 @@ const ATTRS = { url: "url", docUrl: "doc-url", toolId: "tool-id", + path: "path", } as const; export type RegisterPatchworkViewElementParams = { @@ -87,6 +88,8 @@ export type PatchworkViewElementBase = HTMLElement & { repo: Repo; docUrl?: AutomergeUrl | null; toolId?: string | null; + /** The live path the doc was opened at; see `OpenDocumentEventDetail.path`. */ + path?: string | null; }; export type PatchworkViewElement = PatchworkViewElementBase & { @@ -189,6 +192,18 @@ export function registerPatchworkViewElement( else this.removeAttribute(ATTRS.toolId); } + // Not observed: the mounted tool reads it, but a new path for the same + // doc shouldn't remount the tool. + get path(): string | null { + return this.getAttribute(ATTRS.path); + } + + set path(path: string | null) { + if (this.path === path) return; + if (path) this.setAttribute(ATTRS.path, path); + else this.removeAttribute(ATTRS.path); + } + static get observedAttributes() { return [ATTRS.component, ATTRS.url, ATTRS.docUrl, ATTRS.toolId]; } diff --git a/core/elements/test/patchwork-view.test.ts b/core/elements/test/patchwork-view.test.ts index 6c246608c..749485d70 100644 --- a/core/elements/test/patchwork-view.test.ts +++ b/core/elements/test/patchwork-view.test.ts @@ -170,6 +170,43 @@ describe("patchwork-view (legacy mode)", () => { expect(toast?.querySelector("button.pw-error__load")).toBeTruthy(); }); + it("opens part of a document with no type of its own in its document's tool, handing it the part", async () => { + const id = `test-canvas-tool-${++seq}`; + const handed: string[] = []; + const render = (handle: { url: string }) => { + handed.push(handle.url); + return () => {}; + }; + toolRegistry.register( + { + id, + type: "patchwork:tool", + name: id, + supportedDatatypes: ["test-canvas"], + load: async () => render, + module: render, + } as never, + `test://${id}` + ); + registeredTools.push(id); + + const canvas = repo.create({ + "@patchwork": { type: "test-canvas" }, + store: { "shape:a": { id: "shape:a", typeName: "shape" } }, + }); + const part = canvas.sub("store", "shape:a").url; + const provider = document.createElement("repo-provider"); + const view = document.createElement("patchwork-view-legacy"); + view.setAttribute("doc-url", part); + provider.append(view); + document.body.append(provider); + await settle(); + await settle(); + + expect(view.textContent).not.toContain("missing @patchwork metadata"); + expect(handed).toEqual([part]); + }); + it("still mounts the wildcard stopgap when the doc suggests nothing", async () => { const raw = registerWildcardTool(); diff --git a/core/filesystem/package.json b/core/filesystem/package.json index 7f4b71120..d65666eec 100644 --- a/core/filesystem/package.json +++ b/core/filesystem/package.json @@ -23,6 +23,7 @@ "@automerge/automerge-repo": "*" }, "dependencies": { + "bs58": "^6.0.0", "debug": "^4.4.3", "resolve.exports": "^2.0.3" }, diff --git a/core/filesystem/src/content-hash.ts b/core/filesystem/src/content-hash.ts new file mode 100644 index 000000000..01b33fcda --- /dev/null +++ b/core/filesystem/src/content-hash.ts @@ -0,0 +1,17 @@ +// The one way content is hashed, by whoever builds a file list and by the service worker +// checking what it serves: SHA-256 over the bytes, base64url without padding. A file list's own +// hash, the `~listHash` in a pinned path, is this too. + +/** Bytes as served: strings as UTF-8, which is what a Response body of a string sends. */ +export function toBytes(content: string | Uint8Array): Uint8Array { + return typeof content === "string" ? new TextEncoder().encode(content) : content; +} + +export async function contentHash(bytes: Uint8Array): Promise { + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", bytes as BufferSource) + ); + let binary = ""; + for (const byte of digest) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} diff --git a/core/filesystem/src/file-list.ts b/core/filesystem/src/file-list.ts new file mode 100644 index 000000000..33a88ded5 --- /dev/null +++ b/core/filesystem/src/file-list.ts @@ -0,0 +1,116 @@ +// A container's file list: what each file under it is, at the moment it was pinned. A pinned +// path names one by its hash (`@~`), so the service worker checks the +// list against the URL and each file's bytes against the list, and trusts nothing in between. +// No document ids: a resolver finds a file by walking to it by name from the container at its +// pinned heads, and reads it at the heads the list recorded. +import type { DocumentId, Repo } from "@automerge/automerge-repo/slim"; +import { contentHash, toBytes } from "./content-hash.js"; +import { isKeyhiveDocumentId } from "./keyhive.js"; +import { formatHeads, parsePath } from "./paths.js"; +import { listFiles, type WalkOptions } from "./walk.js"; + +export type FileEntry = { + /** SHA-256 of the bytes the file serves as. */ + sha256: string; + type: string; + /** The heads the file's document was read at; absent for a value inside the container. */ + heads?: string; + /** + * The heads of each nested folder doc on the way to the file, outermost first, for a file in + * a package's nested folder: a walk back to it reads them at these, so renaming the file + * after it was pinned doesn't lose it. Absent otherwise, so other lists hash as before. + */ + via?: string[]; + /** Whether the bytes may be kept in the content store. */ + store: boolean; +}; + +export type FileList = { v: 1; files: Record }; + +/** JSON with every object's keys sorted and no whitespace, so equal values are equal bytes. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + const entries = Object.entries(value) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; + } + return JSON.stringify(value); +} + +const isEntry = (value: unknown): value is FileEntry => { + const entry = value as FileEntry; + return ( + !!entry && + typeof entry.sha256 === "string" && + typeof entry.type === "string" && + typeof entry.store === "boolean" && + (entry.heads === undefined || typeof entry.heads === "string") && + (entry.via === undefined || + (Array.isArray(entry.via) && entry.via.every((heads) => typeof heads === "string"))) + ); +}; + +/** A list from its bytes, or undefined if they aren't one. Check the hash before trusting it. */ +export function parseFileList(bytes: Uint8Array): FileList | undefined { + try { + const value = JSON.parse(new TextDecoder().decode(bytes)); + if (value?.v !== 1 || !value.files || typeof value.files !== "object") return undefined; + if (!Object.values(value.files).every(isEntry)) return undefined; + return value as FileList; + } catch { + return undefined; + } +} + +/** + * The file list of the container `path` is in: every file under it, or only the tails in + * `only`. Returns its bytes and hash, and the container's pinned path without `~listHash`. + * Undefined unless `path` is canonical. `missing` names files that couldn't be read, which a + * whole list then leaves out. + * + * What may be stored: content from unprotected documents, and keyhive-protected content only + * when it's reached through `/packages`, since package code needs to load fast at boot and user + * documents don't. Nothing under a share link. + */ +export async function buildFileList( + repo: Repo, + path: string, + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: string[]; + isProtected?: (documentId: DocumentId) => boolean; + } +): Promise< + | { prefix: string; list: FileList; bytes: Uint8Array; listHash: string; missing: string[] } + | undefined +> { + const listed = await listFiles(repo, path, options); + if (!listed) return undefined; + const { root, segments } = parsePath(listed.prefix); + const first = segments[0]; + const throughPackages = first?.kind === "name" && first.name === "packages"; + const isProtected = options.isProtected ?? isKeyhiveDocumentId; + + // Tails are names people chose: "__proto__" must be a key like any other. + const files: Record = Object.create(null); + for (const file of listed.files) { + files[file.tail] = { + sha256: await contentHash(toBytes(file.content)), + type: file.type, + ...(file.heads ? { heads: formatHeads(file.heads) } : {}), + ...(file.via?.length ? { via: file.via.map(formatHeads) } : {}), + store: root.kind === "account" && (throughPackages || !isProtected(file.documentId)), + }; + } + const list: FileList = { v: 1, files }; + const bytes = new TextEncoder().encode(canonicalJson(list)); + return { + prefix: listed.prefix, + list, + bytes, + listHash: await contentHash(bytes), + missing: listed.missing, + }; +} diff --git a/core/filesystem/src/index.ts b/core/filesystem/src/index.ts index c2e1278e8..f647c7c62 100644 --- a/core/filesystem/src/index.ts +++ b/core/filesystem/src/index.ts @@ -1,8 +1,15 @@ +export * from "./content-hash.js"; +export * from "./file-list.js"; export * from "./find-handle.js"; +export * from "./keyhive.js"; export * from "./metadata.js"; export * from "./module-watcher.js"; export * from "./package-tracker.js"; export * from "./packages.js"; +export * from "./packages-folder.js"; +export * from "./paths.js"; +export * from "./providers.js"; export * from "./resolve.js"; export * from "./urls.js"; +export * from "./walk.js"; export type * from "./types.js"; diff --git a/core/filesystem/src/keyhive.ts b/core/filesystem/src/keyhive.ts new file mode 100644 index 000000000..d54518f23 --- /dev/null +++ b/core/filesystem/src/keyhive.ts @@ -0,0 +1,15 @@ +import { + documentIdToBinary, + type DocumentId, +} from "@automerge/automerge-repo/slim"; + +/** + * Whether a document is keyhive-protected: its binary id carries a non-zero + * second half, where a pre-keyhive document's is empty. + */ +export function isKeyhiveDocumentId(documentId: DocumentId): boolean { + const bytes = documentIdToBinary(documentId); + return ( + !!bytes && bytes.length >= 32 && bytes.subarray(16, 32).some((byte) => byte !== 0) + ); +} diff --git a/core/filesystem/src/module-watcher.ts b/core/filesystem/src/module-watcher.ts index 3e46aeb5c..e60aa63d0 100644 --- a/core/filesystem/src/module-watcher.ts +++ b/core/filesystem/src/module-watcher.ts @@ -7,10 +7,11 @@ import { stringifyAutomergeUrl, } from "@automerge/automerge-repo/slim"; import { - importPackageFromFolderDocUrl, + importPackage, importPackageFromHttpUrl, } from "./packages.js"; import { type HasPatchworkMetadata } from "./metadata.js"; +import type { PackageEntry } from "./packages-folder.js"; import { DEFAULT_BRANCH, trackPackagesInRepo, @@ -22,6 +23,13 @@ const log = debug("patchwork:modules"); export type ModuleSettingsDoc = { modules: AutomergeUrl[]; + /** + * Documents whose unpinned entries `/packages` took over (see + * migratePackages): those entries load through `/packages` or not at all, + * so removing a package there removes it. Builds that predate `/packages` + * ignore this and keep loading them. + */ + movedToPackages?: string[]; } & HasPatchworkMetadata & { "@patchwork": { type: "patchwork:module-settings" }; }; @@ -100,6 +108,9 @@ type WatchedModule = { const stripHeads = (url: AutomergeUrl) => stringifyAutomergeUrl({ documentId: parseAutomergeUrl(url).documentId }); +const describeError = (error: unknown) => + error instanceof Error ? error.message : String(error); + function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { console.warn( `module ${branchesDocUrl} is a branches doc. Branches docs are no longer supported as modules; falling back to the "${DEFAULT_BRANCH}" branch. Please let us know you hit this: post in #patchwork-testers or email chee@inkandswitch.com` @@ -109,6 +120,12 @@ function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { // todo this can be a function that takes a plugin system and returns a change // handler +/** How a package is doing: what a sidebar shows beside it. */ +export type PackageStatus = + | { state: "loading" } + | { state: "loaded"; plugins: string[] } + | { state: "failed"; error: string }; + /** * Settings sources are passed in keyed by name (e.g. `{ system, user }`). Each * source URL may be either an Automerge module-settings doc (`automerge:...`, @@ -116,8 +133,11 @@ function warnBranchesUnsupported(branchesDocUrl: AutomergeUrl) { * (fetched once at construction). The two kinds can be freely mixed, and the * module URLs *within* either kind can themselves point at Automerge package * docs (folders or directories) or plain HTTP(S) bundles. + * + * Dispatches a `"status"` event whenever a package's {@link PackageStatus} + * changes; read it with {@link ModuleWatcher.packageStatus}. */ -export class ModuleWatcher { +export class ModuleWatcher extends EventTarget { repo: Repo; urls: Record; handles: Record> | undefined; @@ -130,16 +150,26 @@ export class ModuleWatcher { // eventually succeeding and rolling the registry back to that old version. #announceGenerations = new Map(); #retryTimers = new Map>(); + /** Lookups that failed, waiting to try again. */ + #lookupRetries = new Map>(); #lastAnnounced = new Map(); #activeModules = new Set(); + /** Package folders (`/packages`), by source name: what they list, and how to stop hearing them. */ + #folders = new Map< + string, + { entries: () => PackageEntry[]; unsubscribe: () => void } + >(); #disposed = false; + /** Package status, by the package doc's documentId. */ + #status = new Map(); onLoad: (name: string, mod: any) => void; onUnload?: (name: string) => void; /** - * How an Automerge package doc (pinned to heads) is turned into the - * `{ plugins }` shape `onLoad` consumes. Defaults to importing the package - * entry point directly on this thread. The browser bootloader overrides this + * How an Automerge package doc (pinned to heads, or the pinned `/@/` URL a + * `/packages` lookup answers with) is turned into the `{ plugins }` shape + * `onLoad` consumes. Defaults to importing the package entry point directly + * on this thread. The browser bootloader overrides this * to run the entry point in a worker for descriptor discovery and rebuild the * `{ plugins }` shape with main-thread `load()` functions. */ @@ -158,9 +188,10 @@ export class ModuleWatcher { callback: (name: string, mod: any) => void, onUnload?: (name: string) => void, importAutomergePackage: (urlAtHeads: string) => Promise = (url) => - importPackageFromFolderDocUrl(url as AutomergeUrl), + importPackage(url), tracker: PackageTracker = trackPackagesInRepo(repo) ) { + super(); this.repo = repo; this.urls = { ...urls }; this.onLoad = callback; @@ -238,14 +269,40 @@ export class ModuleWatcher { ); } - private async processModuleEntry(importName: string) { + private async processModuleEntry(importName: string, attempt = 0) { if (isValidAutomergeUrl(importName)) { // Watch before looking up, so a change landing in between still reloads. // Keyed by the settings entry, a branches doc's included, so removing // it from settings unloads this module. + const watched = this.#watchedModules.get(importName); + // Already found: its watch looks it up again when it moves. Looking up + // every package whenever its source's list changes would reload them. + if (watched?.version !== undefined) return; const entry = this.watchPackage(importName, stripHeads(importName)); if (!entry) return; - const latest = await this.findLatestVersion(importName, entry); + let latest: AutomergeUrl | undefined; + try { + latest = await this.findLatestVersion(importName, entry); + } catch (error) { + // Nothing else brings it back until the package moves: back off and + // look again. + const delay = RETRY_DELAYS_MS[attempt]; + console.warn( + `couldn't look up ${importName}${delay === undefined ? "" : `; trying again in ${delay}ms`}`, + error + ); + this.#setStatus(importName, { state: "failed", error: describeError(error) }); + if (delay !== undefined) { + const timer = setTimeout(() => { + this.#lookupRetries.delete(importName); + if (this.#activeModules.has(importName)) { + void this.processModuleEntry(importName, attempt + 1); + } + }, delay); + this.#lookupRetries.set(importName, timer); + } + return; + } // A newer lookup took over, and announces what it finds. if (!latest) return; entry.version = latest; @@ -258,23 +315,62 @@ export class ModuleWatcher { await this.announce(importName); } - private async importPackageSafe(importName: string): Promise { + /** + * How the package doc `urlOrDocumentId` is doing, or undefined if it isn't + * a package this watcher loads (an http module has none). + */ + packageStatus(urlOrDocumentId: string): PackageStatus | undefined { + const documentId = isValidAutomergeUrl(urlOrDocumentId) + ? parseAutomergeUrl(urlOrDocumentId).documentId + : urlOrDocumentId; + return this.#status.get(documentId); + } + + /** + * The package an import url was loaded for, if this watcher loaded it. A + * `/packages` package is imported from its pinned `/@/` url, which is what + * its plugins are registered under; this maps that back to the package doc. + */ + packageFor(importUrl: string): AutomergeUrl | undefined { + for (const [moduleKey, version] of this.#lastAnnounced) { + if (version === importUrl && isValidAutomergeUrl(moduleKey)) return moduleKey; + } + return undefined; + } + + #setStatus(moduleKey: string, status: PackageStatus) { + if (!isValidAutomergeUrl(moduleKey)) return; + this.#status.set(parseAutomergeUrl(moduleKey).documentId, status); + this.dispatchEvent(new Event("status")); + } + + private async importPackageSafe( + importName: string, + moduleKey: string + ): Promise<{ mod: any } | { error: unknown }> { try { if (isValidAutomergeUrl(importName)) { // Pin to heads so descriptor discovery and any later main-thread load // resolve against the exact same version of the package doc. - return await this.importAutomergePackage( - await this.tracker.findLatest(importName) - ); + return { + mod: await this.importAutomergePackage( + await this.tracker.findLatest(importName) + ), + }; } - return await importPackageFromHttpUrl(importName); + // The tracker's answer for a package doc: a /packages package's pinned + // /@/ URL. It's still that doc's package, so it's imported the same way. + if (isValidAutomergeUrl(moduleKey)) { + return { mod: await this.importAutomergePackage(importName) }; + } + return { mod: await importPackageFromHttpUrl(importName) }; } catch (error) { console.error( `%c Failed to import ${importName}`, "color: #000, background: #ffbcef", error ); - return undefined; + return { error }; } } @@ -314,15 +410,27 @@ export class ModuleWatcher { this.#announceGenerations.get(moduleKey) === generation; const attempt = async (attemptIndex: number): Promise => { - const mod = await this.importPackageSafe(importUrl); + if (attemptIndex === 0) this.#setStatus(moduleKey, { state: "loading" }); + const result = await this.importPackageSafe(importUrl, moduleKey); // A newer announce for this module started while we were importing or // backing off — its result wins; announcing ours now could regress the // registry to an older version. if (!current()) return; - if (mod) { + if ("mod" in result && result.mod) { + const mod = result.mod; this.#lastAnnounced.set(moduleKey, versionKey); + this.#setStatus(moduleKey, { + state: "loaded", + plugins: Array.isArray(mod.plugins) + ? mod.plugins.map((plugin: { id?: string }) => plugin?.id).filter(Boolean) + : [], + }); return this.onLoad(importUrl, mod); } + this.#setStatus(moduleKey, { + state: "failed", + error: describeError("error" in result ? result.error : "it exported nothing"), + }); const delay = RETRY_DELAYS_MS[attemptIndex]; if (delay === undefined) return; const timer = setTimeout(() => { @@ -410,6 +518,8 @@ export class ModuleWatcher { ); if (this.#watchedModules.get(importName) !== entry) return undefined; if (lookup !== entry.lookups) return undefined; + // A /packages package comes back as its pinned /@/ URL, imported over http. + if (!isValidAutomergeUrl(latest)) return latest; const packageUrl = stripHeads(latest); if (packageUrl !== entry.packageUrl) { warnBranchesUnsupported(importName as AutomergeUrl); @@ -489,12 +599,22 @@ export class ModuleWatcher { private unloadModule(entry: string) { this.detachDocWatcher(entry); this.cancelAnnounce(entry); + clearTimeout(this.#lookupRetries.get(entry)); + this.#lookupRetries.delete(entry); + // Its plugins were registered under the url it was imported from. + const announced = this.#lastAnnounced.get(entry); this.#lastAnnounced.delete(entry); - this.onUnload?.(entry); + if (isValidAutomergeUrl(entry)) { + this.#status.delete(parseAutomergeUrl(entry).documentId); + this.dispatchEvent(new Event("status")); + } + this.onUnload?.(announced ?? entry); } dispose() { this.#disposed = true; + for (const { unsubscribe } of this.#folders.values()) unsubscribe(); + this.#folders.clear(); for (const handle of Object.values(this.handles ?? {})) { handle.removeListener("change", this.onChange); } @@ -506,14 +626,50 @@ export class ModuleWatcher { } for (const timer of this.#retryTimers.values()) clearTimeout(timer); this.#retryTimers.clear(); + for (const timer of this.#lookupRetries.values()) clearTimeout(timer); + this.#lookupRetries.clear(); this.#activeModules.clear(); } + /** + * Load every package in a `/packages` folder, and keep loading what it lists: + * `subscribe` calls back whenever `entries` may have changed. A package is + * keyed by its document, so a rename reloads nothing, and one that module + * settings also lists loads once, through here. Looking packages up is the + * tracker's job (see trackPackagesFolder). + */ + async addPackagesFolder( + name: string, + entries: () => PackageEntry[], + subscribe: (onChange: () => void) => () => void + ): Promise { + await this.doneLoading; + if (this.#disposed) return; + this.#folders.get(name)?.unsubscribe(); + this.#folders.set(name, { entries, unsubscribe: subscribe(this.onChange) }); + await this.load(); + } + private async load() { if (!this.handles) throw new Error("No handles"); const urls = new Set(); + const inFolders = new Set(); + for (const { entries } of this.#folders.values()) { + for (const entry of entries()) { + urls.add(entry.url); + inFolders.add(parseAutomergeUrl(entry.url).documentId); + } + } for (const { doc } of this.settingsDocs()) { - for (const m of doc?.modules ?? []) urls.add(m); + const moved = new Set(Array.isArray(doc?.movedToPackages) ? doc.movedToPackages : []); + for (const m of doc?.modules ?? []) { + if (isValidAutomergeUrl(m)) { + const { documentId, heads } = parseAutomergeUrl(m); + if (inFolders.has(documentId)) continue; + if (!heads && moved.has(documentId)) continue; + } + urls.add(m); + } } const previousActive = this.#activeModules; this.#activeModules = urls; diff --git a/core/filesystem/src/packages-folder.ts b/core/filesystem/src/packages-folder.ts new file mode 100644 index 000000000..05fc571bd --- /dev/null +++ b/core/filesystem/src/packages-folder.ts @@ -0,0 +1,255 @@ +// Package management is whatever is bound at `/packages`: the account's union +// of package folders. This lists it for the ModuleWatcher, and looks each +// package up as a pinned `/@/` URL whose file list covers the whole package. +import { + isValidAutomergeUrl, + parseAutomergeUrl, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocHandle, + type Repo, +} from "@automerge/automerge-repo/slim"; +import type { PackageTracker } from "./package-tracker.js"; +import { docStep, idhash } from "./paths.js"; +import { accountBindings, providerFor } from "./providers.js"; +import type { AccountNamespaceDoc } from "./types.js"; +import { documentBaseOrigin } from "./urls.js"; + +export type PackageEntry = { + /** The package doc, without heads. */ + url: AutomergeUrl; + name: string; + /** The union member it's in, as a doc step: how a lookup finds it, shadowed or not. */ + memberStep: string; +}; + +const bare = (url: AutomergeUrl) => + stringifyAutomergeUrl({ documentId: parseAutomergeUrl(url).documentId }); + +/** + * Watch an account's `/packages` union. `onChange` gets its packages, one per + * document (the first binding that has it wins), now and whenever the account + * doc or a member folder changes. Only the account and the member folders are + * loaded, never the packages. + */ +export function watchPackagesFolder( + repo: Repo, + account: DocHandle, + onChange: (entries: PackageEntry[]) => void, + /** How long the first listing waits for every member before going without the slow ones. */ + { patienceMs = FIRST_LISTING_PATIENCE_MS }: { patienceMs?: number } = {} +): () => void { + let stopped = false; + let patient = true; + const patience = setTimeout(() => { + patient = false; + if (!stopped) changed(); + }, patienceMs); + let members: DocHandle[] = []; + let last: string | undefined; + let pending: Promise | undefined; + + // Each member is found on its own: one that isn't available yet (or ever) + // mustn't hold up the others. It's listed once it arrives, and one that + // couldn't be found is asked for again a while later. + const found = new Map>(); + const finding = new Set(); + const retries = new Set>(); + const find = (url: AutomergeUrl) => { + if (found.has(url) || finding.has(url)) return; + finding.add(url); + repo + .find(url) + .then( + (handle) => { + found.set(url, handle); + }, + () => { + const retry = setTimeout(() => { + retries.delete(retry); + if (!stopped) changed(); + }, MEMBER_RETRY_MS); + retries.add(retry); + } + ) + .finally(() => { + finding.delete(url); + if (!stopped) changed(); + }); + }; + + const list = async () => { + const urls = accountBindings(account.doc()).get("/packages") ?? []; + for (const url of urls) find(url); + // The first listing waits a while for every member, so packages don't + // load in dribs; after that, a member is listed whenever it arrives. + if (last === undefined && patient && urls.some((url) => finding.has(url))) return; + const handles = urls.flatMap((url) => { + const handle = found.get(url); + return handle ? [handle] : []; + }); + if (stopped) return; + for (const member of members) member.off("change", changed); + members = handles; + for (const member of members) member.on("change", changed); + + const entries: PackageEntry[] = []; + const seen = new Set(); + for (const member of members) { + // The doc and the heads it was read at, together. + const heads = member.heads(); + const doc = member.view(heads).doc(); + const memberStep = docStep(await idhash(member.documentId), heads); + const names = new Set(); + for (const child of providerFor(doc)?.list(doc) ?? []) { + if (!("link" in child)) continue; + // Only the first of two links with one name in a folder is reachable + // by that name, so the second would import the first's code. + if (names.has(child.name)) { + console.warn( + `/packages: skipping a second package named ${JSON.stringify(child.name)} in one folder; rename one` + ); + continue; + } + names.add(child.name); + const { documentId } = parseAutomergeUrl(child.link); + if (seen.has(documentId)) continue; + seen.add(documentId); + entries.push({ url: bare(child.link), name: child.name, memberStep }); + } + } + const key = JSON.stringify(entries); + if (stopped || key === last) return; + last = key; + onChange(entries); + }; + + // Changes arrive in bursts; list once per burst, and once more if one + // landed while listing. + let again = false; + function changed() { + if (pending) { + again = true; + return; + } + pending = Promise.resolve() + .then(list) + .catch((error) => console.warn("couldn't list /packages", error)) + .finally(() => { + pending = undefined; + if (again && !stopped) { + again = false; + changed(); + } + }); + } + + account.on("change", changed); + changed(); + return () => { + stopped = true; + account.off("change", changed); + for (const member of members) member.off("change", changed); + for (const retry of retries) clearTimeout(retry); + clearTimeout(patience); + }; +} + +/** How long the first listing of `/packages` waits for every member folder. */ +const FIRST_LISTING_PATIENCE_MS = 3_000; + +/** How long before a `/packages` member that couldn't be found is asked for again. */ +const MEMBER_RETRY_MS = 10_000; + +/** The live lookup path for a package: through the member that holds it. */ +export function packageLookupPath(acctHash: string, entry: PackageEntry): string { + return `${acctHash}/packages/${entry.memberStep}/${encodeURIComponent(entry.name)}`; +} + +/** + * Follow a live `/@/` path's redirect to its pinned URL. The service worker + * keeps the file list the redirect carries. + */ +async function lookUpViaServiceWorker(path: string): Promise { + const response = await fetch(`${documentBaseOrigin()}/@/${path}`); + if (!response.ok || !response.redirected) { + throw new Error( + `the service worker didn't redirect /@/${path} to a pinned URL (HTTP ${response.status})` + ); + } + return response.url; +} + +/** + * A tracker for the packages under `/packages`: `findLatest` answers with the + * package's pinned `/@/` URL, which the ModuleWatcher imports through its + * package import (in a browser, discovered in the module-loader worker). + * Anything else goes to `fallback`, which also does the watching. + */ +export function trackPackagesFolder( + acctHash: Promise, + entries: () => PackageEntry[], + fallback: PackageTracker, + lookUp: (path: string) => Promise = lookUpViaServiceWorker +): PackageTracker { + // A lookup pins the whole chain, so the URL changes whenever the account or + // the member folder does. A pinned URL stays valid forever, so keep the one + // already imported while the package's own step (its heads and list) is + // the same: a change elsewhere in /packages mustn't reload every package. + const known = new Map(); + const ownStep = (url: string) => new URL(url).pathname.split("/").at(-1); + return { + async findLatest(url) { + const { documentId } = parseAutomergeUrl(url); + const entry = entries().find( + (candidate) => parseAutomergeUrl(candidate.url).documentId === documentId + ); + if (!entry) return fallback.findLatest(url); + const latest = await lookUp(packageLookupPath(await acctHash, entry)); + const previous = known.get(documentId); + if (previous && ownStep(previous) === ownStep(latest)) { + return previous as AutomergeUrl; + } + known.set(documentId, latest); + // Not an automerge URL: the watcher hands it to its package import as it is. + return latest as AutomergeUrl; + }, + watch(url, onChange) { + const documentId = parseAutomergeUrl(url).documentId; + const isEntry = () => + entries().some((entry) => parseAutomergeUrl(entry.url).documentId === documentId); + // A package that's a branches doc stands for its default branch, whose + // code changes without the branches doc changing: watch that too, and + // follow it when the branch is pointed somewhere else. + let stopped = false; + let branch: AutomergeUrl | undefined; + let unwatchBranch = () => {}; + const follow = async () => { + if (!isEntry()) return; + let target: AutomergeUrl | undefined; + try { + const latest = await fallback.findLatest(url); + if (isValidAutomergeUrl(latest)) target = bare(latest); + } catch { + return; // keep the branch already watched; the next change asks again + } + if (stopped) return; + if (target && parseAutomergeUrl(target).documentId === documentId) target = undefined; + if (target === branch) return; + unwatchBranch(); + branch = target; + unwatchBranch = target ? fallback.watch(target, onChange) : () => {}; + }; + const unwatch = fallback.watch(url, (seen) => { + void follow(); + onChange(seen); + }); + void follow(); + return () => { + stopped = true; + unwatch(); + unwatchBranch(); + }; + }, + }; +} diff --git a/core/filesystem/src/packages.ts b/core/filesystem/src/packages.ts index f79f5511c..01a5ac5ac 100644 --- a/core/filesystem/src/packages.ts +++ b/core/filesystem/src/packages.ts @@ -165,9 +165,10 @@ async function httpEntryPointUrl( } /** - * Import the entry point of a package (at `folderDocUrl`, which should be - * pinned to heads) and return the loaded implementation of a single one of the - * plugins it exports, selected by its `pluginType` and `pluginId`. + * Import the entry point of a package (at `packageUrl`: a folder doc pinned + * to heads, or a pinned `/@/` URL) and return the loaded implementation of a + * single one of the plugins it exports, selected by its `pluginType` and + * `pluginId`. * * A plugin `id` is only unique *within* a plugin type (there is a separate * registry per type), so a package can export e.g. a `patchwork:datatype` and @@ -181,25 +182,21 @@ async function httpEntryPointUrl( * with the live descriptor, so the registered description and the loaded * implementation come from the same pinned version. */ -export async function importPluginFromFolderDocUrl( - folderDocUrl: AutomergeUrl, +export async function importPluginFromPackage( + packageUrl: string, pluginType: string, pluginId: string, subpath: string = ".", conditions: string[] = defaultImportConditions ) { - const mod = await importPackageFromFolderDocUrl( - folderDocUrl, - subpath, - conditions - ); + const mod = await importPackage(packageUrl, subpath, conditions); const plugins: any[] = Array.isArray(mod?.plugins) ? mod.plugins : []; const plugin = plugins.find( (p) => p?.type === pluginType && p?.id === pluginId ); if (!plugin) { throw new Error( - `No plugin "${pluginType}:${pluginId}" exported by the package at ${folderDocUrl}` + `No plugin "${pluginType}:${pluginId}" exported by the package at ${packageUrl}` ); } if (typeof plugin.load === "function") { @@ -209,10 +206,19 @@ export async function importPluginFromFolderDocUrl( return import(/* @vite-ignore */ plugin.import); } throw new Error( - `Plugin "${pluginType}:${pluginId}" at ${folderDocUrl} has no load() function or import URL` + `Plugin "${pluginType}:${pluginId}" at ${packageUrl} has no load() function or import URL` ); } +/** @deprecated Use {@link importPluginFromPackage}, which also takes a pinned `/@/` URL. */ +export const importPluginFromFolderDocUrl: ( + folderDocUrl: AutomergeUrl, + pluginType: string, + pluginId: string, + subpath?: string, + conditions?: string[] +) => Promise = importPluginFromPackage; + async function packageJsonContentsFromFolderDocUrl( folderDocUrl: AutomergeUrl ): Promise> { diff --git a/core/filesystem/src/paths.ts b/core/filesystem/src/paths.ts new file mode 100644 index 000000000..7541d77bf --- /dev/null +++ b/core/filesystem/src/paths.ts @@ -0,0 +1,159 @@ +import { + type DocumentId, + type UrlHeads, + isValidDocumentId, +} from "@automerge/automerge-repo/slim"; +import bs58 from "bs58"; + +/** + * A step that enters a document at given heads: `@`, or + * `@~` for a container whose files the URL names by their list (see + * file-list.ts). `raw` is the segment as written, so a walk that finds no link with this idhash + * can read it as a name instead. + */ +export type DocStep = { + kind: "doc"; + idhash: string; + heads: UrlHeads; + listHash?: string; + raw: string; +}; +export type NameStep = { kind: "name"; name: string }; +export type Segment = DocStep | NameStep; + +/** + * Where a path starts: an account the resolver knows by its idhash, or (for a portable share + * link) a document named by its id. A share root is always pinned. + */ +export type PathRoot = + | { kind: "account"; acctHash: string; heads?: UrlHeads; listHash?: string } + | { kind: "share"; documentId: DocumentId; heads: UrlHeads; listHash?: string }; + +export type ParsedPath = { root: PathRoot; segments: Segment[] }; + +export class PathSyntaxError extends Error { + name = "PathSyntaxError"; +} + +const BASE58 = /^[1-9A-HJ-NP-Za-km-z]+$/; +const BASE64URL = /^[A-Za-z0-9_-]+$/; +const SHARE_PREFIX = "automerge:"; + +/** + * The first 12 bytes of SHA-256 over the UTF-8 of a document id, in base58. It tells a document + * from its siblings and keeps the id out of URLs. It is not a secret. + */ +export async function idhash(documentId: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(documentId) + ); + return bs58.encode(new Uint8Array(digest).slice(0, 12)); +} + +/** Heads sorted and joined with `.`, so one set of heads is one string. */ +export function formatHeads(heads: readonly string[]): string { + return [...heads].sort().join("."); +} + +export function parseHeads(segment: string): UrlHeads | undefined { + const parts = segment.split("."); + if (!parts.every((part) => BASE58.test(part))) return undefined; + return parts.sort() as UrlHeads; +} + +export function docStep( + idhash: string, + heads: readonly string[], + listHash?: string +): string { + return `${idhash}@${formatHeads(heads)}${listHash ? `~${listHash}` : ""}`; +} + +export function livePath(acctHash: string, names: string[]): string { + return [acctHash, ...names.map(encodeURIComponent)].join("/"); +} + +function decode(raw: string): string { + try { + return decodeURIComponent(raw); + } catch { + throw new PathSyntaxError(`malformed escape in path segment ${raw}`); + } +} + +/** `` or `~`, split, or undefined if it isn't that shape. */ +function splitHeads( + text: string +): { heads: UrlHeads; listHash?: string } | undefined { + const tilde = text.indexOf("~"); + const heads = parseHeads(tilde < 0 ? text : text.slice(0, tilde)); + if (!heads) return undefined; + if (tilde < 0) return { heads }; + const listHash = text.slice(tilde + 1); + return BASE64URL.test(listHash) ? { heads, listHash } : undefined; +} + +/** `@[~]`, split, or undefined if it isn't that shape. */ +function splitDocStep( + raw: string +): { id: string; heads: UrlHeads; listHash?: string } | undefined { + const at = raw.indexOf("@"); + if (at < 0) return undefined; + const id = raw.slice(0, at); + const rest = splitHeads(raw.slice(at + 1)); + if (!BASE58.test(id) || !rest) return undefined; + return { id, ...rest }; +} + +function parseRoot(raw: string): PathRoot { + const decoded = decode(raw); + if (decoded.startsWith(SHARE_PREFIX)) { + const rest = decoded.slice(SHARE_PREFIX.length); + const at = rest.lastIndexOf("@"); + const documentId = rest.slice(0, at); + const pinned = at < 0 ? undefined : splitHeads(rest.slice(at + 1)); + if (!pinned || !isValidDocumentId(documentId)) { + throw new PathSyntaxError( + `a share link must be automerge:@, got ${decoded}` + ); + } + return { kind: "share", documentId, ...pinned }; + } + if (BASE58.test(raw)) return { kind: "account", acctHash: raw }; + const step = splitDocStep(raw); + if (!step) { + throw new PathSyntaxError( + `a path must start with an account hash or a share link, got ${raw}` + ); + } + return { + kind: "account", + acctHash: step.id, + heads: step.heads, + ...(step.listHash ? { listHash: step.listHash } : {}), + }; +} + +/** + * Read a live or pinned path. Segments are percent-encoded and split on literal slashes; a + * segment with a literal `@` that parses as `@` is a doc step, everything else a + * name. Whether a doc step really is one is the walk's call, since only it can see the links. + */ +export function parsePath(path: string): ParsedPath { + const [first, ...rest] = path.split("/").filter(Boolean); + if (!first) throw new PathSyntaxError("empty path"); + const segments = rest.map((raw): Segment => { + const step = splitDocStep(raw); + return step + ? { + kind: "doc", + idhash: step.id, + heads: step.heads, + ...(step.listHash ? { listHash: step.listHash } : {}), + raw, + } + : { kind: "name", name: decode(raw) }; + }); + return { root: parseRoot(first), segments }; +} diff --git a/core/filesystem/src/providers.ts b/core/filesystem/src/providers.ts new file mode 100644 index 000000000..eb6e83488 --- /dev/null +++ b/core/filesystem/src/providers.ts @@ -0,0 +1,276 @@ +import { + type AutomergeUrl, + type DocHandle, + isImmutableString, + isValidAutomergeUrl, +} from "@automerge/automerge-repo/slim"; +import { getType } from "./metadata.js"; +import { getDefaultBranch } from "./package-tracker.js"; +import type { + AccountNamespaceDoc, + BranchesDoc, + FolderDoc, +} from "./types.js"; + +/** A step into a document: an object key or an array index. */ +export type SubSegment = string | number; + +/** + * What a provider finds under a name: a link to another document, a value inside the same + * document (a nested directory, a file's bytes), an ordered union of documents (a mount), or a + * value offered as a sub-document, opened as `handle.sub(...sub)`. `sub` is relative to the node + * the provider was given. + */ +export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] } + | { name: string; sub: SubSegment[] }; + +/** + * A child and how many of the leading names it used. A step that consumes no name (the `/` + * mount, a branches doc's default) has `consumed: 0`. + */ +export type Lookup = { child: Child; consumed: number }; + +/** + * How to walk one kind of document. `lookup` and `list` take the document or a value nested in + * it, and never leave it: following a link is the walker's job. `lookup` is called with at least + * one name. They read only the node they're given, so a walk at the same heads gives the same + * answer. + * + * The editing methods are optional; the sidebar offers only those a provider has. Each changes + * the document through `handle`; `node` is what `list` was given. + */ +export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; + list(node: unknown): Child[]; + add?( + handle: DocHandle, + node: unknown, + child: { name: string; link: AutomergeUrl }, + index?: number + ): void | Promise; + rename?(handle: DocHandle, node: unknown, name: string, to: string): void | Promise; + remove?(handle: DocHandle, node: unknown, name: string): void | Promise; +} + +/** + * Providers asked after the built-ins, which can't be overridden. A tab passes the ones + * packages register; the automerge worker passes none. + */ +export type ExtraProviders = (doc: unknown) => Promise; + +/** The value at `segments` below `node`, or undefined. */ +export function valueAt(node: unknown, segments: SubSegment[]): unknown { + let at = node; + for (const segment of segments) { + if (!at || typeof at !== "object" || at instanceof Uint8Array) return undefined; + if (!Object.hasOwn(at, segment)) return undefined; + at = (at as Record)[segment]; + } + return at; +} + +/** + * `value` as a link, if it is one. A link written as an `ImmutableString` + * (one op, where a plain string is an op per character) reads back as an + * object, not a string; it's the same url. + */ +export function asLink(value: unknown): AutomergeUrl | undefined { + const url = isImmutableString(value) ? String(value) : value; + return typeof url === "string" && isValidAutomergeUrl(url) ? url : undefined; +} + +const isObject = (node: unknown): node is Record => + !!node && + typeof node === "object" && + !Array.isArray(node) && + !(node instanceof Uint8Array) && + !isImmutableString(node); + +function childOf(name: string, value: unknown): Child { + const link = asLink(value); + return link ? { name, link } : { name, value }; +} + +const MOUNT_POINT = /^\/[^/@]*$/; + +// The same malformed mount is read on every lookup, so say so once. +const warned = new Set(); +const warnOnce = (key: string, message: string) => { + if (warned.has(key)) return; + warned.add(key); + console.warn(message); +}; + +// Bindings are read on every lookup and list, from a doc that doesn't change under them. +const bindingsByDoc = new WeakMap>(); + +/** + * Mount point → its bindings, in order. `/` falls back to `rootFolderUrl` when the namespace + * binds nothing valid there, so older accounts keep working. Malformed mount points and bindings are + * skipped with a warning. + */ +export function accountBindings( + doc: AccountNamespaceDoc +): Map { + const memo = bindingsByDoc.get(doc); + if (memo) return memo; + const bindings = new Map(); + for (const [mount, list] of Object.entries(doc.namespace ?? {})) { + if (!MOUNT_POINT.test(mount) || !Array.isArray(list)) { + warnOnce(mount, `namespace: skipping malformed mount ${JSON.stringify(mount)}`); + continue; + } + const urls: AutomergeUrl[] = []; + for (const binding of list) { + const url = asLink(binding?.url); + if (url) urls.push(url); + else + warnOnce( + `binding ${mount}`, + `namespace: skipping a binding at ${mount} without a valid url` + ); + } + bindings.set(mount, urls); + } + const root = asLink(doc.rootFolderUrl); + if (!bindings.get("/")?.length && root) bindings.set("/", [root]); + bindingsByDoc.set(doc, bindings); + return bindings; +} + +export const accountProvider: PathProvider = { + kind: "account", + matches(doc) { + return getType(doc as AccountNamespaceDoc) === "account"; + }, + lookup(node, names) { + const bindings = accountBindings(node as AccountNamespaceDoc); + const mount = bindings.get(`/${names[0]}`); + if (names[0] && mount) { + return { child: { name: names[0], union: mount }, consumed: 1 }; + } + const root = bindings.get("/"); + if (!root?.length) return undefined; + return { child: { name: "", union: root }, consumed: 0 }; + }, + list(node) { + const children: Child[] = []; + for (const [mount, union] of accountBindings(node as AccountNamespaceDoc)) { + const child = { name: mount.slice(1), union }; + if (mount === "/") children.unshift(child); + else children.push(child); + } + return children; + }, +}; + +export const directoryProvider: PathProvider = { + kind: "directory", + matches(doc) { + return getType(doc as AccountNamespaceDoc) === "directory"; + }, + // Longest-prefix match: try "main/dist/index.js", then "main/dist", then "main" + lookup(node, names) { + if (!isObject(node)) return undefined; + for (let i = names.length; i >= 1; i--) { + const key = names.slice(0, i).join("/"); + if (Object.hasOwn(node, key)) return { child: childOf(key, node[key]), consumed: i }; + } + return undefined; + }, + list(node) { + if (!isObject(node)) return []; + return Object.entries(node) + .filter(([key]) => key !== "@patchwork") + .map(([key, value]) => childOf(key, value)); + }, +}; + +export const folderProvider: PathProvider = { + kind: "folder", + matches(doc) { + return ( + !!doc && + typeof doc === "object" && + "docs" in doc && + Array.isArray((doc as { docs: unknown }).docs) + ); + }, + lookup(node, names) { + const entry = (node as FolderDoc).docs.find( + (doc) => doc && String(doc.name) === names[0] + ); + const link = asLink(entry?.url); + return link ? { child: { name: names[0], link }, consumed: 1 } : undefined; + }, + list(node) { + return (node as FolderDoc).docs.flatMap((doc) => { + const link = asLink(doc?.url); + return link ? [{ name: String(doc.name), link }] : []; + }); + }, +}; + +/** A branches doc's default branch, or undefined if it has none or isn't one. */ +export function defaultBranch(doc: unknown): AutomergeUrl | undefined { + try { + return asLink(getDefaultBranch(doc)); + } catch { + return undefined; + } +} + +/** + * A branches doc stands for its default branch: a name that isn't a branch is looked up in + * `default`, so a package published as a branches doc resolves like the package itself. A branch + * name wins over a same-named file. + */ +export const branchesProvider: PathProvider = { + kind: "branches", + matches(doc) { + return getType(doc as BranchesDoc) === "branches"; + }, + lookup(node, names) { + const named = asLink((node as BranchesDoc).branches?.[names[0]]); + if (named) return { child: { name: names[0], link: named }, consumed: 1 }; + const link = defaultBranch(node); + return link ? { child: { name: "default", link }, consumed: 0 } : undefined; + }, + list(node) { + return Object.entries((node as BranchesDoc).branches ?? {}).flatMap( + ([name, url]) => { + const link = asLink(url); + return link ? [{ name, link }] : []; + } + ); + }, +}; + +/** Checked in order; the first that matches a document walks it. */ +export const PROVIDERS: readonly PathProvider[] = [ + accountProvider, + directoryProvider, + folderProvider, + branchesProvider, +]; + +export function providerFor(doc: unknown): PathProvider | undefined { + return PROVIDERS.find((provider) => provider.matches(doc)); +} + +/** + * Whether a package's provider for `doc`'s type may walk it in place of `builtIn`. Built-ins + * can't be overridden, with one exception: the folder provider matches anything with a `docs` + * array, and a typed document (a tldraw canvas keeps its scripts that way) is its own type + * first. A folder-typed or untyped folder is always the built-in's. + */ +export function yieldsToPackage(builtIn: PathProvider | undefined, doc: unknown): boolean { + if (builtIn !== folderProvider) return false; + const type = getType(doc as AccountNamespaceDoc); + return !!type && type !== "folder"; +} diff --git a/core/filesystem/src/resolve.ts b/core/filesystem/src/resolve.ts index 98a8368ca..71721f145 100644 --- a/core/filesystem/src/resolve.ts +++ b/core/filesystem/src/resolve.ts @@ -1,13 +1,9 @@ import { - type AutomergeUrl, type DocHandle, type Repo, isImmutableString, - isValidAutomergeUrl, - parseAutomergeUrl, } from "@automerge/automerge-repo/slim"; -import type { FolderDoc } from "./types.js"; -import { getType } from "./metadata.js"; +import { type PathProvider, asLink, providerFor } from "./providers.js"; export type Resolved = { content: string | Uint8Array; @@ -37,95 +33,42 @@ type Walk = { /** A URL carries heads when it has a fragment: `automerge:abc#heads`. */ const hasHeads = (url: unknown): boolean => String(url).includes("#"); +/** @deprecated Use {@link PathProvider}, which replaces it. */ +export type FolderStrategy = PathProvider; + /** - * `value` as a link, if it is one. A link written as an `ImmutableString` - * (one op, where a plain string is an op per character) reads back as an - * object, not a string; it's the same url. + * Resolve `parts` against `node`, a document or a value inside one, using the provider of the + * document it belongs to. Following a link hands the rest to that document's own provider. */ -function asLink(value: unknown): AutomergeUrl | undefined { - const url = isImmutableString(value) ? String(value) : value; - return typeof url === "string" && isValidAutomergeUrl(url) ? url : undefined; -} - -export interface FolderStrategy { - matches(doc: unknown): boolean; - resolve( - repo: Repo, - handle: DocHandle, - parts: string[], - walk: Walk - ): Promise; -} - -const folderStrategy: FolderStrategy = { - matches(doc) { - return ( - !!doc && - typeof doc === "object" && - "docs" in doc && - Array.isArray((doc as { docs: unknown }).docs) - ); - }, - async resolve(repo, handle, parts, walk) { - const folder = handle.doc() as FolderDoc | undefined; - if (!folder?.docs) return undefined; - - const part = parts[0]; - const docLink = folder.docs.find((doc) => String(doc.name) === part); - const url = asLink(docLink?.url); - if (!url) return undefined; - - // Following a bare link means everything below this point is current, not fixed. - walk.pinned &&= hasHeads(url); - const next = await repo.find(url); - return resolvePathInternal(repo, next, parts.slice(1), walk); - }, -}; - -const directoryStrategy: FolderStrategy = { - matches(doc) { - return getType(doc as Parameters[0]) === "directory"; - }, - async resolve(repo, handle, parts, walk) { - return walkDirectoryDoc(repo, handle.doc(), parts, walk); - }, -}; - -async function walkDirectoryDoc( +async function resolveIn( repo: Repo, + provider: PathProvider, node: unknown, parts: string[], walk: Walk ): Promise { - const link = asLink(node); - if (link) { - // Following a url consumes no parts, so revisiting the same url with the - // same remaining parts is a cycle. - const key = `${link}|${parts.join("/")}`; - if (walk.visited.has(key)) return undefined; - walk.visited.add(key); - walk.pinned &&= hasHeads(link); - const next = await repo.find(link); - return resolvePathInternal(repo, next, parts, walk); - } - - if (parts.length === 0) { - return materialize(repo, node, undefined, walk); - } - - if (!node || typeof node !== "object" || node instanceof Uint8Array) { - return undefined; - } - - // Longest-prefix match: try "main/dist/index.js", then "main/dist", then "main" - const obj = node as Record; - for (let i = parts.length; i >= 1; i--) { - const key = parts.slice(0, i).join("/"); - if (key in obj) { - return walkDirectoryDoc(repo, obj[key], parts.slice(i), walk); - } - } - return undefined; + if (parts.length === 0) return materialize(repo, node, undefined, walk); + + const found = provider.lookup(node, parts); + if (!found) return undefined; + const { child, consumed } = found; + const rest = parts.slice(consumed); + if ("value" in child) return resolveIn(repo, provider, child.value, rest, walk); + // Only built-ins walk here, and none hands out sub-documents. + if ("sub" in child) return undefined; + + // A union (an account's mount) is read through its first binding. + const link = "link" in child ? child.link : child.union[0]; + if (!link) return undefined; + // A step that consumes no parts can revisit the same url with the same remaining parts: a + // cycle. + const key = `${link}|${rest.join("/")}`; + if (walk.visited.has(key)) return undefined; + walk.visited.add(key); + // Following a bare link means everything below this point is current, not fixed. + walk.pinned &&= hasHeads(link); + const next = await repo.find(link); + return resolvePathInternal(repo, next, rest, walk); } // rule of thumb: bytes pass through, anything else gets JSON.stringify'd unless @@ -207,8 +150,6 @@ async function materialize( return undefined; } -const STRATEGIES: FolderStrategy[] = [directoryStrategy, folderStrategy]; - async function resolvePathInternal( repo: Repo, handle: DocHandle, @@ -220,12 +161,8 @@ async function resolvePathInternal( } const doc = handle.doc(); - for (const strategy of STRATEGIES) { - if (strategy.matches(doc)) { - return strategy.resolve(repo, handle, parts, walk); - } - } - return undefined; + const provider = providerFor(doc); + return provider ? resolveIn(repo, provider, doc, parts, walk) : undefined; } export async function resolvePath( @@ -240,3 +177,15 @@ export async function resolvePath( pinned: hasHeads(rootHandle.url), }); } + +/** + * What `node` serves as, the way {@link resolvePath} serves the end of a path: a file doc's + * content with its mime type, bytes as they are, anything else as JSON. + */ +export async function readNode( + repo: Repo, + node: unknown +): Promise<{ content: string | Uint8Array; type: string } | undefined> { + const read = await materialize(repo, node); + return read && { content: read.content, type: read.type }; +} diff --git a/core/filesystem/src/types.ts b/core/filesystem/src/types.ts index 45898012b..836e70885 100644 --- a/core/filesystem/src/types.ts +++ b/core/filesystem/src/types.ts @@ -33,3 +33,18 @@ export type UnixFileEntry = { mimeType: string; name: string; }; + +/** One document bound at a mount point. Only automerge documents can be bound. */ +export type NamespaceBinding = { url: AutomergeUrl }; + +/** + * An account's mount table, after plan9: each mount point is `/` or a top-level `/`, bound + * to an ordered union of documents. Lookup tries each binding in order; listing concatenates. + */ +export type Namespace = { [mountPoint: string]: NamespaceBinding[] }; + +/** The parts of an account doc a path walk reads. */ +export type AccountNamespaceDoc = HasPatchworkMetadata<"account"> & { + namespace?: Namespace; + rootFolderUrl?: AutomergeUrl; +}; diff --git a/core/filesystem/src/walk.ts b/core/filesystem/src/walk.ts new file mode 100644 index 000000000..12110287e --- /dev/null +++ b/core/filesystem/src/walk.ts @@ -0,0 +1,782 @@ +import { + type AutomergeUrl, + type DocHandle, + type DocumentId, + type Repo, + type UrlHeads, + decodeHeads, + isImmutableString, + parseAutomergeUrl, + stringifyAutomergeUrl, +} from "@automerge/automerge-repo/slim"; +import { hasHeads } from "@automerge/automerge/slim"; +import { + type DocStep, + docStep, + formatHeads, + idhash, + parsePath, +} from "./paths.js"; +import { + type Child, + type ExtraProviders, + type Lookup, + type SubSegment, + directoryProvider, + folderProvider, + type PathProvider, + accountProvider, + branchesProvider, + defaultBranch, + providerFor, + valueAt, + yieldsToPackage, +} from "./providers.js"; +import { buildFileList } from "./file-list.js"; +import { readNode } from "./resolve.js"; +import { documentBaseOrigin } from "./urls.js"; + +export type WalkOptions = { + /** acctHash → account doc url. The roots a live or pinned account path may start at. */ + accounts: ReadonlyMap; + /** + * Find a document, given its bare url and the heads the walk will read it at (none: current). + * The walk views what it returns at those heads. A resolver passes one that waits for the + * heads to arrive, or for peers to catch up; the default is `repo.find`. A rejection fails + * the walk. + */ + find?: (url: AutomergeUrl, heads?: UrlHeads) => Promise>; + /** Providers for documents no built-in walks. A tab passes the ones packages register. */ + providers?: ExtraProviders; +}; + +/** The built-in provider for `doc`, else one of `extra`'s; one that fails is none. */ +async function providerOf( + doc: unknown, + extra?: ExtraProviders +): Promise { + const builtIn = providerFor(doc); + if (!extra || (builtIn && !yieldsToPackage(builtIn, doc))) return builtIn; + try { + return (await extra(doc)) ?? builtIn; + } catch { + return builtIn; + } +} + +// A package's provider is package code: one that throws has no children here. +function lookUp(provider: PathProvider, node: unknown, names: string[]): Lookup | undefined { + try { + return provider.lookup(node, names); + } catch { + return undefined; + } +} + +function listOf(provider: PathProvider | undefined, node: unknown): Child[] { + try { + const children = provider?.list(node); + return Array.isArray(children) ? children : []; + } catch { + return []; + } +} + +/** + * The path segments a child's name is written as: one, except a directory key, which spans + * several. The walk and file lists both write it this way, so a name reached by several + * spellings ("a%2Fb", or "a/b" consumed together) pins to the one tail its list has. + */ +const nameSegments = (provider: PathProvider | undefined, name: string) => + provider === directoryProvider ? name.split("/") : [name]; + +// idhashes never change, and every walk hashes the same few documents again. +const idhashes = new Map>(); +function memoIdhash(documentId: string): Promise { + let hash = idhashes.get(documentId); + if (!hash) { + if (idhashes.size >= 10_000) idhashes.clear(); + idhashes.set(documentId, (hash = idhash(documentId))); + } + return hash; +} + +export type WalkEnd = { + /** The last document the walk entered, as a view at the heads it was read at. */ + handle: DocHandle; + /** Where the walk stopped: the document itself, or a value nested in it. */ + node: unknown; + /** False when the path ends at a value inside the document rather than at the document. */ + atDocument: boolean; + /** The pinned path, without a leading slash. */ + pinned: string; + /** + * True when every document the pinned path names was entered at heads the input gave: the + * input was already pinned and means what `pinned` means. A pinned root followed by names is + * not, since the names resolve at whatever the documents below it hold now. + */ + canonical: boolean; + /** + * The last document the walk entered that has a provider (a folder, a directory, …): its + * pinned path, without its own `~listHash`, and that list hash if the input gave one. Its file + * list covers the files around the end of the path. + */ + container?: { handle: DocHandle; pinned: string; listHash?: string }; + /** The encoded names after the container, joined with "/": "" at the container itself. */ + tail: string; + /** Where the walk ended inside the document, when that is a sub-document: `handle.sub(...sub)`. */ + sub?: SubSegment[]; +}; + +/** Where a walk is: inside a document, or at a mount whose binding the next step picks. */ +type At = + | { + kind: "doc"; + handle: DocHandle; + node: unknown; + provider?: PathProvider; + inside: boolean; + /** The segments from the document to `node`, while every step inside was a sub-document. */ + sub?: SubSegment[]; + } + | { kind: "union"; members: AutomergeUrl[] }; + +type AtDoc = Extract; + +/** + * Walk a live or pinned path and write its pinned form as it goes. + * + * Every document that has a provider (a container) is written as `@` in place of + * the name that led to it. A step that consumes no name (a union member, a branches doc's + * default) adds its own doc step. Steps inside a document, and the names that lead from the last + * container to a document without a provider (a file), stay as names: that tail is what lets a + * module's relative imports resolve. + * + * Returns undefined when the path doesn't resolve. Throws only for a path that doesn't parse. + */ +export async function walkPath( + repo: Repo, + path: string, + options: WalkOptions +): Promise { + const { root, segments } = parsePath(path); + const hashOf = (url: AutomergeUrl) => + memoIdhash(parseAutomergeUrl(url).documentId); + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const stepOf = async (handle: DocHandle) => + docStep(await hashOf(handle.url), handle.heads()); + + // Package providers apply to a document the walk goes on through, and to one named by a doc + // step (only a container is ever written as one, so a pinned prefix stays its container). A + // document a path ends at *by name* is a container or a file by the built-ins' call, so a path + // pins the same with or without a package's provider, and a plugin installed later doesn't + // change what an existing pinned URL means. + async function enter( + url: AutomergeUrl, + heads: UrlHeads | undefined, + through: boolean + ): Promise { + try { + const parsed = parseAutomergeUrl(url); + // A link to a sub-document: walking it as the whole document would name the wrong thing. + if (parsed.segments?.length) return undefined; + const at = heads ?? parsed.heads; + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + at + ); + const handle = live.view(at ?? live.heads()); + const node = handle.doc(); + if (node === undefined) return undefined; + const provider = through ? await providerOf(node, options.providers) : providerFor(node); + return { kind: "doc", handle, node, provider, inside: false }; + } catch { + return undefined; + } + } + + async function matchLink(links: AutomergeUrl[], step: DocStep) { + for (const link of links) { + if ((await hashOf(link)) === step.idhash) return link; + } + return undefined; + } + + // Steps that consume no name can loop (a branches doc whose default is itself, `/` bound to the + // account); revisiting a document with as many segments left is a cycle. + const seen = new Set(); + const guard = (url: AutomergeUrl, remaining: number) => { + const key = `${parseAutomergeUrl(url).documentId}|${remaining}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }; + + // The names from i on, for a lookup that may consume several (a directory key like + // "vendor/react@18.3.1/index.js"). A doc step reads as a name here: one is only taken as a step + // at the front, and only when it matches a link. + const namesFrom = (i: number) => + segments + .slice(i) + .map((segment) => + segment.kind === "name" ? segment.name : decodeURIComponent(segment.raw) + ); + + const out: string[] = []; + // Index in `out` → the file list hash the input gave that doc step. + const lists = new Map(); + const render = (from: number, to: number) => + out + .slice(from, to) + .map((step, j) => (lists.has(from + j) ? `${step}~${lists.get(from + j)}` : step)) + .join("/"); + // Set once a document the pinned path names was entered at its current heads. + let live = !root.heads; + let at: At | undefined; + if (root.kind === "account") { + const url = options.accounts.get(root.acctHash); + at = url && (await enter(url, root.heads, true)); + if (!at) return undefined; + out.push(docStep(root.acctHash, at.handle.heads())); + if (root.listHash) lists.set(0, root.listHash); + } else { + at = await enter( + stringifyAutomergeUrl({ documentId: root.documentId }), + root.heads, + true + ); + if (!at) return undefined; + out.push(`automerge:${root.documentId}@${formatHeads(root.heads)}`); + if (root.listHash) lists.set(0, root.listHash); + } + + let container: { handle: DocHandle; index: number } | undefined; + const noteContainer = () => { + if (at?.kind === "doc" && at.provider && !at.inside) { + container = { handle: at.handle, index: out.length - 1 }; + } + }; + + let i = 0; + while (true) { + noteContainer(); + const segment = segments[i] as (typeof segments)[number] | undefined; + + if (at.kind === "union") { + let member: AutomergeUrl | undefined; + let next: AtDoc | undefined; + let listHash: string | undefined; + if (segment?.kind === "doc") { + member = await matchLink(at.members, segment); + if (member) { + listHash = segment.listHash; + i++; + if (!guard(member, segments.length - i)) return undefined; + next = await enter(member, segment.heads, true); + } + } + if (!member) { + const names = namesFrom(i); + for (const url of at.members) { + const candidate = await enter(url, undefined, names.length > 0); + if ( + !names.length || + (candidate?.provider && lookUp(candidate.provider, candidate.node, names)) + ) { + member = url; + next = candidate; + break; + } + } + if (!member || !guard(member, segments.length - i)) return undefined; + live = true; + } + if (!next) return undefined; + out.push(await stepOf(next.handle)); + if (listHash) lists.set(out.length - 1, listHash); + at = next; + continue; + } + + if (!segment) { + // A path that ends at a branches doc stands for its default branch. + if (at.provider !== branchesProvider || at.inside) break; + const link = defaultBranch(at.node); + if (!link || !guard(link, 0)) return undefined; + const next = await enter(link, undefined, false); + if (!next) return undefined; + out.push(await stepOf(next.handle)); + live = true; + at = next; + continue; + } + + if (segment.kind === "doc") { + const links = listOf(at.provider, at.node).flatMap((child) => + "link" in child ? [child.link] : "union" in child ? child.union : [] + ); + const link = await matchLink(links, segment); + if (link) { + const next = await enter(link, segment.heads, true); + if (!next) return undefined; + out.push(await stepOf(next.handle)); + if (segment.listHash) lists.set(out.length - 1, segment.listHash); + at = next; + i++; + continue; + } + } + + // A document without a provider has no children. + if (!at.provider) return undefined; + const names = namesFrom(i); + const found: Lookup | undefined = lookUp(at.provider, at.node, names); + if (!found) return undefined; + const { child, consumed }: Lookup = found; + // A step inside the document must use a name, or the walk would never move on; and none + // can use more names than there are. + if (consumed > names.length || (consumed < 1 && ("value" in child || "sub" in child))) { + return undefined; + } + const written = + consumed === 0 + ? [] + : typeof child.name === "string" && child.name + ? nameSegments(at.provider, child.name).map(encodeURIComponent) + : names.slice(0, consumed).map(encodeURIComponent); + i += consumed; + + if ("value" in child) { + out.push(...written); + at = { ...at, node: child.value, inside: true, sub: undefined }; + } else if ("sub" in child) { + const node = valueAt(at.node, child.sub); + if (node === undefined) return undefined; + out.push(...written); + const sub: SubSegment[] | undefined = at.inside && !at.sub ? undefined : [...(at.sub ?? []), ...child.sub]; + at = { ...at, node, inside: true, sub }; + } else if ("union" in child) { + out.push(...written); + at = { kind: "union", members: child.union }; + } else { + if (!consumed && !guard(child.link, segments.length - i)) return undefined; + const next = await enter(child.link, undefined, i < segments.length); + if (!next) return undefined; + if (next.provider || !consumed) { + out.push(await stepOf(next.handle)); + live = true; + } else out.push(...written); + at = next; + } + } + + noteContainer(); + return { + handle: at.handle, + node: at.node, + atDocument: !at.inside, + pinned: render(0, out.length), + canonical: !live, + container: container && { + handle: container.handle, + pinned: [render(0, container.index), out[container.index]] + .filter(Boolean) + .join("/"), + listHash: lists.get(container.index), + }, + tail: container ? out.slice(container.index + 1).join("/") : "", + ...(at.sub ? { sub: at.sub } : {}), + }; +} + +/** + * The document a live or pinned path names. For a canonical pinned path (see + * {@link WalkEnd.canonical}) that is a view at the heads it was read at; a file reached by name + * after the last pinned document is read at its current heads, or the heads its link carries. + * Anything else gets the live handle. A path that ends at a sub-document gets that handle's + * sub-handle. Undefined if the path doesn't resolve, or ends at any other value inside a + * document. + */ +export async function resolveLivePath( + repo: Repo, + path: string, + options: WalkOptions +): Promise | undefined> { + const end = await walkPath(repo, path, options); + if (!end || (!end.atDocument && !end.sub)) return undefined; + const handle = end.canonical + ? end.handle + : await repo.find(stringifyAutomergeUrl({ documentId: end.handle.documentId })); + return end.sub ? handle.sub(...end.sub) : handle; +} + +/** + * A file list couldn't read every file it covers. A pinned URL never changes meaning, so a + * list missing a file would make it answer 404 for good; pin again once they can be read. + */ +export class IncompleteFileListError extends Error { + name = "IncompleteFileListError"; + constructor(readonly missing: string[]) { + super(`couldn't read ${missing.join(", ")} to pin them`); + } +} + +/** + * The pinned form of a path, with a file list on the last container it enters: every file + * under that container, or with `only: "end"`, only the one the path ends at (what a live → + * pinned redirect hands out). Returns the path and the list's bytes, which whoever pins hands to + * the service worker. No list if the path enters no container. Throws + * {@link IncompleteFileListError} rather than pin a list missing a file it should cover. + */ +export async function pinnedPath( + repo: Repo, + path: string, + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: "end"; + isProtected?: (documentId: DocumentId) => boolean; + } +): Promise<{ path: string; list?: Uint8Array } | undefined> { + const end = await walkPath(repo, path, options); + if (!end) return undefined; + if (!end.container) return { path: end.pinned }; + // A path that ends at a container lists the whole container; one that ends + // at a file, just that file. + const atContainer = end.tail === "" && end.atDocument; + const built = await buildFileList(repo, end.container.pinned, { + ...options, + only: options.only === "end" && !atContainer ? [end.tail] : undefined, + }); + if (!built) return { path: end.pinned }; + if (built.missing.length) throw new IncompleteFileListError(built.missing); + const listed = `${end.container.pinned}~${built.listHash}`; + return { path: end.tail ? `${listed}/${end.tail}` : listed, list: built.bytes }; +} + +/** The pinned service worker URL for a path, for fetching or importing, and its file list. */ +export async function pinPath( + repo: Repo, + path: string, + options: Parameters[2] +): Promise<{ url: string; list?: Uint8Array } | undefined> { + const pinned = await pinnedPath(repo, path, options); + return pinned && { url: `${documentBaseOrigin()}/@/${pinned.path}`, list: pinned.list }; +} + +/** Every binding's children, concatenated in binding order. */ +export async function listUnion( + repo: Repo, + urls: AutomergeUrl[], + providers?: ExtraProviders +): Promise { + const lists = await Promise.all( + urls.map(async (url) => { + try { + const doc = (await repo.find(url)).doc(); + return listOf(await providerOf(doc, providers), doc); + } catch { + return []; + } + }) + ); + return lists.flat(); +} + +/** + * A document's children. For an account, `/`'s union is expanded and each mount shadows a + * same-named entry in it; mounts stay as union children. + */ +export async function listChildren( + repo: Repo, + handle: DocHandle, + providers?: ExtraProviders +): Promise { + const doc = handle.doc(); + const provider = await providerOf(doc, providers); + if (!provider) return []; + const children = listOf(provider, doc); + if (provider !== accountProvider) return children; + + const mounts = children.filter((child) => child.name !== ""); + const root = children.find((child) => child.name === ""); + const shadowed = new Set(mounts.map((child) => child.name)); + const rootChildren = + root && "union" in root ? await listUnion(repo, root.union, providers) : []; + return [ + ...rootChildren.filter((child) => !shadowed.has(child.name)), + ...mounts, + ]; +} + +export type FileContent = { content: string | Uint8Array; type: string }; + +/** + * What a path names, read: its pinned form, whether the input was canonical, and the bytes. + * With `heads`, the document the path ends at is read at those heads instead: how a file is + * read at the heads its file list recorded, after it has changed. With `via`, the heads its + * list recorded for the nested folders on the way: a document the walk reaches by a link + * without heads is read at whichever of these it has, so a file renamed since is still found. + */ +export async function readPath( + repo: Repo, + path: string, + options: WalkOptions & { heads?: UrlHeads; via?: UrlHeads[] } +): Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined> { + const recorded = [...(options.via ?? []), ...(options.heads ? [options.heads] : [])]; + const find0 = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const end = await walkPath(repo, path, { + ...options, + find: recorded.length + ? async (url, at) => { + const handle = await find0(url, at); + if (at) return handle; + // A document's own change hashes: only it, or a copy of it, has them. + const doc = handle.doc(); + const mine = doc && recorded.find((heads) => hasHeads(doc, decodeHeads(heads))); + return mine ? handle.view(mine) : handle; + } + : options.find, + }); + if (!end) return undefined; + let node = end.node; + if (options.heads) { + const heads = options.heads; + const find = options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const holder = end.handle.documentId; + try { + if (end.atDocument) { + const bare = stringifyAutomergeUrl({ documentId: holder }); + node = (await find(bare, heads)).view(heads).doc(); + } else { + // A value inside a document: walk again with that document read at the heads. + const atHeads = await walkPath(repo, path, { + ...options, + find: async (url, at) => + parseAutomergeUrl(url).documentId === holder + ? (await find(url, heads)).view(heads) + : find(url, at), + }); + if (!atHeads) return undefined; + node = atHeads.node; + } + } catch { + return undefined; + } + } + const read = await readNode(repo, node); + return read && { ...read, pinned: end.pinned, canonical: end.canonical }; +} + +/** + * A file under a pinned document, the document that holds its bytes, and the heads that + * document was read at if it was reached by a link. `via`: the heads each nested folder doc + * on the way to it was read at, outermost first, when it's in a package's nested folder. + */ +export type ListedFile = FileContent & { + tail: string; + documentId: DocumentId; + heads?: UrlHeads; + via?: UrlHeads[]; +}; + +const isNested = (value: unknown): value is Record => + !!value && + typeof value === "object" && + !Array.isArray(value) && + !(value instanceof Uint8Array) && + !isImmutableString(value) && + !("content" in value); + +// Which of two spellings of one tail the walk finds: at the first level where their directory +// keys differ in length, the longer key, since lookup tries the longest prefix first. +function walkPrefers(a: number[], b: number[]): boolean { + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (a[i] !== b[i]) return a[i] > b[i]; + } + return false; +} + +// How many file docs a listing reads at once. +const LIST_CONCURRENCY = 8; + +/** + * Every file under the last container a canonical pinned path enters (see + * {@link WalkEnd.container}), keyed by tail: "" for the container itself, encoded names joined + * with "/" below it. Folders below it are listed into only when it's a package (see `deep`); + * otherwise they have their own doc steps. `path` may name the container or anything in it. + * Undefined unless `path` is canonical. + * + * File docs are read with `findFile` (default: `find`), several at once. One that can't be read + * is left out and its tail named in `missing`, so a caller can tell a partial list from a whole + * one. A file reached by a link without heads is read as it is now, so the list is a snapshot. + */ +export async function listFiles( + repo: Repo, + path: string, + options: WalkOptions & { + findFile?: WalkOptions["find"]; + only?: string[]; + /** List into nested folders. Default: when the container has a `package.json`. */ + deep?: boolean; + } +): Promise<{ prefix: string; files: ListedFile[]; missing: string[] } | undefined> { + const end = await walkPath(repo, path, options); + if (!end?.canonical || !end.container) return undefined; + const find = + options.findFile ?? options.find ?? ((url: AutomergeUrl) => repo.find(url)); + const root = end.container.handle.doc(); + const rootProvider = await providerOf(root, options.providers); + const rootId = end.container.handle.documentId; + // A package's relative imports reach into its nested folders by name, so a + // package lists everything under it. Anything else lists its own files. + const deep = + options.deep ?? + listOf(rootProvider, root).some((child) => child.name === "package.json"); + + const found = new Map(); + const missing: string[] = []; + const tailOf = (parts: string[]) => parts.map(encodeURIComponent).join("/"); + const wanted = (parts: string[]) => !options.only || options.only.includes(tailOf(parts)); + // A folder is worth entering if anything wanted could be under it. + const wantedBelow = (parts: string[]) => + !options.only || options.only.some((tail) => tail.startsWith(`${tailOf(parts)}/`)); + const add = async ( + parts: string[], + keys: number[], + node: unknown, + holder: DocumentId, + heads?: UrlHeads, + via: UrlHeads[] = [] + ) => { + if (!wanted(parts)) return; + const tail = tailOf(parts); + const existing = found.get(tail); + if (existing && !walkPrefers(keys, existing.keys)) return; + const read = await readNode(repo, node); + if (read) { + found.set(tail, { + keys, + file: { + ...read, + tail, + documentId: holder, + ...(heads ? { heads } : {}), + ...(via.length ? { via } : {}), + }, + }); + } + }; + + // Values are read in place; links are gathered and read below. `via`: the heads of the + // nested folder docs on the way to the link, outermost first. + type Link = { parts: string[]; keys: number[]; link: AutomergeUrl; via: UrlHeads[] }; + const entered = new Set([rootId]); + // `heads`: what the holder was read at, when it's a doc below the container (reached by a + // link, so not pinned by the container's step): its values are read back at those heads. + // `via`: the heads of the nested folder docs above the holder, so a walk back to a file can + // read each at what it was, after a file in one is renamed. + const visit = async ( + node: unknown, + provider: PathProvider | undefined, + holder: DocumentId, + parts: string[], + keys: number[], + heads?: UrlHeads, + via: UrlHeads[] = [] + ): Promise => { + const links: Link[] = []; + for (const child of listOf(provider, node)) { + const names = nameSegments(provider, child.name); + const childParts = [...parts, ...names]; + const childKeys = [...keys, names.length]; + const builtIn = provider === directoryProvider || provider === folderProvider; + if ("sub" in child || "value" in child) { + const value = "sub" in child ? valueAt(node, child.sub) : child.value; + if (value === undefined) continue; + await add(childParts, childKeys, value, holder, heads, via); + // A package's provider walks on into the values it hands out, so list what's in them. + if (!builtIn && value && typeof value === "object" && !(value instanceof Uint8Array)) { + links.push(...(await visit(value, provider, holder, childParts, childKeys, heads, via))); + continue; + } + } + if ("value" in child) { + // Nested values are addressable by path only inside a directory doc. + if (provider === directoryProvider && isNested(child.value)) { + links.push( + ...(await visit(child.value, provider, holder, childParts, childKeys, heads, via)) + ); + } + } else if ( + "link" in child && + (wanted(childParts) || (deep && wantedBelow(childParts))) + ) { + links.push({ + parts: childParts, + keys: childKeys, + link: child.link, + via: heads ? [...via, heads] : via, + }); + } + } + return links; + }; + + // At most LIST_CONCURRENCY file docs read at once, nested folders included. + let active = 0; + const queue: Array<() => void> = []; + const acquire = async () => { + if (active >= LIST_CONCURRENCY) await new Promise((resolve) => queue.push(resolve)); + active++; + }; + const release = () => { + active--; + queue.shift()?.(); + }; + + const read = async ({ parts, keys, link, via }: Link): Promise => { + let below: Link[] = []; + await acquire(); + try { + const parsed = parseAutomergeUrl(link); + // A link to a sub-document isn't a file a walk can reach (see walkPath). + if (parsed.segments?.length) return; + const live = await find( + stringifyAutomergeUrl({ documentId: parsed.documentId }), + parsed.heads + ); + const heads = parsed.heads ?? live.heads(); + const doc = live.view(heads).doc(); + if (doc === undefined) return; + // Built-ins decide what's a file, as the walk does for a path that ends here. + const provider = providerFor(doc); + if (!provider) { + await add(parts, keys, doc, parsed.documentId, heads, via); + } else if ( + deep && + (provider === folderProvider || provider === directoryProvider) && + !entered.has(parsed.documentId) + ) { + // Into a nested folder, by name: its files are under its name here. + entered.add(parsed.documentId); + await add(parts, keys, doc, parsed.documentId, heads, via); + below = await visit(doc, provider, parsed.documentId, parts, keys, heads, via); + } + } catch { + missing.push(tailOf(parts)); + } finally { + release(); + } + // Read what's below after letting go of the slot, so nesting can't deadlock. + await Promise.all(below.map(read)); + }; + + await add([], [], root, rootId); + await Promise.all((await visit(root, rootProvider, rootId, [], [])).map(read)); + + return { + prefix: end.container.pinned, + files: [...found.values()].map(({ file }) => file), + missing, + }; +} diff --git a/core/filesystem/test/file-list.test.ts b/core/filesystem/test/file-list.test.ts new file mode 100644 index 000000000..a2998856e --- /dev/null +++ b/core/filesystem/test/file-list.test.ts @@ -0,0 +1,252 @@ +import { describe, it, expect } from "vitest"; +import { parseAutomergeUrl } from "@automerge/automerge-repo"; +import { formatHeads, livePath, parseHeads } from "../src/paths.js"; +import { IncompleteFileListError, pinnedPath, readPath } from "../src/walk.js"; +import { contentHash, toBytes } from "../src/content-hash.js"; +import { buildFileList, canonicalJson, parseFileList } from "../src/file-list.js"; +import { chain, file, fixture, folder } from "./namespace-fixture.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +describe("canonicalJson", () => { + it("sorts keys at every level and leaves no whitespace", () => { + expect(canonicalJson({ b: 1, a: { d: 1, c: [2, { f: 1, e: 2 }] } })).toBe( + '{"a":{"c":[2,{"e":2,"f":1}],"d":1},"b":1}' + ); + }); +}); + +describe("buildFileList", () => { + it("is canonical: building it twice gives the same bytes and hash", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const first = await buildFileList(repo, prefix!, { accounts }); + const second = await buildFileList(repo, prefix!, { accounts }); + expect(first?.bytes).toEqual(second?.bytes); + expect(first?.listHash).toBe(await contentHash(first!.bytes)); + expect(parseFileList(first!.bytes)).toEqual(first?.list); + }); + + it("records each file's hash, type and heads, and no document ids", async () => { + const { repo, a, accounts, minePkg, mine, account } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const fileUrl = (minePkg.doc() as any)["dist/index.js"] as string; + const file = await repo.find(fileUrl as any); + expect(built?.list.files["dist/index.js"]).toEqual({ + sha256: await contentHash(toBytes("mine")), + type: "text/plain", + heads: formatHeads(file.heads()), + store: true, + }); + const text = new TextDecoder().decode(built!.bytes); + for (const doc of [minePkg, mine, account, file]) { + expect(text).not.toContain(doc.documentId); + } + expect(text).not.toContain(parseAutomergeUrl(fileUrl as any).documentId); + }); + + it("lists a file named __proto__ like any other", async () => { + const { repo, a, accounts, foo } = await fixture(); + foo.change((d: any) => d.docs.push({ name: "__proto__", type: "x", url: file(repo, "p").url })); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(Object.hasOwn(built!.list.files, "__proto__")).toBe(true); + expect(new TextDecoder().decode(built!.bytes)).toContain('"__proto__":'); + expect(Object.hasOwn(parseFileList(built!.bytes)!.files, "__proto__")).toBe(true); + }); + + it("lists only what it's asked to", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts, only: ["dist/index.js"] }); + expect(Object.keys(built!.list.files)).toEqual(["dist/index.js"]); + }); + + it("stores protected content only through /packages", async () => { + const { repo, a, accounts } = await fixture(); + const isProtected = () => true; + const projects = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const pkg = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const userList = await buildFileList(repo, projects!, { accounts, isProtected }); + const pkgList = await buildFileList(repo, pkg!, { accounts, isProtected }); + expect(userList?.list.files.notes.store).toBe(false); + expect(pkgList?.list.files["dist/index.js"].store).toBe(true); + }); + + it("reads a file at the heads its list recorded, after it changed", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dist/index.js"]; + const file = await repo.find((minePkg.doc() as any)["dist/index.js"]); + (file as any).change((d: any) => (d.content = "changed")); + + const now = await readPath(repo, `${prefix}/dist/index.js`, { accounts }); + expect(decode(now!.content)).toBe("changed"); + const then = await readPath(repo, `${prefix}/dist/index.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("mine"); + }); +}); + +describe("pinnedPath", () => { + it("refuses to pin a list that's missing files", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const bad = (minePkg.doc() as any)["package.json"] as string; + const find = (url: any) => + url === bad ? Promise.reject(new Error("unavailable")) : repo.find(url); + const pinning = pinnedPath(repo, livePath(a, ["packages", "foo"]), { accounts, find }); + await expect(pinning).rejects.toBeInstanceOf(IncompleteFileListError); + await expect(pinning).rejects.toMatchObject({ missing: ["package.json"] }); + }); +}); + +describe("package lists", () => { + /** A folder package under /packages: package.json, index.js, and sub/x.js. */ + async function folderPackage() { + const f = await fixture(); + const x = file(f.repo, "x"); + const sub = folder(f.repo, { "x.js": x.url }); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"index.js"}').url, + "index.js": file(f.repo, "export * from './sub/x.js'").url, + sub: sub.url, + }); + f.mine.change((d: any) => d.docs.push({ name: "fpkg", type: "folder", url: pkg.url })); + const prefix = await chain(f.repo, livePath(f.a, ["packages", "fpkg"]), f.accounts); + return { ...f, x, sub, pkg, prefix: prefix! }; + } + + it("lists a package's nested folders deep", async () => { + const { repo, accounts, prefix } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + expect(Object.keys(built!.list.files)).toEqual( + expect.arrayContaining(["package.json", "index.js", "sub/x.js"]) + ); + }); + + it("lists a user folder shallow", async () => { + const { repo, a, accounts, foo } = await fixture(); + const sub = folder(repo, { inner: file(repo, "inner").url }); + foo.change((d: any) => d.docs.push({ name: "sub", type: "folder", url: sub.url })); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(Object.keys(built!.list.files).some((tail) => tail.startsWith("sub/"))).toBe(false); + }); + + it("reads a nested file at its recorded heads", async () => { + const { repo, accounts, prefix, x } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + const entry = built!.list.files["sub/x.js"]; + x.change((d: any) => (d.content = "changed")); + const then = await readPath(repo, `${prefix}/sub/x.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("x"); + }); + + it("records the heads of a nested doc that holds values, and reads them there", async () => { + const { repo, a, accounts, pkg } = await folderPackage(); + const dir = repo.create({ + "@patchwork": { type: "directory" }, + "a.js": { content: "old", mimeType: "text/javascript" }, + }); + pkg.change((d: any) => d.docs.push({ name: "dir", type: "directory", url: dir.url })); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dir/a.js"]; + expect(entry?.heads).toBeDefined(); + dir.change((d: any) => (d["a.js"].content = "new")); + const then = await readPath(repo, `${built!.prefix}/dir/a.js`, { + accounts, + heads: parseHeads(entry!.heads!), + }); + expect(decode(then!.content)).toBe("old"); + }); + + it("still reads a nested file renamed since it was pinned", async () => { + const { repo, accounts, prefix, sub } = await folderPackage(); + const built = await buildFileList(repo, prefix, { accounts }); + const entry = built!.list.files["sub/x.js"]; + expect(entry.via).toEqual([formatHeads(sub.heads())]); + sub.change((d: any) => (d.docs[0].name = "y.js")); + const then = await readPath(repo, `${built!.prefix}/sub/x.js`, { + accounts, + heads: parseHeads(entry.heads!), + via: entry.via!.map((heads) => parseHeads(heads)!), + }); + expect(decode(then!.content)).toBe("x"); + }); + + it("still reads a value renamed in a nested directory doc since it was pinned", async () => { + const { repo, a, accounts, pkg } = await folderPackage(); + const dir = repo.create({ + "@patchwork": { type: "directory" }, + "a.js": { content: "old", mimeType: "text/javascript" }, + }); + pkg.change((d: any) => d.docs.push({ name: "dir", type: "directory", url: dir.url })); + const prefix = await chain(repo, livePath(a, ["packages", "fpkg"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + const entry = built!.list.files["dir/a.js"]; + expect(entry.via).toBeUndefined(); + dir.change((d: any) => { + d["b.js"] = { content: "old", mimeType: "text/javascript" }; + delete d["a.js"]; + }); + const then = await readPath(repo, `${built!.prefix}/dir/a.js`, { + accounts, + heads: parseHeads(entry.heads!), + }); + expect(decode(then!.content)).toBe("old"); + }); + + it("leaves via out of a list with no nested folders, so it hashes as before", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const built = await buildFileList(repo, prefix!, { accounts }); + expect(new TextDecoder().decode(built!.bytes)).not.toContain('"via"'); + }); + + it("pins a live path to a folder with the folder's whole list", async () => { + const { repo, a, accounts } = await folderPackage(); + const pinned = await pinnedPath(repo, livePath(a, ["packages", "fpkg"]), { + accounts, + only: "end", + }); + const list = parseFileList(pinned!.list!)!; + expect(Object.keys(list.files)).toEqual(expect.arrayContaining(["package.json", "sub/x.js"])); + }); +}); + +describe("listFiles concurrency", () => { + it("reads a nested folder's files several at a time", async () => { + const f = await fixture(); + const files = Object.fromEntries( + Array.from({ length: 20 }, (_, i) => [`f${i}.js`, file(f.repo, `${i}`).url]) + ); + const dist = folder(f.repo, files); + const pkg = folder(f.repo, { + "package.json": file(f.repo, '{"main":"dist/f0.js"}').url, + dist: dist.url, + }); + f.mine.change((d: any) => d.docs.push({ name: "big", type: "folder", url: pkg.url })); + const prefix = await chain(f.repo, livePath(f.a, ["packages", "big"]), f.accounts); + let active = 0; + let peak = 0; + const findFile = async (url: any) => { + active++; + peak = Math.max(peak, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active--; + return f.repo.find(url); + }; + const built = await buildFileList(f.repo, prefix!, { accounts: f.accounts, findFile }); + expect(Object.keys(built!.list.files).filter((t) => t.startsWith("dist/"))).toHaveLength(20); + expect(peak).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/core/filesystem/test/files.test.ts b/core/filesystem/test/files.test.ts new file mode 100644 index 000000000..b0d42e3a5 --- /dev/null +++ b/core/filesystem/test/files.test.ts @@ -0,0 +1,143 @@ +import { describe, it, expect } from "vitest"; +import { parseAutomergeUrl, type AutomergeUrl } from "@automerge/automerge-repo"; +import { livePath } from "../src/paths.js"; +import { listFiles, readPath, type ListedFile } from "../src/walk.js"; +import { chain, file, fixture, folder } from "./namespace-fixture.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +async function collect(files: AsyncIterable) { + const out: ListedFile[] = []; + for await (const f of files) out.push(f); + return out.sort((a, b) => a.tail.localeCompare(b.tail)); +} + +const idOf = (url: AutomergeUrl) => parseAutomergeUrl(url).documentId; + +describe("readPath", () => { + it("reads a file and its pinned form", async () => { + const { repo, a, accounts } = await fixture(); + const path = livePath(a, ["packages", "foo", "dist", "index.js"]); + const read = await readPath(repo, path, { accounts }); + expect(read && decode(read.content)).toBe("mine"); + expect(read?.type).toBe("text/plain"); + expect(read?.canonical).toBe(false); + expect(read?.pinned).toBe(await chain(repo, path, accounts)); + }); + + it("reads nothing for a path that doesn't resolve", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await readPath(repo, livePath(a, ["packages", "nope"]), { accounts }) + ).toBeUndefined(); + }); +}); + +describe("listFiles", () => { + it("lists a directory package's files", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const listed = await listFiles(repo, prefix!, { accounts }); + expect(listed?.prefix).toBe(prefix); + const files = await collect(listed!.files); + expect(files.map((f) => f.tail)).toEqual(["", "dist/index.js", "package.json"]); + const index = files.find((f) => f.tail === "dist/index.js")!; + expect(index.documentId).toBe(idOf((minePkg.doc() as any)["dist/index.js"])); + expect(files[0].documentId).toBe(minePkg.documentId); + }); + + it("lists a folder's leaves and skips its subfolders", async () => { + const { repo, a, accounts, foo } = await fixture(); + const sub = folder(repo, { inner: file(repo, "inner").url }); + foo.change((d: any) => d.docs.push({ name: "sub", type: "folder", url: sub.url })); + const prefix = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toEqual(["", "notes"]); + }); + + it("lists nested values inside a directory doc", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.dist = { "a.js": "x" })); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toContain("dist/a.js"); + expect(files.find((f) => f.tail === "dist/a.js")?.documentId).toBe(minePkg.documentId); + }); + + it("encodes names in tails", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d["my file@1.js"] = file(repo, "spaced").url)); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.map((f) => f.tail)).toContain("my%20file%401.js"); + }); + + it("lists a flat key over a nested one at the same path, as the walk finds it", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => { + d["x/y"] = file(repo, "flat").url; + d.x = { y: "nested" }; + }); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + const xy = files.filter((f) => f.tail === "x/y"); + expect(xy).toHaveLength(1); + expect(decode(xy[0].content)).toBe("flat"); + }); + + it("agrees with readPath for every file it lists", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => { + d.dist = { "a.js": "x" }; + d["x/y"] = file(repo, "flat").url; + d.x = { y: "nested" }; + }); + for (const names of [["packages", "foo"], ["Projects", "foo"]]) { + const prefix = await chain(repo, livePath(a, names), accounts); + const files = await collect((await listFiles(repo, prefix!, { accounts }))!.files); + expect(files.length).toBeGreaterThan(1); + for (const f of files) { + const read = await readPath(repo, f.tail ? `${prefix}/${f.tail}` : prefix!, { accounts }); + expect([f.tail, read && decode(read.content), read?.type]).toEqual([ + f.tail, + decode(f.content), + f.type, + ]); + } + } + }); + + it("lists the package a file path is in", async () => { + const { repo, a, accounts } = await fixture(); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const listed = await listFiles(repo, `${prefix}/dist/index.js`, { accounts }); + expect(listed?.prefix).toBe(prefix); + expect((await collect(listed!.files)).map((f) => f.tail)).toContain("dist/index.js"); + }); + + it("lists the package for a path that ends inside it", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.dist = { "a.js": "x" })); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + expect((await listFiles(repo, `${prefix}/dist`, { accounts }))?.prefix).toBe(prefix); + }); + + it("says which files it couldn't read", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const bad = (minePkg.doc() as any)["package.json"] as string; + const find = (url: AutomergeUrl) => + url === bad ? Promise.reject(new Error("unavailable")) : repo.find(url); + const prefix = await chain(repo, livePath(a, ["packages", "foo"]), accounts); + const listed = await listFiles(repo, prefix!, { accounts, find }); + expect(listed?.missing).toEqual(["package.json"]); + expect(listed?.files.map((f) => f.tail)).not.toContain("package.json"); + }); + + it("lists nothing for a path that isn't canonical", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await listFiles(repo, livePath(a, ["packages", "foo"]), { accounts }) + ).toBeUndefined(); + }); +}); diff --git a/core/filesystem/test/module-watcher.test.ts b/core/filesystem/test/module-watcher.test.ts index 7f97a832c..10171ef91 100644 --- a/core/filesystem/test/module-watcher.test.ts +++ b/core/filesystem/test/module-watcher.test.ts @@ -318,7 +318,8 @@ describe("ModuleWatcher module lifecycle", () => { d.modules = []; }); await pause(100); - expect(unloaded).toEqual([folder.url]); + // Under the url it was imported from, which its plugins are registered under. + expect(unloaded).toEqual([importCalls[0]]); folder.change((d: any) => { d.rev = 1; @@ -770,3 +771,354 @@ describe("importPackageFromHttpUrl error handling", () => { expect(err.cause).toBe(parseError); }); }); + +describe("ModuleWatcher /packages folder", () => { + const pause = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + /** Packages as a /packages folder lists them, and a tracker that looks them up as http modules. */ + function folderSource(repo: InstanceType) { + const docs = [0, 1].map(() => repo.create({ "@patchwork": { type: "directory" } })); + let entries = docs.map((doc, i) => ({ + url: doc.url, + name: `pkg${i}`, + memberStep: "m@h", + })); + let changed: () => void = () => {}; + const lookups: string[] = []; + const tracker: PackageTracker = { + async findLatest(url) { + lookups.push(url); + return dataModule(`export const plugins = []; // ${url}`) as any; + }, + watch: () => () => {}, + }; + return { + docs, + tracker, + lookups, + entries: () => entries, + subscribe: (onChange: () => void) => { + changed = onChange; + return () => {}; + }, + set(next: typeof entries) { + entries = next; + changed(); + }, + }; + } + + it("loads every package in the folder", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const loaded: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, source.tracker); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(loaded).toHaveLength(2); + expect(source.lookups.sort()).toEqual(source.docs.map((d) => d.url).sort()); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("imports a package's pinned url, first and on reload, through the package import", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create({ "@patchwork": { type: "directory" } }); + let version = 0; + let moved: (seen?: string) => void = () => {}; + const tracker: PackageTracker = { + findLatest: async () => `https://patchwork.test/@/acct/packages/p@v${version}~list` as any, + watch: (_url, onChange) => { + moved = onChange; + return () => {}; + }, + }; + const imported: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + () => {}, + undefined, + async (url) => { + imported.push(url); + return { plugins: [] }; + }, + tracker + ); + await watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + expect(imported).toEqual(["https://patchwork.test/@/acct/packages/p@v0~list"]); + version = 1; + moved(); + await pause(400); + expect(imported).toEqual([ + "https://patchwork.test/@/acct/packages/p@v0~list", + "https://patchwork.test/@/acct/packages/p@v1~list", + ]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("says which package a loaded import url belongs to", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const announced: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => announced.push(name), undefined, undefined, source.tracker); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(announced).toHaveLength(2); + const packages = announced.map((url) => watcher.packageFor(url)).sort(); + expect(packages).toEqual(source.docs.map((d) => d.url).sort()); + expect(watcher.packageFor("https://elsewhere.test/x.js")).toBeUndefined(); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("loads a package in both module settings and /packages once", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const atHeads = source.docs[0].view(source.docs[0].heads()).url; + const settings = repo.create({ + "@patchwork": { type: "patchwork:module-settings" }, + modules: [atHeads], + }); + const loaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + undefined, + undefined, + source.tracker + ); + // As patchwork wires them: /packages first, then the user's module settings. + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + await watcher.addUrl("user", settings.url); + await pause(50); + expect(loaded).toHaveLength(2); + expect(source.lookups).not.toContain(atHeads); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("doesn't load a module-settings entry migration moved to /packages, once it's removed there", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const [moved, kept] = source.docs; + const settings = repo.create({ + "@patchwork": { type: "patchwork:module-settings" }, + modules: [moved.url, kept.url], + movedToPackages: [moved.documentId], + }); + const loaded: string[] = []; + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, source.tracker); + // The moved package was removed from /packages; the other was never in it. + source.set([]); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + await watcher.addUrl("user", settings.url); + await pause(50); + expect(source.lookups).not.toContain(moved.url); + expect(source.lookups).toContain(kept.url); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("doesn't reload a renamed package, and unloads a removed one", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const source = folderSource(repo); + const loaded: string[] = []; + const unloaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + (name) => unloaded.push(name), + undefined, + source.tracker + ); + await watcher.addPackagesFolder("packages", source.entries, source.subscribe); + expect(loaded).toHaveLength(2); + + source.set(source.entries().map((e, i) => (i === 0 ? { ...e, name: "renamed" } : e))); + await pause(50); + expect(loaded).toHaveLength(2); + + source.set([source.entries()[0]]); + await pause(50); + expect(unloaded).toHaveLength(1); + expect(loaded).toContain(unloaded[0]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); +}); + +describe("ModuleWatcher /packages, through the real lookup chain", () => { + const pause = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + async function packagesAccount() { + const { idhash } = await import("../src/paths.js"); + const { pinnedPath } = await import("../src/walk.js"); + const { trackPackagesFolder, watchPackagesFolder } = await import("../src/packages-folder.js"); + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const pkg = () => { + const json = repo.create({ "@patchwork": { type: "file" }, content: "{}", mimeType: "application/json" }); + return repo.create({ "@patchwork": { type: "directory" }, "package.json": json.url }); + }; + const [p1, p2, p3] = [pkg(), pkg(), pkg()]; + const member = repo.create({ + title: "packages", + docs: [ + { name: "one", type: "directory", url: p1.url }, + { name: "two", type: "directory", url: p2.url }, + ], + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: member.url }] }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + // The pinned path a lookup would redirect to, as a module that says which. + const lookUp = async (path: string) => { + const pinned = await pinnedPath(repo, path, { accounts, only: "end" }); + return `data:text/javascript,export const plugins = []//${pinned!.path}`; + }; + let entries: any[] = []; + const listeners = new Set<() => void>(); + const first = Promise.withResolvers(); + watchPackagesFolder(repo, account, (next) => { + entries = next; + first.resolve(); + for (const listener of listeners) listener(); + }); + await first.promise; + const fallback: PackageTracker = { findLatest: async (u) => u, watch: () => () => {} }; + const tracker = trackPackagesFolder(Promise.resolve(a), () => entries, fallback, lookUp); + const loaded: string[] = []; + const unloaded: string[] = []; + const watcher = new ModuleWatcher( + repo, + {}, + (name) => loaded.push(name), + (name) => unloaded.push(name), + undefined, + tracker + ); + await watcher.addPackagesFolder("packages", () => entries, (onChange) => { + listeners.add(onChange); + return () => listeners.delete(onChange); + }); + return { repo, member, p3, watcher, loaded, unloaded }; + } + + it("adding a package to the folder loads only that package", async () => { + const { repo, member, p3, watcher, loaded } = await packagesAccount(); + expect(loaded).toHaveLength(2); + member.change((d: any) => d.docs.push({ name: "three", type: "directory", url: p3.url })); + await pause(300); + expect(loaded).toHaveLength(3); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("renaming a package reloads nothing", async () => { + const { repo, member, watcher, loaded } = await packagesAccount(); + member.change((d: any) => (d.docs[0].name = "uno")); + await pause(300); + expect(loaded).toHaveLength(2); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("removing a package unloads it under the name it was loaded with", async () => { + const { repo, member, watcher, loaded, unloaded } = await packagesAccount(); + member.change((d: any) => d.docs.splice(1, 1)); + await pause(300); + expect(unloaded).toHaveLength(1); + expect(loaded).toContain(unloaded[0]); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); +}); + +describe("ModuleWatcher lookup retries", () => { + it("retries a lookup that failed", async () => { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create({ "@patchwork": { type: "directory" } }); + let calls = 0; + const tracker: PackageTracker = { + async findLatest() { + if (calls++ === 0) throw new Error("not yet"); + return dataModule("export const plugins = []") as any; + }, + watch: () => () => {}, + }; + const loaded: string[] = []; + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const watcher = new ModuleWatcher(repo, {}, (name) => loaded.push(name), undefined, undefined, tracker); + await watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + expect(loaded).toHaveLength(0); + await vi.waitFor(() => expect(loaded).toHaveLength(1), { timeout: 3000 }); + watcher.dispose(); + errors.mockRestore(); + await repo.shutdown().catch(() => {}); + }); +}); + +describe("ModuleWatcher package status", () => { + function setup(module: string) { + const repo = new AutomergeRepo({ peerId: "tab" as PeerId }); + const doc = repo.create({ "@patchwork": { type: "directory" } }); + const tracker: PackageTracker = { + findLatest: async () => dataModule(module) as any, + watch: () => () => {}, + }; + const watcher = new ModuleWatcher(repo, {}, () => {}, undefined, undefined, tracker); + const events: string[] = []; + watcher.addEventListener("status", () => + events.push(watcher.packageStatus(doc.url)?.state ?? "none") + ); + const add = () => + watcher.addPackagesFolder( + "packages", + () => [{ url: doc.url, name: "p", memberStep: "m@h" }], + () => () => {} + ); + return { repo, doc, watcher, events, add }; + } + + it("says a package loaded, and which plugins it brought", async () => { + const { repo, doc, watcher, events, add } = setup( + "export const plugins = [{ type: 'patchwork:tool', id: 'demo' }]" + ); + await add(); + expect(watcher.packageStatus(doc.url)).toEqual({ state: "loaded", plugins: ["demo"] }); + expect(watcher.packageStatus(doc.documentId)?.state).toBe("loaded"); + expect(events).toContain("loading"); + expect(events.at(-1)).toBe("loaded"); + watcher.dispose(); + await repo.shutdown().catch(() => {}); + }); + + it("says a package failed, and why", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { repo, doc, watcher, add } = setup("throw new Error('kaboom')"); + await add(); + expect(watcher.packageStatus(doc.url)).toMatchObject({ state: "failed" }); + expect(watcher.packageStatus(doc.url)?.error).toContain("kaboom"); + watcher.dispose(); + errors.mockRestore(); + await repo.shutdown().catch(() => {}); + }); + + it("knows nothing of a document that isn't a package", async () => { + const { repo, watcher } = setup("export const plugins = []"); + expect(watcher.packageStatus("automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu")).toBeUndefined(); + await repo.shutdown().catch(() => {}); + }); +}); diff --git a/core/filesystem/test/namespace-fixture.ts b/core/filesystem/test/namespace-fixture.ts new file mode 100644 index 000000000..1f4da5603 --- /dev/null +++ b/core/filesystem/test/namespace-fixture.ts @@ -0,0 +1,80 @@ +// A small account to walk: / is a folder of Projects, /packages a union of two folders. +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { docStep, idhash } from "../src/paths.js"; +import { walkPath } from "../src/walk.js"; + +export const step = async (handle: DocHandle) => + docStep(await idhash(handle.documentId), handle.heads()); + +export const folder = (repo: Repo, entries: Record) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ + name, + type: "x", + url, + })), + }); + +export const file = (repo: Repo, content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + +// A package the way pushwork writes one: a directory doc of path → file doc url. +export const pkg = (repo: Repo, main: string) => + repo.create({ + "@patchwork": { type: "directory" }, + "package.json": file(repo, "{}").url, + "dist/index.js": file(repo, main).url, + }); + +export async function fixture() { + const repo = new Repo(); + const notes = file(repo, "hi"); + const foo = folder(repo, { notes: notes.url }); + const projects = folder(repo, { foo: foo.url }); + const decoy = folder(repo, { foo: file(repo, "decoy").url }); + const loop = repo.create<{ "@patchwork": { type: string }; branches: Record }>({ + "@patchwork": { type: "branches" }, + branches: {}, + }); + loop.change((d) => (d.branches.default = loop.url)); + const root = folder(repo, { + Projects: projects.url, + packages: decoy.url, + loop: loop.url, + }); + const minePkg = pkg(repo, "mine"); + const mine = folder(repo, { foo: minePkg.url }); + const branchyPkg = pkg(repo, "branchy"); + const branchy = repo.create({ + "@patchwork": { type: "branches" }, + branches: { default: branchyPkg.url }, + }); + const team = folder(repo, { + foo: pkg(repo, "team foo").url, + bar: pkg(repo, "team bar").url, + branchy: branchy.url, + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { + "/": [{ url: root.url }], + "/packages": [{ url: mine.url }, { url: team.url }], + }, + }); + const a = await idhash(account.documentId); + const accounts = new Map([[a, account.url]]); + return { + repo, a, accounts, account, root, projects, foo, notes, mine, minePkg, + team, branchy, branchyPkg, + }; +} + +export const text = (handle: DocHandle | undefined) => handle?.doc()?.content; + +/** A path's chain of doc steps, without file lists: what the walk writes. */ +export const chain = async ( + repo: Repo, + path: string, + accounts: ReadonlyMap +) => (await walkPath(repo, path, { accounts }))?.pinned; diff --git a/core/filesystem/test/package-providers.test.ts b/core/filesystem/test/package-providers.test.ts new file mode 100644 index 000000000..6e2e4119e --- /dev/null +++ b/core/filesystem/test/package-providers.test.ts @@ -0,0 +1,294 @@ +// Providers a tab passes in (`WalkOptions.providers`), after the built-ins, and +// children offered as sub-documents. +import { describe, it, expect } from "vitest"; +import type { AutomergeUrl, DocHandle } from "@automerge/automerge-repo"; +import { livePath } from "../src/paths.js"; +import { getType } from "../src/metadata.js"; +import type { Child, PathProvider } from "../src/providers.js"; +import { listChildren, listFiles, pinnedPath, resolveLivePath, walkPath } from "../src/walk.js"; +import { buildFileList, parseFileList } from "../src/file-list.js"; +import { file, fixture } from "./namespace-fixture.js"; + +type BoardDoc = { + "@patchwork": { type: "board" }; + links: { name: string; url: AutomergeUrl }[]; + notes?: { content: string; mimeType: string }; +}; + +const listBoard = (node: unknown): Child[] => { + const board = node as BoardDoc; + const children: Child[] = board.links.map(({ name, url }) => ({ name, link: url })); + if (board.notes) children.push({ name: "notes", sub: ["notes"] }); + return children; +}; + +const board: PathProvider = { + kind: "board", + matches: (doc) => getType(doc as BoardDoc) === "board", + list: listBoard, + lookup(node, names) { + const child = listBoard(node).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, +}; + +const providers = async (doc: unknown) => (board.matches(doc) ? board : undefined); + +async function boardFixture() { + const base = await fixture(); + const child = file(base.repo, "child"); + const b = base.repo.create({ + "@patchwork": { type: "board" }, + links: [{ name: "child", url: child.url }], + notes: { content: "n", mimeType: "text/plain" }, + }); + base.root.change((d: any) => d.docs.push({ name: "b", type: "board", url: b.url })); + return { ...base, b, child }; +} + +describe("extra providers", () => { + it("walks through a provider it's given, and not without it", async () => { + const { repo, a, accounts, child } = await boardFixture(); + const path = livePath(a, ["b", "child"]); + expect((await resolveLivePath(repo, path, { accounts, providers }))?.url).toBe(child.url); + expect(await resolveLivePath(repo, path, { accounts })).toBeUndefined(); + }); + + it("asks the built-ins first", async () => { + const { repo, a, accounts, notes } = await fixture(); + const greedy: PathProvider = { kind: "greedy", matches: () => true, list: () => [], lookup: () => undefined }; + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes"]), { + accounts, + providers: async () => greedy, + }); + expect(handle?.url).toBe(notes.url); + }); + + it("treats a provider that throws as having no children", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const broken: PathProvider = { + ...board, + list: () => { + throw new Error("list"); + }, + lookup: () => { + throw new Error("lookup"); + }, + }; + const options = { accounts, providers: async () => broken }; + expect(await walkPath(repo, livePath(a, ["b", "child"]), options)).toBeUndefined(); + expect(await listChildren(repo, b as DocHandle, options.providers)).toEqual([]); + }); + + it("lists a provider's children when given it", async () => { + const { repo, b } = await boardFixture(); + const names = (await listChildren(repo, b as DocHandle, providers)).map((c) => c.name); + expect(names).toEqual(["child", "notes"]); + }); +}); + +describe("sub-documents", () => { + it("ends at a sub-document as a name after the doc's step", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const end = await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); + expect(end?.atDocument).toBe(false); + expect(end?.sub).toEqual(["notes"]); + expect(end?.handle.documentId).toBe(b.documentId); + expect(end?.pinned.endsWith("/notes")).toBe(true); + }); + + it("resolves to the sub-handle", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const handle = await resolveLivePath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); + expect(handle?.url).toBe(b.sub("notes").url); + expect(handle?.url).toMatch(new RegExp(`^automerge:${b.documentId}/notes`)); + expect((handle?.doc() as any)?.content).toBe("n"); + }); + + it("names nothing once the value is gone", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const sub: PathProvider = { + ...board, + list: (node) => [...board.list(node), { name: "gone", sub: ["missing"] }], + lookup: (node, names) => + names[0] === "gone" + ? { child: { name: "gone", sub: ["missing"] }, consumed: 1 } + : board.lookup(node, names), + }; + expect( + await walkPath(repo, livePath(a, ["b", "gone"]), { accounts, providers: async () => sub }) + ).toBeUndefined(); + b.change((d) => delete d.notes); + expect(await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers })).toBeUndefined(); + }); + + it("lists a sub-document in the container's file list", async () => { + const { repo, a, accounts } = await boardFixture(); + // A path through the board: one that ends at it treats it as a file (see below). + const end = await walkPath(repo, livePath(a, ["b", "notes"]), { accounts, providers }); + const listed = await listFiles(repo, end!.pinned, { accounts, providers }); + expect(listed?.files.map((f) => f.tail).sort()).toEqual(["", "child", "notes"]); + }); +}); + +describe("review fixes", () => { + it("treats a value or sub child that consumes no name as no child, rather than spinning", async () => { + const { repo, a, accounts } = await boardFixture(); + for (const child of [{ name: "x", sub: ["notes"] }, { name: "x", value: 1 }]) { + const spin: PathProvider = { ...board, lookup: () => ({ child, consumed: 0 }) }; + expect( + await walkPath(repo, livePath(a, ["b", "x"]), { accounts, providers: async () => spin }) + ).toBeUndefined(); + } + const greedy: PathProvider = { + ...board, + lookup: (node, names) => ({ child: board.lookup(node, names)!.child, consumed: names.length + 1 }), + }; + expect( + await walkPath(repo, livePath(a, ["b", "child"]), { accounts, providers: async () => greedy }) + ).toBeUndefined(); + }, 5000); + + it("lists nested sub-documents a walk can reach", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const nested: PathProvider = { + ...board, + list: (node: any) => + node?.links ? board.list(node) : node?.inner ? [{ name: "inner", sub: ["inner"] }] : [], + lookup(node, names) { + const child = nested.list(node).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, + }; + b.change((d: any) => (d.notes.inner = { content: "i", mimeType: "text/plain" })); + const options = { accounts, providers: async (doc: unknown) => (board.matches(doc) ? nested : undefined) }; + const end = await walkPath(repo, livePath(a, ["b", "notes", "inner"]), options); + expect(end?.sub).toEqual(["notes", "inner"]); + const listed = await listFiles(repo, end!.pinned, options); + expect(listed?.files.map((f) => f.tail)).toContain("notes/inner"); + }); + + it("pins and lists a doc the path ends at the same way, plugin or not", async () => { + const { repo, a, accounts, b } = await boardFixture(); + // A board that reads as a file too, so a listing has something to show for it. + b.change((d: any) => { + d.content = "board"; + d.mimeType = "text/plain"; + }); + const path = livePath(a, ["b"]); + const without = await walkPath(repo, path, { accounts }); + const withIt = await walkPath(repo, path, { accounts, providers }); + expect(withIt?.pinned).toBe(without?.pinned); + expect(withIt?.canonical).toBe(without?.canonical); + + // The root folder's list, from a path that ends at the board. + const tails = async (options: object) => + (await listFiles(repo, without!.pinned, { accounts, ...options }))?.files.map((f) => f.tail).sort(); + expect(await tails({ providers })).toEqual(await tails({})); + expect(await tails({ providers })).toContain("b"); + }); + + it("doesn't walk a link to a sub-document as if it were the whole document", async () => { + const { repo, a, accounts, root, b } = await boardFixture(); + const subUrl = b.sub("notes").url; + root.change((d: any) => d.docs.push({ name: "subby", type: "x", url: subUrl })); + expect(await walkPath(repo, livePath(a, ["subby"]), { accounts })).toBeUndefined(); + const listed = await listFiles(repo, (await walkPath(repo, livePath(a, ["b"]), { accounts }))!.pinned, { + accounts, + }); + expect(listed?.files.map((f) => f.tail)).not.toContain("subby"); + expect(listed?.missing).not.toContain("subby"); + }); +}); + +describe("deferred review fixes", () => { + it("writes a name a provider spreads over several segments as the list does", async () => { + const { repo, a, accounts, b, child } = await boardFixture(); + const slashy: PathProvider = { + ...board, + list: () => [{ name: "a/b", link: child.url }], + lookup: (_node, names) => + names[0] === "a/b" + ? { child: { name: "a/b", link: child.url }, consumed: 1 } + : names[0] === "a" && names[1] === "b" + ? { child: { name: "a/b", link: child.url }, consumed: 2 } + : undefined, + }; + const options = { accounts, providers: async (doc: unknown) => (board.matches(doc) ? slashy : undefined) }; + const spread = await walkPath(repo, `${livePath(a, ["b"])}/a/b`, options); + const encoded = await walkPath(repo, livePath(a, ["b", "a/b"]), options); + expect(spread?.tail).toBe("a%2Fb"); + expect(encoded?.tail).toBe("a%2Fb"); + const listed = await listFiles(repo, spread!.pinned, options); + expect(listed?.files.map((f) => f.tail)).toContain(spread!.tail); + expect(b).toBeDefined(); + }); + + it("treats a list that isn't an array as no children", async () => { + const { repo, a, accounts, b } = await boardFixture(); + const odd: PathProvider = { ...board, list: () => ({}) as unknown as Child[] }; + const providers = async (doc: unknown) => (board.matches(doc) ? odd : undefined); + expect(await listChildren(repo, b as DocHandle, providers)).toEqual([]); + const end = await walkPath(repo, livePath(a, ["b", "child"]), { accounts, providers }); + await expect(listFiles(repo, end!.pinned, { accounts, providers })).resolves.toBeDefined(); + }); +}); + +describe("pinning through a package's container", () => { + it("pins a path through it to a URL whose list covers the file", async () => { + const { repo, a, accounts } = await boardFixture(); + for (const name of ["child", "notes"]) { + const pinned = await pinnedPath(repo, livePath(a, ["b", name]), { accounts, providers }); + const [prefix, tail] = [pinned!.path.slice(0, pinned!.path.lastIndexOf("/")), name]; + const list = parseFileList(pinned!.list!)!; + expect(Object.keys(list.files), name).toContain(tail); + // The list rebuilds from the container's own prefix, as the service worker asks for it. + const rebuilt = await buildFileList(repo, prefix.replace(/~[^/]*$/, ""), { accounts, providers }); + expect(rebuilt?.listHash).toBe(prefix.match(/~([^/]*)$/)![1]); + } + }); +}); + +describe("a typed doc that's also folder-shaped", () => { + it("is walked by its type's provider when given one, and as a folder otherwise", async () => { + const { repo, a, accounts, root, child } = await boardFixture(); + // A canvas keeps a folder-compatible `docs` (its scripts) beside its own content. + const canvas = repo.create({ + "@patchwork": { type: "canvas" }, + docs: [{ name: "script", type: "file", url: child.url }], + shapes: { s1: { content: "shape", mimeType: "text/plain" } }, + }); + root.change((d: any) => d.docs.push({ name: "c", type: "canvas", url: canvas.url })); + const canvasProvider: PathProvider = { + kind: "canvas", + matches: (doc: any) => doc?.["@patchwork"]?.type === "canvas", + list: (doc: any) => [ + ...doc.docs.map((e: any) => ({ name: e.name, link: e.url })), + ...Object.keys(doc.shapes).map((id) => ({ name: id, sub: ["shapes", id] })), + ], + lookup(doc, names) { + const found = canvasProvider.list(doc).find((c) => c.name === names[0]); + return found && { child: found, consumed: 1 }; + }, + }; + const providers = async (doc: unknown) => (canvasProvider.matches(doc) ? canvasProvider : undefined); + const shape = await walkPath(repo, livePath(a, ["c", "s1"]), { accounts, providers }); + expect(shape?.sub).toEqual(["shapes", "s1"]); + // Its folder entries resolve the same either way, so pinned paths agree. + const viaPlugin = await walkPath(repo, livePath(a, ["c", "script"]), { accounts, providers }); + const viaFolder = await walkPath(repo, livePath(a, ["c", "script"]), { accounts }); + expect(viaPlugin?.pinned).toBe(viaFolder?.pinned); + // Without the plugin, it's a folder: no shapes. + expect(await walkPath(repo, livePath(a, ["c", "s1"]), { accounts })).toBeUndefined(); + }); + + it("never hands a folder-typed or untyped folder to a package", async () => { + const { repo, a, accounts, notes } = await fixture(); + const greedy: PathProvider = { kind: "greedy", matches: () => true, list: () => [], lookup: () => undefined }; + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes"]), { + accounts, + providers: async () => greedy, + }); + expect(handle?.url).toBe(notes.url); + }); +}); diff --git a/core/filesystem/test/packages-folder.test.ts b/core/filesystem/test/packages-folder.test.ts new file mode 100644 index 000000000..0117f0328 --- /dev/null +++ b/core/filesystem/test/packages-folder.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, vi } from "vitest"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; +import { docStep, idhash } from "../src/paths.js"; +import { + packageLookupPath, + trackPackagesFolder, + watchPackagesFolder, + type PackageEntry, +} from "../src/packages-folder.js"; +import type { PackageTracker } from "../src/package-tracker.js"; + +const folder = (repo: Repo, entries: Array<[string, AutomergeUrl]>) => + repo.create({ + title: "packages", + docs: entries.map(([name, url]) => ({ name, type: "x", url })), + }); + +async function fixture() { + const repo = new Repo(); + const pkg = () => repo.create({ "@patchwork": { type: "directory" } }).url; + const [p1, p2, p3] = [pkg(), pkg(), pkg()]; + const mine = folder(repo, [["foo", p1], ["bar", p2]]); + const team = folder(repo, [["foo", p3], ["bar", p2]]); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: mine.url }, { url: team.url }] }, + }); + return { repo, p1, p2, p3, mine, team, account }; +} + +describe("watchPackagesFolder", () => { + it("lists every package in the union, once per document, first binding first", async () => { + const { repo, p1, p2, p3, mine, team, account } = await fixture(); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBeGreaterThan(0)); + const entries = seen.at(-1)!; + expect(entries.map((e) => [e.name, e.url])).toEqual([ + ["foo", p1], + ["bar", p2], + ["foo", p3], + ]); + expect(entries[0].memberStep).toBe(docStep(await idhash(mine.documentId), mine.heads())); + expect(entries[2].memberStep).toBe(docStep(await idhash(team.documentId), team.heads())); + stop(); + }); + + it("tells again when a member folder gains a package", async () => { + const { repo, mine, account } = await fixture(); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBe(1)); + const added = repo.create({ "@patchwork": { type: "directory" } }).url; + mine.change((d: any) => d.docs.push({ name: "new", type: "x", url: added })); + await vi.waitFor(() => expect(seen.at(-1)!.map((e) => e.url)).toContain(added)); + stop(); + }); + + it("tells nothing for an account with no /packages", async () => { + const repo = new Repo(); + const account = repo.create({ "@patchwork": { type: "account" } }); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen).toEqual([[]])); + stop(); + }); +}); + +describe("packageLookupPath", () => { + it("goes through the member that holds the package", () => { + const entry = { url: "automerge:x" as AutomergeUrl, name: "my pkg", memberStep: "m@h" }; + expect(packageLookupPath("acct", entry)).toBe("acct/packages/m@h/my%20pkg"); + }); +}); + +describe("trackPackagesFolder", () => { + it("looks a /packages package up through /@/, and anything else through the fallback", async () => { + const entry = { url: "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl, name: "foo", memberStep: "m@h" }; + const fallback: PackageTracker = { + findLatest: vi.fn(async (url) => `${url}#heads` as AutomergeUrl), + watch: vi.fn(() => () => {}), + }; + const lookUp = vi.fn(async (path: string) => `https://p.test/@/${path}~L`); + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, lookUp); + + expect(await tracker.findLatest(entry.url)).toBe("https://p.test/@/acct/packages/m@h/foo~L"); + const other = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + expect(await tracker.findLatest(other)).toBe(`${other}#heads`); + tracker.watch(entry.url, () => {}); + expect(fallback.watch).toHaveBeenCalled(); + }); +}); + +describe("watchPackagesFolder: duplicate names", () => { + it("skips, and warns about, a second package with one name in one member", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const repo = new Repo(); + const pkg = () => repo.create({ "@patchwork": { type: "directory" } }).url; + const [a, b] = [pkg(), pkg()]; + const mine = folder(repo, [["same", a], ["same", b]]); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/packages": [{ url: mine.url }] }, + }); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries)); + await vi.waitFor(() => expect(seen.length).toBe(1)); + expect(seen[0].map((e) => e.url)).toEqual([a]); + expect(warn).toHaveBeenCalled(); + stop(); + warn.mockRestore(); + }); +}); + +describe("trackPackagesFolder: stable URLs", () => { + it("keeps a package's URL while its own step is unchanged", async () => { + const entry = { url: "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl, name: "foo", memberStep: "m@h1" }; + const fallback: PackageTracker = { findLatest: vi.fn(), watch: vi.fn(() => () => {}) }; + let root = "acct@r1"; + const lookUp = vi.fn(async () => `https://p.test/@/${root}/packages/m@h1/pkg@p1~L1`); + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, lookUp); + const first = await tracker.findLatest(entry.url); + // the account and the member moved; the package didn't + root = "acct@r2"; + expect(await tracker.findLatest(entry.url)).toBe(first); + lookUp.mockResolvedValue("https://p.test/@/acct@r2/packages/m@h1/pkg@p2~L2"); + expect(await tracker.findLatest(entry.url)).toBe("https://p.test/@/acct@r2/packages/m@h1/pkg@p2~L2"); + }); +}); + +describe("trackPackagesFolder: a branches package", () => { + it("watches the branch it stands for, and follows the branch when it moves", async () => { + const BRANCHES = "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ" as AutomergeUrl; + const MAIN = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + const NEXT = "automerge:3tmVsDzywTYecevUsD21YABfXKon" as AutomergeUrl; + const entry = { url: BRANCHES, name: "foo", memberStep: "m@h" }; + let current = MAIN; + const watchers = new Map void>>(); + const fire = (url: string) => watchers.get(url)?.forEach((cb) => cb()); + const fallback: PackageTracker = { + findLatest: vi.fn(async (url) => (url === BRANCHES ? current : url)), + watch: vi.fn((url, onChange) => { + if (!watchers.has(url)) watchers.set(url, new Set()); + watchers.get(url)!.add(onChange); + return () => watchers.get(url)!.delete(onChange); + }), + }; + const tracker = trackPackagesFolder(Promise.resolve("acct"), () => [entry], fallback, vi.fn()); + const changes = vi.fn(); + const stop = tracker.watch(BRANCHES, changes); + + // New code on the default branch, with the branch pointer unchanged. + await vi.waitFor(() => expect(watchers.get(MAIN)?.size).toBe(1)); + fire(MAIN); + expect(changes).toHaveBeenCalledTimes(1); + + // The branch moves: the watch follows it. + current = NEXT; + fire(BRANCHES); + await vi.waitFor(() => expect(watchers.get(NEXT)?.size).toBe(1)); + expect(watchers.get(MAIN)?.size).toBe(0); + + stop(); + expect(watchers.get(NEXT)?.size).toBe(0); + expect(watchers.get(BRANCHES)?.size).toBe(0); + }); +}); + +describe("watchPackagesFolder: a member that can't be found yet", () => { + it("lists the members it has, and the slow one once it arrives", async () => { + const { repo, p1, p2, p3, mine, team, account } = await fixture(); + let release: (() => void) | undefined; + const find = repo.find.bind(repo); + (repo as any).find = (url: AutomergeUrl, ...rest: any[]) => + url === mine.url + ? new Promise((resolve) => (release = () => resolve(find(url, ...rest)))) + : find(url, ...rest); + const seen: PackageEntry[][] = []; + const stop = watchPackagesFolder(repo, account, (entries) => seen.push(entries), { + patienceMs: 100, + }); + await vi.waitFor(() => expect(seen.length).toBeGreaterThan(0)); + expect(seen.at(-1)!.map((e) => e.url).sort()).toEqual([p3, p2].sort()); + release!(); + await vi.waitFor(() => expect(seen.at(-1)!.map((e) => e.url)).toEqual([p1, p2, p3])); + stop(); + expect(team).toBeDefined(); + }); +}); diff --git a/core/filesystem/test/paths.test.ts b/core/filesystem/test/paths.test.ts new file mode 100644 index 000000000..484e4db93 --- /dev/null +++ b/core/filesystem/test/paths.test.ts @@ -0,0 +1,131 @@ +import { describe, it, expect } from "vitest"; +import { + docStep, + formatHeads, + idhash, + livePath, + parseHeads, + parsePath, + PathSyntaxError, +} from "../src/paths.js"; + +describe("idhash", () => { + it("is base58 of the first 12 bytes of sha-256 over the document id", async () => { + expect(await idhash("2j9knpCseyhnK8izDmLpGP5WMdZQ")).toBe( + "vjRneszhMkDwr5tb" + ); + }); +}); + +describe("heads", () => { + it("sorts so one set of heads is one string", () => { + expect(formatHeads(["c", "a", "b"])).toBe("a.b.c"); + expect(formatHeads(["b", "a"])).toBe(formatHeads(["a", "b"])); + }); + + it("parses only base58 parts", () => { + expect(parseHeads("a.b")).toEqual(["a", "b"]); + expect(parseHeads("a.0")).toBeUndefined(); + expect(parseHeads("a..b")).toBeUndefined(); + expect(parseHeads("")).toBeUndefined(); + }); +}); + +describe("parsePath", () => { + it("round-trips a live path, encoding each name", () => { + const names = ["Projects", "a/b", "x@y", "50%", "my notes", "日本"]; + const path = livePath("acct", names); + // a slash inside a name is encoded, so it doesn't split the name + expect(path.split("/")).toHaveLength(1 + names.length); + expect(parsePath(path)).toEqual({ + root: { kind: "account", acctHash: "acct" }, + segments: names.map((name) => ({ kind: "name", name })), + }); + }); + + it("reads a pinned path's doc steps and plain names", () => { + const p = parsePath("acct@h2.h1/packages/mine@h3/foo@h4/dist/index.js"); + expect(p.root).toEqual({ + kind: "account", + acctHash: "acct", + heads: ["h1", "h2"], + }); + expect(p.segments.map((s) => s.kind)).toEqual([ + "name", + "doc", + "doc", + "name", + "name", + ]); + expect(p.segments[1]).toEqual({ + kind: "doc", + idhash: "mine", + heads: ["h3"], + raw: "mine@h3", + }); + }); + + it("reads a share root", () => { + expect( + parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h1/x").root + ).toEqual({ + kind: "share", + documentId: "2j9knpCseyhnK8izDmLpGP5WMdZQ", + heads: ["h1"], + }); + // the router and the service worker hand over encoded segments + expect( + parsePath("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h1").root.kind + ).toBe("share"); + }); + + it("treats an encoded @ as part of a name", () => { + expect(parsePath("acct/a%40b").segments).toEqual([ + { kind: "name", name: "a@b" }, + ]); + }); + + it("ignores a leading slash and empty segments", () => { + expect(parsePath("/acct//a/").segments).toEqual([ + { kind: "name", name: "a" }, + ]); + }); + + it("rejects a share root without heads, an empty path, and a bad root", () => { + expect(() => parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")).toThrow( + PathSyntaxError + ); + expect(() => parsePath("")).toThrow(PathSyntaxError); + expect(() => parsePath("acct@0")).toThrow(PathSyntaxError); + expect(() => parsePath("automerge:not-an-id@h1")).toThrow(PathSyntaxError); + }); +}); + +describe("file list hashes", () => { + it("reads a doc step's file list hash", () => { + expect(parsePath("acct@h/foo@h~AbC-_9/dist/x.js").segments[0]).toEqual({ + kind: "doc", + idhash: "foo", + heads: ["h"], + listHash: "AbC-_9", + raw: "foo@h~AbC-_9", + }); + expect(docStep("foo", ["b", "a"], "L")).toBe("foo@a.b~L"); + expect(docStep("foo", ["a"])).toBe("foo@a"); + }); + + it("reads a root's file list hash", () => { + expect(parsePath("acct@h~L").root).toEqual({ + kind: "account", + acctHash: "acct", + heads: ["h"], + listHash: "L", + }); + }); + + it("takes a malformed list hash as a name, and refuses it at the root", () => { + expect(parsePath("acct@h/foo@h~/x").segments[0]).toEqual({ kind: "name", name: "foo@h~" }); + expect(parsePath("acct@h/foo@h~a.b/x").segments[0].kind).toBe("name"); + expect(() => parsePath("acct@h~")).toThrow(PathSyntaxError); + }); +}); diff --git a/core/filesystem/test/providers.test.ts b/core/filesystem/test/providers.test.ts new file mode 100644 index 000000000..1d12a3e2f --- /dev/null +++ b/core/filesystem/test/providers.test.ts @@ -0,0 +1,192 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { ImmutableString } from "@automerge/automerge"; +import { generateAutomergeUrl } from "@automerge/automerge-repo"; +import { + accountBindings, + accountProvider, + branchesProvider, + directoryProvider, + folderProvider, + providerFor, +} from "../src/providers.js"; + +const [R, P1, P2, A, B] = Array.from({ length: 5 }, generateAutomergeUrl); +const account = (rest: object) => ({ + "@patchwork": { type: "account" as const }, + ...rest, +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("accountProvider", () => { + const doc = account({ + namespace: { + "/": [{ url: R }], + "/packages": [{ url: P1 }, { url: P2 }], + }, + }); + + it("looks a mount up by name and consumes it", () => { + expect(accountProvider.lookup(doc, ["packages", "foo"])).toEqual({ + child: { name: "packages", union: [P1, P2] }, + consumed: 1, + }); + }); + + it("sends any other name to / without consuming it", () => { + expect(accountProvider.lookup(doc, ["Projects"])).toEqual({ + child: { name: "", union: [R] }, + consumed: 0, + }); + }); + + it("falls back to rootFolderUrl with no namespace, or no / key", () => { + for (const d of [ + account({ rootFolderUrl: R }), + account({ rootFolderUrl: R, namespace: { "/packages": [{ url: P1 }] } }), + ]) { + expect(accountProvider.lookup(d, ["Projects"])?.child).toEqual({ + name: "", + union: [R], + }); + } + }); + + it("finds nothing with neither namespace nor rootFolderUrl", () => { + expect(accountProvider.lookup(account({}), ["x"])).toBeUndefined(); + expect(accountProvider.list(account({}))).toEqual([]); + }); + + it("skips malformed mounts and bindings with a warning", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const bindings = accountBindings( + account({ + namespace: { + "/": [{ url: R }, { url: "nope" }], + packages: [{ url: P1 }], + "/a/b": [{ url: P1 }], + "/x@y": [{ url: P1 }], + "/ok": "not an array", + }, + }) + ); + expect([...bindings]).toEqual([["/", [R]]]); + expect(warn).toHaveBeenCalled(); + }); + + it("falls back to rootFolderUrl when every / binding is malformed", () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const bindings = accountBindings( + account({ rootFolderUrl: R, namespace: { "/": [{ url: "nope" }] } }) + ); + expect(bindings.get("/")).toEqual([R]); + }); + + it("warns about a malformed mount once, however often it's read", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doc = account({ + namespace: { "/once@only": [{ url: R }], "/": [{ url: R }] }, + }); + for (let i = 0; i < 5; i++) accountProvider.lookup(doc, ["a"]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("lists every mount as a union child", () => { + expect(accountProvider.list(doc)).toEqual([ + { name: "", union: [R] }, + { name: "packages", union: [P1, P2] }, + ]); + }); +}); + +describe("directoryProvider", () => { + it("matches the longest prefix", () => { + const doc = { dist: { other: A }, "dist/index.js": B }; + expect(directoryProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "dist/index.js", link: B }, + consumed: 2, + }); + }); + + it("returns a nested object as a value child", () => { + const doc = { dist: { "index.js": A } }; + expect(directoryProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "dist", value: { "index.js": A } }, + consumed: 1, + }); + }); + + it("reads ImmutableString links", () => { + const doc = { "a.js": new ImmutableString(A) }; + expect(directoryProvider.lookup(doc, ["a.js"])?.child).toEqual({ + name: "a.js", + link: A, + }); + }); + + it("does not list @patchwork", () => { + const doc = { "@patchwork": { type: "directory" }, "a.js": A }; + expect(directoryProvider.list(doc)).toEqual([{ name: "a.js", link: A }]); + }); +}); + +describe("folderProvider", () => { + it("first entry with the name wins", () => { + const doc = { + docs: [ + { name: "a", type: "x", url: A }, + { name: "a", type: "x", url: B }, + ], + }; + expect(folderProvider.lookup(doc, ["a", "b"])).toEqual({ + child: { name: "a", link: A }, + consumed: 1, + }); + }); + + it("misses an unknown name", () => { + expect(folderProvider.lookup({ docs: [] }, ["a"])).toBeUndefined(); + }); +}); + +describe("branchesProvider", () => { + const doc = { + "@patchwork": { type: "branches" }, + branches: { default: A, feature: B }, + }; + + it("follows a named branch", () => { + expect(branchesProvider.lookup(doc, ["feature", "x"])).toEqual({ + child: { name: "feature", link: B }, + consumed: 1, + }); + }); + + it("falls through to default for any other name", () => { + expect(branchesProvider.lookup(doc, ["dist", "index.js"])).toEqual({ + child: { name: "default", link: A }, + consumed: 0, + }); + }); + + it("finds nothing with no default branch", () => { + const none = { "@patchwork": { type: "branches" }, branches: { x: B } }; + expect(branchesProvider.lookup(none, ["y"])).toBeUndefined(); + }); +}); + +describe("providerFor", () => { + it("checks account, directory, folder, branches in order", () => { + expect(providerFor(account({ docs: [] }))).toBe(accountProvider); + expect( + providerFor({ "@patchwork": { type: "directory" }, docs: [] }) + ).toBe(directoryProvider); + expect(providerFor({ docs: [] })).toBe(folderProvider); + expect( + providerFor({ "@patchwork": { type: "branches" }, branches: {} }) + ).toBe(branchesProvider); + expect( + providerFor({ "@patchwork": { type: "file" }, content: "x" }) + ).toBeUndefined(); + }); +}); diff --git a/core/filesystem/test/resolve.test.ts b/core/filesystem/test/resolve.test.ts index eb05a3d53..ab6020c50 100644 --- a/core/filesystem/test/resolve.test.ts +++ b/core/filesystem/test/resolve.test.ts @@ -45,6 +45,17 @@ describe("resolvePath", () => { }); } + it("follows a branches doc's default branch", async () => { + const repo = new Repo(); + const pkg = publish(repo, (url) => url); + const branches = repo.create({ + "@patchwork": { type: "branches" }, + branches: { default: pkg.url }, + }); + const index = await resolvePath(repo, branches, ["dist", "index.js"]); + expect(index && decode(index.content)).toBe("export default 1"); + }); + it("follows ImmutableString links in a folder doc", async () => { const repo = new Repo(); const file = repo.create({ diff --git a/core/filesystem/test/walk.test.ts b/core/filesystem/test/walk.test.ts new file mode 100644 index 000000000..5437c087b --- /dev/null +++ b/core/filesystem/test/walk.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { Repo, type AutomergeUrl } from "@automerge/automerge-repo"; +import { docStep, idhash, livePath } from "../src/paths.js"; +import { chain, file, fixture, step, text } from "./namespace-fixture.js"; +import { contentHash } from "../src/content-hash.js"; +import { + listChildren, + listUnion, + pinnedPath, + pinPath, + resolveLivePath, + walkPath, +} from "../src/walk.js"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("resolveLivePath", () => { + it("resolves a live path through / and returns the live handle", async () => { + const { repo, a, accounts, notes } = await fixture(); + const handle = await resolveLivePath( + repo, + livePath(a, ["Projects", "foo", "notes"]), + { accounts } + ); + expect(handle?.url).toBe(notes.url); + }); + + it("union: the first binding with the name wins", async () => { + const { repo, a, accounts, minePkg, team } = await fixture(); + const foo = await resolveLivePath(repo, livePath(a, ["packages", "foo"]), { accounts }); + expect(foo?.url).toBe(minePkg.url); + const bar = await resolveLivePath(repo, livePath(a, ["packages", "bar"]), { accounts }); + expect(bar?.url).toBe(team.doc().docs[1].url); + }); + + it("a mount shadows a same-named entry in /'s folder", async () => { + const { repo, a, accounts } = await fixture(); + const index = await resolveLivePath( + repo, + livePath(a, ["packages", "foo", "dist", "index.js"]), + { accounts } + ); + expect(text(index)).toBe("mine"); + }); + + it("a pinned path keeps resolving to the same version after a rename", async () => { + const { repo, a, accounts, projects, foo } = await fixture(); + const pinned = await chain(repo, livePath(a, ["Projects", "foo"]), accounts); + const before = foo.heads(); + + projects.change((d: any) => (d.docs[0].name = "bar")); + foo.change((d: any) => (d.title = "renamed")); + + const old = await resolveLivePath(repo, pinned!, { accounts }); + expect(old?.documentId).toBe(foo.documentId); + expect(old?.heads()).toEqual(before); + expect((old?.doc() as any).title).toBe("folder"); + + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts }) + ).toBeUndefined(); + const bar = await resolveLivePath(repo, livePath(a, ["Projects", "bar"]), { accounts }); + expect(bar?.url).toBe(foo.url); + }); + + it("resolves a name that looks like a doc step but matches no link", async () => { + const { repo, a, accounts, projects } = await fixture(); + const lookalike = `${await idhash("someOtherDocumentId")}@h1`; + const target = file(repo, "lookalike"); + projects.change((d: any) => + d.docs.push({ name: lookalike, type: "x", url: target.url }) + ); + const handle = await resolveLivePath(repo, `${a}/Projects/${lookalike}`, { accounts }); + expect(handle?.url).toBe(target.url); + }); + + it("resolves a name with reserved characters", async () => { + const { repo, a, accounts, projects } = await fixture(); + const name = "a/b c@%日本"; + const target = file(repo, "reserved"); + projects.change((d: any) => d.docs.push({ name, type: "x", url: target.url })); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", name]), { accounts }); + expect(handle?.url).toBe(target.url); + }); + + it("returns undefined for an unknown account, missing names, and heads the repo doesn't have", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await resolveLivePath(repo, livePath("unknown", ["Projects"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "nope"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["Projects", "foo", "notes", "deeper"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, `${a}@3vQB7B6MrGQZaxCuFg4oh/Projects`, { accounts }) + ).toBeUndefined(); + }); + + it("reaches a directory key whose later segments look like doc steps", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + const vendored = file(repo, "react"); + minePkg.change((d: any) => (d["vendor/react@18.3.1/index.js"] = vendored.url)); + const handle = await resolveLivePath( + repo, + `${a}/packages/foo/vendor/react@18.3.1/index.js`, + { accounts } + ); + expect(handle?.url).toBe(vendored.url); + }); + + it("skips a null folder entry", async () => { + const { repo, a, accounts, projects, foo } = await fixture(); + projects.change((d: any) => d.docs.unshift(null)); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + expect(handle?.url).toBe(foo.url); + }); + + it("treats a pinned root followed by live steps as live, not pinned", async () => { + const { repo, a, accounts, account, foo } = await fixture(); + const mixed = `${docStep(a, account.heads())}/Projects/foo`; + const end = await walkPath(repo, mixed, { accounts }); + expect(end?.canonical).toBe(false); + // a live path's answer: the live handle, not a view whose heads look pinned + expect((await resolveLivePath(repo, mixed, { accounts }))?.url).toBe(foo.url); + + const canonical = await walkPath(repo, end!.pinned, { accounts }); + expect(canonical?.canonical).toBe(true); + }); + + it("walks through the find it's given", async () => { + const { repo, a, accounts } = await fixture(); + const seen: string[] = []; + const find = (url: AutomergeUrl) => (seen.push(url), repo.find(url)); + const handle = await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { + accounts, + find, + }); + expect(handle).toBeDefined(); + // the account, /, Projects, foo + expect(seen.length).toBeGreaterThanOrEqual(4); + }); + + it("fails, not throws, when find rejects", async () => { + const { repo, a, accounts } = await fixture(); + const find = () => Promise.reject(new Error("offline")); + expect( + await resolveLivePath(repo, livePath(a, ["Projects"]), { accounts, find }) + ).toBeUndefined(); + }); + + it("returns undefined for a path that ends inside a document", async () => { + const { repo, a, accounts, minePkg } = await fixture(); + minePkg.change((d: any) => (d.nested = { x: "y" })); + expect( + await resolveLivePath(repo, livePath(a, ["packages", "foo", "nested"]), { accounts }) + ).toBeUndefined(); + }); + + it("stops on a branches doc whose default is itself", async () => { + const { repo, a, accounts } = await fixture(); + expect( + await resolveLivePath(repo, livePath(a, ["loop", "x"]), { accounts }) + ).toBeUndefined(); + expect( + await resolveLivePath(repo, livePath(a, ["loop"]), { accounts }) + ).toBeUndefined(); + }); + + it("stops when / is bound to the account itself", async () => { + const repo = new Repo(); + const account = repo.create({ "@patchwork": { type: "account" } }); + account.change((d) => (d.namespace = { "/": [{ url: account.url }] })); + const a = await idhash(account.documentId); + expect( + await resolveLivePath(repo, livePath(a, ["x"]), { + accounts: new Map([[a, account.url]]), + }) + ).toBeUndefined(); + }); +}); + +describe("pinnedPath", () => { + const listHashOf = async (list: Uint8Array | undefined) => contentHash(list!); + + it("pins every container the walk enters, lists the last, and keeps the tail as names", async () => { + const { repo, a, accounts, account, mine, minePkg } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "foo", "dist", "index.js"]), + { accounts } + ); + const listed = docStep( + await idhash(minePkg.documentId), + minePkg.heads(), + await listHashOf(pinned?.list) + ); + expect(pinned?.path).toBe( + `${docStep(a, account.heads())}/packages/${await step(mine)}/${listed}/dist/index.js` + ); + const index = await resolveLivePath(repo, pinned!.path, { accounts }); + expect(text(index)).toBe("mine"); + }); + + it("writes the / binding as a doc step with no name", async () => { + const { repo, a, accounts, account, root, projects, foo } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["Projects", "foo"]), { accounts }); + const listed = docStep(await idhash(foo.documentId), foo.heads(), await listHashOf(pinned?.list)); + expect(pinned?.path).toBe( + [docStep(a, account.heads()), await step(root), await step(projects), listed].join("/") + ); + }); + + it("walks into a branches doc and follows its default", async () => { + const { repo, a, accounts, account, team, branchy, branchyPkg } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "branchy", "dist", "index.js"]), + { accounts } + ); + const listed = docStep( + await idhash(branchyPkg.documentId), + branchyPkg.heads(), + await listHashOf(pinned?.list) + ); + expect(pinned?.path).toBe( + `${docStep(a, account.heads())}/packages/${await step(team)}/${await step(branchy)}/${listed}/dist/index.js` + ); + expect(text(await resolveLivePath(repo, pinned!.path, { accounts }))).toBe("branchy"); + // a path that ends at a branches doc stands for its default branch + const end = await resolveLivePath(repo, livePath(a, ["packages", "branchy"]), { accounts }); + expect(end?.url).toBe(branchyPkg.url); + }); + + it("re-pins a pinned path to the same string", async () => { + const { repo, a, accounts } = await fixture(); + const pinned = await pinnedPath( + repo, + livePath(a, ["packages", "bar", "dist", "index.js"]), + { accounts } + ); + expect((await pinnedPath(repo, pinned!.path, { accounts }))?.path).toBe(pinned?.path); + }); +}); + +describe("pinPath", () => { + it("prefixes the origin and /@/", async () => { + vi.stubGlobal("location", { origin: "https://patchwork.test" }); + const { repo, a, accounts } = await fixture(); + const pinned = await pinnedPath(repo, livePath(a, ["Projects"]), { accounts }); + expect((await pinPath(repo, livePath(a, ["Projects"]), { accounts }))?.url).toBe( + `https://patchwork.test/@/${pinned?.path}` + ); + }); +}); + +describe("listing", () => { + it("lists an account's children: mounts shadow / entries", async () => { + const { repo, account } = await fixture(); + const names = (await listChildren(repo, account)).map((c) => c.name); + expect(names.filter((n) => n === "packages")).toHaveLength(1); + expect(names).toContain("Projects"); + }); + + it("concatenates a union's bindings without deduplicating", async () => { + const { repo, mine, team } = await fixture(); + const names = (await listUnion(repo, [mine.url, team.url])).map((c) => c.name); + expect(names).toEqual(["foo", "foo", "bar", "branchy"]); + }); +}); diff --git a/core/patchwork/src/createAccount.ts b/core/patchwork/src/createAccount.ts index be3f3d72a..a3679d678 100644 --- a/core/patchwork/src/createAccount.ts +++ b/core/patchwork/src/createAccount.ts @@ -1,11 +1,11 @@ -import type { Repo } from "@automerge/automerge-repo/slim"; +import type { AutomergeUrl, Repo } from "@automerge/automerge-repo/slim"; import type { AccountCreator, AccountDoc, } from "@inkandswitch/patchwork-plugins"; import type { HasPatchworkMetadata } from "@inkandswitch/patchwork-filesystem"; -async function createSubdoc( +export async function createSubdoc( repo: Repo, type: string, init: (doc: D) => void @@ -18,15 +18,39 @@ async function createSubdoc( return handle; } +/** + * Show `/packages` in the account's tree: a link named `packages` in its root + * folder, to the packages folder, unless the root folder already has an entry + * by that name. The `/packages` mount shadows a same-named entry in `/`, so the + * link means the same thing; renaming or removing it changes only the view. + */ +export async function linkPackagesIntoRoot( + repo: Repo, + rootFolderUrl: AutomergeUrl | undefined, + packagesFolderUrl: AutomergeUrl +): Promise { + if (!rootFolderUrl) return; + const root = await repo.find<{ docs?: Array<{ name: string }> }>(rootFolderUrl); + if (!Array.isArray(root.doc()?.docs)) return; + if (root.doc()!.docs!.some((entry) => String(entry?.name) === "packages")) return; + root.change((doc) => { + doc.docs!.push({ name: "packages", type: "folder", url: packagesFolderUrl } as never); + }); +} + export const createDefaultAccount: AccountCreator = async ( accountHandle, repo ) => { - const [rootFolder, moduleSettings, contact] = await Promise.all([ + const [rootFolder, packagesFolder, moduleSettings, contact] = await Promise.all([ createSubdoc<{ title: string; docs: unknown[] }>(repo, "folder", (doc) => { doc.title = "New Folder"; doc.docs = []; }), + createSubdoc<{ title: string; docs: unknown[] }>(repo, "folder", (doc) => { + doc.title = "Packages"; + doc.docs = []; + }), createSubdoc<{ modules: string[] }>( repo, "patchwork:module-settings", @@ -41,7 +65,14 @@ export const createDefaultAccount: AccountCreator = async ( accountHandle.change((doc) => { doc.rootFolderUrl = rootFolder.url; + doc.namespace = { + "/": [{ url: rootFolder.url }], + // Package management: whatever is bound here. + "/packages": [{ url: packagesFolder.url }], + }; + // Still made: the Packages UI installs into it. doc.moduleSettingsUrl = moduleSettings.url; doc.contactUrl = contact.url; }); + await linkPackagesIntoRoot(repo, rootFolder.url, packagesFolder.url); }; diff --git a/core/patchwork/src/hash-params.ts b/core/patchwork/src/hash-params.ts new file mode 100644 index 000000000..dfe5a6361 --- /dev/null +++ b/core/patchwork/src/hash-params.ts @@ -0,0 +1,59 @@ +// The router's hash: `#doc=…&path=…&tool=…`, read and written the same way +// everywhere so re-serializing the same params is byte-identical. +import { + isValidAutomergeUrl, + isValidDocumentId, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocumentId, +} from "@automerge/vanillajs/slim"; + +// `doc=` and `draft=` are automerge URLs, kept literal rather than +// percent-encoded so links stay readable. `path=` is a live path whose +// segments are already encoded, with literal slashes between them. +const RAW_HASH_KEYS = new Set(["doc", "path", "draft"]); +// A stable order means re-serializing the same logical params is +// byte-identical, avoiding spurious `hashchange` round-trips. +export const HASH_KEY_ORDER = ["doc", "path", "tool", "type", "title", "frame", "draft"]; + +/** + * The hash's params. `URLSearchParams` decodes every value, which would turn + * an encoded slash in a path segment (`a%2Fb`) into a real one, so `path` is + * put back as its raw text. + */ +export function parseHashParams(hash: string): URLSearchParams { + const params = new URLSearchParams(hash); + const raw = /(?:^|&)path=([^&]*)/.exec(hash)?.[1]; + if (raw !== undefined) params.set("path", raw); + return params; +} + +export function serializeHashParams(params: URLSearchParams): string { + const keys = [...HASH_KEY_ORDER, ...params.keys()]; + const parts: string[] = []; + const emitted = new Set(); + for (const key of keys) { + if (emitted.has(key)) continue; + const value = params.get(key); + if (!value) continue; + emitted.add(key); + parts.push( + `${key}=${RAW_HASH_KEYS.has(key) ? value : encodeURIComponent(value)}` + ); + } + return parts.join("&"); +} + +/** + * Coerce a `doc=` hash param to a full automerge URL. Accepts a full URL + * (`automerge:[#heads]`) or a bare document id, for older links. + */ +export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined { + if (!docParam) return undefined; + if (isValidAutomergeUrl(docParam as AutomergeUrl)) { + return docParam as AutomergeUrl; + } + const documentId = docParam.replace(/^automerge:/, ""); + if (!isValidDocumentId(documentId)) return undefined; + return stringifyAutomergeUrl({ documentId: documentId as DocumentId }); +} diff --git a/core/patchwork/src/index.ts b/core/patchwork/src/index.ts index b3b784b47..0810fddc3 100644 --- a/core/patchwork/src/index.ts +++ b/core/patchwork/src/index.ts @@ -31,7 +31,11 @@ import type { AutomergeRepoKeyhiveBase as AutomergeRepoKeyhive } from "@automerg import { ModuleWatcher, + trackPackagesFolder, trackPackagesViaServiceWorker, + watchPackagesFolder, + type AccountNamespaceDoc, + type PackageEntry, } from "@inkandswitch/patchwork-filesystem"; import { importAutomergePackageViaWorker } from "@inkandswitch/patchwork-bootloader/module-loader"; import { registerPatchworkViewElement } from "@inkandswitch/patchwork-elements"; @@ -49,6 +53,7 @@ import { import * as plugins from "@inkandswitch/patchwork-plugins"; import setupServiceWorker, { connectClassicSync, + registerAccount, resolveHandoffsWith, } from "@inkandswitch/patchwork-bootloader"; import debug from "debug"; @@ -62,6 +67,7 @@ import type { } from "./types.js"; import { createRepo, initWasm } from "./repo.js"; import { watchDocument } from "./watch-document.js"; +import { caughtUpWithServer, migratePackages } from "./migrate-packages.js"; import { createRouter, type Router } from "./router.js"; import { createDefaultAccount } from "./createAccount.js"; @@ -179,6 +185,11 @@ async function doSetup(options: PatchworkOptions): Promise { // Started with the site bundle alone so resolveAccountHandle has something to // await on — the `account` datatype lives there. The user's own // module-settings URL is added lazily once it appears on the account doc. + // Packages under /packages are looked up as pinned /@/ URLs, once the + // account is known; anything else (module settings, the site's manifest) + // through the service worker's legacy redirect. + const account = Promise.withResolvers(); + let packageEntries: PackageEntry[] = []; const moduleWatcher = new ModuleWatcher( repo, nameSources(moduleSources), @@ -189,11 +200,15 @@ async function doSetup(options: PatchworkOptions): Promise { options.importModulesInWorker === false ? undefined : importAutomergePackageViaWorker, - // Look packages up through the automerge worker and watch them through - // this tab's Subduction subscriptions, so their docs never load into this - // tab's Repo. - trackPackagesViaServiceWorker((url, onChange) => - watchDocument(repo, url, onChange) + // Look packages up through the automerge worker (tabs leave /packages + // lookups to it) and watch them through this tab's Subduction + // subscriptions, so their docs never load into this tab's Repo. + trackPackagesFolder( + account.promise, + () => packageEntries, + trackPackagesViaServiceWorker((url, onChange) => + watchDocument(repo, url, onChange) + ) ) ); @@ -202,6 +217,24 @@ async function doSetup(options: PatchworkOptions): Promise { hive, createAccount: options.createAccount ?? createDefaultAccount, })) as DocHandle; + // So /@//… paths under it resolve, here and in the automerge worker. + account.resolve(await registerAccount(accountDocHandle.url)); + const migration = migratePackages(repo, accountDocHandle, { + // Another device may already have migrated this account. + caughtUp: () => caughtUpWithServer(repo, accountDocHandle), + }).catch((error) => + console.warn("couldn't move module settings into /packages", error) + ); + // Waiting to hear from the server can take up to 10s, and offline it never + // says yes, so boot doesn't wait long for it. A migration that finishes + // later binds /packages, and the watcher moves each package over once. + await Promise.race([ + migration, + new Promise((resolve) => setTimeout(resolve, MIGRATION_WAIT_MS)), + ]); + await wirePackagesFolder(repo, accountDocHandle, moduleWatcher, (entries) => { + packageEntries = entries; + }); wireModuleSettings(accountDocHandle, moduleWatcher); @@ -267,6 +300,7 @@ async function doSetup(options: PatchworkOptions): Promise { toolId: openOptions.tool, type: openOptions.type, title: openOptions.title, + path: openOptions.path, }, }) ); @@ -351,6 +385,50 @@ function onModuleLoaded(name: string, mod: any): void { registerPlugins(mod.plugins, name); } +// How long boot waits for /packages' first listing before going on without it. +const FIRST_LISTING_MS = 5_000; +// How long boot waits for the move from module settings into /packages. +const MIGRATION_WAIT_MS = 1_000; + +/** + * Feed the account's /packages folder to the ModuleWatcher. Waits for its + * first listing, so a package that module settings also lists is loaded once, + * through /packages, and adds the folder before module settings is wired. + */ +async function wirePackagesFolder( + repo: Repo, + accountDocHandle: DocHandle, + moduleWatcher: ModuleWatcher, + setEntries: (entries: PackageEntry[]) => void +): Promise { + let current: PackageEntry[] = []; + const listeners = new Set<() => void>(); + const first = Promise.withResolvers(); + watchPackagesFolder(repo, accountDocHandle as unknown as DocHandle, (entries) => { + current = entries; + setEntries(entries); + first.resolve(); + for (const listener of listeners) listener(); + }); + await Promise.race([ + first.promise, + new Promise((resolve) => setTimeout(resolve, FIRST_LISTING_MS)), + ]); + // Not awaited: it loads every package. It registers the folder before + // wireModuleSettings' addUrl does its own load, since both wait on the same + // doneLoading and this one asked first. + moduleWatcher + .addPackagesFolder( + "packages", + () => current, + (onChange) => { + listeners.add(onChange); + return () => listeners.delete(onChange); + } + ) + .catch((error) => console.warn("couldn't load /packages", error)); +} + /** * Feed the account's module settings document to the ModuleWatcher. */ diff --git a/core/patchwork/src/migrate-packages.ts b/core/patchwork/src/migrate-packages.ts new file mode 100644 index 000000000..332673e09 --- /dev/null +++ b/core/patchwork/src/migrate-packages.ts @@ -0,0 +1,171 @@ +// Once per account: an account from before `/packages` gets a packages folder +// holding a link to each automerge package its module-settings doc lists, +// bound at `/packages`. The module-settings doc keeps its entries: the +// Packages UI still installs into it, and builds before this one still read +// it. It records which it moved (`movedToPackages`), which this build then +// loads only through `/packages`. +import { + isValidAutomergeUrl, + parseAutomergeUrl, + stringifyAutomergeUrl, + type AutomergeUrl, + type DocHandle, + type Repo, + type StorageId, +} from "@automerge/automerge-repo/slim"; +import { + getType, + resolvePath, + type HasPatchworkMetadata, +} from "@inkandswitch/patchwork-filesystem"; +import type { AccountDoc } from "@inkandswitch/patchwork-plugins"; +import { createSubdoc, linkPackagesIntoRoot } from "./createAccount.js"; + +const decode = (content: string | Uint8Array) => + typeof content === "string" ? content : new TextDecoder().decode(content); + +const within = (work: Promise, ms: number) => + Promise.race([ + work, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`no answer in ${ms}ms`)), ms) + ), + ]); + +/** A package's name and type: its package.json's name, else the doc's title. */ +async function describePackage(repo: Repo, url: AutomergeUrl) { + const handle = await repo.find & { title?: string }>(url); + const type = getType(handle.doc() ?? {}) ?? "folder"; + try { + const manifest = await resolvePath(repo, handle, ["package.json"]); + const name = manifest && JSON.parse(decode(manifest.content))?.name; + if (typeof name === "string" && name) return { name, type }; + } catch { + // no package.json, or not JSON: fall back below + } + return { name: handle.doc()?.title || handle.documentId, type }; +} + +/** + * Whether `handle` has caught up with every sync server this repo is + * connected to, waiting up to `timeoutMs`. False when there's no server to + * ask: offline, the account may be behind another device's. + */ +export async function caughtUpWithServer( + repo: Repo, + handle: DocHandle, + timeoutMs = 10_000 +): Promise { + if (!repo.isSubductionConnected()) return false; + const peers = (await repo.connectedSubductionPeerIds()) as StorageId[]; + if (!peers.length) return false; + const done = () => peers.every((peer) => handle.isCaughtUpWith(peer) === true); + if (done()) return true; + return new Promise((resolve) => { + const finish = (caughtUp: boolean) => { + clearTimeout(timer); + handle.off("remote-heads", check); + handle.off("heads-changed", check); + resolve(caughtUp); + }; + const check = () => { + if (done()) finish(true); + }; + const timer = setTimeout(() => finish(done()), timeoutMs); + handle.on("remote-heads", check); + handle.on("heads-changed", check); + }); +} + +/** + * Bind `/packages` for an account that has module settings and has never + * had a `/packages` binding (an empty one was emptied on purpose). True if it + * did. + * + * - `caughtUp` is asked first: another device may have migrated already, and + * two migrations would each set `/packages`, one losing. If it says no, the + * migration waits for a later boot. + * - A package it can't read is linked under its document id. + * - Entries pinned to heads, and http entries, stay where they are, in module + * settings, which is still read; a `/packages` link would float to latest. + * - Two packages with one name get distinct names, since only the first of a + * name in a folder is reachable by it. + */ +export async function migratePackages( + repo: Repo, + account: DocHandle, + options: { caughtUp?: () => Promise; readTimeoutMs?: number } = {} +): Promise { + const unmigrated = () => { + const doc = account.doc(); + return !!doc?.moduleSettingsUrl && !(doc.namespace && "/packages" in doc.namespace); + }; + if (!unmigrated()) return false; + if (options.caughtUp && !(await options.caughtUp())) return false; + if (!unmigrated()) return false; + + const settings = await repo.find<{ modules?: string[] }>(account.doc().moduleSettingsUrl!); + const seen = new Set(); + const urls = (settings.doc()?.modules ?? []).flatMap((entry) => { + if (!isValidAutomergeUrl(entry)) return []; + const { documentId, heads } = parseAutomergeUrl(entry); + if (heads || seen.has(documentId)) return []; + seen.add(documentId); + return [stringifyAutomergeUrl({ documentId })]; + }); + + const described = await Promise.all( + urls.map(async (url) => { + try { + return { url, ...(await within(describePackage(repo, url), options.readTimeoutMs ?? 10_000)) }; + } catch (error) { + console.warn(`linking package ${url} into /packages by its id: couldn't read it`, error); + return { url, name: parseAutomergeUrl(url).documentId as string, type: "folder" }; + } + }) + ); + // "alpha", "alpha (2)", "alpha": the third is "alpha (3)", not a second "alpha (2)". + const used = new Set(); + const links = described.map(({ url, name, type }) => { + let unique = name; + for (let n = 2; used.has(unique); n++) unique = `${name} (${n})`; + used.add(unique); + return { name: unique, type, url }; + }); + + // Another tab may have migrated while this one read the packages. + if (!unmigrated()) return false; + const folder = await createSubdoc<{ title: string; docs: unknown[] }>( + repo, + "folder", + (folder) => { + folder.title = "Packages"; + folder.docs = links; + } + ); + // A binding that arrived while the folder was made (another device's + // migration, synced in) wins: this folder is new and holds only links, so + // dropping it loses nothing. Two migrations that never see each other's + // binding before they merge still conflict, and Automerge keeps one; both + // folders hold the same links, read from the same module settings. + let bound = false; + account.change((doc) => { + if (doc.namespace && "/packages" in doc.namespace) return; + doc.namespace ??= {}; + doc.namespace["/packages"] = [{ url: folder.url }]; + bound = true; + }); + if (!bound) return false; + // What /packages now holds: the watcher leaves those entries to it, so + // removing a package there removes it. The entries themselves stay, for + // builds that predate /packages. + settings.change((doc) => { + const moved = new Set([...((doc as { movedToPackages?: string[] }).movedToPackages ?? [])]); + for (const url of urls) moved.add(parseAutomergeUrl(url).documentId); + (doc as { movedToPackages?: string[] }).movedToPackages = [...moved]; + }); + await linkPackagesIntoRoot(repo, account.doc()?.rootFolderUrl, folder.url).catch( + (error) => console.warn("couldn't show /packages in the root folder", error) + ); + return true; +} diff --git a/core/patchwork/src/router.ts b/core/patchwork/src/router.ts index e425696f4..c33b9bb5f 100644 --- a/core/patchwork/src/router.ts +++ b/core/patchwork/src/router.ts @@ -9,12 +9,19 @@ import { type Repo, } from "@automerge/vanillajs/slim"; import { openDocument } from "@inkandswitch/patchwork-elements"; +import { idhash, resolveLivePath } from "@inkandswitch/patchwork-filesystem"; +import { + docParamToUrl, + parseHashParams, + serializeHashParams, +} from "./hash-params.js"; import { type AccountDoc, type DatatypeDescription, type DatatypeImplementation, type ToolDescription, getRegistry, + packagePathProviders, } from "@inkandswitch/patchwork-plugins"; // Legacy big-patchwork hash shape: `--[?…]`. The slug can @@ -24,42 +31,7 @@ import { const BIG_PATCHWORK_HASH_REGEX = /^(?[^=&?/#]*)--(?<docId>[1-9A-HJ-NP-Za-km-z]+)/; -// The `doc=` and `draft=` values are automerge URLs, kept literal rather than -// percent-encoded so links stay readable. -const RAW_HASH_KEYS = new Set(["doc", "draft"]); -// A stable order means re-serializing the same logical params is -// byte-identical, avoiding spurious `hashchange` round-trips. -const HASH_KEY_ORDER = ["doc", "tool", "type", "title", "frame", "draft"]; - -function serializeHashParams(params: URLSearchParams): string { - const keys = [...HASH_KEY_ORDER, ...params.keys()]; - const parts: string[] = []; - const emitted = new Set<string>(); - for (const key of keys) { - if (emitted.has(key)) continue; - const value = params.get(key); - if (!value) continue; - emitted.add(key); - parts.push( - `${key}=${RAW_HASH_KEYS.has(key) ? value : encodeURIComponent(value)}` - ); - } - return parts.join("&"); -} - -/** - * Coerce a `doc=` hash param to a full automerge URL. Accepts a full URL - * (`automerge:<id>[#heads]`) or a bare document id, for older links. - */ -export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined { - if (!docParam) return undefined; - if (isValidAutomergeUrl(docParam as AutomergeUrl)) { - return docParam as AutomergeUrl; - } - const documentId = docParam.replace(/^automerge:/, ""); - if (!isValidDocumentId(documentId)) return undefined; - return stringifyAutomergeUrl({ documentId: documentId as DocumentId }); -} +export { docParamToUrl } from "./hash-params.js"; export interface RouterParams { rootElement: HTMLElement; @@ -87,11 +59,56 @@ export function createRouter({ siteTitle, frameToolId, }: RouterParams): Router { + // A path in the hash resolves only under this site's own account. + const accounts = idhash(accountDocHandle.documentId).then( + (acctHash) => new Map([[acctHash, accountDocHandle.url]]) + ); + const resolvePathParam = async (path: string) => { + try { + // Package code runs here, so paths go through the providers packages register. + return await resolveLivePath(repo, path, { + accounts: await accounts, + providers: packagePathProviders, + }); + } catch { + return undefined; // a path that doesn't parse names nothing + } + }; + // The document, and the part of it (`automerge:<id>/<path>`), without heads. + const named = (url: string) => url.split("#")[0]; + /** + * Take `path=` out of the hash, if it and `doc=` are still the ones that were + * checked: a check that resolves after a navigation is about the old one. + */ + const dropPath = (path: string, docUrl: AutomergeUrl, reason: string) => { + const params = parseHashParams(window.location.hash.slice(1)); + if (params.get("path") !== path) return; + const doc = docParamToUrl(params.get("doc")); + if (!doc || named(doc) !== named(docUrl)) return; + console.warn(`patchwork: dropping path=${path} from the URL: ${reason}`); + params.delete("path"); + window.location.hash = serializeHashParams(params); + }; + /** + * `doc=` says what is open; when `path=` names something else, `doc=` wins. + * A path that doesn't resolve stays: it may name something not synced yet. + */ + const checkPath = async (docUrl: AutomergeUrl, path: string) => { + const handle = await resolvePathParam(path); + if (!handle) { + console.warn(`patchwork: path=${path} doesn't resolve here (yet)`); + return; + } + if (named(handle.url) !== named(docUrl)) { + dropPath(path, docUrl, `it names ${handle.url}, not ${docUrl}`); + } + }; + const route = async () => { // The first call seeds the root view's tool/doc so it can mount; later // calls reconcile the mounted view with the hash. if (!rootElement.hasAttribute("tool-id")) { - const params = new URLSearchParams(location.hash.slice(1)); + const params = parseHashParams(location.hash.slice(1)); const frame = params.get("frame"); const toolId = frame ?? @@ -131,8 +148,9 @@ export function createRouter({ return; } - const params = new URLSearchParams(hash); + const params = parseHashParams(hash); const docUrl = docParamToUrl(params.get("doc")); + const path = params.get("path") ?? undefined; const frame = params.get("frame"); if (frame) { @@ -146,29 +164,81 @@ export function createRouter({ } } - if (docUrl) { + const open = (url: AutomergeUrl) => rootElement.dispatchEvent( new CustomEvent("patchwork:open-document", { detail: { - url: docUrl, + url, toolId: params.get("tool"), title: params.get("title"), type: params.get("type"), + path, }, }) ); + + if (docUrl) { + open(docUrl); + if (path) void checkPath(docUrl, path); + } else if (path) { + // Just a path: open what it names, unless the hash has moved on while + // it was walked. One that doesn't resolve is left in the address bar, + // to be fixed or retried. + await openPath(path, open, true); } }; + // A path-only hash that didn't resolve may go through a datatype whose + // package hasn't registered its path provider yet (on a cold boot, say). + // It's walked again whenever path providers change, until it resolves or + // the hash moves on. + let stopRetrying: (() => void) | undefined; + const pathOnly = (path: string) => { + const now = parseHashParams(window.location.hash.slice(1)); + return now.get("path") === path && !now.get("doc"); + }; + async function openPath( + path: string, + open: (url: AutomergeUrl) => void, + first: boolean + ) { + const handle = await resolvePathParam(path); + if (!pathOnly(path)) return; + if (handle) { + stopRetrying?.(); + open(handle.url); + return; + } + if (!first) return; + console.warn(`patchwork: path=${path} doesn't resolve here (yet); opening nothing`); + stopRetrying?.(); + const stop = () => { + off(); + if (stopRetrying === stop) stopRetrying = undefined; + }; + // Runs inside whoever registered a plugin, so it mustn't throw at them. + const off = getRegistry("patchwork:path-provider").on("changed", () => { + try { + if (pathOnly(path)) void openPath(path, open, false).catch(stop); + else stop(); + } catch { + stop(); + } + }); + stopRetrying = stop; + stopRetrying = off; + } + rootElement.addEventListener("patchwork:open-document", async (event) => { - const { url, toolId, type, title } = event.detail as { + const { url, toolId, type, title, path } = event.detail as { url: AutomergeUrl; toolId?: string; type?: string; title?: string; + path?: string; }; - const params = new URLSearchParams(window.location.hash.slice(1)); + const params = parseHashParams(window.location.hash.slice(1)); // `doc` is the full automerge URL, so heads live inside it and the separate // `heads=` param is gone. params.delete("heads"); @@ -176,14 +246,18 @@ export function createRouter({ // navigating to a different document invalidates the selection, while // same-doc navigation (tool switches, heads changes) keeps it. const prevDoc = docParamToUrl(params.get("doc")); - if ( - prevDoc && - parseAutomergeUrl(prevDoc).documentId !== parseAutomergeUrl(url).documentId - ) { - params.delete("draft"); - } + const sameDoc = + !!prevDoc && + parseAutomergeUrl(prevDoc).documentId === parseAutomergeUrl(url).documentId; + if (prevDoc && !sameDoc) params.delete("draft"); + // Where the doc was opened from survives a tool switch or heads change on + // the same doc, as `draft` does. + // Only on the same part of it, though: another sub-document is elsewhere. + const samePart = !!prevDoc && named(prevDoc) === named(url); + const keptPath = !path && samePart ? params.get("path") ?? undefined : path; params.set("doc", url); for (const [key, value] of [ + ["path", keptPath], ["tool", toolId], ["title", title], ["type", type], diff --git a/core/patchwork/src/types.ts b/core/patchwork/src/types.ts index 1ecc33f32..829b31129 100644 --- a/core/patchwork/src/types.ts +++ b/core/patchwork/src/types.ts @@ -39,6 +39,8 @@ export interface OpenOptions { tool?: string; type?: string; title?: string; + /** The live path the document was opened at; see `OpenDocumentEventDetail.path`. */ + path?: string; } export interface PatchworkOptions { diff --git a/core/patchwork/test/hash-params.test.ts b/core/patchwork/test/hash-params.test.ts new file mode 100644 index 000000000..d5bf4bafe --- /dev/null +++ b/core/patchwork/test/hash-params.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { parseHashParams, serializeHashParams } from "../src/hash-params.js"; + +describe("hash params", () => { + it("keeps a path's segments as they're encoded", () => { + const hash = "doc=automerge:abc&path=acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"; + expect(parseHashParams(hash).get("path")).toBe( + "acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1" + ); + expect(serializeHashParams(parseHashParams(hash))).toBe(hash); + }); + + it("puts path right after doc", () => { + const params = new URLSearchParams({ tool: "t", doc: "automerge:abc" }); + params.set("path", "acct/x"); + expect(serializeHashParams(params)).toBe("doc=automerge:abc&path=acct/x&tool=t"); + }); + + it("reads a hash with no path as before", () => { + const params = parseHashParams("doc=automerge:abc&title=a%20b"); + expect(params.get("title")).toBe("a b"); + expect(params.has("path")).toBe(false); + }); +}); diff --git a/core/patchwork/test/migrate-packages.test.ts b/core/patchwork/test/migrate-packages.test.ts new file mode 100644 index 000000000..e58ce2d99 --- /dev/null +++ b/core/patchwork/test/migrate-packages.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { migratePackages } from "../src/migrate-packages.js"; +import { createDefaultAccount } from "../src/createAccount.js"; + +async function account(repo: Repo, modules: string[]) { + const settings = repo.create({ "@patchwork": { type: "patchwork:module-settings" }, modules }); + const acct = repo.create<any>({ + "@patchwork": { type: "account" }, + moduleSettingsUrl: settings.url, + }); + return acct as DocHandle<any>; +} + +function packages(repo: Repo) { + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "application/json" }).url; + const alpha = repo.create({ + "@patchwork": { type: "directory" }, + "package.json": file('{"name":"alpha","main":"index.js"}'), + }); + const bee = repo.create({ "@patchwork": { type: "folder" }, title: "Bee", docs: [] }); + return { alpha, bee }; +} + +const linked = async (repo: Repo, acct: DocHandle<any>) => { + const binding = acct.doc().namespace?.["/packages"]; + if (!binding) return undefined; + const folder = await repo.find<any>(binding[0].url as AutomergeUrl); + return folder.doc().docs.map((d: any) => [d.name, d.url]); +}; + +describe("migratePackages", () => { + it("links each automerge entry by its package.json name, and binds /packages", async () => { + const repo = new Repo(); + const { alpha, bee } = packages(repo); + const acct = await account(repo, [ + alpha.url, + bee.url, + "https://cdn.example.test/tool/index.js", + ]); + expect(await migratePackages(repo, acct)).toBe(true); + expect(await linked(repo, acct)).toEqual([ + ["alpha", alpha.url], + ["Bee", bee.url], + ]); + }); + + it("records in module settings which entries it moved", async () => { + const repo = new Repo(); + const { alpha, bee } = packages(repo); + const pinned = alpha.view(alpha.heads()).url; + const acct = await account(repo, [alpha.url, bee.url, pinned, "https://cdn.example.test/x.js"]); + await migratePackages(repo, acct); + const settings = await repo.find<any>(acct.doc().moduleSettingsUrl); + expect([...settings.doc().movedToPackages].sort()).toEqual( + [alpha.documentId, bee.documentId].sort() + ); + expect(settings.doc().modules).toHaveLength(4); + }); + + it("does nothing the second time", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + await migratePackages(repo, acct); + const bound = acct.doc().namespace["/packages"][0].url; + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"][0].url).toBe(bound); + }); + + it("leaves an account that already binds /packages alone", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const existing = repo.create({ title: "mine", docs: [] }).url; + acct.change((d: any) => (d.namespace = { "/packages": [{ url: existing }] })); + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"]).toEqual([{ url: existing }]); + }); + + it("does nothing for an account without module settings", async () => { + const repo = new Repo(); + const acct = repo.create<any>({ "@patchwork": { type: "account" } }); + expect(await migratePackages(repo, acct as DocHandle<any>)).toBe(false); + }); +}); + +describe("migratePackages: what it can't simply link", () => { + it("names two packages with one package.json name apart", async () => { + const repo = new Repo(); + const twin = () => { + const json = repo.create({ "@patchwork": { type: "file" }, content: '{"name":"alpha"}', mimeType: "application/json" }); + return repo.create({ "@patchwork": { type: "directory" }, "package.json": json.url }); + }; + const [a, b] = [twin(), twin()]; + const acct = await account(repo, [a.url, b.url]); + await migratePackages(repo, acct); + expect(await linked(repo, acct)).toEqual([ + ["alpha", a.url], + ["alpha (2)", b.url], + ]); + }); + + it("links a package it can't read under its document id", async () => { + const repo = new Repo(); + const missing = "automerge:4NMNnkMhL8jXrdJ9jamS58PAVdXu" as AutomergeUrl; + const acct = await account(repo, [missing]); + await migratePackages(repo, acct, { readTimeoutMs: 200 }); + expect(await linked(repo, acct)).toEqual([["4NMNnkMhL8jXrdJ9jamS58PAVdXu", missing]]); + }); + + it("leaves an entry pinned to heads in module settings", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.view(alpha.heads()).url]); + await migratePackages(repo, acct); + expect(await linked(repo, acct)).toEqual([]); + }); + + it("waits for the account to catch up, and leaves it for next boot if it can't", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + expect(await migratePackages(repo, acct, { caughtUp: async () => false })).toBe(false); + expect(acct.doc().namespace).toBeUndefined(); + }); + + it("doesn't migrate again after /packages was emptied on purpose", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + acct.change((d: any) => (d.namespace = { "/packages": [] })); + expect(await migratePackages(repo, acct)).toBe(false); + }); +}); + +describe("migratePackages: the packages folder in the tree", () => { + it("links the new folder into the root folder as packages", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const root = repo.create<any>({ title: "root", docs: [] }); + acct.change((d: any) => (d.rootFolderUrl = root.url)); + await migratePackages(repo, acct); + expect(root.doc().docs).toEqual([ + { name: "packages", type: "folder", url: acct.doc().namespace["/packages"][0].url }, + ]); + }); + + it("leaves a root folder that already has something named packages alone", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const mine = repo.create({ title: "mine", docs: [] }).url; + const root = repo.create<any>({ title: "root", docs: [{ name: "packages", type: "folder", url: mine }] }); + acct.change((d: any) => (d.rootFolderUrl = root.url)); + await migratePackages(repo, acct); + expect(root.doc().docs).toHaveLength(1); + expect(root.doc().docs[0].url).toBe(mine); + }); +}); + +describe("createDefaultAccount", () => { + it("binds / to the root folder and /packages to a new one", async () => { + const repo = new Repo(); + const acct = repo.create<any>({ "@patchwork": { type: "account" } }); + await createDefaultAccount(acct as DocHandle<any>, repo as any); + const doc = acct.doc(); + expect(doc.namespace["/"]).toEqual([{ url: doc.rootFolderUrl }]); + const packagesFolder = await repo.find<any>(doc.namespace["/packages"][0].url); + expect(packagesFolder.doc()).toMatchObject({ title: "Packages", docs: [] }); + const root = await repo.find<any>(doc.rootFolderUrl); + expect(root.doc().docs).toEqual([ + { name: "packages", type: "folder", url: packagesFolder.url }, + ]); + expect(doc.moduleSettingsUrl).toBeDefined(); + }); +}); + +describe("migratePackages: copilot review", () => { + const named = (repo: Repo, name: string) => { + const manifest = repo.create({ + "@patchwork": { type: "file" }, + content: JSON.stringify({ name }), + mimeType: "application/json", + }).url; + return repo.create({ "@patchwork": { type: "directory" }, "package.json": manifest }); + }; + + it("never gives two packages one name, whatever they're called", async () => { + const repo = new Repo(); + const one = named(repo, "alpha"); + const two = named(repo, "alpha (2)"); + const three = named(repo, "alpha"); + const acct = await account(repo, [one.url, two.url, three.url]); + await migratePackages(repo, acct); + const names = (await linked(repo, acct))!.map(([name]: string[]) => name); + expect(new Set(names).size).toBe(3); + expect(names.slice(0, 2)).toEqual(["alpha", "alpha (2)"]); + }); + + it("leaves a /packages binding that arrives while its folder is made", async () => { + const repo = new Repo(); + const { alpha } = packages(repo); + const acct = await account(repo, [alpha.url]); + const theirs = repo.create({ title: "theirs", docs: [] }); + // Another device's migration syncs in just as this one creates its folder. + const create = repo.create.bind(repo); + (repo as any).create = (...args: any[]) => { + if (!acct.doc().namespace) { + acct.change((d: any) => (d.namespace = { "/packages": [{ url: theirs.url }] })); + } + return create(...args); + }; + for (const name of ["create2"] as const) { + const original = (repo as any)[name]?.bind(repo); + if (original) { + (repo as any)[name] = (...args: any[]) => { + if (!acct.doc().namespace) { + acct.change((d: any) => (d.namespace = { "/packages": [{ url: theirs.url }] })); + } + return original(...args); + }; + } + } + expect(await migratePackages(repo, acct)).toBe(false); + expect(acct.doc().namespace["/packages"]).toEqual([{ url: theirs.url }]); + }); +}); diff --git a/core/patchwork/test/router.test.ts b/core/patchwork/test/router.test.ts new file mode 100644 index 000000000..ed6cd4750 --- /dev/null +++ b/core/patchwork/test/router.test.ts @@ -0,0 +1,371 @@ +// The router in node: a fake window, location and root element, a real +// in-memory Repo, and an account whose / holds Projects/foo/notes. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Repo, type AutomergeUrl, type DocHandle } from "@automerge/automerge-repo"; +import { livePath } from "@inkandswitch/patchwork-filesystem"; +import { registerPlugins, unregisterPlugins } from "@inkandswitch/patchwork-plugins"; + +vi.mock("@inkandswitch/patchwork-elements", () => ({ + openDocument: (element: EventTarget, url: string) => + element.dispatchEvent(new CustomEvent("patchwork:open-document", { detail: { url } })), +})); + +const { createRouter } = await import("../src/router.js"); +const { idhash } = await import("@inkandswitch/patchwork-filesystem"); + +class FakeElement extends EventTarget { + attrs = new Map<string, string>(); + setAttribute(name: string, value: string) { + this.attrs.set(name, value); + } + getAttribute(name: string) { + return this.attrs.get(name) ?? null; + } + hasAttribute(name: string) { + return this.attrs.has(name); + } +} + +/** + * A window whose location's hash reads back with its "#", and which fires + * hashchange (after the current task) only when the hash really changes, as a + * browser does. + */ +function fakeWindow(initial: string) { + let hash = initial; + const target = new EventTarget(); + const location = { + get hash() { + return hash ? `#${hash}` : ""; + }, + set hash(value: string) { + const next = value.replace(/^#/, ""); + if (next === hash) return; + hash = next; + setTimeout(() => target.dispatchEvent(new Event("hashchange"))); + }, + }; + const window = { + location, + addEventListener: (type: string, listener: EventListener) => + target.addEventListener(type, listener), + }; + return { window, location }; +} + +async function setup( + hash: (a: string, notes: AutomergeUrl) => string, + extra: (repo: Repo) => Record<string, AutomergeUrl> = () => ({}) +) { + const repo = new Repo(); + const file = (content: string) => + repo.create({ "@patchwork": { type: "file" }, content, mimeType: "text/plain" }); + const folder = (entries: Record<string, AutomergeUrl>) => + repo.create({ + title: "folder", + docs: Object.entries(entries).map(([name, url]) => ({ name, type: "x", url })), + }); + const notes = file("hi"); + const other = file("other"); + const root = folder({ + Projects: folder({ foo: folder({ notes: notes.url }).url }).url, + ...extra(repo), + }); + const account = repo.create({ + "@patchwork": { type: "account" }, + namespace: { "/": [{ url: root.url }] }, + }); + const a = await idhash(account.documentId); + + const { window, location } = fakeWindow(hash(a, notes.url)); + vi.stubGlobal("location", location); + vi.stubGlobal("window", window); + vi.stubGlobal("document", { title: "" }); + + const rootElement = new FakeElement(); + rootElement.setAttribute("tool-id", "frame"); + const opened: Array<{ url: string; path?: string }> = []; + rootElement.addEventListener("patchwork:open-document", (event) => + opened.push((event as CustomEvent).detail) + ); + const router = createRouter({ + rootElement: rootElement as unknown as HTMLElement, + repo, + accountDocHandle: account as unknown as DocHandle<any>, + siteTitle: "test", + }); + const live = livePath(a, ["Projects", "foo", "notes"]); + return { a, repo, live, notes, other, location, rootElement, opened, router }; +} + +const hashParam = (location: { hash: string }, key: string) => + new RegExp(`(?:^|[#&])${key}=([^&]*)`).exec(location.hash)?.[1]; + +// A document this repo doesn't have, for a doc= the path disagrees with. +const OTHER = "automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ"; + +/** Let the router's background work (path walks, hashchanges) settle. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 300)); + +afterEach(async () => { + await settle(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("router: path=", () => { + it("writes an opened document's path to the hash", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + expect(hashParam(location, "doc")).toBe(notes.url); + expect(hashParam(location, "path")).toBe(live); + }); + + it("drops path from the hash when another document is opened without one", async () => { + const { live, notes, other, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: other.url } }) + ); + expect(hashParam(location, "path")).toBeUndefined(); + }); + + it("opens what a path names when the hash has no doc", async () => { + const { live, notes, opened, location } = await setup((a) => + `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0]).toMatchObject({ url: notes.url, path: live }); + expect(hashParam(location, "doc")).toBe(notes.url); + }); + + it("keeps path when the same document is opened without one, as on a tool switch", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, toolId: "other" } }) + ); + expect(hashParam(location, "path")).toBe(live); + expect(hashParam(location, "tool")).toBe("other"); + }); + + it("warns about a path that names nothing, opens nothing, and leaves the hash", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { a, opened, location } = await setup((a) => `path=${livePath(a, ["nope"])}`); + await vi.waitFor(() => expect(warn).toHaveBeenCalled()); + await settle(); + expect(opened).toHaveLength(0); + expect(hashParam(location, "path")).toBe(livePath(a, ["nope"])); + }); + + it("keeps a path beside doc when the path can't be resolved", async () => { + const { a, location } = await setup((a, n) => `doc=${n}&path=${livePath(a, ["nope"])}`); + await settle(); + expect(hashParam(location, "path")).toBe(livePath(a, ["nope"])); + }); + + it("doesn't open a path's document after the hash has moved on", async () => { + const { notes, opened, location } = await setup( + (a) => `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + // a click lands while the path is still being walked + location.hash = `doc=${OTHER}`; + await settle(); + expect(opened.map((o) => o.url)).not.toContain(notes.url); + }); + + it("settles after opening from a path alone: doc= and path= stay, nothing loops", async () => { + const { live, notes, opened, location } = await setup( + (a) => `path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(hashParam(location, "doc")).toBe(notes.url)); + await settle(); + const settledHash = location.hash; + const count = opened.length; + await settle(); + expect(location.hash).toBe(settledHash); + expect(opened.length).toBe(count); + expect(hashParam(location, "path")).toBe(live); + // the open, then the hashchange it caused: no more than that + expect(count).toBeLessThanOrEqual(2); + }); + + it("keeps a path that agrees with doc, opening once", async () => { + const { live, opened, location, notes } = await setup( + (a, n) => `doc=${n}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + await vi.waitFor(() => expect(opened).toHaveLength(1)); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(opened).toHaveLength(1); + expect(opened[0]).toMatchObject({ url: notes.url, path: live }); + expect(hashParam(location, "path")).toBe(live); + }); + + it("drops a path that disagrees with doc, keeping doc", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { other, location } = await setup( + (a) => `doc=${OTHER}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + void other; + await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined()); + expect(hashParam(location, "doc")).toBe(OTHER); + expect(warn).toHaveBeenCalled(); + }); +}); + +describe("router: paths through package providers", () => { + const PLUGIN_URL = "https://example.test/board.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [ + { name: "child", link: doc.child }, + { name: "notes", sub: ["notes"] }, + ], + lookup(doc: any, names: string[]) { + const child = board.list(doc).find((c) => c.name === names[0]); + return child && { child, consumed: 1 }; + }, + }; + let b: DocHandle<any> | undefined; + const withBoard = (repo: Repo) => { + const child = repo.create({ "@patchwork": { type: "file" }, content: "c", mimeType: "text/plain" }); + b = repo.create({ + "@patchwork": { type: "board" }, + child: child.url, + notes: { content: "n", mimeType: "text/plain" }, + }); + return { b: b.url }; + }; + + afterEach(() => unregisterPlugins(PLUGIN_URL)); + const register = () => + registerPlugins( + [{ id: "board", name: "board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + + it("opens a document a package's provider names", async () => { + register(); + const { opened } = await setup((a) => `path=${livePath(a, ["b", "child"])}`, withBoard); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(b!.doc().child); + }); + + it("opens a sub-document at its sub-handle's url", async () => { + register(); + const { opened } = await setup((a) => `path=${livePath(a, ["b", "notes"])}`, withBoard); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(b!.sub("notes").url); + }); +}); + +describe("router: paths to part of a document", () => { + const PLUGIN_URL = "https://example.test/board2.js"; + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: () => [{ name: "notes", sub: ["notes"] }], + lookup: (_doc: any, names: string[]) => + names[0] === "notes" ? { child: { name: "notes", sub: ["notes"] }, consumed: 1 } : undefined, + }; + let b: DocHandle<any> | undefined; + const withBoard = (repo: Repo) => { + b = repo.create({ "@patchwork": { type: "board" }, notes: { content: "n" } }); + return { b: b.url }; + }; + afterEach(() => unregisterPlugins(PLUGIN_URL)); + + it("drops a path that names part of doc when doc is the whole document", async () => { + registerPlugins( + [{ id: "board2", name: "board2", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let boardUrl = ""; + const { location } = await setup((a) => { + return `path=${livePath(a, ["b", "notes"])}`; + }, (repo) => { + const entries = withBoard(repo); + boardUrl = entries.b; + return entries; + }); + location.hash = `doc=${boardUrl}&path=${hashParam(location, "path")}`; + await vi.waitFor(() => expect(hashParam(location, "path")).toBeUndefined(), { timeout: 3000 }); + expect(hashParam(location, "doc")).toBe(boardUrl); + warn.mockRestore(); + }); +}); + +describe("router: copilot review", () => { + const PLUGIN_URL = "https://example.test/late-board.js"; + afterEach(() => unregisterPlugins(PLUGIN_URL)); + + it("doesn't let a check for an earlier doc= drop the path from a newer one", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { live, notes, location } = await setup((a) => + `doc=${OTHER}&path=${livePath(a, ["Projects", "foo", "notes"])}` + ); + // Before the first check resolves, navigate to the doc the path names. + location.hash = `doc=${notes.url}&path=${live}`; + await settle(); + expect(hashParam(location, "doc")).toBe(notes.url); + expect(hashParam(location, "path")).toBe(live); + warn.mockRestore(); + }); + + it("drops the path when another part of the same document is opened without one", async () => { + const { live, notes, location, rootElement } = await setup(() => ""); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: `${notes.url}/content` } }) + ); + expect(hashParam(location, "path")).toBeUndefined(); + // A tool switch on the same part keeps it. + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, path: live } }) + ); + rootElement.dispatchEvent( + new CustomEvent("patchwork:open-document", { detail: { url: notes.url, toolId: "other" } }) + ); + expect(hashParam(location, "path")).toBe(live); + }); + + it("opens a path-only hash once the provider it needs registers", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let childUrl = ""; + const { opened } = await setup( + (a) => `path=${livePath(a, ["b", "child"])}`, + (repo) => { + const child = repo.create({ "@patchwork": { type: "file" }, content: "c" }); + childUrl = child.url; + const b = repo.create({ "@patchwork": { type: "board" }, child: child.url }); + return { b: b.url }; + } + ); + await settle(); + expect(opened).toHaveLength(0); + const board = { + kind: "board", + matches: (doc: any) => doc?.["@patchwork"]?.type === "board", + list: (doc: any) => [{ name: "child", link: doc.child }], + lookup: (doc: any, names: string[]) => + names[0] === "child" ? { child: { name: "child", link: doc.child }, consumed: 1 } : undefined, + }; + registerPlugins( + [{ id: "late-board", name: "late-board", type: "patchwork:path-provider", supportedDatatypes: ["board"], load: async () => board }], + PLUGIN_URL + ); + await vi.waitFor(() => expect(opened.length).toBeGreaterThanOrEqual(1)); + expect(opened[0].url).toBe(childUrl); + warn.mockRestore(); + }); +}); diff --git a/core/plugins/package.json b/core/plugins/package.json index a374e17c0..80ee2f246 100644 --- a/core/plugins/package.json +++ b/core/plugins/package.json @@ -13,7 +13,7 @@ "scripts": { "build": "tsc", "dev": "tsc -w --preserveWatchOutput", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run" }, "keywords": [], "license": "ISC", diff --git a/core/plugins/src/account.ts b/core/plugins/src/account.ts index f3b1f264d..ef356437c 100644 --- a/core/plugins/src/account.ts +++ b/core/plugins/src/account.ts @@ -4,7 +4,10 @@ import { type DocHandle, type Repo, } from "@automerge/automerge-repo/slim"; -import type { HasPatchworkMetadata } from "@inkandswitch/patchwork-filesystem"; +import type { + HasPatchworkMetadata, + Namespace, +} from "@inkandswitch/patchwork-filesystem"; import type { AutomergeRepoKeyhiveBase as AutomergeRepoKeyhive } from "@automerge/automerge-repo-keyhive"; import { isKeyhiveDoc } from "./keyhive.js"; @@ -20,7 +23,16 @@ export type AccountDoc = { contextToolIds: string[]; documentToolbarToolIds: string[]; + /** + * Mount point → ordered bindings. Without one, `/` is {@link AccountDoc.rootFolderUrl}. + */ + namespace?: Namespace; rootFolderUrl?: AutomergeUrl; + /** + * @deprecated Package management is whatever is bound at `/packages` in + * {@link AccountDoc.namespace}. Still read, since the Packages UI installs + * into it. + */ moduleSettingsUrl?: AutomergeUrl; contactUrl?: AutomergeUrl; }; diff --git a/core/plugins/src/index.ts b/core/plugins/src/index.ts index ce36107fc..a714f5153 100644 --- a/core/plugins/src/index.ts +++ b/core/plugins/src/index.ts @@ -4,3 +4,4 @@ export * from "./datatypes.js"; export * from "./registry/guards.js"; export * from "./account.js"; export * from "./keyhive.js"; +export * from "./path-providers.js"; diff --git a/core/plugins/src/keyhive.ts b/core/plugins/src/keyhive.ts index c95700af3..029cfafeb 100644 --- a/core/plugins/src/keyhive.ts +++ b/core/plugins/src/keyhive.ts @@ -2,14 +2,11 @@ import { parseAutomergeUrl, type AutomergeUrl, } from "@automerge/automerge-repo/slim"; +import { isKeyhiveDocumentId } from "@inkandswitch/patchwork-filesystem"; export function isKeyhiveDoc(url: AutomergeUrl): boolean { try { - const { binaryDocumentId } = parseAutomergeUrl(url); - return ( - binaryDocumentId.length >= 32 && - binaryDocumentId.subarray(16, 32).some((byte) => byte !== 0) - ); + return isKeyhiveDocumentId(parseAutomergeUrl(url).documentId); } catch { return false; } diff --git a/core/plugins/src/path-providers.ts b/core/plugins/src/path-providers.ts new file mode 100644 index 000000000..0449fecc1 --- /dev/null +++ b/core/plugins/src/path-providers.ts @@ -0,0 +1,90 @@ +// A datatype package says what a document of its type contains by registering +// a `patchwork:path-provider` plugin. Tabs walk paths through them; the +// automerge worker never loads package code, so it walks built-ins only. +import { + getType, + providerFor, + yieldsToPackage, + type ExtraProviders, + type HasPatchworkMetadata, + type PathProvider, +} from "@inkandswitch/patchwork-filesystem"; +import { getRegistry } from "./registry/index.js"; +import type { LoadablePlugin, PluginDescription } from "./registry/types.js"; + +export const PATH_PROVIDER = "patchwork:path-provider"; + +export type PathProviderDescription = PluginDescription & { + type: typeof PATH_PROVIDER; + /** The datatypes it walks. `"*"` isn't honored: built-ins aside, a provider says what it walks. */ + supportedDatatypes: string[]; +}; + +export type PathProviderPlugin = LoadablePlugin<PathProviderDescription, PathProvider>; + +const registry = () => getRegistry<PathProviderDescription>(PATH_PROVIDER); + +const warned = new Set<string>(); +const warnOnce = (key: string, ...message: unknown[]) => { + if (warned.has(key)) return; + warned.add(key); + console.warn(...message); +}; + +const datatypesOf = (plugin: PathProviderDescription): string[] => { + const types = Array.isArray(plugin.supportedDatatypes) ? plugin.supportedDatatypes : []; + if (types.includes("*")) { + warnOnce(`${plugin.id} *`, `path provider ${plugin.id}: "*" in supportedDatatypes is ignored`); + } + return types.filter((type) => typeof type === "string" && type !== "*"); +}; + +const isProvider = (module: unknown): module is PathProvider => + !!module && + typeof module === "object" && + ["matches", "lookup", "list"].every( + (method) => typeof (module as Record<string, unknown>)[method] === "function" + ); + +/** + * The plugin provider for `doc`: the first, in registration order, that supports its type, + * loads, is shaped like a provider, and `matches` it. What a tab passes walks as `providers`. + */ +export const packagePathProviders: ExtraProviders = async (doc) => { + const type = getType((doc ?? {}) as HasPatchworkMetadata); + if (!type) return undefined; + for (const plugin of registry().filter((p) => datatypesOf(p).includes(type))) { + let module: unknown; + try { + module = (await registry().load(plugin.id))?.module; + } catch (error) { + warnOnce(`${plugin.id} load`, `path provider ${plugin.id} didn't load`, error); + continue; + } + if (!isProvider(module)) { + warnOnce(`${plugin.id} shape`, `path provider ${plugin.id} lacks matches, lookup or list`); + continue; + } + try { + if (module.matches(doc)) return module; + } catch { + // a provider whose matches throws walks nothing + } + } + return undefined; +}; + +/** + * Built-ins first, which a plugin can't override, then {@link packagePathProviders}. The one + * exception is a typed document that's also folder-shaped (see `yieldsToPackage`). + */ +export async function pathProviderFor(doc: unknown): Promise<PathProvider | undefined> { + const builtIn = providerFor(doc); + if (builtIn && !yieldsToPackage(builtIn, doc)) return builtIn; + return (await packagePathProviders(doc)) ?? builtIn; +} + +/** Every type some path-provider plugin supports, without loading any. */ +export function pathProviderTypes(): Set<string> { + return new Set(registry().all().flatMap((plugin) => datatypesOf(plugin))); +} diff --git a/core/plugins/src/registry/types.ts b/core/plugins/src/registry/types.ts index 93f130472..3e35ee2b5 100644 --- a/core/plugins/src/registry/types.ts +++ b/core/plugins/src/registry/types.ts @@ -1,5 +1,6 @@ import { DatatypeDescription } from "../datatypes.js"; import { ToolDescription } from "../tools.js"; +import type { PathProviderDescription } from "../path-providers.js"; export interface PluginRegistryEvents<D extends PluginDescription, I = any> { registered: (plugin: Plugin<D, I>) => void; @@ -16,6 +17,7 @@ export interface PluginRegistryEvents<D extends PluginDescription, I = any> { export type RegistryTypeMap = { "patchwork:tool": ToolDescription; "patchwork:datatype": DatatypeDescription; + "patchwork:path-provider": PathProviderDescription; }; /** diff --git a/core/plugins/test/path-providers.test.ts b/core/plugins/test/path-providers.test.ts new file mode 100644 index 000000000..8e8c1e6a5 --- /dev/null +++ b/core/plugins/test/path-providers.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { folderProvider, type PathProvider } from "@inkandswitch/patchwork-filesystem"; +import { getRegistry, registerPlugins, unregisterPlugins } from "../src/registry/index.js"; +import { + packagePathProviders, + pathProviderFor, + pathProviderTypes, + type PathProviderDescription, + type PathProviderPlugin, +} from "../src/path-providers.js"; + +const boardDoc = { "@patchwork": { type: "board" }, links: [] }; +const folderDoc = { title: "f", docs: [] }; + +const provider = (kind: string, extra: Partial<PathProvider> = {}): PathProvider => ({ + kind, + matches: () => true, + lookup: () => undefined, + list: () => [], + ...extra, +}); + +const plugin = ( + id: string, + supportedDatatypes: string[], + load: () => Promise<unknown> +): PathProviderPlugin => + ({ + id, + name: id, + type: "patchwork:path-provider", + supportedDatatypes, + load, + }) as PathProviderPlugin; + +const URL_ = "https://example.test/pkg.js"; + +afterEach(() => { + unregisterPlugins(URL_); + vi.restoreAllMocks(); +}); + +describe("pathProviderFor", () => { + it("finds a plugin for the doc's type", async () => { + const board = provider("board"); + registerPlugins([plugin("board", ["board"], async () => board)], URL_); + expect(await pathProviderFor(boardDoc)).toBe(board); + expect(await packagePathProviders(boardDoc)).toBe(board); + }); + + it("asks the built-ins first, whatever a plugin claims", async () => { + registerPlugins([plugin("greedy", ["folder"], async () => provider("greedy"))], URL_); + expect(await pathProviderFor(folderDoc)).toBe(folderProvider); + }); + + it("only gives plugins documents of a type they support", async () => { + registerPlugins([plugin("board", ["board"], async () => provider("board"))], URL_); + expect(await pathProviderFor({ "@patchwork": { type: "other" } })).toBeUndefined(); + expect(await packagePathProviders(folderDoc)).toBeUndefined(); + }); + + it("takes the first registered, skipping one that fails to load or isn't a provider", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const second = provider("second"); + registerPlugins( + [ + plugin("broken", ["board"], () => Promise.reject(new Error("nope"))), + plugin("shapeless", ["board"], async () => ({ kind: "shapeless" })), + plugin("not-matching", ["board"], async () => provider("x", { matches: () => false })), + plugin("second", ["board"], async () => second), + plugin("third", ["board"], async () => provider("third")), + ], + URL_ + ); + expect(await pathProviderFor(boardDoc)).toBe(second); + expect(warn).toHaveBeenCalled(); + }); + + it("doesn't honor \"*\"", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + registerPlugins([plugin("star", ["*"], async () => provider("star"))], URL_); + expect(await pathProviderFor(boardDoc)).toBeUndefined(); + expect(pathProviderTypes().has("*")).toBe(false); + expect(warn).toHaveBeenCalled(); + }); +}); + +describe("pathProviderTypes", () => { + it("is every type some plugin supports", () => { + registerPlugins( + [ + plugin("a", ["board", "canvas"], async () => provider("a")), + plugin("b", ["board"], async () => provider("b")), + ], + URL_ + ); + expect(pathProviderTypes()).toEqual(new Set(["board", "canvas"])); + expect(getRegistry<PathProviderDescription>("patchwork:path-provider").all()).toHaveLength(2); + }); +}); + +describe("pathProviderFor: a typed doc that's also folder-shaped", () => { + it("prefers the plugin for its declared type over the folder built-in", async () => { + const canvas = provider("canvas"); + registerPlugins([plugin("canvas", ["canvas"], async () => canvas)], URL_); + expect(await pathProviderFor({ "@patchwork": { type: "canvas" }, docs: [] })).toBe(canvas); + expect(await pathProviderFor({ "@patchwork": { type: "folder" }, docs: [] })).toBe(folderProvider); + expect(await pathProviderFor({ "@patchwork": { type: "other" }, docs: [] })).toBe(folderProvider); + }); +}); diff --git a/core/plugins/vitest.config.ts b/core/plugins/vitest.config.ts new file mode 100644 index 000000000..ed8bf7739 --- /dev/null +++ b/core/plugins/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["test/**/*.test.ts"], + }, +}); diff --git a/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md b/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md new file mode 100644 index 000000000..1e4d320e0 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-path-provider-plugins.md @@ -0,0 +1,187 @@ +# Package-supplied path providers — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** A datatype package can register a `patchwork:path-provider` plugin that says what a document of its type contains. Tabs walk paths through it. The sidebar opens those documents up, offers whatever editing the provider offers, and opens sub-document children as automerge sub-handles. + +**Architecture:** This spans two repos. +- **system** (`paths-design`): + - `patchwork-filesystem` gains the optional editing methods and the `sub` child on `PathProvider`. Walks take extra providers as an option, checked after the built-ins. + - `patchwork-plugins` owns the plugin type, and `pathProviderFor(doc)`: built-ins first, then plugins. filesystem can't import plugins, since plugins depends on filesystem. + - The tab resolver and the router pass plugin providers into their walks. The automerge worker passes none. +- **base** (`paths`): the sidebar finds providers through the host's `patchwork-plugins` (an importmap external), feature-detected. On an older host it behaves as it does now. + +**Tech stack:** TypeScript, automerge-repo 2.6 sub-handles (`handle.sub(...segments)`; `repo.find` of `automerge:<id>/<path>` returns the sub-handle), vitest, Solid + happy-dom in base. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 6. + +## Global Constraints + +- Built-ins (account, directory, folder, branches) are checked first and can't be overridden. Package providers come after, in registration order. +- A provider is a pure function of the node it's given: `list` and `lookup` read nothing else. +- Package providers run in tabs only. Nothing on the automerge worker's path imports `patchwork-plugins` provider code or passes `providers`. +- `/packages` stays the worker's. Tabs already decline it, so no change is needed there. +- The sidebar enables only the editing a provider offers. A child of a provider that offers none is read-only. +- A changeset for every published package touched (filesystem, plugins, bootloader, patchwork). `patch` level. + +## Decisions + +1. **`supportedDatatypes: string[]`**, with no `"*"`. A provider that claims every type would be asked about every document the sidebar shows. `"*"` is ignored, with a warning. +2. **Sub-document segments are relative to the node passed to `lookup`/`list`.** A walk accumulates them across consecutive `sub` steps. A plain `value` step ends the chain: what's below it isn't openable as a sub-document. +3. **Sidebar drops into a provider container link, never move.** A document dropped on a row whose provider offers `add` calls `add(handle, doc, { name, link })`. Its source is left alone, as in copy mode. File drops and new-document drops aren't accepted there; this first cut doesn't try to map folder semantics onto arbitrary providers. +4. **The folder provider gets no editing methods yet.** The spec says it *would* offer all three, and that there's no need to move folder editing yet. +5. **A loaded provider is checked for shape.** `matches`, `lookup` and `list` must be functions; one that isn't is skipped, with a warning. So is one that fails to load. + +## Review Focus + +1. **A plugin that registers after the sidebar has rendered.** A row of its type should become expandable without a reload. Test: B1 `canExpand` follows the registry's `"changed"` event. +2. **A provider that throws in `list` or `lookup`.** The walk or listing treats that document as having no children; it doesn't fail the page. Test: S1 `walkPath` with a throwing provider returns undefined, and B2 renders no children. +3. **A sub child whose segments don't resolve** (the value was deleted). The walk returns undefined, and the sidebar hides the row. Test: S1. +4. **Automerge worker isolation.** The worker's resolver must never consult plugins. Test: S3 asserts that a resolver created without `providers` can't walk through a plugin-typed doc that a tab resolver can. +5. **Built-in precedence.** A plugin whose `supportedDatatypes` includes `"folder"`, or whose `matches` returns true for a folder doc, never walks one. Test: S2. + +--- + +## Tasks + +### system + +### Task S1: filesystem — `sub` children, editing methods, extra providers in walks + +**Files:** modify `core/filesystem/src/providers.ts`, `core/filesystem/src/walk.ts`; test `core/filesystem/test/walk.test.ts` (or the existing walk test file; same fixtures). + +**Interfaces — produces:** +```ts +// providers.ts +export type Segment = string | number; +export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] } + | { name: string; sub: Segment[] }; +export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; + list(node: unknown): Child[]; + add?(handle: DocHandle<unknown>, node: unknown, child: { name: string; link: AutomergeUrl }, index?: number): void | Promise<void>; + rename?(handle: DocHandle<unknown>, node: unknown, name: string, to: string): void | Promise<void>; + remove?(handle: DocHandle<unknown>, node: unknown, name: string): void | Promise<void>; +} +/** Extra providers, asked after the built-ins (a tab passes the plugin ones). */ +export type ExtraProviders = (doc: unknown) => Promise<PathProvider | undefined>; +export function valueAt(node: unknown, segments: Segment[]): unknown; +// walk.ts +WalkOptions.providers?: ExtraProviders; +WalkEnd.sub?: Segment[]; // set when the walk ends at a sub-document +``` + +- [ ] **Step 1: Write failing tests** with a fake provider `{ kind: "board", matches: d => getType(d) === "board", list: d => [...d.links as {name, link}, { name: "notes", sub: ["notes"] }], lookup: (d, [n]) => … }` over a doc of `@patchwork.type: "board"`: + - `walkPath(repo, "<acct>/b/child", { accounts, providers })` reaches the linked doc; without `providers` it returns undefined. + - `walkPath(..., "<acct>/b/notes")` ends with `atDocument: false` and `sub: ["notes"]`, and its pinned path ends `<boardStep>/notes`. + - `resolveLivePath` of that path returns a handle whose `url` is `stringifyAutomergeUrl({ documentId: board, segments: ["notes"] })`. + - Deleting `notes` from the doc makes the same walk return undefined. + - A provider whose `list`/`lookup` throws: the walk returns undefined, and it doesn't throw. + - `listFiles` on a board container lists `notes` (read from `valueAt`) next to its linked files. `listChildren` returns the board's children when `providers` is passed. +- [ ] **Step 2: Run them** with `pnpm vitest run core/filesystem`. Expected: the new tests fail (no `providers` option; `sub` unknown). +- [ ] **Step 3: Implement.** + - In `walk.ts`, `const providerOf = async (doc) => providerFor(doc) ?? (await safe(options.providers?.(doc)))`. Use it everywhere `providerFor` is called today (`enter`, `listFiles` root and `read`). `listUnion` and `listChildren` take an optional `providers` argument. + - A `sub` child is walked like `value`, with `node = valueAt(node, sub)`. Undefined means the walk returns undefined. It extends `at.sub`; a `value` step clears it. + - A provider's `lookup`/`list` that throws is caught: the walk ends undefined, and the listing gives `[]`. + - `resolveLivePath`: when `!atDocument && end.sub`, return the handle (canonical view, or live) `.sub(...end.sub)`. +- [ ] **Step 4: Run the tests.** Expected: the whole filesystem suite passes (exit 0). +- [ ] **Step 5: Commit** with message `filesystem: sub-document children, provider editing methods, extra providers in walks`. + +### Task S2: plugins — the `patchwork:path-provider` plugin type + +**Files:** create `core/plugins/src/path-providers.ts`; modify `core/plugins/src/registry/types.ts` (add it to `RegistryTypeMap`) and `core/plugins/src/index.ts` (export it); test `core/plugins/test/path-providers.test.ts`. + +**Interfaces — produces:** +```ts +export type PathProviderDescription = PluginDescription & { + type: "patchwork:path-provider"; + supportedDatatypes: string[]; +}; +export type PathProviderPlugin = LoadablePlugin<PathProviderDescription, PathProvider>; +/** Built-ins first, then the first plugin, in registration order, that supports the doc's type, loads, has the right shape, and `matches` the doc. */ +export function pathProviderFor(doc: unknown): Promise<PathProvider | undefined>; +/** Only the plugin step: what walks get as `providers`. */ +export const packagePathProviders: ExtraProviders; +/** Every type some path-provider plugin supports (never "*"). */ +export function pathProviderTypes(): Set<string>; +``` + +- [ ] **Step 1: Write failing tests:** + - For a registered `board` provider, `pathProviderFor(boardDoc)` returns it. + - For a folder doc, `pathProviderFor` returns `folderProvider`, even when a plugin lists `"folder"` and its `matches` returns true. + - Two plugins for one type: the first registered wins. If the first fails to load, or its module lacks `list`, the second is used, and a warning is logged. + - `"*"` in `supportedDatatypes` isn't honored, and is warned about. + - `pathProviderTypes()` gives `{"board"}`. +- [ ] **Step 2: Run** `pnpm vitest run core/plugins`. Expected: FAIL (the module doesn't exist). +- [ ] **Step 3: Implement** on `getRegistry<PathProviderDescription>("patchwork:path-provider")`, with `.filter` and `.load(id)`. +- [ ] **Step 4: Run the tests.** Expected: the plugins suite passes (exit 0). +- [ ] **Step 5: Commit** with message `plugins: patchwork:path-provider plugins`. + +### Task S3: tabs walk through plugin providers; the worker doesn't + +**Files:** modify `core/bootloader/src/resolver.ts` (an option `providers?: ExtraProviders` on `createResolver`, threaded into `resolveWalkPath`'s walk options), `core/bootloader/src/tab-resolver.ts` (passes `packagePathProviders`), `core/patchwork/src/router.ts` (`resolveLivePath(repo, path, { accounts, providers: packagePathProviders })`); tests in `core/bootloader/test/resolver.test.ts` and `core/patchwork/test/router.test.ts`. + +**Consumes:** S1 `WalkOptions.providers`; S2 `packagePathProviders`. + +- [ ] **Step 1: Write failing tests:** + - A resolver created with `providers` answers a `/@/` live path through a board doc with a 307. One created without it (the worker's) answers 404. + - The router opens `&path=<acct>/b/child` at the linked doc, and `&path=<acct>/b/notes` at `automerge:<board>/notes`, with a board path-provider plugin registered. +- [ ] **Step 2: Run** `pnpm vitest run core/bootloader core/patchwork`. Expected: FAIL. +- [ ] **Step 3: Implement** the threading. Confirm that `automerge-protocol-handler.ts` (the worker) doesn't pass `providers`. +- [ ] **Step 4: Run the tests.** Expected: the bootloader and patchwork suites pass (exit 0). +- [ ] **Step 5: Changeset, spec, commit.** + - Changeset `.changeset/path-provider-plugins.md` for filesystem, plugins, bootloader and patchwork, saying what was added: `Child`'s `sub`, the editing methods, `WalkOptions.providers`, and `pathProviderFor`/`packagePathProviders`/`pathProviderTypes`. + - Spec section 6: "Approved … Not implemented" becomes "Implemented"; drop the non-goal's "until section 6 is built" clause. + - Build `core/filesystem`, `core/plugins`, `core/bootloader` and `core/patchwork` (tsc only, no shell build). Commit. + +### base (`paths` branch) + +### Task B1: the sidebar finds providers through the host + +**Files:** modify `sideboard/src/sideboard/container.ts`; test `sideboard/src/sideboard/container.test.ts`. + +**Interfaces — produces:** +```ts +/** Host's plugins.pathProviderFor, else filesystem.providerFor, else nothing. */ +export function providerOf(doc: unknown, host?: HostPaths): Promise<PathProvider | undefined>; +/** Reactive: true for directory/branches, or a type a path-provider plugin supports. */ +export function useExpandable(host?: HostPaths): (type: string) => boolean; +/** Link and sub children; values without `sub` stay hidden. */ +export function containerChildren(doc: unknown, provider: PathProvider | undefined): Array<{ name: string; link: AutomergeUrl } | { name: string; sub: (string | number)[] }>; +``` +`HostPaths` is the feature-detected subset of the host's `patchwork-plugins` and `patchwork-filesystem` modules. Tests inject it, as `live-path.ts` does. + +- [ ] **Step 1: Write failing tests:** + - `providerOf` prefers `pathProviderFor`, falls back to `providerFor`, and gives undefined when neither exists. + - `useExpandable` turns `"board"` true after a fake registry fires `"changed"`. + - `containerChildren` returns `sub` children and drops bare values. +- [ ] **Step 2: Run** `pnpm vitest run src/sideboard/container.test.ts` in `sideboard/`. Expected: FAIL. +- [ ] **Step 3: Implement.** `canExpand` stays exported (now a non-reactive check) for the existing callers. +- [ ] **Step 4: Run.** Expected: PASS. +- [ ] **Step 5: Commit.** + +### Task B2: provider children, sub-documents and editing in `ContainerItem` + +**Files:** modify `sideboard/src/sideboard/document-list/container-item.tsx`, `item.tsx` (an optional `onProviderDrop(items: DndPayloadItem[])`: when set on a read-only row, it accepts document drags as copy and calls it), `item-menu.tsx`/`state.ts` (`MenuTarget` gets `canRename?`/`canRemove?`, which default to `!readOnly`), `document-list.tsx` (uses `useExpandable`); test `sideboard/src/sideboard/document-list/container-item.test.tsx`. + +**Consumes:** B1. + +- [ ] **Step 1: Write failing tests** with a fake board provider: + - Expanding a board row lists its link child and its `notes` sub child. + - Opening `notes` dispatches `{ url: "automerge:<board>/notes", path: "<acct>/…/b/notes" }`. + - With `rename` and `remove` offered, the child's menu target has `canRename`/`canRemove`, and invoking them calls `provider.rename(handle, doc, "child", "new")` and `provider.remove(handle, doc, "child")`. + - With none offered, the child is read-only, as now. + - Dropping a document payload on the board row calls `provider.add(handle, doc, { name, link })` when `add` is offered, and does nothing when it isn't. + - A provider whose `list` throws renders no children. +- [ ] **Step 2: Run** `pnpm vitest run src/sideboard/document-list/container-item.test.tsx`. Expected: FAIL. +- [ ] **Step 3: Implement.** + - `ContainerChildren` resolves `providerOf(doc())` (a resource keyed on the doc) and renders `ChildRow` for links and `SubRow` for subs. + - A `SubRow` opens `stringifyAutomergeUrl({ documentId, segments })`. Its type is the value's `@patchwork.type`, or empty. + - Rename and remove go to the provider, with the container's handle and current doc. +- [ ] **Step 4: Run all sideboard tests, then tsc and lint.** Expected: everything passes. tsc shows only the 2–3 errors that were already there (`create-new.tsx`, `document-list-panel.tsx`). +- [ ] **Step 5: Commit, and push `paths`** (PR #94 is open on it). diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md b/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md new file mode 100644 index 000000000..d115a14e5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md @@ -0,0 +1,513 @@ +# Paths phase 1: grammar, providers, namespace — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give `@inkandswitch/patchwork-filesystem` the path grammar, the built-in path providers, the account namespace, and the helpers that walk live and pinned paths. Nothing calls them yet except `resolvePath`, which moves onto the providers. + +**Architecture:** Three new modules in `core/filesystem/src`: `paths.ts` (pure string grammar, no repo), `providers.ts` (sync, one doc or node at a time: "which child does this name pick?"), and `walk.ts` (async, follows links through a `Repo`, emits the pinned form as it goes). `resolve.ts` keeps its public `resolvePath` but dispatches through `providers.ts` instead of `FolderStrategy`. + +**Tech Stack:** TypeScript, `@automerge/automerge-repo` (catalog pin 2.6.0-subduction.48), vitest in node, `bs58@^6.0.0`, `crypto.subtle`. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. This plan covers **Implementation phases → 1**, plus the section 3 filesystem helpers (`idhash`, `livePath`, `parsePath`, `resolveLivePath`, `pinPath`), because they are pure filesystem code and phases 2–4 all consume them. Phases 2–4 get their own plans. Phase 2 waits on the two spikes. + +## Global Constraints + +- idhash = first 12 bytes of SHA-256 over the UTF-8 of the `documentId`, base58 (bitcoin alphabet). Not a secret. +- heads segment = the doc's heads in base58, **sorted**, joined with `.`. +- Pinned first segment: `<acctHash>@<heads>` or `automerge:<documentId>@<heads>`. The `automerge:` prefix tells them apart. +- Doc step syntax `<base58>@<heads>`. A segment is a doc step only if it parses that way **and** a link reachable from the current node has that idhash; otherwise it is a name. +- Provider order, first match wins: account, directory, folder, branches. Built-in only; the interface is exported. +- Namespace: `namespace?: { [mountPoint: string]: Array<{ url: AutomergeUrl }> }` on the account doc. Mount points are `/` or a top-level `/<name>` with no `/` or `@` in `<name>`. A mount shadows a same-named entry in `/`'s folder. Lookup: first binding with the name wins. Listing: concatenation. +- No `namespace` → `/` is `[{ url: rootFolderUrl }]`. +- Path segments are `encodeURIComponent`-encoded and slashes stay literal (same encoding the router will use in phase 3). +- Service worker URLs live under `/@/`. `pinPath` returns `<origin>/@/<pinned path>`; the service worker starts serving it in phase 2. +- No behaviour change at runtime: `createDefaultAccount`, the bootloader resolver, `ModuleWatcher` and the router are untouched in this phase. +- One changeset for the whole phase: `@inkandswitch/patchwork-filesystem: patch`, `@inkandswitch/patchwork-plugins: patch`. + +## Decisions this plan makes that the spec leaves open + +1. **The tail after the last container stays as names.** A link followed *by name* to a doc that has no provider (a file doc, a markdown doc) is written as its name(s), not `<idhash>@<heads>`. This is what the spec's example `<fooHash>@h/dist/index.js` shows, and the only form under which relative imports (`./util.js`) inside a package resolve. The cost: a leaf file doc reached by a bare link can still change under a pinned path. Phase 2's manifest, memoized by `container@heads`, is what freezes leaves. Doc steps that consume no name (union members, a branches doc's default) are always written as doc steps. +2. **A branches doc's lookup falls through to `default`.** If the next name isn't a branch name, the branches provider follows `default` without consuming the name. Packages published as a branches doc then resolve as `/packages/foo/dist/index.js`, the way the bootloader resolves them today. A branch name wins over a same-named file. +3. **The `/` fallback also applies when `namespace` exists but has no `/` key.** +4. **Child gets a third variant, `union`**, for a mount's bindings. The spec's `{ name, link | value }` can't express an ordered union. +5. **`resolveLivePath` and `pinPath` take an options bag** `{ accounts }` mapping acctHash → account url. In phase 2 the automerge worker fills this map from what tabs register. +6. **`FolderStrategy` is removed.** Its signature referenced the private `Walk` type, so nothing outside the package could implement it. `PathProvider` replaces it. + +## Review Focus + +- **Names with reserved characters:** a folder entry called `a/b`, `x@y`, `50%`, `my notes`, or `日本` should round-trip through `livePath` → `parsePath` and resolve to that entry. Pinned in Task 1 and Task 3. +- **A name that looks like a doc step:** a folder entry literally named `<some base58>@<base58>` should resolve as a name when no reachable link has that idhash. Pinned in Task 3. +- **Malformed or partial namespaces:** keys `packages` (no slash), `/a/b`, `/x@y`, a binding without a valid url, an account with neither `namespace` nor `rootFolderUrl`. Bad entries should be skipped with a `console.warn`. Nothing should throw. Pinned in Task 2. +- **Pinned heads the repo doesn't have, or malformed heads:** `resolveLivePath` should return `undefined`, not throw. Pinned in Task 3. +- **Cycles through steps that consume no name:** a branches doc whose `default` is itself, or a `/` binding that is the account doc itself. The walk should stop and return `undefined`. Pinned in Task 3. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `core/filesystem/src/paths.ts` (create) | `idhash`, heads segment, live/pinned path parse and format. No `Repo`. | +| `core/filesystem/src/providers.ts` (create) | `Child`, `PathProvider`, the four providers, `providerFor`, `accountBindings`, `asLink` (moved from `resolve.ts`). | +| `core/filesystem/src/walk.ts` (create) | `walkPath`, `resolveLivePath`, `pinnedPath`, `pinPath`, `listChildren`. | +| `core/filesystem/src/resolve.ts` (modify) | `resolvePath` dispatches through `providerFor`; `FolderStrategy` deleted. | +| `core/filesystem/src/types.ts` (modify) | `NamespaceBinding`, `Namespace`, `AccountNamespaceDoc`. | +| `core/filesystem/src/index.ts` (modify) | export the three new modules. | +| `core/filesystem/package.json` (modify) | add `"bs58": "^6.0.0"` to `dependencies`. | +| `core/plugins/src/account.ts` (modify) | `AccountDoc.namespace?: Namespace`. | +| `core/filesystem/test/paths.test.ts`, `providers.test.ts`, `walk.test.ts` (create) | unit tests, in-memory `new Repo()` as in `test/resolve.test.ts`. | +| `.changeset/paths-grammar-providers.md` (create) | the changeset. | + +--- + +### Task 1: Path grammar + +**Files:** +- Create: `core/filesystem/src/paths.ts` +- Modify: `core/filesystem/package.json` (add `bs58`), `core/filesystem/src/index.ts` +- Test: `core/filesystem/test/paths.test.ts` + +**Interfaces:** +- Produces: + ```ts + export type DocStep = { kind: "doc"; idhash: string; heads: UrlHeads; raw: string }; + export type NameStep = { kind: "name"; name: string }; + export type Segment = DocStep | NameStep; + export type PathRoot = + | { kind: "account"; acctHash: string; heads?: UrlHeads } + | { kind: "share"; documentId: DocumentId; heads: UrlHeads }; + export type ParsedPath = { root: PathRoot; segments: Segment[] }; + export class PathSyntaxError extends Error {} + + export function idhash(documentId: string): Promise<string>; + export function formatHeads(heads: readonly string[]): string; // sorted copy, "."-joined + export function parseHeads(segment: string): UrlHeads | undefined; // undefined unless every part is base58 and non-empty + export function docStep(idhash: string, heads: readonly string[]): string; // `${idhash}@${formatHeads(heads)}` + export function livePath(acctHash: string, names: string[]): string; + export function parsePath(path: string): ParsedPath; // throws PathSyntaxError + ``` +- `raw` on `DocStep` is the undecoded segment, so the walker can fall back to treating it as a name (`decodeURIComponent(raw)`). + +- [ ] **Step 1: Write the failing tests** + +```ts +describe("idhash", () => { + it("is base58 of the first 12 bytes of sha-256 over the document id", async () => { + expect(await idhash("2j9knpCseyhnK8izDmLpGP5WMdZQ")).toBe("vjRneszhMkDwr5tb"); + }); +}); + +describe("heads", () => { + it("sorts so one set of heads is one string", () => { + expect(formatHeads(["c", "a", "b"])).toBe("a.b.c"); + expect(formatHeads(["b", "a"])).toBe(formatHeads(["a", "b"])); + }); + it("parses only base58 parts", () => { + expect(parseHeads("a.b")).toEqual(["a", "b"]); + expect(parseHeads("a.0")).toBeUndefined(); // 0 isn't base58 + expect(parseHeads("a..b")).toBeUndefined(); + expect(parseHeads("")).toBeUndefined(); + }); +}); + +describe("parsePath", () => { + it("round-trips a live path, encoding each name", () => { + const names = ["Projects", "a/b", "x@y", "50%", "my notes", "日本"]; + const path = livePath("acct", names); + expect(path.split("/")).toHaveLength(1 + names.length); // a slash inside a name is encoded + expect(parsePath(path)).toEqual({ + root: { kind: "account", acctHash: "acct" }, + segments: names.map((name) => ({ kind: "name", name })), + }); + }); + it("reads a pinned path's doc steps and plain names", () => { + const p = parsePath("acct@h2.h1/packages/mine@h3/foo@h4/dist/index.js"); + expect(p.root).toEqual({ kind: "account", acctHash: "acct", heads: ["h1", "h2"] }); + expect(p.segments.map((s) => s.kind)).toEqual(["name", "doc", "doc", "name", "name"]); + expect(p.segments[1]).toEqual({ kind: "doc", idhash: "mine", heads: ["h3"], raw: "mine@h3" }); + }); + it("reads a share root", () => { + expect(parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ@h1/x").root).toEqual({ + kind: "share", documentId: "2j9knpCseyhnK8izDmLpGP5WMdZQ", heads: ["h1"], + }); + // the router and service worker hand over encoded segments + expect(parsePath("automerge%3A2j9knpCseyhnK8izDmLpGP5WMdZQ@h1").root.kind).toBe("share"); + }); + it("treats an encoded @ as part of a name", () => { + expect(parsePath("acct/a%40b").segments).toEqual([{ kind: "name", name: "a@b" }]); + }); + it("ignores a leading slash and empty segments", () => { + expect(parsePath("/acct//a/").segments).toEqual([{ kind: "name", name: "a" }]); + }); + it("rejects a share root without heads, an empty path, and a bad root", () => { + expect(() => parsePath("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")).toThrow(PathSyntaxError); + expect(() => parsePath("")).toThrow(PathSyntaxError); + expect(() => parsePath("acct@0")).toThrow(PathSyntaxError); + expect(() => parsePath("automerge:not-an-id@h1")).toThrow(PathSyntaxError); + }); +}); +``` + +(`h1`…`h4`, `acct`, `mine`, `foo` are all valid base58. Keep test fixtures base58.) + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts` +Expected: FAIL, cannot resolve `../src/paths.js`. + +- [ ] **Step 3: Add `"bs58": "^6.0.0"` to `dependencies` and run `pnpm install`, then implement `paths.ts`** + +- Base58: `/^[1-9A-HJ-NP-Za-km-z]+$/`. +- `idhash`: `crypto.subtle.digest("SHA-256", new TextEncoder().encode(id))`, take `.slice(0, 12)`, `bs58.encode`. +- Root segment: if `decodeURIComponent(raw)` starts with `automerge:`, it's a share root. Split on the **last** `@`, check the id with `isValidDocumentId`, and require heads. Otherwise `<base58>` or `<base58>@<heads>`; anything else throws. +- Other segments: a raw segment containing a literal `@` whose two halves are base58 and `parseHeads` is a `DocStep`. Everything else becomes `{ kind: "name", name: decodeURIComponent(raw) }`. +- Export it from `index.ts`. + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/package.json pnpm-lock.yaml core/filesystem/src/paths.ts core/filesystem/src/index.ts core/filesystem/test/paths.test.ts +git commit -m "filesystem: path grammar (idhash, heads, live and pinned paths)" +``` + +--- + +### Task 2: Providers and the account namespace + +**Files:** +- Create: `core/filesystem/src/providers.ts` +- Modify: `core/filesystem/src/types.ts`, `core/filesystem/src/index.ts`, `core/plugins/src/account.ts`, `core/filesystem/src/resolve.ts` (only to import `asLink` from `providers.ts` instead of defining it) +- Test: `core/filesystem/test/providers.test.ts` + +**Interfaces:** +- Consumes: `getType`, `getDefaultBranch`, `BranchesDoc`, `FolderDoc` (existing). +- Produces: + ```ts + // types.ts + export type NamespaceBinding = { url: AutomergeUrl }; + export type Namespace = { [mountPoint: string]: NamespaceBinding[] }; + export type AccountNamespaceDoc = HasPatchworkMetadata<"account"> & { + namespace?: Namespace; + rootFolderUrl?: AutomergeUrl; + }; + + // providers.ts + export type Child = + | { name: string; link: AutomergeUrl } + | { name: string; value: unknown } + | { name: string; union: AutomergeUrl[] }; + /** `consumed` is how many leading names the step used: 0 for a step that consumes none (the `/` mount, a branches doc's default). */ + export type Lookup = { child: Child; consumed: number }; + export interface PathProvider { + readonly kind: string; + matches(doc: unknown): boolean; + lookup(node: unknown, names: string[]): Lookup | undefined; // names.length >= 1 + list(node: unknown): Child[]; + } + export const accountProvider: PathProvider; // kind "account" + export const directoryProvider: PathProvider; // kind "directory" + export const folderProvider: PathProvider; // kind "folder" + export const branchesProvider: PathProvider; // kind "branches" + export const PROVIDERS: readonly PathProvider[]; // [account, directory, folder, branches] + export function providerFor(doc: unknown): PathProvider | undefined; + /** Mount point ("/" or "/name") → bindings, with "/" falling back to rootFolderUrl. Invalid keys and bindings are skipped with a warning. */ + export function accountBindings(doc: AccountNamespaceDoc): Map<string, AutomergeUrl[]>; + export function asLink(value: unknown): AutomergeUrl | undefined; // moved verbatim from resolve.ts + ``` +- `AccountDoc` in plugins gains `namespace?: Namespace` (imported as a type from `@inkandswitch/patchwork-filesystem`). + +Provider behaviour: +- **account**: `lookup` returns the union of `/<names[0]>` with `consumed: 1` if that mount exists (and isn't `/`). Otherwise it returns `{ child: { name: "", union: bindings("/") }, consumed: 0 }`, or `undefined` if `/` has no bindings. `list` returns a union child per mount: `name` is the mount without its slash, and `/` is `""`. +- **directory**: on the node it's given (the doc or an object nested in it), the longest prefix of `names` joined with `/` that is a key. A link value gives `{ link }`, anything else `{ value }`. `consumed` = prefix length. `list` returns every key except `@patchwork`. +- **folder**: first `.docs` entry whose `String(name) === names[0]` with a valid link, `consumed: 1`. `list` returns each entry with a valid link. +- **branches**: `names[0]` names a branch → that link, `consumed: 1`. Otherwise `default`, `consumed: 0`; `undefined` if there is no default (`getDefaultBranch` throws, so catch it). `list` returns each branch. + +- [ ] **Step 1: Write the failing tests** + +All tests use plain objects. Providers are synchronous and never touch a repo. Use real automerge urls from `generateAutomergeUrl()`. + +```ts +describe("accountProvider", () => { + it("looks a mount up by name and consumes it", ...) + // namespace { "/": [{url: R}], "/packages": [{url: P1}, {url: P2}] } + // lookup(doc, ["packages", "foo"]) → { child: { name: "packages", union: [P1, P2] }, consumed: 1 } + it("sends any other name to / without consuming it", ...) + // lookup(doc, ["Projects"]) → { child: { name: "", union: [R] }, consumed: 0 } + it("falls back to rootFolderUrl with no namespace, or no / key", ...) + // { rootFolderUrl: R } and { rootFolderUrl: R, namespace: { "/packages": [...] } } both → "/" union [R] + it("finds nothing with neither namespace nor rootFolderUrl", ...) + // lookup(doc, ["x"]) → undefined; list(doc) → [] + it("skips malformed mounts and bindings with a warning", ...) + // keys "packages", "/a/b", "/x@y" and a binding { url: "nope" } are absent from accountBindings; + // console.warn spied, called; no throw + it("lists every mount as a union child", ...) + // list → [{ name: "", union: [R] }, { name: "packages", union: [P1, P2] }] (order: "/" first, then object order) +}); + +describe("directoryProvider", () => { + it("matches the longest prefix", ...) + // { "dist": {..}, "dist/index.js": L } lookup ["dist","index.js"] → { child: { name: "dist/index.js", link: L }, consumed: 2 } + it("returns a nested object as a value child", ...) + it("reads ImmutableString links", ...) + it("does not list @patchwork", ...) +}); + +describe("folderProvider", () => { + it("first entry with the name wins", ...) + // docs [{name:"a", url: A1}, {name:"a", url: A2}] → link A1 + it("misses an unknown name", ...) +}); + +describe("branchesProvider", () => { + it("follows a named branch", ...) // consumed 1 + it("falls through to default for any other name", ...) // consumed 0, link = default + it("finds nothing with no default branch", ...) // undefined, no throw +}); + +describe("providerFor", () => { + it("checks account, directory, folder, branches in order", ...) + // an object with @patchwork.type "account" and a docs array → accountProvider + // a directory-typed doc with docs: [] → directoryProvider + // a file doc → undefined +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/providers.test.ts` +Expected: FAIL, cannot resolve `../src/providers.js`. + +- [ ] **Step 3: Implement `providers.ts`, the namespace types, and `AccountDoc.namespace`** + +Mount key regex: `/^\/[^/@]*$/`. `accountBindings` builds the map from `namespace`, then sets `/` from `rootFolderUrl` if `/` is absent. `account` matches on `getType(doc) === "account"`. + +- [ ] **Step 4: Run them to see them pass, along with the existing resolve tests** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/providers.test.ts test/resolve.test.ts test/resolve-pinning.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/providers.ts core/filesystem/src/types.ts core/filesystem/src/index.ts core/filesystem/src/resolve.ts core/plugins/src/account.ts core/filesystem/test/providers.test.ts +git commit -m "filesystem: path providers and the account namespace" +``` + +--- + +### Task 3: Walking live and pinned paths + +**Files:** +- Create: `core/filesystem/src/walk.ts` +- Modify: `core/filesystem/src/index.ts` +- Test: `core/filesystem/test/walk.test.ts` + +**Interfaces:** +- Consumes: Task 1's `parsePath`, `idhash`, `docStep`, `formatHeads`, `ParsedPath`. Task 2's `providerFor`, `PathProvider`, `Child`, `accountBindings`. `documentBaseOrigin` from `urls.ts`. +- Produces: + ```ts + export type WalkOptions = { + /** acctHash → account doc url. The roots a live or pinned account path may start at. */ + accounts: ReadonlyMap<string, AutomergeUrl>; + }; + export type WalkEnd = { + /** The last document the walk entered, as a view at the heads it was read at. */ + handle: DocHandle<unknown>; + /** Where the walk stopped inside it: handle.doc(), or a value nested in it. */ + node: unknown; + /** The pinned path, without a leading slash. */ + pinned: string; + }; + export function walkPath(repo: Repo, path: string, options: WalkOptions): Promise<WalkEnd | undefined>; + /** The document a live or pinned path names. undefined if it doesn't resolve or ends at a value inside a doc. */ + export function resolveLivePath(repo: Repo, path: string, options: WalkOptions): Promise<DocHandle<unknown> | undefined>; + export function pinnedPath(repo: Repo, path: string, options: WalkOptions): Promise<string | undefined>; + /** `${documentBaseOrigin()}/@/${pinnedPath}` */ + export function pinPath(repo: Repo, path: string, options: WalkOptions): Promise<string | undefined>; + /** The children of a doc, with unions expanded: an account's `/` listing concatenated and shadowed by its mounts; any other doc's provider list. */ + export function listChildren(repo: Repo, handle: DocHandle<unknown>): Promise<Child[]>; + /** Every binding's children, concatenated in binding order. Phase 4 lists `/packages` with this. */ + export function listUnion(repo: Repo, urls: AutomergeUrl[]): Promise<Child[]>; + ``` +- `resolveLivePath` returns the live handle (`repo.find(url)`) when the input was a live path, and the view when it was pinned. To decide, check whether the parsed root has heads. + +Walk algorithm. The signatures don't determine this: + +``` +root: + account → url = accounts.get(acctHash) ?? fail; handle = find(url); heads = root.heads ?? handle.heads() + view = handle.view(heads); emit docStep(acctHash, heads) + share → handle = find(documentId).view(heads); emit `automerge:${documentId}@${formatHeads(heads)}` +state: { handle, node = handle.doc(), provider = providerFor(node) } + +loop while segments remain: + if next is DocStep: + for each link reachable from node (provider.list(node): link children and every union member, + plus the branches default): + if await idhash(documentIdOf(link)) == step.idhash → enter(link, step.heads), emit step.raw, consume 1, continue loop + otherwise fall through and treat it as NameStep(decodeURIComponent(step.raw)) + names = the run of leading name segments + { child, consumed } = provider.lookup(node, names) ?? fail + value child → node = value; emit names[0..consumed] encoded; stay with the same provider + link child → doc = find(link); if providerFor(doc): enter at current heads, emit docStep + else (leaf): emit names[0..consumed] encoded; if segments remain after, fail; end here + union child → emit names[0..consumed] encoded (none for "/"); + rest = names after consumed + member = rest.length ? first member whose providerFor(doc).lookup(doc, rest) is defined + : first member + enter(member, current heads), emit docStep +at the end: if provider is branches and nothing remains → enter default, emit docStep +``` + +- "enter" = `find(url)`, a `view(heads)`, and `providerFor(doc)`. If `view` throws (heads the repo doesn't have), fail. +- "fail" returns `undefined`. The only thing that throws is `parsePath` on syntax. +- Cycle guard: keep a `Set` of `documentId|segmentsRemaining` for steps that consume nothing (union member, branches default). Revisiting one fails. +- `idhash` results are memoized per walk in a `Map`. + +- [ ] **Step 1: Write the failing tests** + +Fixture: build one in-memory account with `new Repo()`: +`account { @patchwork: account, namespace: { "/": [root], "/packages": [mine, team] } }`, where +- root is a folder `Projects`, and `Projects` is a folder containing `foo` (a folder holding `notes` → a file doc), +- mine is a folder with `foo` → a pushwork-shaped directory doc (`package.json`, `dist/index.js` file links), +- team is a folder with `foo` (a different package) and `bar`. + +Set `accounts = new Map([[await idhash(account.documentId), account.url]])`. + +```ts +it("resolves a live path through / and returns the live handle", ...) + // resolveLivePath(repo, livePath(a, ["Projects","foo","notes"]), {accounts}) → handle.url === notes.url (no heads) +it("pins every container the walk enters, and keeps the tail after the last one as names", async () => { + const pinned = await pinnedPath(repo, livePath(a, ["packages", "foo", "dist", "index.js"]), { accounts }); + expect(pinned).toBe( + `${docStep(a, account.heads())}/packages/${docStep(await idhash(mine.documentId), mine.heads())}` + + `/${docStep(await idhash(minePkg.documentId), minePkg.heads())}/dist/index.js` + ); +}); +it("writes the / binding as a doc step with no name", ...) + // pinnedPath(a/Projects/foo) === `${acct}/${root}/${projects}/${foo}` (doc steps only) +it("union: the first binding with the name wins", ...) + // packages/foo → mine's foo; packages/bar → team's bar +it("a mount shadows a same-named entry in /'s folder", ...) + // root folder also has an entry "packages"; a/packages/foo still goes through the mount +it("a pinned path keeps resolving to the same version after a rename", async () => { + // pin a/Projects/foo/notes, then rename foo → bar and change notes' content + // resolveLivePath(pinned) → the foo doc's view at its old heads (the same documentId as foo) + // resolveLivePath(live a/Projects/foo/notes) → undefined; a/Projects/bar/notes → resolves +}); +it("a pinned path that walks into a branches doc follows its default", ...) + // packages entry "branchy" → a branches doc whose default is a directory doc; packages/branchy/dist/index.js resolves, + // and the pinned form has a doc step for the branches doc and another for its default +it("resolves a name that looks like a doc step but matches no link", ...) + // a folder entry named `${await idhash(someOtherId)}@h1`; the literal segment resolves to that entry +it("resolves a name with reserved characters", ...) + // entry "a/b c@%" under Projects; resolveLivePath(livePath(a, ["Projects","a/b c@%"])) finds it +it("returns undefined for an unknown account, missing names, and heads the repo doesn't have", ...) +it("stops on a branches doc whose default is itself", ...) // resolves undefined, doesn't hang +it("stops when / is bound to the account itself", ...) // a/x → undefined +it("pinPath prefixes the origin and /@/", ...) // stub globalThis.location for node +it("lists an account's children: mounts shadow / entries, unions concatenate", ...) + // listChildren(repo, account) → names include "Projects" and "packages", each once + // listUnion(repo, [mine.url, team.url]) → names ["foo", "foo", "bar"]: concatenated, not deduplicated +``` + +`listChildren` on an account keeps each mount as one `{ name, union }` entry. It replaces the `/` union with its members' children, minus any name a mount shadows. + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts` +Expected: FAIL, cannot resolve `../src/walk.js`. + +- [ ] **Step 3: Implement `walk.ts` per the algorithm above, and export it from `index.ts`** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/walk.ts core/filesystem/src/index.ts core/filesystem/test/walk.test.ts +git commit -m "filesystem: resolve and pin live paths through the namespace" +``` + +--- + +### Task 4: `resolvePath` on providers + +**Files:** +- Modify: `core/filesystem/src/resolve.ts` +- Test: `core/filesystem/test/resolve.test.ts` (add one case) + +**Interfaces:** +- Consumes: `providerFor`, `Lookup`, `asLink` from Task 2. +- Produces: unchanged `resolvePath(repo, rootHandle, parts): Promise<Resolved | undefined>` and `Resolved`. `FolderStrategy` is deleted. + +`resolvePath` keeps its contract: it serves bytes, follows bare links, and reports `pinned` honestly. It does not go through `walkPath`, which pins and doesn't materialize. Replace `STRATEGIES` with `providerFor(doc)`, and `walkDirectoryDoc`'s prefix loop with `provider.lookup(node, parts)`. A value child keeps walking in the same provider, a link child keeps `walk.pinned &&= hasHeads(link)` and the visited-set cycle check, and a union child (only an account doc yields one) follows its first member. `materialize` is unchanged. + +- [ ] **Step 1: Write the failing test** + +```ts +it("follows a branches doc's default branch", async () => { + const repo = new Repo(); + const pkg = publish(repo, (url) => url); + const branches = repo.create({ "@patchwork": { type: "branches" }, branches: { default: pkg.url } }); + const index = await resolvePath(repo, branches, ["dist", "index.js"]); + expect(decode(index!.content)).toBe("export default 1"); +}); +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/resolve.test.ts` +Expected: FAIL. `index` is undefined, because no strategy matches a branches doc. + +- [ ] **Step 3: Rewrite the dispatch in `resolve.ts` as described** + +- [ ] **Step 4: Run the whole filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS, including `resolve-pinning.test.ts` unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add core/filesystem/src/resolve.ts core/filesystem/test/resolve.test.ts +git commit -m "filesystem: resolvePath dispatches through path providers" +``` + +--- + +### Task 5: Typecheck downstream and add the changeset + +**Files:** +- Create: `.changeset/paths-grammar-providers.md` + +- [ ] **Step 1: Build the two touched packages and typecheck their dependents** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins build && pnpm --filter @inkandswitch/patchwork-bootloader --filter @inkandswitch/patchwork exec tsc --noEmit` +Expected: no errors. (Don't run e2e or site builds locally; CI does.) + +- [ ] **Step 2: Write the changeset** + +```markdown +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-plugins": patch +--- + +Add paths: `idhash`, `livePath`, `parsePath`, `resolveLivePath`, `pinnedPath`, `pinPath` and `listChildren` walk an account's namespace by name and write the pinned `<idhash>@<heads>` form that imports and share links will use. Path providers (`accountProvider`, `directoryProvider`, `folderProvider`, `branchesProvider`, `PathProvider`) replace `FolderStrategy`, which is removed; `resolvePath` now also follows a branches doc to its default branch. `AccountDoc` gains an optional `namespace` mount table; without one, `/` is `rootFolderUrl`. Nothing calls the new path helpers yet. +``` + +- [ ] **Step 3: Commit** + +```bash +git add .changeset/paths-grammar-providers.md +git commit -m "changeset: paths grammar and providers" +``` diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md b/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md new file mode 100644 index 000000000..43d850ad8 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md @@ -0,0 +1,621 @@ +# Paths phase 2: service worker, content store, index — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** The service worker serves `/@/` URLs: live ones redirect (307) to their pinned form, and pinned ones come from a content-addressed store. Every read from that store is checked against an index that only the automerge worker can fill. + +**Architecture:** +- `patchwork-filesystem` learns to read one file at a path (`readPath`) and to list every file under a pinned document (`listFiles`). +- The bootloader gets four small modules the service worker wires together, unit-tested under vitest: + - `content-hash.ts`: the one hash function. + - `content-store.ts`: the store. + - `pinned.ts`: key normalization, the index, and the request flow. + - `index-channel.ts`: checks who sent an answer and posts answers. +- The automerge worker builds manifests (`manifest.ts`) and answers them with an authenticated `fetch()`. +- Resolvers, in tabs and in the worker, stop writing the cache. They return bytes and the service worker decides what to keep. + +**Tech Stack:** TypeScript; `@automerge/automerge-repo` 2.6.0-subduction.48; the service worker and Cache Storage APIs; `crypto.subtle`; `isUnprotectedDocId` from `@automerge/automerge-repo-keyhive`; vitest in node; Playwright e2e and bench in CI only. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. This plan covers section 3 *URLs*, section 5 in full, and **Implementation phases → 2**. Read the spike results under *Spikes before planning*: they decide the channel. Phase 1's plan is `docs/superpowers/plans/2026-10-28-paths-phase-1-grammar-providers-namespace.md`, and its walker is in `core/filesystem/src/walk.ts`. + +## Global Constraints + +- Store: a CacheStorage cache named `patchwork-content-<acctHash>`. Key `/.content/<sha256 base64url>`, on this origin, never fetched. Value: the bytes only. +- Store reads and writes go through `get(hash)` / `put(hash, bytes)` (plus `delete(hash)`), so they can be encrypted later. +- Index: a `Map` in service worker memory from pinned URL to `{ sha256, contentType }` (plus `store`; see Decisions). Only the automerge worker fills it. +- A store hit is hashed with `crypto.subtle` SHA-256 **on every read**. A mismatch deletes the entry, warns loudly, and goes back to a resolver. +- Bytes a resolver returns are hashed in the service worker: + - matching the index entry: serve, and store if the entry allows; + - not matching: a network error, nothing stored; + - no index entry (no automerge worker): serve inline, store nothing. +- What gets stored: content from unprotected documents (`isUnprotectedDocId`), and keyhive-protected content only when it's reached through `/packages`. +- Index answers count only from a `Client` with `type === "sharedworker"` and the automerge worker's script URL. The worker sends them as `fetch()` requests to a reserved path, and the service worker looks up `event.clientId`. Requests may ride the BroadcastChannel. +- A live `/@/` URL answers 307 to its pinned form. A pinned URL never redirects. +- Legacy `/automerge%3A…/` URLs keep working, keyed by URL and unchecked. +- `clearAccountStore(acctHash)` deletes the account's store. +- Stale workers outlive deploys, so `HANDOFF_CHANNEL` messages and handoff replies only change additively (`core/bootloader/src/types.ts`, `HandoffRequest`). +- No local e2e, `vite preview` or shell builds. Unit tests and typechecks only; CI runs the rest. + +## Decisions this plan makes that the spec leaves open + +1. **The worker's script URL is fixed, not hashed.** The vite plugin serves `/automerge-protocol-handler-worker.js` under that name in every build (`core/patchwork/src/vite/service-worker-plugin.ts`, `workers`). The service worker compares the client's origin and pathname with a constant `AUTOMERGE_WORKER_PATH`, and ignores the query. A site that overrides `workerPath` in `setupServiceWorker` gets its answers rejected: everything is then served inline and nothing is stored. The spec's word "hashed" is corrected in Task 9. +2. **Answers go to `POST /@/.index`.** `.` isn't base58, so it can't be an acctHash. The service worker replies `204` with `x-patchwork-index: accepted`. A reply without that header means the fetch skipped the service worker, as the spike saw in Firefox. The worker then retries 5 times, 200ms apart. +3. **Manifest requests ride `HANDOFF_CHANNEL`** as a new message type, `manifest`. Old workers ignore it, since they only read `type: "request"`. +4. **The manifest covers the last document the URL enters at its heads.** Its entries are keyed by the tail (the names after the last doc step, each `encodeURIComponent`'d, joined with `/`). The document itself is the empty tail `""`. Containers below it aren't listed: they have their own doc steps. The worker returns the **canonical** prefix, and the service worker indexes `prefix + (tail ? "/" + tail : "")`. If the prefix the service worker guessed isn't canonical (a name that looks like a doc step), the request misses the index and is served inline, unstored. That's safe. +5. **The store flag is the worker's call**, made per entry: + - `store = rootIsAccount && (unprotected(contentDocId) || prefixSegments[1] === "packages")`. + - A share root (`automerge:<id>@h`) never stores. +6. **The tab tells its account to the resolvers.** `registerAccount(url)` in the bootloader adds the account to the tab's own resolver and posts `{ type: "register-account", url }` to the automerge worker. It posts again whenever the worker respawns. The worker computes the acctHash itself, so a tab can't map a hash to the wrong document. This settles phase 1's carry-over "the accounts map isn't checked". +7. **Waiting for documents moves into the walk.** `WalkOptions` gains `find?: (url, heads?) => Promise<DocHandle>`. The bootloader passes one that waits for heads (pinned) or for peers to catch up (live), within the resolver's 30s signal. This settles phase 1's carry-over "no signal / waitForHeads". +8. **Legacy replies are inline.** For `automerge:` URLs the resolver now always answers `type: "response"`. The service worker caches it by URL itself when it's immutable (status 200/203/204, no `no-store`). The service worker still accepts `type: "cached"` from older workers. +9. **The manifest wait is bounded at 10s** (`MANIFEST_TIMEOUT_MS`). After that the request is served as if there were no worker. +10. **Cold start cost, accepted and measured.** The index lives in memory, so after a browser restart every package needs a manifest, and building one loads and hashes every file document in the package. Today the URL-keyed cache serves those without loading any document. Task 8's bench measures it. If it's too slow, persisting manifests is the follow-up, and it reopens a gap. That trade-off isn't this plan's to make. + +## Review Focus + +- **Request URL spellings:** browsers send `@`, `:` and non-ASCII in path segments in different encodings (`a@b.js` vs `a%40b.js`, `日本` vs `%E6%97%A5%E6%9C%AC`). Every spelling of one file must hit one index key. Pinned in Task 3 (`normalizePinned`). +- **The service worker stops mid-session:** the index is empty again and the worker's LRU answers. Content is identical, nothing is re-stored that was already there, and no request hangs. Pinned in Task 3 (fresh index, warm store). +- **No worker, or a worker that is slow to boot:** the manifest times out, the request is served inline within the timeout, and nothing is stored. Pinned in Task 3. +- **Many files of one package requested at once:** exactly one manifest request. Pinned in Task 3. +- **Non-ASCII string content:** the worker hashes the same UTF-8 bytes the service worker receives. Pinned in Task 4. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `core/filesystem/src/walk.ts` (modify) | `WalkOptions.find`; module-level idhash memo; `readPath`; `listFiles` | +| `core/filesystem/src/providers.ts` (modify) | `accountBindings` memoized per doc object, warning once per bad mount | +| `core/filesystem/src/resolve.ts` (modify) | export `materialize` for `walk.ts` | +| `core/bootloader/vitest.config.ts`, `core/bootloader/test/setup.ts` (create), `package.json` (modify) | test infra (same shape as `core/filesystem`) | +| `core/bootloader/src/content-hash.ts` (create) | `contentHash(bytes)`, `toBytes(content)` | +| `core/bootloader/src/content-store.ts` (create) | `ContentStore`, `cacheStore`, `contentCacheName`, `clearAccountStore` | +| `core/bootloader/src/pinned.ts` (create) | `normalizePinned`, `indexManifest`, `servePinned` | +| `core/bootloader/src/index-channel.ts` (create) | `AUTOMERGE_WORKER_PATH` check, `acceptIndexAnswer`, `postIndexAnswer` | +| `core/bootloader/src/manifest.ts` (create) | `buildManifest`: the worker's side | +| `core/bootloader/src/types.ts` (modify) | `Manifest`, `IndexEntry`, `ManifestRequestMessage`, `IndexAnswer`, `HandoffRequest.path`, `RegisterAccountMessage` | +| `core/bootloader/src/resolver.ts` (modify) | answers `path` requests (307 or bytes), legacy inline, no `cache.put`, `find` with waits | +| `core/bootloader/src/tab-resolver.ts`, `setup.ts` (modify) | `registerAccount`, account map, re-register on respawn | +| `core/bootloader/src/automerge-protocol-handler.ts` (modify) | accounts, manifest requests, LRU, answers via `postIndexAnswer` | +| `core/bootloader/src/service-worker.ts` (modify) | `/@/` routing, `POST /@/.index`, legacy caching moved in, content caches survive `clearOtherCaches` | +| `core/patchwork/src/index.ts` (modify) | call `registerAccount(accountDocHandle.url)` | +| `sites/bench/tests/content-hash.spec.ts` (create) | hashing cost on a hit | +| `packages/e2e/tests/content-store.spec.ts` (create) | forged store entry and forged answer | + +--- + +### Task 1: Walk carry-overs and `WalkOptions.find` + +**Files:** Modify `core/filesystem/src/walk.ts`, `core/filesystem/src/providers.ts`. Test `core/filesystem/test/walk.test.ts`, `core/filesystem/test/providers.test.ts`. + +**Interfaces:** +- Produces: `WalkOptions.find?: (url: AutomergeUrl, heads?: UrlHeads) => Promise<DocHandle<unknown>>`. It returns a handle that `enter` then `.view()`s at `heads ?? link heads ?? current`. The default is `repo.find` of the bare documentId url, as `enter` does now. `idhash` results are memoized in a module-level `Map<string, Promise<string>>`, cleared when it passes 10_000 entries. + +- [ ] **Step 1: Write the failing tests** + +```ts +it("walks through the find it's given", async () => { + const { repo, a, accounts } = await fixture(); + const seen: string[] = []; + const find = (url: AutomergeUrl) => (seen.push(url), repo.find(url)); + await resolveLivePath(repo, livePath(a, ["Projects", "foo"]), { accounts, find }); + expect(seen.length).toBeGreaterThanOrEqual(4); // account, /, Projects, foo +}); +it("fails, not throws, when find rejects", ...) + // find = () => Promise.reject(new Error("offline")) → resolveLivePath → undefined +``` + +```ts +it("warns about a malformed mount once, however often it's read", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doc = account({ namespace: { "/x@y": [{ url: R }], "/": [{ url: R }] } }); + for (let i = 0; i < 5; i++) accountProvider.lookup(doc, ["a"]); + expect(warn).toHaveBeenCalledTimes(1); +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/walk.test.ts test/providers.test.ts` +Expected: FAIL. `find` is never called (seen is empty), and warn is called 5 times. + +- [ ] **Step 3: Implement** + +- `accountBindings`: memoize in a `WeakMap<object, Map<…>>` keyed by the doc object. +- Warn once per malformed mount key, using a module-level `Set<string>`. +- `enter` calls `(options.find ?? defaultFind)(bareUrl, heads)` inside its existing `try`. + +- [ ] **Step 4: Run them to see them pass, plus the whole filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS. + +- [ ] **Step 5: Commit** `filesystem: walk takes a find, and remembers idhashes and warnings` + +--- + +### Task 2: `readPath` and `listFiles` + +**Files:** Modify `core/filesystem/src/walk.ts`, and `core/filesystem/src/resolve.ts` (export `materialize`). Test `core/filesystem/test/files.test.ts`. + +**Interfaces:** +- Consumes: `walkPath`, `WalkEnd`, `providerFor`, `Child` (phase 1); `materialize(repo, node, typeHint?, walk?)` from `resolve.ts`. +- Produces: + ```ts + export type FileContent = { content: string | Uint8Array; type: string }; + /** What a path names, read: its pinned form, whether the input was canonical, and the bytes. */ + export function readPath(repo: Repo, path: string, options: WalkOptions): + Promise<(FileContent & { pinned: string; canonical: boolean }) | undefined>; + export type ListedFile = FileContent & { tail: string; documentId: DocumentId }; + /** + * Every file under the last document a canonical pinned path enters, keyed by tail: "" for the + * document itself, encoded names joined with "/" below it. Containers aren't listed; they have + * their own doc steps. Undefined unless `path` is canonical and ends at a document. + */ + export function listFiles(repo: Repo, path: string, options: WalkOptions): + Promise<{ prefix: string; files: AsyncIterable<ListedFile> } | undefined>; + ``` +- `documentId` is the document holding the bytes: the leaf doc for a link, the enclosing doc for a value. + +`listFiles` enumeration (the walk doesn't determine this): + +``` +emit("", end.handle.doc(), end.handle.documentId) +visit(node, names, docId) for each child of provider.list(node): + value child, object (not bytes, not ImmutableString) → visit(value, [...names, ...name.split("/")], docId) + value child, otherwise → emit(join(names + name parts), value, docId) + link child → doc = find(link) at link heads or current; providerFor(doc) ? skip : emit(tail, doc, doc's id) + union child → skip +tail = parts.map(encodeURIComponent).join("/") +If two entries produce the same tail, the one whose directory key had more "/"-separated parts wins, matching the walk's longest-prefix lookup. +``` + +- [ ] **Step 1: Write the failing tests** + +```ts +it("reads a file and its pinned form", ...) + // readPath(repo, livePath(a, ["packages","foo","dist","index.js"])) → + // content "mine", type "text/plain", canonical false, pinned === pinnedPath(...) +it("lists a directory package's files", ...) + // tails sorted → ["", "dist/index.js", "package.json"]; each documentId is the file doc's +it("lists a folder's leaves and skips its subfolders", ...) + // Projects/foo has notes (file) and sub (folder) → tails ["", "notes"] +it("lists nested values inside a directory doc", ...) + // d.dist = { "a.js": "x" } with mimeType hint absent → tail "dist/a.js" +it("encodes names in tails", ...) // key "my file@1.js" → tail "my%20file%401.js" +it("agrees with readPath for every file it lists", async () => { + // for the directory package and the folder: for each listed file f, + // readPath(prefix + (f.tail ? "/" + f.tail : "")) has the same content and type +}); +it("lists nothing for a path that isn't canonical", ...) // live path → undefined +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/files.test.ts` +Expected: FAIL, no export `readPath`. + +- [ ] **Step 3: Implement both in `walk.ts`, exported from `index.ts` through the existing `export *`** + +- [ ] **Step 4: Run the filesystem suite** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem test` +Expected: PASS. + +- [ ] **Step 5: Commit** `filesystem: readPath and listFiles` + +--- + +### Task 3: Store, index, and the request flow (bootloader, pure) + +**Files:** +- Create `core/bootloader/vitest.config.ts` and `core/bootloader/test/setup.ts`, copied from `core/filesystem`. +- Add `"test": "vitest run"` to `core/bootloader/package.json`. +- Create `src/content-hash.ts`, `src/content-store.ts` and `src/pinned.ts`; modify `src/types.ts`. +- Test `core/bootloader/test/pinned.test.ts` and `test/content-store.test.ts`. + +**Interfaces:** +- Produces: + ```ts + // types.ts + export type IndexEntry = { sha256: string; contentType: string; store: boolean }; + export type Manifest = { prefix: string; entries: Record<string, IndexEntry> }; + + // content-hash.ts + export function toBytes(content: string | Uint8Array): Uint8Array; // strings → UTF-8 + export function contentHash(bytes: Uint8Array): Promise<string>; // SHA-256, base64url, no padding + + // content-store.ts + export interface ContentStore { + get(hash: string): Promise<Uint8Array | undefined>; + put(hash: string, bytes: Uint8Array): Promise<void>; + delete(hash: string): Promise<void>; + } + export const contentCacheName = (acctHash: string) => `patchwork-content-${acctHash}`; + export const CONTENT_CACHE_PREFIX = "patchwork-content-"; + export function cacheStore(cache: Cache, origin: string): ContentStore; // key `${origin}/.content/${hash}` + export function clearAccountStore(acctHash: string): Promise<boolean>; // caches.delete + + // pinned.ts + /** The `/@/` path, normalized so every spelling of one file is one key. */ + export function normalizePinned(path: string): + { key: string; prefix: string; tail: string; acctHash?: string; pinned: boolean } | undefined; + export function indexManifest(index: Map<string, IndexEntry>, manifest: Manifest): void; + export class PinnedAbort extends Error {} + export type PinnedDeps = { + index: Map<string, IndexEntry>; + store: ContentStore | undefined; // undefined: nothing is read or written (share roots) + hash(bytes: Uint8Array): Promise<string>; + manifest(key: string): Promise<Manifest | undefined>; // deduped by the caller, see below + resolve(key: string): Promise<{ body: Uint8Array; contentType: string } | undefined>; + warn(message: string): void; + keep(work: Promise<unknown>): void; // fetchEvent.waitUntil + }; + export function servePinned(key: string, deps: PinnedDeps): Promise<Response>; // throws PinnedAbort + /** Wrap a manifest fetcher so concurrent calls for one prefix share a request. */ + export function sharedManifests(fetch: (key: string) => Promise<Manifest | undefined>, prefixOf: (key: string) => string): + (key: string) => Promise<Manifest | undefined>; + ``` + +`normalizePinned` rules: +- Split on `/`, dropping empty segments. +- The root is pinned if it has a literal `@` or decodes to `automerge:`. A share root is written `automerge:` with the colon decoded. +- `prefix` runs up to and including the last segment that matches `/^[1-9A-HJ-NP-Za-km-z]+@[1-9A-HJ-NP-Za-km-z.]+$/`. +- In the prefix, plain names are re-encoded (`encodeURIComponent(decodeURIComponent(s))`) and doc steps are kept verbatim. +- Every tail segment is decoded and re-encoded the same way. +- `key = prefix + (tail ? "/" + tail : "")`. +- Undefined on a malformed escape. + +`servePinned` follows spec section 5's *Request flow* exactly. The response headers are `content-type: entry.contentType` and `cache-control: public, max-age=31536000, immutable`. Without an index entry it's `no-store`. + +- [ ] **Step 1: Write the failing tests** (fake store: a `Map`, real `contentHash`) + +```ts +describe("normalizePinned", () => { + it("gives one key for every spelling of a file", () => { + const keys = ["acct@h/x@h/a@b.js", "acct@h/x@h/a%40b.js", "/acct@h//x@h/a%40b.js/"] + .map((p) => normalizePinned(p)!.key); + expect(new Set(keys).size).toBe(1); + expect(normalizePinned("acct@h/x@h/日本")!.key).toBe(normalizePinned("acct@h/x@h/%E6%97%A5%E6%9C%AC")!.key); + }); + it("splits at the last doc step", () => { + expect(normalizePinned("acct@h/packages/m@h/f@h/dist/index.js")).toMatchObject({ + prefix: "acct@h/packages/m@h/f@h", tail: "dist/index.js", acctHash: "acct", pinned: true, + }); + }); + it("says a live path is live", () => expect(normalizePinned("acct/Projects")!.pinned).toBe(false)); + it("reads a share root", ...) // "automerge%3A<id>@h/x" → prefix starts "automerge:", acctHash undefined +}); + +describe("servePinned", () => { + it("serves a store hit whose bytes match, without asking a resolver", ...) + it("deletes a forged store entry, warns, and serves the resolver's real bytes", ...) + // store.put(realHash, forged); index has realHash → body === real; warn called; store now holds real + it("asks for one manifest on an index miss, then serves", ...) + it("stores resolver bytes that match an entry that allows it", ...) + it("never stores when the entry says not to", ...) + it("fails as a network error when resolver bytes don't match, storing nothing", ...) + // rejects with PinnedAbort; store empty + it("serves inline and stores nothing with no manifest", ...) + // manifest → undefined; resolve → bytes; response cache-control no-store; store empty; index empty + it("rebuilds the index after a restart and re-uses the store", ...) + // second deps with empty index, same store: one manifest, zero resolves, same bytes + it("shares one manifest among concurrent requests for a package", ...) + // sharedManifests: 5 parallel servePinned for 5 files of one prefix → fetch called once +}); + +describe("cacheStore", () => { + it("round-trips bytes under /.content/<hash>", ...) // fake Cache with put/match/delete over a Map keyed by url +}); +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, missing modules. + +- [ ] **Step 3: Implement the three modules and the types** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: content store, index and the pinned request flow` + +--- + +### Task 4: Manifests and the authenticated answer channel + +**Files:** Create `core/bootloader/src/manifest.ts` and `core/bootloader/src/index-channel.ts`; modify `src/types.ts`. Test `test/manifest.test.ts` and `test/index-channel.test.ts`. + +**Interfaces:** +- Consumes: `listFiles`, `WalkOptions` (Task 2); `contentHash`, `toBytes`, `Manifest` (Task 3). +- Produces: + ```ts + // types.ts + export const AUTOMERGE_WORKER_PATH = "/automerge-protocol-handler-worker.js"; // setup.ts's default uses it + export const INDEX_ANSWER_PATH = "/@/.index"; + export interface ManifestRequestMessage { type: "manifest"; id: string; path: string } + export type IndexAnswer = { id: string; manifest?: Manifest; error?: string }; + export interface RegisterAccountMessage { type: "register-account"; url: AutomergeUrl } + + // manifest.ts + export function buildManifest(repo: Repo, path: string, options: WalkOptions & { + isProtected?: (documentId: DocumentId) => boolean; // default: !isUnprotectedDocId(documentIdToBinary(id)) + }): Promise<Manifest | undefined>; + + // index-channel.ts + /** True only for the automerge worker: a sharedworker on this origin at AUTOMERGE_WORKER_PATH, any query. */ + export function isAutomergeWorker(client: Pick<Client, "type" | "url"> | undefined, origin: string): boolean; + /** The service worker's side of POST /@/.index: 403 unless the client is the worker, else parse, hand the answer to onAnswer, and reply 204 with x-patchwork-index: accepted. */ + export function acceptIndexAnswer(request: Request, client: Pick<Client, "type" | "url"> | undefined, + origin: string, onAnswer: (answer: IndexAnswer) => void): Promise<Response>; + /** The worker's side: POST the answer, retrying (5 × 200ms) until a reply carries x-patchwork-index: accepted. */ + export function postIndexAnswer(answer: IndexAnswer, fetcher?: typeof fetch): Promise<boolean>; + ``` +- `buildManifest`: + - Takes `listFiles(path)` and hashes `toBytes(content)` for each file. + - Sets `store` per Decision 5. `prefixSegments[1]` is read from `parsePath(prefix).segments[0]`, which must be the name `packages`. + - Returns `{ prefix, entries }`. + +- [ ] **Step 1: Write the failing tests** + +```ts +describe("buildManifest", () => { + it("hashes each file as the service worker will", ...) + // a file doc with content "héllo ✓" → entries["…"].sha256 === await contentHash(new TextEncoder().encode("héllo ✓")) + it("stores unprotected content and, through /packages, protected content", ...) + // isProtected: () => true → entries under a/Projects/... store false; under a/packages/... store true + it("never stores under a share root", ...) + it("returns nothing for a live path", ...) +}); + +describe("isAutomergeWorker", () => { + const o = "https://p.test"; + it.each([ + [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js` }, true], + [{ type: "sharedworker", url: `${o}/automerge-protocol-handler-worker.js?x=1` }, true], + [{ type: "worker", url: `${o}/automerge-protocol-handler-worker.js` }, false], // same url, dedicated + [{ type: "window", url: `${o}/` }, false], + [{ type: "sharedworker", url: `${o}/@/acct@h/evil.js` }, false], + [{ type: "sharedworker", url: `https://evil.test/automerge-protocol-handler-worker.js` }, false], + [undefined, false], + ])("%j → %s", (client, ok) => expect(isAutomergeWorker(client as any, o)).toBe(ok)); +}); + +describe("acceptIndexAnswer", () => { + it("refuses a window with 403 and hands nothing on", ...) + it("accepts the worker with 204 and the header", ...) +}); + +describe("postIndexAnswer", () => { + it("retries a reply that skipped the service worker", async () => { + const replies = [new Response("", { status: 404 }), new Response(null, { status: 204, headers: { "x-patchwork-index": "accepted" } })]; + const fetcher = vi.fn(async () => replies.shift()!); + expect(await postIndexAnswer({ id: "1" }, fetcher)).toBe(true); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + it("gives up after 5 tries", ...) +}); +``` + +Bootloader tests that use a `Repo` need the wasm setup from Task 3's `test/setup.ts`. + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, missing modules. + +- [ ] **Step 3: Implement** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: manifests, and the worker's authenticated answers` + +--- + +### Task 5: Resolvers answer paths and stop writing the cache + +**Files:** Modify `core/bootloader/src/resolver.ts`, `src/types.ts` (`HandoffRequest.path?: string`) and `src/tab-resolver.ts`. Test `test/resolver.test.ts`. + +**Interfaces:** +- Consumes: `readPath`, `pinnedPath`, `walkPath`, `WalkOptions` (filesystem); `toBytes`. +- Produces: + - `HandoffRequest.path?: string`: the `/@/`-stripped path. For these requests the service worker sets `handoffURL` to `patchwork-path:` + path, so an old resolver parses it, sees a protocol that isn't `automerge:`, and declines. + - `createResolver(getRepo, log?, accounts?: () => ReadonlyMap<string, AutomergeUrl>)`: the new third parameter defaults to an empty map. + - `registerAccount(url: AutomergeUrl): Promise<string>` in `tab-resolver.ts` returns the acctHash. It adds the account to the map the tab's resolver reads. Task 6 makes it also post to the worker. +- `answer()`: + - A `path` request that isn't pinned (`parsePath(path).root.heads` is missing), or isn't canonical: + - 307 to `/@/` + `pinnedPath(...)`, with `cache-control: no-store`; + - 404 if it doesn't resolve. + - A canonical pinned `path`: inline `{ status: 200, headers: { "content-type": type }, body: toBytes(content) }`; 404 if it doesn't resolve. + - An `automerge:` URL: as now, but always inline. The `cache-control` header is kept so the service worker can decide. + - No `caches` calls remain in `resolver.ts`. +- The resolver's `find` for the walk: + - With heads: `waitForHeads` (abort → `AbortHandoff`). + - Without heads: `caughtUpWithPeers`. + - Both use `AbortSignal.timeout(RESOLVE_TIMEOUT_MS)`. + +- [ ] **Step 1: Write the failing tests** (node, in-memory `Repo`, no `caches` global: any call would throw) + +```ts +it("redirects a live path to its pinned form", ...) + // answer({ request: { path: livePath(a, ["Projects","foo","notes"]), handoffURL: "patchwork-path:…", … } }) + // → type "response", status 307, headers.location === "/@/" + await pinnedPath(...), cache-control no-store +it("answers a pinned path with its bytes inline", ...) + // status 200, content-type text/plain, body bytes === UTF-8 of "hi" +it("redirects a pinned-looking path that isn't canonical", ...) +it("answers an unknown path with 404", ...) +it("answers legacy automerge URLs inline, without touching caches", ...) + // pinned automerge:<id>#heads/dist/index.js → type "response", cache-control includes immutable +it("registerAccount makes the tab's resolver answer for that account", ...) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL. Path requests return undefined, and the legacy test throws `caches is not defined`. + +- [ ] **Step 3: Implement** + +- [ ] **Step 4: Run them to see them pass** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: resolvers answer /@/ paths and leave caching to the service worker` + +--- + +### Task 6: The automerge worker builds and posts manifests; tabs register accounts + +**Files:** +- Modify `core/bootloader/src/automerge-protocol-handler.ts`, `src/setup.ts`, `src/tab-resolver.ts`. +- Modify `core/patchwork/src/index.ts`. +- Test `core/bootloader/test/worker-manifests.test.ts`. + +**Interfaces:** +- Consumes: `buildManifest`, `postIndexAnswer` (Task 4); `ManifestRequestMessage`, `RegisterAccountMessage`. +- Produces: + - `createManifestHandler(getRepo, accounts, post = postIndexAnswer)` in `manifest.ts`. It returns `(message: ManifestRequestMessage) => Promise<void>`, and keeps an LRU of 64 manifests keyed by canonical prefix. Requests already in flight for the same `path` share one build. A failed build posts `{ id, error }`. + - The worker wires it to `HANDOFF_CHANNEL` messages of `type: "manifest"`. It handles `register-account` port messages through `startWorkerControl`'s `onMessage`, adding `[await idhash(documentId), url]` to its accounts map. It counts manifest work in `handoffStarted`/`handoffFinished`, so a recycle waits for it. + - `setup.ts`: `registerAccount` posts `{ type: "register-account", url }` to `automergeProtocolHandlerWorker`. It remembers the url and posts it again from the handle's `onSpawn`. Re-export `registerAccount` and `clearAccountStore` from `setup.ts`, the package's `.` entry. + - `core/patchwork/src/index.ts`: after `resolveAccountHandle`, `await registerAccount(accountDocHandle.url)`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it("builds a manifest once per prefix and posts it", ...) + // two messages for files of one package → post called twice with the same manifest, listFiles walked once (spy on find) +it("posts an error for a path it can't list", ...) +it("evicts the oldest manifest after 64", ...) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, no `createManifestHandler`. + +- [ ] **Step 3: Implement and wire, then typecheck the dependents** + +Run: `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins --filter @inkandswitch/patchwork-bootloader build && pnpm --filter @inkandswitch/patchwork exec tsc --noEmit` +Expected: no errors. + +- [ ] **Step 4: Run the tests** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: PASS. + +- [ ] **Step 5: Commit** `bootloader: the automerge worker answers manifests; tabs register their account` + +--- + +### Task 7: The service worker serves `/@/` + +**Files:** Modify `core/bootloader/src/service-worker.ts`. Test `core/bootloader/test/pinned.test.ts` (one case for the request builder) and a typecheck. + +**Interfaces:** +- Consumes: everything from Tasks 3–5. +- Wiring, in order, in the `fetch` listener: + 1. `POST` to `INDEX_ANSWER_PATH`: `respondWith(acceptIndexAnswer(request, await clients.get(event.clientId), origin, settle))`. `settle` resolves the pending manifest request by `id`, and `indexManifest`s the answer. + 2. `GET` whose pathname starts with `/@/` and isn't `/@/.index`: + - `normalizePinned(path)`; on undefined, 400. + - Live → `handoff()` with `path`. The resolver's 307 is served as is (never cached). + - Pinned → `servePinned(key, deps)`, with: + - `store`: `cacheStore(await caches.open(contentCacheName(acctHash)), origin)`, or undefined for a share root; + - `manifest`: `sharedManifests(requestManifest, prefixOf)`. `requestManifest` posts `{ type: "manifest", id, path: key }` on `HANDOFF_CHANNEL`, calls `startWorkerIfMissing`, and resolves on `settle` or after `MANIFEST_TIMEOUT_MS = 10_000` with undefined; + - `resolve`: `handoff()` with `path: key`, read inline; + - `keep`: `event.waitUntil`. + - `PinnedAbort` → rethrow (a network error). + 3. Everything else, as today. +- `serveHandoff`, for legacy URLs: an inline `response` that is immutable (status 200/203/204, no `no-store` in `cache-control`) is `cacheInBackground` by its request URL. The `cached` branch stays for old workers. +- `clearOtherCaches` keeps every cache whose name starts with `CONTENT_CACHE_PREFIX`. +- Move the handoff-message construction into `pinned.ts` as `handoffRequest(request: Request, path?: string): HandoffRequestMessage` so it's testable. + +- [ ] **Step 1: Write the failing test** + +```ts +it("builds a path handoff that old resolvers decline", () => { + const m = handoffRequest(new Request("https://p.test/@/acct/Projects"), "acct/Projects"); + expect(m.request.path).toBe("acct/Projects"); + expect(new URL(m.request.handoffURL).protocol).not.toBe("automerge:"); +}); +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test` +Expected: FAIL, no export `handoffRequest`. + +- [ ] **Step 3: Wire the service worker as above** + +- [ ] **Step 4: Run the bootloader tests and typecheck** + +Run: `pnpm --filter @inkandswitch/patchwork-bootloader test && pnpm --filter @inkandswitch/patchwork-bootloader exec tsc --noEmit` +Expected: PASS, no type errors. + +- [ ] **Step 5: Commit** `service worker: serve /@/ paths from the checked content store` + +--- + +### Task 8: E2E and bench (CI) + +**Files:** Create `packages/e2e/tests/content-store.spec.ts` and `sites/bench/tests/content-hash.spec.ts`. + +**Interfaces:** The page exposes `window.patchwork.repo` and `window.patchwork.account`, as the existing e2e helpers use them. The acctHash is base58 of the first 12 bytes of SHA-256 over the account's `documentId`; the test computes it in the page with `crypto.subtle` and a short base58 loop. + +- [ ] **Step 1: Write the e2e spec** (two tests) + +```ts +test("a forged store entry is never served", async ({ page }) => { + // boot; create a file doc "forge.js" (content "export default 42", mimeType text/javascript) + // in the account's root folder; pinned = (await fetch(`/@/${acct}/forge.js`, { redirect: "manual" }) …) + // follow the 307 to get the pinned URL; fetch it once (stores it) + // in the page: cache.put(`/.content/${sha256(real)}`, new Response("export default 666")) + // into caches.open(`patchwork-content-${acct}`) + // reload; expect((await import(pinned)).default).toBe(42) +}); +test("an index answer from a page is refused", async ({ page }) => { + // fetch("/@/.index", { method: "POST", body: JSON.stringify({ id: "x", manifest: {...} }) }) → status 403 +}); +``` + +- [ ] **Step 2: Write the bench** `content-hash.spec.ts`: in `patchwork` mode, time `crypto.subtle.digest("SHA-256", …)` over 10 KB, 100 KB and 1 MB, 200 runs each. `record({ metric: "sha256 <size>", mode: "patchwork", value: median, unit: "ms" })`. + +- [ ] **Step 3: Typecheck both** + +Run: `pnpm --filter @inkandswitch/patchwork-e2e exec tsc --noEmit && pnpm --filter patchwork-bench exec tsc --noEmit` +Expected: no errors. Don't run them locally. CI's e2e matrix runs the spec on the PR. + +- [ ] **Step 4: Commit** `e2e: forged store entries and forged answers; bench: hashing a hit` + +--- + +### Task 9: Spec corrections and the changeset + +**Files:** Modify `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`. Create `.changeset/paths-content-store.md`. + +- [ ] **Step 1: Correct the spec** + +- Section 5, *The index channel is authenticated*: "the automerge worker's script URL is fixed (`/automerge-protocol-handler-worker.js`), not hashed. A site that sets `workerPath` gets inline, unstored serving." +- *Remaining gaps*: add the cold-start cost from Decision 10, and say that Task 8's bench measures it. + +- [ ] **Step 2: Write the changeset** + +```markdown +--- +"@inkandswitch/patchwork-filesystem": patch +"@inkandswitch/patchwork-bootloader": patch +"@inkandswitch/patchwork": patch +"@inkandswitch/patchwork-e2e": patch +--- + +The service worker serves `/@/` paths. A live path (`/@/<acctHash>/Projects/foo/x.js`) redirects to its pinned form; a pinned one is served from a content-addressed store (`patchwork-content-<acctHash>`) whose every read is hashed and checked against an index that only the automerge worker fills, over `fetch()` requests the service worker checks are really from it. Resolvers no longer write the cache: they hand bytes back and the service worker decides what to keep, so legacy `/automerge%3A…` URLs are now cached by the service worker. New: `registerAccount` and `clearAccountStore` from `@inkandswitch/patchwork-bootloader`; `readPath`, `listFiles` and `WalkOptions.find` from `@inkandswitch/patchwork-filesystem`. The e2e suite gains a forged-store-entry test. +``` + +- [ ] **Step 3: Commit** `spec and changeset: content store` diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md b/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md new file mode 100644 index 000000000..6dcecf99e --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-2b-file-lists.md @@ -0,0 +1,280 @@ +# Paths phase 2b: pinned URLs name their file list — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rework phase 2 so a pinned `/@/` URL is checked against itself. The last container step carries `~<listHash>`, the hash of a canonical file list. The service worker checks the list against the URL and the bytes against the list, and trusts nothing in between. + +**Architecture:** +- `patchwork-filesystem` learns three things: + - the `~listHash` grammar; + - how to build a file list (`buildFileList`); + - how to read a file at the heads a list recorded (`readPath` with `heads`). +- `pinPath` returns the URL and the list it names. +- The bootloader's request flow (`servePinned`) is rewritten around lists. +- Resolvers answer three kinds of request: + - a live path: a 307, whose body is the one-file list; + - a pinned file at given heads: its bytes; + - a container: its full list. +- The in-memory index, the worker's manifests and the authenticated `/@/.index` channel are deleted. + +**Tech Stack:** As in phase 2. TypeScript; automerge-repo 2.6.0-subduction.48; Cache Storage; `crypto.subtle`; vitest in node, including the service worker harness (`core/bootloader/test/service-worker-harness.ts`). + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, revised in f9446695. Section 1 covers the `~listHash` grammar and rule 4; section 5 covers *File lists*, *What gets stored*, *Request flow* and *Who is trusted for what*. The plan being reworked is `docs/superpowers/plans/2026-10-28-paths-phase-2-service-worker.md`. + +## Global Constraints + +- The pinned container step is `<idhash>@<heads>~<listHash>`. `listHash` is SHA-256 of the list's bytes, in base64url without padding: the same function as content (`contentHash`). +- A list is canonical JSON: keys sorted at every level, no whitespace. + - Shape: `{"files":{"<tail>":{"heads"?:"a.b","sha256":"…","store":bool,"type":"…"}},"v":1}`. + - `heads` is absent for a value inside the container. + - The container itself is the tail `""`. +- No document ids in lists or URLs (except a share root). +- Store a list if and only if every entry in it has `store: true`. Otherwise the service worker keeps it in memory. +- What gets stored is unchanged: unprotected documents, or protected ones reached through `/packages`, never under a share root. +- `pinPath` lists every file under the last container. A live → pinned redirect lists only the file asked for. A tail a list doesn't cover answers 404. +- A container step without a list leads to files served live: `no-store`, unstored, unchecked. +- Request flow, exactly as the spec's section 5. A list or bytes that don't match fail as a network error, and the service worker never stores anything that doesn't match. +- Phases 1 and 2 aren't released, so their changesets are rewritten, not added to. +- No local e2e, `vite preview` or site builds. + +## Decisions this plan makes + +1. **Hashing moves to filesystem** (`core/filesystem/src/content-hash.ts`: `contentHash`, `toBytes`). `pinPath` needs it, and the bootloader imports it from there. The bootloader's `content-hash.ts` is deleted. +2. **Keyhive protection moves to filesystem too:** `isKeyhiveDocumentId(documentId)`. `@inkandswitch/patchwork-plugins`'s `isKeyhiveDoc` calls it, so the rule lives once. +3. **`pinPath` and `pinnedPath` return `{ url | path, list }`** instead of a string. They're unreleased phase 1 API. +4. **A live → pinned 307 carries the one-file list as its body.** The service worker hashes it, keeps it if it matches the `~listHash` in `Location` (stored or in memory, by the rule above), and serves the 307 without the body. A list `pinPath` builds for imports (phase 4) is handed over the same way. Phase 4 adds the store write, or a message to the service worker; nothing calls it before then. +5. **Rebuilding a missing list** is a handoff with `request.list = true` on the container's pinned path, without `~`. The resolver answers the full list's bytes. Only a list that hashes to the URL's `listHash` is kept. +6. **Key normalization changes.** `normalizePinned` returns candidates of `{ listHash?, prefix, tail }`, latest doc-step-shaped segment first, as before, plus the share-root fix the review deferred. The first candidate that has a `listHash` is used. +7. **The worker keeps account registration**, since it still resolves paths for bytes and lists. Manifests, `index-channel.ts`, `INDEX_ANSWER_PATH`, the Firefox probe and the `/@/.index` route are deleted. `AUTOMERGE_WORKER_PATH` stays as `setup.ts`'s default. + +## Review Focus + +- **A file changed after pinning:** the pinned URL still serves the old bytes. The resolver reads the file at the list's `heads`, and they match the list. Pinned in Task 2 (`readPath` with `heads`) and Task 5 (the harness round trip). +- **A list in the store forged or truncated:** it's deleted and rebuilt. With nothing changed the rebuild matches; with something changed, the request fails as a network error, and no forged bytes are ever served. Pinned in Tasks 3 and 5. +- **Canonical bytes:** the same files in a different insertion order give the same `listHash`. Pinned in Task 2. +- **A redirect's one-file list:** a sibling requested under it answers 404, not someone else's bytes. Pinned in Task 3. +- **No document ids:** the list bytes contain no documentId. Pinned in Task 2. + +--- + +## File structure + +| File | Change | +|---|---| +| `core/filesystem/src/paths.ts` | `DocStep.listHash?`, `docStep(idhash, heads, listHash?)`, parse `~` | +| `core/filesystem/src/content-hash.ts` (create) | `contentHash`, `toBytes`, `isKeyhiveDocumentId` | +| `core/filesystem/src/file-list.ts` (create) | `FileList`, `FileEntry`, `canonicalJson`, `buildFileList` | +| `core/filesystem/src/walk.ts` | the walk keeps `~listHash` and exposes `container.listHash` and `tail`; `readPath(…, { heads })`; `pinPath`/`pinnedPath` return the list | +| `core/plugins/src/keyhive.ts` | `isKeyhiveDoc` via `isKeyhiveDocumentId` | +| `core/bootloader/src/pinned.ts` | candidates with `listHash`; `servePinned` rewritten; manifest functions deleted | +| `core/bootloader/src/resolver.ts` | heads reads, `list` requests, a 307 with its list | +| `core/bootloader/src/types.ts` | `HandoffRequest.heads?`, `HandoffRequest.list?`; manifest/index types deleted | +| `core/bootloader/src/service-worker.ts` | wiring; index, manifests and `/@/.index` deleted | +| `core/bootloader/src/automerge-protocol-handler.ts` | manifest wiring deleted | +| delete | `core/bootloader/src/{manifest,index-channel,content-hash}.ts` and their tests | +| `packages/e2e/tests/content-store.spec.ts` | drop the forged-answer test | +| `.changeset/paths-*.md` | rewrite | + +--- + +### Task 1: Grammar and walk carry `~listHash` + +**Files:** `core/filesystem/src/paths.ts`, `src/walk.ts`; tests `test/paths.test.ts`, `test/walk.test.ts`. + +**Interfaces:** +- Produces: + - `DocStep = { kind: "doc"; idhash; heads; listHash?: string; raw }`. + - `docStep(idhash, heads, listHash?)`. + - `parsePath` reads `<base58>@<heads>~<base64url>`: `/^[A-Za-z0-9_-]+$/` after the `~`, and a `~` with nothing after it is a syntax error for the root and a name otherwise. + - `WalkEnd.container` gains `listHash?: string`. The walk writes a matched doc step back with its `listHash`, so re-pinning a listed path gives the same string. + - `WalkEnd.tail: string`: the encoded names after the container, joined with `/`. + +- [ ] **Step 1: Failing tests** + +```ts +// paths.test.ts +it("reads a doc step's file list hash", () => { + expect(parsePath("acct@h/foo@h~AbC-_9/dist/x.js").segments[0]).toEqual({ + kind: "doc", idhash: "foo", heads: ["h"], listHash: "AbC-_9", raw: "foo@h~AbC-_9", + }); + expect(docStep("foo", ["b", "a"], "L")).toBe("foo@a.b~L"); +}); +// walk.test.ts +it("keeps a container's list hash and says which tail follows it", async () => { + // pinned = (await pinnedPath(repo, live(packages/foo), {accounts}))!.path, then append "~L" by hand + // walkPath(`${pinned}~L/dist/index.js`) → container.listHash "L", tail "dist/index.js", + // pinned === `${pinned}~L/dist/index.js`, canonical true +}); +``` + +- [ ] **Step 2: Run, expect FAIL** (`listHash` undefined): `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/paths.test.ts test/walk.test.ts` +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run the filesystem suite, expect PASS:** `pnpm --filter @inkandswitch/patchwork-filesystem test` +- [ ] **Step 5: Commit** `filesystem: doc steps carry a file list hash` + +### Task 2: File lists, reading at heads, `pinPath` returns the list + +**Files:** create `core/filesystem/src/content-hash.ts` and `src/file-list.ts`; modify `src/walk.ts`, `src/index.ts` and `core/plugins/src/keyhive.ts`; test `test/file-list.test.ts`. + +**Interfaces:** +- Produces: + ```ts + // content-hash.ts + export function toBytes(content: string | Uint8Array): Uint8Array; + export function contentHash(bytes: Uint8Array): Promise<string>; // moved from bootloader, same output + export function isKeyhiveDocumentId(documentId: DocumentId): boolean; + // file-list.ts + export type FileEntry = { sha256: string; type: string; heads?: string; store: boolean }; + export type FileList = { v: 1; files: Record<string, FileEntry> }; + export function canonicalJson(value: unknown): string; + export function parseFileList(bytes: Uint8Array): FileList | undefined; + /** The list for the container `path` is in (all its files, or only `only`'s tails), its bytes, hash, and the container's pinned path without `~`. */ + export function buildFileList(repo: Repo, path: string, options: WalkOptions & { + findFile?: WalkOptions["find"]; only?: string[]; isProtected?: (id: DocumentId) => boolean; + }): Promise<{ prefix: string; list: FileList; bytes: Uint8Array; listHash: string; missing: string[] } | undefined>; + // walk.ts + export function readPath(repo, path, options: WalkOptions & { heads?: UrlHeads }): … // heads: read the final doc at these + export function pinnedPath(repo, path, options): Promise<{ path: string; list: Uint8Array } | undefined>; + export function pinPath(repo, path, options): Promise<{ url: string; list: Uint8Array } | undefined>; + ``` +- `buildFileList` uses `listFiles` (phase 2), drops `documentId` from each entry, and records `heads` (`formatHeads` of the heads the file doc was read at) for files reached by a link. `ListedFile` gains `heads?: UrlHeads`. `store` follows the spec rule, with `isProtected` defaulting to `isKeyhiveDocumentId`. +- `pinnedPath` builds the full list for the container the path ends in, or the one-file list when called with `{ only: [tail] }`, and writes `~listHash` on that container's step. + +- [ ] **Step 1: Failing tests** + +```ts +it("is canonical: the same files in another order hash the same", ...) + // two directory docs with the same keys inserted in opposite order and equal file contents/heads? — use one doc, + // build twice → identical bytes; canonicalJson({b:1,a:{d:1,c:2}}) === '{"a":{"c":2,"d":1},"b":1}' +it("records each file's hash, type and heads, and no document ids", ...) + // bytes as text contains no documentId of any fixture doc; entry for dist/index.js has heads === formatHeads(file.heads()) +it("lists only what it's asked to", ...) // only: ["dist/index.js"] → keys ["dist/index.js"] +it("stores protected content only through /packages", ...) // as phase 2's manifest test +it("reads a file at the heads its list recorded, after it changed", async () => { + // pin; change the file doc's content; readPath(pinned path, {heads: parseHeads(entry.heads)}) → old content +}); +it("pins a path with its container's list hash", ...) + // pinnedPath(live packages/foo/dist/index.js) → path ends `~${await contentHash(list)}/dist/index.js` +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork-filesystem exec vitest run test/file-list.test.ts` +- [ ] **Step 3: Implement.** Update existing callers of `pinnedPath` and `pinPath` in the filesystem tests and in `core/bootloader/src/resolver.ts` (it reads `.path`). +- [ ] **Step 4: Run the filesystem suite and typecheck the dependents, expect PASS:** `pnpm --filter @inkandswitch/patchwork-filesystem test && pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-plugins build && pnpm --filter @inkandswitch/patchwork-bootloader exec tsc --noEmit` +- [ ] **Step 5: Commit** `filesystem: file lists, reading at recorded heads, pinPath names its list` + +### Task 3: The request flow around lists (bootloader, pure) + +**Files:** `core/bootloader/src/pinned.ts`, `src/types.ts`; tests `test/pinned.test.ts`. Delete `src/manifest.ts`, `src/index-channel.ts`, `src/content-hash.ts` and their tests. Change imports to `@inkandswitch/patchwork-filesystem`. + +**Interfaces:** +- Produces: + ```ts + export type Candidate = { prefix: string; tail: string; listHash?: string }; + export function normalizePinned(path: string): + { candidates: Candidate[]; acctHash?: string; pinned: boolean; share: boolean } | undefined; + export type PinnedDeps = { + lists: Map<string, FileList>; // in memory, by listHash + store: ContentStore | undefined; + hash(bytes: Uint8Array): Promise<string>; + /** Ask for the full list of the container at `prefix`; its bytes, or undefined. */ + buildList(prefix: string): Promise<Uint8Array | undefined>; + /** Ask for the file at `path`, read at `heads` if given. */ + resolve(path: string, heads?: string): Promise<{ body: Uint8Array; contentType: string } | Response | undefined>; + warn(message: string): void; + keep(work: Promise<unknown>): void; + }; + /** Check `bytes` against `listHash` and keep the list: stored if every entry allows it, else in memory. False if it doesn't match. */ + export function keepFileList(bytes: Uint8Array, listHash: string, deps: PinnedDeps): Promise<boolean>; + export function servePinned(normalized: { candidates: Candidate[] }, deps: PinnedDeps): Promise<Response>; // throws PinnedAbort + ``` +- `handoffRequest(request, cachename, to: URL | { path: string; heads?: string; list?: boolean })`. +- Flow: the spec's *Request flow for a pinned URL*. A candidate with no `listHash` anywhere is served live through `resolve(path)`: `no-store`, unchecked. Concurrent `buildList` calls for one `listHash` share a request (`sharedRequests(fn)`, keyed by its argument). + +- [ ] **Step 1: Failing tests** (fake store, real `contentHash`, lists built with `canonicalJson`) + +```ts +it("serves a file whose list and bytes are in the store, asking no resolver", ...) +it("deletes a forged list, rebuilds it, and serves", ...) // store has garbage under listHash; buildList returns real bytes +it("fails when the rebuilt list doesn't match the URL", ...) // buildList returns a different list → PinnedAbort +it("deletes a forged file, rereads it at the list's heads, and serves", ...) // resolve called with entry.heads +it("fails on a resolver's bytes that don't match, storing nothing", ...) +it("answers 404 for a tail the list doesn't cover", ...) +it("keeps a list in memory when an entry may not be stored", ...) // keepFileList → lists has it, store empty +it("serves a path with no list live and unchecked", ...) // cache-control no-store +it("normalizes a share root's doc-step-shaped name", ...) // the deferred review minor +it("shares one rebuild among concurrent requests", ...) +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork-bootloader exec vitest run test/pinned.test.ts` +- [ ] **Step 3: Implement**, and delete the manifest and index-channel code. +- [ ] **Step 4: Typecheck, expect a clean run.** Expect errors only in `service-worker.ts`, `resolver.ts` and `automerge-protocol-handler.ts`, which Tasks 4–5 fix. Run the pinned tests, expect PASS. +- [ ] **Step 5: Commit** `bootloader: check pinned URLs against their own file list` + +### Task 4: Resolvers answer lists and reads at heads; the worker sheds manifests + +**Files:** `core/bootloader/src/resolver.ts`, `src/automerge-protocol-handler.ts`, `src/types.ts`; test `test/resolver.test.ts`. + +**Interfaces:** +- `HandoffRequest.heads?: string` and `HandoffRequest.list?: boolean`, both additive. +- `answer()` for a `path`: + - **`list`:** `buildFileList(path)`, which must be canonical. Answers `200 application/json` with the list's bytes, or 404. + - **`heads`:** `readPath(path, { heads: parseHeads(heads) })` → 200 with the bytes. + - **Live, or not canonical:** `pinnedPath(path, { only: [tail] })` → 307. `Location` is `/@/` plus the pinned path, and the body is the one-file list. + - **Canonical pinned, no heads:** the bytes, as now. +- The worker: drop `createManifestHandler` and the `manifest` channel branch. Keep `register-account`. + +- [ ] **Step 1: Failing tests** + +```ts +it("answers a live path with a 307 whose body is the list it names", ...) + // contentHash(body) === the ~listHash in location; parseFileList(body) has one entry +it("answers a container's full list", ...) +it("reads a file at given heads, after it changed", ...) +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** `pnpm --filter @inkandswitch/patchwork-bootloader exec vitest run test/resolver.test.ts` +- [ ] **Step 5: Commit** `bootloader: resolvers answer file lists and reads at heads` + +### Task 5: Service worker wiring, through the harness + +**Files:** `core/bootloader/src/service-worker.ts`; test `test/service-worker.test.ts`. + +**Interfaces:** +- `servePath`: + - Live → `handoff({ path })`. On a 307 with a body, `keepFileList(body, listHash from location)`, then serve the 307 without the body. + - Pinned → `servePinned` with: + - `buildList = handoff({ path: prefix, list: true })`; + - `resolve = handoff({ path, heads })`; + - `lists`: a module-level `Map`. +- Delete: `contentIndex`, manifest requests, `settleIndexAnswer`, `answerIndex`, the `/@/.index` route, and the manifest re-broadcast on `online`. + +- [ ] **Step 1: Failing harness tests** (a fake tab running the real resolver, a real in-memory Repo, an account registered in the tab) + +```ts +it("redirects a live path, then serves the pinned file from its list", ...) + // fetch live → 307; fetch location → bytes; content store now holds the list and the bytes +it("serves the old bytes for a pinned URL after the file changed", ...) +it("never serves a forged store entry", ...) + // put forged bytes under the file's sha256 → fetch again → real bytes +it("fails, rather than serving, when a tab answers with other bytes", ...) + // swap the tab's postMessage to answer with forged bytes → event.response() rejects +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** `pnpm --filter @inkandswitch/patchwork-bootloader test`, plus `pnpm --filter @inkandswitch/patchwork exec tsc --noEmit` after building filesystem, plugins and bootloader. +- [ ] **Step 5: Commit** `service worker: serve pinned paths checked against their file list` + +### Task 6: e2e, changesets, cleanup + +- [ ] **Step 1:** In `packages/e2e/tests/content-store.spec.ts`, delete the forged-answer test. The forged-store test stays as is, since the live → 307 → pinned flow still holds. Typecheck: `pnpm --filter @inkandswitch/patchwork-e2e exec tsc --noEmit`. +- [ ] **Step 2: Rewrite `.changeset/paths-content-store.md`** for what ships: + - `/@/` paths, file lists, the store; + - `registerAccount` and `clearAccountStore`; + - no index and no answer channel. + + Rewrite `.changeset/paths-grammar-providers.md` for `pinPath` and `pinnedPath` now returning `{ url | path, list }`. Add `@inkandswitch/patchwork-plugins` for `isKeyhiveDoc` now going through filesystem. +- [ ] **Step 3: Run the full suites and lint.** `pnpm --filter @inkandswitch/patchwork-filesystem --filter @inkandswitch/patchwork-bootloader test` should pass. In `pnpm lint`, ignore hits under the git-ignored `sites/bench/bench-results`. +- [ ] **Step 4: Commit** `content store: changesets and e2e for file lists` diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md b/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md new file mode 100644 index 000000000..20142fe57 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-3-routing.md @@ -0,0 +1,119 @@ +# Paths phase 3: routing — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Clicking into a document can say where it was opened from: the live path lands in `&path=` in the hash, a hash with only `&path=` opens what the path names, and a `doc=` that disagrees with its `path=` wins. + +**Architecture:** Bring in the `open-document-path` branch (48f4ab98), which threads `path` through the `patchwork:open-document` event, `patchwork.open` and `<patchwork-view path>`. Then redefine that `path` as the **live path** (`<acctHash>/Projects/foo`), not a service worker path. The router reads and writes `path` raw, since its segments are already percent-encoded, and resolves it with `resolveLivePath` from phase 1. + +**Tech Stack:** TypeScript; `core/patchwork/src/router.ts`; `@inkandswitch/patchwork-filesystem` (`idhash`, `resolveLivePath`); vitest in node against a fake `window`, `location` and root element. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 3, *Hash routing* and *Opening at a path*. + +## Global Constraints + +- `path` joins `HASH_KEY_ORDER` right after `doc`. Each segment is `encodeURIComponent`-encoded and the slashes stay literal: the value goes into the hash exactly as `livePath` wrote it. +- `doc=` says what is open; `path=` says where it was opened from. +- A hash with `path=` and no `doc=` resolves the path and opens the result. +- If `doc=` and `path=` disagree, `doc=` wins and `path=` is dropped with a console warning. +- `patchwork:open-document`'s `path` holds the live path, and the router writes it to `&path=`. That reverses the branch's choice to keep it out of the hash. +- `<patchwork-view path>` hands the path to the tool without remounting, as on the branch. + +## Decisions this plan makes + +1. **`path` is read raw from the hash.** `URLSearchParams` decodes `%2F` to `/`, which would split a name containing a slash into two. A single `parseHashParams(hash)` reads every key through `URLSearchParams` and then puts `path`'s raw text back. Every place the router reads the hash uses it. +2. **The router resolves only the signed-in account's paths.** Its `accounts` map is the account doc's own acctHash. A path under any other account doesn't resolve: it's dropped with a warning if there's a `doc=`, or ignored if there isn't. +3. **The agreement check runs after the document is opened**, so a slow path lookup never delays opening it. If the path names a different document, or none at all, the router rewrites the hash without `path=`. +4. **The router is tested in node.** `@inkandswitch/patchwork-elements` is mocked down to an `openDocument` that dispatches the event, and the DOM is replaced with small fakes: an `EventTarget` with attributes, plus `window`, `location` and `document`. + +## Review Focus + +- **Reserved characters in a path segment** (`a%2Fb`, `%26`, `%23`, a space as `%20`, `+`) survive a round trip through the hash, including through an open-document event that re-serializes the other params. Pinned in Task 2. +- **Path only, unresolvable** (a typo, another account): no exception, a warning, and no document opened. Pinned in Task 3. +- **The same `path=` arriving again on every hashchange** doesn't re-open anything or loop. Only a hash without `doc=` opens from the path. Pinned in Task 3. + +--- + +### Task 1: Bring in `open-document-path`, as a live path + +**Files:** cherry-pick 48f4ab98 (`core/elements/src/{events,index,patchwork-view}.ts`, `core/patchwork/src/{index,types}.ts`, `.changeset/open-document-sw-path.md`). Then edit the doc comments and the changeset. + +- [ ] **Step 1:** `git cherry-pick 48f4ab98`. Expected: it applies cleanly, since those files aren't touched on `paths-design`. Verify with `git diff HEAD~1 --stat`. +- [ ] **Step 2: Redefine `path`.** + - `OpenDocumentEventDetail.path` becomes: "The live path the document was opened at, like a cwd: `<acctHash>/Projects/foo`, each segment `encodeURIComponent`-encoded. The router writes it to `&path=`." + - Point the `OpenOptions.path` and `PatchworkViewElementBase.path` comments at it. + - Rewrite the changeset (rename it `open-document-path.md`) to say the same, listing `@inkandswitch/patchwork-elements` and `@inkandswitch/patchwork`. +- [ ] **Step 3: Typecheck:** `pnpm --filter @inkandswitch/patchwork-elements build && pnpm --filter @inkandswitch/patchwork exec tsc --noEmit`. Expected: no errors. +- [ ] **Step 4: Commit** `elements: open-document's path is the live path it was opened at`. + +### Task 2: Hash params keep `path` raw + +**Files:** create `core/patchwork/src/hash-params.ts` (moving `serializeHashParams`, `HASH_KEY_ORDER`, `RAW_HASH_KEYS` and `docParamToUrl` out of `router.ts`); modify `router.ts` to import them. Test `core/patchwork/test/hash-params.test.ts`. + +**Interfaces:** +```ts +export const HASH_KEY_ORDER = ["doc", "path", "tool", "type", "title", "frame", "draft"]; +/** `URLSearchParams` over a hash, except that `path`'s value is its raw, still-encoded text. */ +export function parseHashParams(hash: string): URLSearchParams; +export function serializeHashParams(params: URLSearchParams): string; // doc, draft and path emitted raw +export function docParamToUrl(docParam: string | null): AutomergeUrl | undefined; // moved unchanged +``` + +- [ ] **Step 1: Failing tests** + +```ts +it("keeps a path's segments as they're encoded", () => { + const hash = "doc=automerge:abc&path=acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"; + expect(parseHashParams(hash).get("path")).toBe("acct/a%2Fb/x%26y/%23z/my%20notes/1%2B1"); + expect(serializeHashParams(parseHashParams(hash))).toBe(hash); +}); +it("puts path right after doc", () => { + const params = new URLSearchParams({ tool: "t", doc: "automerge:abc" }); + params.set("path", "acct/x"); + expect(serializeHashParams(params)).toBe("doc=automerge:abc&path=acct/x&tool=t"); +}); +it("reads a hash with no path as before", () => { + expect(parseHashParams("doc=automerge:abc&title=a%20b").get("title")).toBe("a b"); +}); +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork exec vitest run test/hash-params.test.ts`. +- [ ] **Step 3: Implement.** Use `parseHashParams` in place of `new URLSearchParams(hash)` everywhere in `router.ts`. +- [ ] **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `patchwork: hash params keep a path's encoding`. + +### Task 3: The router writes, opens from, and checks `path=` + +**Files:** `core/patchwork/src/router.ts`; tests `core/patchwork/test/router.test.ts`. + +**Interfaces:** +- `createRouter` is unchanged on the outside. Internally it: + - computes `accounts = new Map([[await idhash(accountDocHandle.documentId), accountDocHandle.url]])` once; + - in the open-document listener, sets `path` from `detail.path`, or deletes it if there's none; + - in `route()`, for a hash with `path` and no `doc`: `resolveLivePath(repo, path, { accounts })`. If that finds a document, it dispatches `patchwork:open-document` with `{ url: handle.url, path }`; if not, it warns and rewrites the hash without `path`; + - in `route()`, for a hash with both: dispatches open-document as now, with `path` in its detail. Then it resolves the path, and if the result's `documentId` differs from `doc`'s, or nothing resolves, it warns and rewrites the hash without `path`. + +- [ ] **Step 1: Failing tests** (fake DOM, real in-memory `Repo`, an account whose `/` holds `Projects/foo/notes`) + +```ts +it("writes an opened document's path to the hash", ...) + // dispatch open-document {url: notes.url, path: livePath(a, ["Projects","foo","notes"])} → location.hash has doc=…&path=<that> +it("drops path from the hash when a document is opened without one", ...) +it("opens what a path names when the hash has no doc", ...) + // location.hash = `path=${live}`; await router.route() → an open-document event with url === notes.url and path === live +it("warns and drops a path that names nothing, opening nothing", ...) +it("keeps a path that agrees with doc", ...) +it("drops a path that disagrees with doc, keeping doc", ...) // console.warn called; hash keeps doc, loses path +it("doesn't open anything twice when the same path arrives with its doc", ...) + // hash doc=notes&path=live → route() → exactly one open-document event +``` + +- [ ] **Step 2: Run, expect FAIL:** `pnpm --filter @inkandswitch/patchwork exec vitest run test/router.test.ts`. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run the patchwork suite, expect PASS:** `pnpm --filter @inkandswitch/patchwork test`, plus `pnpm --filter @inkandswitch/patchwork exec tsc --noEmit`. +- [ ] **Step 5: Commit** `patchwork: route &path= — write it, open from it, drop it when it disagrees`. + +### Task 4: Changeset + +- [ ] Add `.changeset/paths-routing.md` (`@inkandswitch/patchwork: patch`) describing `&path=`: written when a document is opened at a path, opened from when there's no `doc=`, and dropped with a warning when it disagrees. The e2e test the spec lists ("open a document with `&path=` alone") needs a full-UI boot. Add it to `packages/e2e/tests/content-store.spec.ts`'s Chromium-only pattern as `path-routing.spec.ts`, typecheck it, and let CI run it. +- [ ] Commit `changeset: path routing; e2e: open with &path= alone`. diff --git a/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md b/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md new file mode 100644 index 000000000..31daaea1e --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-phase-4-packages.md @@ -0,0 +1,189 @@ +# Paths phase 4: packages via `/packages` — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Package management is whatever is bound at `/packages`. The ModuleWatcher loads every package in that union, looks each one up as a pinned `/@/` URL whose list covers the whole package, and imports it from there, checked against the list. New accounts get `/` and `/packages` bound. Old accounts are migrated once. + +**Architecture:** +- **Filesystem:** lists of packages go deep, and the walk exposes the union member a package was found in. +- **Bootloader resolver:** + - a redirect to a container carries its whole list; + - reads at `heads` ignore whether the path is canonical. +- **ModuleWatcher:** a `folder` source lists `/packages` and watches it. A composite tracker sends `/packages` packages through `/@/` lookups, and everything else through the legacy tracker. +- **Patchwork:** binds `/packages` on new accounts, migrates old ones, and wires the source. + +**Tech Stack:** TypeScript, automerge-repo, vitest (node, plus the bootloader's service worker harness). e2e runs in CI only. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 4, revised in 274dc28d, and section 5 for lists. + +## Global Constraints + +- The module list is the `/packages` union listing, deduplicated by document id. Two packages with the same name both load. +- Packages are tracked by document id; a rename reloads nothing. +- When a package's own heads move, the watcher looks it up again and imports the new pinned URL. Other packages keep their URLs. +- A link under `/packages` that isn't a package (no `package.json`) is warned about and skipped. +- Lookup: `/@/<acctHash>/packages/<memberIdhash>@<memberHeads>/<encoded name>` → 307 → the pinned URL. The watcher imports with `importPackageFromHttpUrl(pinnedUrl)`. +- A container with a `package.json` file lists every file under it, through nested folders. Others list only their own files. +- A redirect to a container carries the container's whole list; a redirect to a file, the one-file list. +- The module-settings doc stays a source; its automerge entries are deduplicated against `/packages` by document id. +- `createDefaultAccount` binds `/` to the root folder and `/packages` to a new folder. +- Migration happens once at boot, when the account has `moduleSettingsUrl` and no `/packages` binding: + - create a folder; + - link each automerge entry, named by its `package.json` `name`, falling back to the doc title, then to its documentId; + - bind the folder at `/packages`, leaving `/` to fall back to `rootFolderUrl` as before; + - leave the module-settings doc untouched. +- `AccountDoc.moduleSettingsUrl` is `@deprecated`. + +## Decisions this plan makes + +1. **Package lists recurse into nested folders,** but only for a container that has a `package.json`, so fetching a user folder never lists the whole tree. Tails cross folders by name (`sub/x.js`). Each nested file records its own `heads`, so a resolver walks to it by name and reads it at those heads. +2. **Nested folders aren't pinned by heads in a package's list.** A rebuild walks them at their current heads. If a nested folder was restructured, a rebuild of an old list won't match, and that old URL fails as a network error. The live lookup gives a new URL, so imports recover. +3. **Lookups always use the member step.** The tab lists the union, so it knows each member's idhash and heads. The member step finds shadowed packages too, and works the same for unique names. +4. **Composite tracker.** The ModuleWatcher keeps one tracker: + - `findLatest(url)` answers with the `/@/` lookup for a document id the `/packages` index knows, and with the legacy service worker redirect otherwise; + - `watch` is shared. + + Its `version` strings can be `/@/` URLs. `importPackageSafe` already sends anything that isn't an automerge URL to `importPackageFromHttpUrl`. +5. **Discovery off the main thread is skipped** for `/@/` packages: the module-loader worker path takes automerge URLs only. Moving it over is a follow-up. +6. **The `/packages` index runs in the tab.** It loads the account doc (already loaded) and each member folder doc (small), never the package docs. Each package's `package.json` presence is checked through the lookup instead: a lookup whose list has no `package.json` entry is warned about and skipped. + +## Review Focus + +- **A package that appears both in module settings and `/packages`** is imported once, through `/packages`. Pinned in Task 4. +- **A rename inside `/packages`** reloads nothing. Removing the link unloads the package. Pinned in Task 4. +- **The same name in two union members:** both load. Pinned in Task 4. +- **A folder package with `./sub/x.js`** imports through its pinned URL. Pinned in Tasks 1 and 2 (filesystem and harness). +- **Migration runs once.** A second boot changes nothing, and an account with `/packages` already bound isn't migrated. Pinned in Task 5. + +--- + +### Task 1: Deep lists for packages; the redirect to a container carries its list + +**Files:** +- `core/filesystem/src/walk.ts` (`listFiles` gains `deep`) and `src/file-list.ts`. +- `core/bootloader/src/resolver.ts`. +- Tests: `core/filesystem/test/file-list.test.ts` and `core/bootloader/test/resolver.test.ts`. + +**Interfaces:** +- `listFiles(repo, path, options & { deep?: boolean })`. With `deep`, a link child whose document has a provider is listed into, with its name as a tail prefix, instead of being skipped. Branches and account docs are still skipped. Each recursion guards against cycles by documentId. +- `buildFileList` sets `deep` when the container's own listing has a `package.json` entry. That check runs on the shallow pass, then the list is redone deep. +- `pinnedPath(…, { only: "end" })`: + - when the path ends at a container (`end.tail === ""` and `end.atDocument`), it builds the container's full list; + - otherwise it builds the one-file list. +- In the resolver's `resolveWalkPath`, a request with `heads` reads with `readPath(…, { heads })` whether or not the path is canonical. Only a request without heads and without `list` redirects when the path isn't canonical. + +- [ ] **Step 1: Failing tests** + +```ts +// file-list.test.ts +it("lists a package's nested folders deep", ...) + // a folder package {package.json, sub → folder {x.js}} → tails include "package.json", "sub/x.js" +it("lists a user folder shallow", ...) // Projects/foo has sub folder, no package.json → no "sub/…" +it("reads a nested file at its recorded heads", ...) // readPath(`${prefix}/sub/x.js`, {heads}) after x changed → old +// resolver.test.ts +it("redirects a live path to a container with its whole list", ...) // body list keys include every file +it("reads at heads through a folder reached by name", ...) // path pkg@h/sub/x.js with heads → 200 old bytes +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run, expect PASS:** both suites. +- [ ] **Step 5: Commit** `paths: package lists go deep; a redirect to a folder carries its list` + +### Task 2: The service worker serves a folder package through the harness + +**Files:** `core/bootloader/test/service-worker.test.ts` only, unless the test exposes a bug. + +- [ ] **Step 1: Test.** An account binds `/packages` to a folder holding a folder package: `package.json` → `{"main":"index.js"}`, `index.js` → `export * from "./sub/x.js"`, and `sub/x.js`. Then: + - fetch the live `/@/<a>/packages/pkg` → 307; + - fetch `<location>/package.json` and `<location>/sub/x.js` → both 200, `immutable`; + - store holds the list and both files. +- [ ] **Step 2: Run.** It should pass if Task 1 is right; a failure here is a bug to fix with systematic debugging. +- [ ] **Step 3: Commit** `service worker: test a folder package end to end` + +### Task 3: The `/packages` index and a composite tracker + +**Files:** create `core/filesystem/src/packages-folder.ts`; modify `src/index.ts`; test `core/filesystem/test/packages-folder.test.ts`. + +**Interfaces:** +```ts +export type PackageEntry = { + url: AutomergeUrl; // the package doc, without heads + name: string; + memberStep: string; // docStep(memberIdhash, memberHeads) +}; +/** Watch an account's /packages union: call onChange with its entries (deduplicated by documentId, first binding wins) now and whenever the account doc or a member folder changes. */ +export function watchPackagesFolder(repo: Repo, account: DocHandle<AccountNamespaceDoc>, + onChange: (entries: PackageEntry[]) => void): () => void; +/** The live lookup path for an entry. */ +export function packageLookupPath(acctHash: string, entry: PackageEntry): string; + // `${acctHash}/packages/${entry.memberStep}/${encodeURIComponent(entry.name)}` +/** A tracker that answers findLatest through /@/ for the urls `entries()` knows, and through `fallback` otherwise. */ +export function trackPackagesFolder(acctHash: Promise<string>, entries: () => PackageEntry[], + fallback: PackageTracker, lookUp?: (path: string) => Promise<string>): PackageTracker; + // lookUp default: fetch(`/@/${path}`) → response.url (redirected) → the pinned URL; throws if not redirected +``` + +- [ ] **Step 1: Failing tests:** entries from a two-member union, deduplicated, with the same name in both; onChange after a member folder gains an entry; `packageLookupPath`; the tracker routes a known url to `lookUp` and an unknown one to `fallback`. +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement.** **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `filesystem: watch /packages and look packages up through it` + +### Task 4: The ModuleWatcher loads `/packages` + +**Files:** `core/filesystem/src/module-watcher.ts`; test `core/filesystem/test/module-watcher.test.ts`. + +**Interfaces:** +- `ModuleWatcher.addPackagesFolder(name: string, entries: () => PackageEntry[], subscribe: (onChange: () => void) => () => void): Promise<void>`. +- The module list is the settings docs' modules plus the folder's entry urls, deduplicated by `stripHeads` (document id). An entry present in `/packages` is keyed by its bare url. The tracker (composite, constructed by the caller) decides how it's looked up. + +- [ ] **Step 1: Failing tests** (existing fakes in module-watcher.test.ts): + +```ts +it("loads every package in the folder", ...) +it("loads a package in both settings and /packages once", ...) +it("doesn't reload a renamed package", ...) +it("unloads a package removed from the folder", ...) +it("loads both packages with one name in two members", ...) +``` + +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement.** **Step 4: Run the filesystem suite, expect PASS.** +- [ ] **Step 5: Commit** `filesystem: ModuleWatcher loads the /packages folder` + +### Task 5: Accounts bind `/packages`; migration; wiring + +**Files:** +- `core/patchwork/src/createAccount.ts`. +- Create `core/patchwork/src/migrate-packages.ts`. +- `core/patchwork/src/index.ts` (wire the folder source and the composite tracker). +- `core/plugins/src/account.ts` (`@deprecated` on `moduleSettingsUrl`). +- Test `core/patchwork/test/migrate-packages.test.ts`. + +**Interfaces:** +- `migratePackages(repo, account): Promise<boolean>`: + - true if it migrated; + - a no-op when `/packages` is bound or there's no `moduleSettingsUrl`; + - names come from `package.json` `name` (read with `resolvePath(repo, pkg, ["package.json"])`), then the doc's `title`, then its documentId. +- `createDefaultAccount`: `doc.namespace = { "/": [{ url: rootFolder.url }], "/packages": [{ url: packagesFolder.url }] }`. The packages folder is titled "Packages". `moduleSettingsUrl` is still created, for the Packages UI. + +- [ ] **Step 1: Failing tests:** + - migration links each automerge entry by its `package.json` name and binds `/packages`; + - a second run is a no-op; + - an http entry is left to module settings; + - an account already bound isn't touched; + - `createDefaultAccount` binds both mounts. +- [ ] **Step 2: Run, expect FAIL.** **Step 3: Implement**, then wire in `index.ts`: + - after `registerAccount`, `await migratePackages`; + - `watchPackagesFolder` feeds `moduleWatcher.addPackagesFolder("packages", …)`; + - the tracker passed to the ModuleWatcher is `trackPackagesFolder(acctHash, () => latestEntries, trackPackagesViaServiceWorker(…))`; + - `acctHash` comes from `registerAccount`'s return. The watcher is built before the account resolves, so the tracker takes promises/getters. +- [ ] **Step 4: Run the patchwork suite and typecheck, expect PASS.** +- [ ] **Step 5: Commit** `patchwork: /packages on new accounts, migration, and wiring` + +### Task 6: Changeset + +- [ ] **One changeset** for the filesystem, bootloader, plugins and patchwork packages, saying: + - `/packages` is package management; + - packages are imported through checked `/@/` URLs; + - module settings is deprecated but still read; + - what moved: `moduleSettingsUrl` is `@deprecated`. +- [ ] Commit. diff --git a/docs/superpowers/plans/2026-10-28-paths-sidebar.md b/docs/superpowers/plans/2026-10-28-paths-sidebar.md new file mode 100644 index 000000000..b61d1da42 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-paths-sidebar.md @@ -0,0 +1,64 @@ +# Paths in the sidebar: packages as a real folder, status, open at a path — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** In the shell, `/packages` shows up as a real `packages` folder in the sidebar tree. Each package row shows whether it loaded. Opening anything from the sidebar opens it at its live path: `&path=` is set, and the tool gets `<patchwork-view path>`. The account bar's Packages link goes once `/packages` is bound. + +**Architecture:** This spans two repos. +- **system** (`paths-design`): + - the ModuleWatcher reports each package's status; + - accounts get a root-folder link to their `/packages` folder. +- **base** (`paths`), each tool standalone, with no build-time links between them: + - `providers`: `selected-view` carries `path`; + - `threepane`: passes it to the main view, and hides the Packages link; + - `sideboard`: rows open with their live path and show package status, and the separate Packages section is removed. + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 3 (*Opening at a path*: "the sidebar dispatches `{ url, path }` on click, using the names it walked") and section 4. + +## Decisions + +1. **The `packages` folder is real.** It's a link named `packages` in the root folder, to the first folder bound at `/packages`. The mount shadows a same-named entry in `/`, so `<acct>/packages/x` means the same thing either way. It's added only if the root folder has no entry by that name. Removing or renaming the link changes only the view. +2. **Status belongs to the document, not the folder.** + - `ModuleWatcher.packageStatus(urlOrDocumentId)` returns `{ state: "loading" | "loaded" | "failed", error?, plugins?: string[] }`, or undefined for anything that isn't a package. + - The watcher dispatches a `"status"` event when one changes. It's an `EventTarget`. + - Rows read it late-bound through `globalThis.patchwork?.packages`, and look as they do today when it's missing. +3. **A row's live path is built from the names walked from the root folder.** + - Each `DocumentList` passes `names` down. + - The acctHash is idhash(account documentId), copied into sideboard (SHA-256, first 12 bytes, base58, about 15 lines), since tools can't import across repos at build time. + - The account comes from `globalThis.patchwork?.account`. A panel whose folder isn't that account's `rootFolderUrl` opens without a path. +4. **`selected-view` carries `path`.** threepane passes it to `<patchwork-view path>` on the main view. On boot, the provider reads `path=` raw from the hash, since `URLSearchParams` decodes `%2F`. + +## Tasks + +### system + +- [ ] **S1: Package status on the ModuleWatcher.** Test (`module-watcher.test.ts`): + - a package goes `loading` → `loaded` with its plugin ids; + - a failing one goes to `failed` with the error; + - `"status"` fires; + - an http module has no status. + + Implement: a `#status` map keyed by documentId, updated in `announce`. `importPackageSafe` returns the error instead of swallowing it. +- [ ] **S2: The root-folder `packages` link.** Test (`migrate-packages.test.ts`): + - `createDefaultAccount` adds `{ name: "packages", type: "folder", url: <packages folder> }` to the root folder; + - migration adds it when there's no `packages` entry, and leaves an existing one alone. +- [ ] **S3: Changeset**, then build, test (by exit code), and commit. + +### base (`paths` branch) + +- [ ] **B1: providers.** `SelectedView.path`, set from `detail.path` and from the raw `path=` in the hash on boot. Test it. +- [ ] **B2: threepane.** + - `MainDocumentView` sets `path` on the main `<patchwork-view>`. + - The account bar's Packages button shows only when `/packages` isn't bound; the namespace is read from the account doc the frame already holds. + - Test the bar. +- [ ] **B3: sideboard paths.** + - `DocumentList` threads `names`. + - Rows and folders open with `path` when the panel is the account's root. + - `idhash` gets a known-vector test. + - Test that a nested row's open event carries `<acctHash>/a/b`. +- [ ] **B4: sideboard status.** + - A row whose document has a package status shows a dot (`data-package-state`) with a tooltip. + - It re-renders on `"status"`. + - Test it with a fake `globalThis.patchwork.packages`. +- [ ] **B5: Remove the Packages section** from `sideboard.tsx`, and `account-bar.tsx`'s use of it. threepane inlines its own bar, and `chee/account-bar` stays as it was. Keep the panel's menu-scoping fix, which still matters for two list widgets. +- [ ] **B6: Build, test and lint** each touched tool, then commit and push `paths`. diff --git a/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md b/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md new file mode 100644 index 000000000..872773289 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-packages-tool-installs-into-packages.md @@ -0,0 +1,97 @@ +# The Packages tool installs into /packages — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** On a host that loads `/packages`, the Packages tool installs a package by linking it into the account's own packages folder, and uninstalls one by removing that link. Module settings stops growing. Once it holds nothing `/packages` doesn't cover, the Packages button retires itself; the rule for that is already built. + +**Architecture:** All in base, in `packages/` (the tool). The tool keeps its handle, the module-settings doc. It finds the account through `globalThis.patchwork.account`, as the sidebar does, and the repo through its element. It writes to the folder bound first at `/packages`. The host's ModuleWatcher already loads, unloads and dedupes whatever that folder lists, so nothing changes in system except the spec. + +**Tech stack:** Solid, vitest with happy-dom, base's standalone-package rules (no build-time imports between tools; `@inkandswitch/patchwork-*` are host externals, feature-detected). + +**Spec:** `docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md`, section 4 ("module settings still read; the Packages UI writes it" becomes "the Packages UI writes `/packages`"). + +## Decisions + +1. **Where an install goes.** + - An unpinned automerge URL goes into **your** packages folder: the first binding at `/packages`. + - Pinned URLs (with heads) and http URLs still go into module settings. A folder link floats to the latest version, and a folder can't hold an http module. + - On a host that doesn't load `/packages`, or an account without it bound, everything goes into module settings, as now. +2. **What a link looks like.** `{ name, type, url }`: + - `url` is the bare automerge URL. + - `type` is the package doc's `@patchwork.type`, or `folder`. + - `name` is its package.json `name`, else its title, else its document id. The same allocator as migration keeps names unique (`alpha`, `alpha (2)`, `alpha (3)`), since only the first of a name in a folder is reachable. +3. **"Installed" means in your packages folder, or listed in module settings.** + - A package linked from another `/packages` member (a team folder) is shown as installed, from that folder's title, with no Remove: removing the link there would uninstall it for everyone who shares the folder. + - Module-settings entries recorded in `movedToPackages` count as installed only if the folder still links them, matching what the host loads. +4. **Uninstall:** + - removes the package's link from your folder (every link to that document, in case of duplicates); + - removes an unmoved, unpinned module-settings entry for the same document; + - pinned and http entries are removed from module settings as now; + - `deactivateModule` still runs, so the plugins go away immediately. +5. **No move button for leftovers.** Bare entries an older host adds to module settings later still load, through module settings, and stay removable there. Moving them over can come later if it's wanted. + +## Review focus + +1. **Two tabs installing the same package at once.** Both push a link; the second is a duplicate, and only the first of a name is reachable. The install checks for the document inside the `change` callback, so concurrent installs in one tab can't double up. Across devices it's a merge; the host dedupes by document. Test: installing an already-linked document is a no-op. +2. **A package doc that can't be read yet** (unsynced). It's linked under its document id rather than failing the install. Test. +3. **Your folder not loaded yet when the tool opens.** The install waits for it rather than falling back to module settings, which would put it in the wrong place. Test with a slow `find`. +4. **An older host.** No `watchPackagesFolder` on the host's filesystem: every install goes to module settings. Test. +5. **A link from someone else's folder.** It's never removed by Uninstall. Test. + +--- + +## Tasks + +### Task 1: `packages/src/packages-folder.ts` — where installs go, and the folder edits + +**Files:** create `packages/src/packages-folder.ts` and `packages/src/packages-folder.test.ts`. + +**Produces:** +```ts +export type AccountLike = { doc(): { namespace?: Record<string, { url?: unknown }[]> } | undefined }; +/** Your packages folder (the first /packages binding), on a host that loads /packages; else undefined. */ +export function installFolderUrl(account: AccountLike | undefined, hostLoadsPackages: boolean): AutomergeUrl | undefined; +/** Whether `url` goes into the packages folder: an unpinned automerge url. */ +export function installsIntoFolder(url: string): boolean; +/** A name not in `taken`: `name`, `name (2)`, … */ +export function uniqueName(name: string, taken: Iterable<string>): string; +/** Link `url` into the folder under a unique name; a no-op if it already links that document. */ +export function linkPackage(folder: DocHandle<FolderDoc>, link: { url: AutomergeUrl; name: string; type: string }): void; +/** Remove every link to `documentId` from the folder. True if any went. */ +export function unlinkPackage(folder: DocHandle<FolderDoc>, documentId: string): boolean; +``` + +- [ ] Tests first, covering: + - `installFolderUrl`: the first binding; undefined without `/packages` or on an older host; + - `installsIntoFolder`: true for a bare automerge url; false with heads, and false for http; + - `uniqueName("alpha", ["alpha", "alpha (2)"])` → `"alpha (3)"`; + - `linkPackage`: adds `{name, type, url}`; a second call for the same document adds nothing; + - `unlinkPackage`: removes duplicates too, and returns false when there's nothing to remove. +- [ ] Run `npx vitest run src/packages-folder.test.ts` in `packages/`. It should fail first, then pass once implemented. +- [ ] Commit. + +### Task 2: the tool installs and uninstalls through it + +**Files:** modify `packages/src/packages.tsx` (install, uninstall), `packages/src/use-package-data.ts` (installed keys, and where each package is from), `packages/src/origin.ts` (an origin hint for "in your packages folder" vs "from <folder>"); tests in `packages/src/packages.test.tsx`. + +**Consumes:** Task 1. + +- [ ] Tests first, with a fake account and fake folder handles on a fake repo: + - Installing a bare automerge URL links it into your folder, named from package.json, and leaves module settings alone. + - Installing a pinned or http URL writes module settings. + - On an older host (the host check returns false), everything goes to module settings. + - Uninstalling a package in your folder removes its link and calls `deactivateModule`. + - A package linked only from a second binding shows as installed, with no Remove button. + - A package whose doc can't be read is linked under its document id. + - Installing while your folder is still loading waits for it. +- [ ] Implement: + - `use-package-data` reads the account's `/packages` members (repo `find`, plus their `change` events) and merges their document ids into `installedKeys`, tagging each with its member. + - `install`/`uninstall` follow the decisions above. + - The host check is `typeof filesystem.watchPackagesFolder === "function"`. +- [ ] Run every `packages/` test, `tsc -p` and the package's build. They should pass, with no new tsc errors. +- [ ] Commit. + +### Task 3: spec and push + +- [ ] System spec section 4: the Packages UI installs into `/packages`; module settings keeps pinned and http entries and what older hosts add. Commit to `paths-design`. +- [ ] Push base `paths`. diff --git a/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md new file mode 100644 index 000000000..389974387 --- /dev/null +++ b/docs/superpowers/specs/2026-10-28-paths-namespace-content-cache-design.md @@ -0,0 +1,622 @@ +# Paths, namespace, and a content-addressed package cache + +Status: design, approved in conversation 2026-10-28. Revised the same day: +pinned URLs now name a hash of their folder's file list, so a pinned URL is +checked against itself and nothing else (sections 1 and 5). Phase 1 is +implemented. Phase 2 was implemented against the earlier section 5 (an index +in service worker memory, filled by the automerge worker over an +authenticated channel) and gets reworked to this one. + +## Goals + +- Clicking an item in the sidebar sets `&path=` in the hash to a readable path + rooted at the account, and a tool can be opened at a path. +- Paths are real: the service worker serves them, packages are imported + through them, and the filesystem is a namespace rather than only the + `.docs` walk of the root folder. +- Package management is a folder: whatever is bound at `/packages`. +- URLs don't carry raw document ids unless a portable share link is wanted, + and neither does anything in the content store. +- The package cache can't be poisoned by a crafted link, by code running on + the origin writing to CacheStorage, or by a stale name resolution. + +## Non-goals + +- Cache isolation between two accounts in the same browser profile. +- Running package code on a separate origin. This design is meant not to + preclude it (the store and file lists work the same there), but doesn't do it. +- Mount points below the top level of the namespace. +- Garbage collection of the content store. + +## Threat model + +| Attacker | In scope | What stops it | +|---|---|---| +| A crafted link makes the service worker cache content under a key your own imports later hit | yes | Content is stored only under its own hash, after hashing it; a crafted link can only name other content | +| Code on the origin (a package or tool) writes to CacheStorage | yes | Every store read is hashed and compared with a hash that comes from the URL, directly or through a file list that is itself checked against the URL | +| A name path resolved once keeps serving old content | yes | Names are never stored; a pinned URL carries the heads of each folder and the hash of its file list, so it can't change meaning | +| Another account on the same origin | no | — | + +Everything on the origin shares one realm boundary, so this raises the bar +rather than closing it. What stays open is listed under +[Remaining gaps](#remaining-gaps). + +## 1. Path grammar + +### Identifiers + +- **idhash** of a document: the first 12 bytes of SHA-256 over the UTF-8 of + its `documentId`, in base58. It only distinguishes a doc from its siblings + and keeps the id out of URLs. It is not a secret and not a security + boundary. +- **acctHash**: the idhash of the account document. +- **heads**: the doc's heads in base58, sorted, joined with `.`. Sorting + makes one set of heads one URL. Base58 has no `.`, so it's unambiguous. +- **listHash**: the SHA-256, in base64url, of a container's **file list** + (section 5): every file the URL may reach below that container, with the + hash of its bytes and the heads it was read at. base64url has no `@`, `.` + or `~`. + +### Live path + +What the address bar holds and what people read. Names resolve at the +current heads of everything along the way. + +``` +&path=<acctHash>/Projects/foo/index.js +``` + +### Pinned path + +What imports and share links use. Every document the walk enters is pinned: + +1. A step that follows a link **by name** is written `<idhash>@<heads>` in + place of the name. +2. A step that follows a link **without consuming a name** (a union member, + a link that points at a link) adds its own `<idhash>@<heads>` segment. +3. A step **inside** a document (a directory key, a mount name in the + account doc) stays a plain name, since the enclosing doc's heads already + pin it. +4. The names after the last container (a folder, a directory, …) lead to a + **file**: a value inside the container, or a document reached by a link. + They stay names, so a module's relative imports (`./util.js`, `../x.js`) + resolve against them. A link without heads doesn't pin the file it leads + to, so the container's step carries `~<listHash>`, and the list records + each file's hash and heads. Links don't need to carry heads. + +``` +live: <acctHash>/Projects/foo/dist/index.js +pinned: <acctHash>@h/<rootHash>@h/<projectsHash>@h/<fooHash>@h~<listHash>/dist/index.js + +live: <acctHash>/packages/foo/dist/index.js +pinned: <acctHash>@h/packages/<myPackagesHash>@h/<fooHash>@h~<listHash>/dist/index.js +``` + +Which containers carry a list: + +- `pinPath` puts one on the last container. +- Pinning a package (phase 4) puts one on every container from the + package's own step down, so `../` between a folder package's subfolders + stays pinned. +- A file reached under a container step with no list is resolved live, + served `no-store`, and never stored or checked. + +When resolving, a segment is a doc step if it parses as +`<base58>@<heads>` or `<base58>@<heads>~<listHash>` and a link reachable +from the current node hashes to that idhash; otherwise it's a name. + +The first segment of a pinned path is either an acctHash the resolver knows +locally, or `automerge:<documentId>@<heads>` for a portable share link (safe +to hand out once the doc is on keyhive, since the id no longer grants +access). The `automerge:` prefix is what tells the two apart. + +A pinned path never changes meaning, so whoever builds an import URL builds +the whole chain, and its file lists, first. + +No raw document id appears in a live or pinned path: every step is an +idhash. The only exception is a share link's root, which carries one on +purpose. Nothing in a module graph goes through a redirect: the +ES module map keys an entry by the URL requested, so a redirect is memoized +for the life of the realm (the same reason `importModule` retries with +`?retry=1`). + +## 2. Path providers and the namespace + +### Providers + +A provider matches a document and lists its children as +`{ name, link | value }`. Walking a path asks the current doc's provider for +the next child. This generalizes `FolderStrategy` in +`core/filesystem/src/resolve.ts`. + +Built-in providers, checked in order, first match wins: + +- **account**: a doc whose `@patchwork.type` is `account`; its children are + the namespace (below). +- **directory**: keys, longest prefix first, as `walkDirectoryDoc` does now. +- **folder**: `.docs` by `name`. +- **branches**: each branch by name. A path that ends at a branches doc + stands for its `default` branch. This replaces the special case in + `core/bootloader/src/resolver.ts`. + +These built-ins run in every resolver, the automerge worker's included. +Nothing trusts a provider's answers, since every byte is checked against the +URL. Packages can supply providers too (section 6), but they run only in +tabs, after the built-ins, and only for documents a walk goes through. + +### Namespace + +The account doc gets an ordered mount table: + +```ts +namespace?: { + [mountPoint: string]: Array<{ url: AutomergeUrl }>; +}; +// e.g. +// "/": [{ url: rootFolderUrl }] +// "/packages": [{ url: myPackagesUrl }, { url: teamPackagesUrl }] +``` + +Union semantics, after plan9: + +- Lookup tries each binding of a mount in order; the first that has the name + wins. +- Listing concatenates every binding's children. +- Array position takes the place of `bind -b` / `bind -a`. +- A mount shadows an entry of the same name in `/`'s folder. +- Mount points are top-level names only, and may not contain `@`. +- Only automerge documents can be bound. + +If `namespace` is missing, `/` falls back to `rootFolderUrl`, so older +accounts and older readers keep working. + +The automerge worker needs to know acctHash → account url to resolve a path's +root. Tabs register it with the worker at boot. The worker computes the +acctHash from the url itself, so no tab can register an account under +another's hash. + +## 3. Service worker URLs and routing + +### URLs + +All new forms live under a reserved prefix, so no site file can be mistaken +for a path: + +``` +/@/<live path> e.g. /@/<acctHash>/Projects/foo/x.js +/@/<pinned path> e.g. /@/<acctHash>@h/<rootHash>@h/.../<fooHash>@h~<listHash>/x.js +``` + +- A **live** URL answers with a 307 to its pinned form, computed by a + resolver. Fine for `fetch()`; never use one for `import()`. The pinned + form's list covers only the file asked for, so a redirect costs one file + read, not a folder's worth (section 5, *File lists*). +- A **pinned** URL is served from the content store, checked against its + own list (section 5). +- Existing `/automerge%3A…/` URLs keep working as they do today, keyed by URL + and unchecked. They are deprecated. + +### Hash routing + +- `path` joins `HASH_KEY_ORDER` in `core/patchwork/src/router.ts`. Each + segment is `encodeURIComponent`-encoded and the slashes stay literal. +- `doc=` says what is open; `path=` says where it was opened from. +- A hash with `path=` and no `doc=` resolves the path and opens the result. +- If `doc=` and `path=` disagree (something moved or was renamed), `doc=` + wins and `path=` is dropped with a console warning. + +### Opening at a path + +Builds on the `open-document-path` branch (48f4ab98): + +- `patchwork:open-document`'s `path` field holds the **live path** (it was a + service-worker path on that branch), and the router writes it to `&path=`. + This reverses that branch's choice to keep it out of the hash. +- `<patchwork-view path>` hands it to the tool without remounting, as on that + branch. +- The sidebar (an installed tool, not in this repo) dispatches + `{ url, path }` on click, using the names it walked. + +### Helpers in `@inkandswitch/patchwork-filesystem` + +- `idhash(documentId)` +- `livePath(acctHash, names)` +- `parsePath(path)`: live or pinned, into segments +- `resolveLivePath(repo, path)` → handle +- `pinPath(repo, path)` → pinned service-worker URL, for fetching siblings or + importing, and the file list it names (section 5) + +## 4. Packages via `/packages` + +- Package management is whatever is bound at `/packages`. The folder can live + anywhere in the tree, or nowhere. +- `ModuleWatcher` gets a third source kind, `folder` (alongside automerge + module-settings docs and HTTP manifests), pointed at the `/packages` union: + - The module list is the union listing, deduplicated by document id. Two + packages with the same name both load; the name only decides which one a + path lookup finds. + - Packages are tracked by document id, so a rename doesn't reload anything. + - When a package's own heads move, the watcher looks it up again, gets a + new pinned URL, and imports that. A push that changes only files behind + links without heads must still touch the package doc; pushwork already + bumps `lastSyncAt` for that. + - The lookup is a live URL through the union member that holds the + package: `/@/<acctHash>/packages/<memberIdhash>@<heads>/<name>`. That + finds a shadowed package too. A resolver answers with a 307 to the + package's pinned URL. The watcher imports the package as an http + package from that URL: `package.json`, then its entry, both checked + against the list. + - A package's list is **deep**: a container with a `package.json` file + lists every file under it, through nested folders. A folder package's + `./sub/x.js` is reached by name, not by a doc step, so it has to be in + the package's own list. Other containers list only their own files. + - A live → pinned redirect to a **container** carries that container's + whole list (deep, for a package). A redirect to a file carries only that + file's. A file in a nested folder is read through a link without heads, + so a resolver reads at a list's `heads` whether or not the path is + canonical. Other packages keep their existing URLs: a + pinned URL stays valid forever, so a change elsewhere in the chain + doesn't reload them. + - `PackageTracker.findLatest` keeps asking the service worker, in the new + grammar. +- The site's `system` HTTP manifest stays a separate source. +- So does the module-settings doc, for now. The watcher loads both it and + `/packages`, deduplicated by document id; a package in both is imported + through `/packages`. Entries migration moved (`movedToPackages`) load only + through `/packages`, so removing a package there removes it. +- The Packages tool (base `packages/`), on a host that loads `/packages` and + viewing the account's own module-settings doc, installs an automerge + package by linking it into your packages folder (the first `/packages` + binding) and uninstalls by removing the link. It tells which package a + plugin came from with `ModuleWatcher.packageFor`, since a `/packages` + package's plugins are registered under its pinned `/@/` url. Pinned and + http modules still go into module settings, since a folder link follows + the latest version and a folder can't hold an http module. A package + linked from a folder you share (even if yours links it too) can't be + uninstalled from the tool: removing it there would uninstall it for + everyone who shares it, and leaving it there keeps it loaded. +- A link under `/packages` that isn't a package (no `package.json`) is + warned about and skipped. + +### Accounts + +- `createDefaultAccount` binds `/` to the root folder and `/packages` to a new + folder. +- Migration, once at boot: if the account has `moduleSettingsUrl` and no + `/packages` binding, create a folder, add a link for each automerge entry + (named by `package.json` `name`, falling back to the doc title), and bind + it at `/packages`. The module-settings doc keeps its entries, and records + which it moved in `movedToPackages`. +- Packages installed by an older host after the migration still land in the + module-settings doc, which this build keeps reading, but not in + `/packages`. Builds older than this one never see `/packages`. +- `AccountDoc.moduleSettingsUrl` is deprecated in favour of the `/packages` + binding. + +## 5. Content store and file lists + +### Store + +- A CacheStorage cache named `patchwork-content-<acctHash>`, separate from + the `patchwork` cache that holds site pages and assets. +- Key: `/.content/<sha256 base64url>`, an internal key that is never fetched. +- Value: the bytes only. The content type lives in the file list, so + identical bytes are stored once whatever their type. +- Reads and writes go through one small interface (`get(hash)`, + `put(hash, bytes)`), so entries can later be encrypted under an at-rest key. +- Anything on the origin can write it. That's fine: every read is hashed and + compared with a hash the URL vouches for, and a mismatch is deleted. +- `clearAccountStore(acctHash)` deletes the cache. There is no sign-out flow + today; a site calls this from its own. + +### File lists + +A container's file list says what each file under it is, at the moment it +was pinned. It is canonical JSON: keys sorted, no whitespace. + +```json +{"files":{"dist/index.js":{"heads":"a.b","sha256":"…","store":true,"type":"text/javascript"},"package.json":{…}},"v":1} +``` + +- **Keys** are tails: the names below the container, each + `encodeURIComponent`-encoded and joined with `/`. The container itself is + `""`. +- **Entries:** + - `sha256` is the hash of the bytes the file serves as. + - `type` is its content type. + - `heads` is the heads its document was read at. It's absent for a value + inside the container, which the container's own heads pin. + - `via`, for a file in a package's nested folder, is the heads of each + nested folder doc on the way to it, outermost first. Those folders are + reached by links without heads, so a walk back to the file reads each + at the heads it has from `via`: renaming the file, or a folder above + it, doesn't lose it. Absent everywhere else, so lists without nested + folders hash as they did before it. + - `store` says whether the bytes may be kept (see + [What gets stored](#what-gets-stored)). +- **No document ids, by design.** A resolver that needs a file's bytes walks + to it by name from the container at its pinned heads, through nested + folders at their `via` heads, and reads it at the recorded heads. Automerge keeps history, so an old version stays readable + wherever the document is. +- **The listHash is the SHA-256 of these bytes.** The list is content like + any other: stored under its hash if its container's content may be stored, + and otherwise held in service worker memory. +- **A list may cover only some of a container's files.** A package's list + covers all of them, so relative imports work. The list behind a live → + pinned redirect covers only the file asked for. A tail a list doesn't + cover answers 404. +- **Whoever pins builds the list** (`pinPath`): it loads each file once and + hashes it. A tab or the automerge worker, it doesn't matter which: a list + is checked against the URL, not trusted for who made it. + - The pinner writes the list into the store itself when it may be stored. + - Otherwise it hands the list to the service worker in a message, or with + the 307 it answers. + - The service worker hashes the list before keeping it in memory. + +### What gets stored + +- Content, file lists included, from unprotected documents + (`isUnprotectedDocId`). +- Content from keyhive-protected documents only when reached through + `/packages`. Package code needs to load fast at boot; user documents don't. +- Everything else is served after checking, never stored. + +### Request flow for a pinned URL + +1. **The list:** + - Read the last container's list by its listHash: from the store, or + from memory. + - Hash it. + - On a miss, or a mismatch (delete the store entry and warn), ask a + resolver to build that container's list at its pinned heads. Keep the + answer only if it hashes to the listHash. + - No list: fail as a network error. +2. **The entry:** look the tail up in the list; not there, 404. +3. **The bytes:** + - Read the store by the entry's `sha256` and hash the bytes. A match + serves them; a mismatch deletes the entry, warns, and goes on. + - Ask a resolver for the file at its pinned path and the entry's `heads`, + and hash its bytes. + - A match serves them, and stores them if the entry allows it. A + mismatch fails as a network error and stores nothing. + +A container step with no list leads to files resolved live: served +`no-store` from a resolver, never stored, not checked. + +Hashing is `crypto.subtle` SHA-256 on every hit, since a store entry can +change between two reads. `sites/bench` gets a case measuring it. + +Cold start needs no document: a package's list and its files are both in the +store, and both are checked against the URL the watcher pinned. + +### Who is trusted for what + +- **Nothing between the URL and the bytes is trusted.** Lists, store + entries, and resolver answers from a tab or the automerge worker are all + checked against hashes that start at the URL. A forged answer can fail a + request, but it can't change what's served. +- **Whoever builds a URL chooses the version it names.** For imports that's + the tab's own ModuleWatcher. For a live → pinned redirect it's whichever + resolver answered (see [Remaining gaps](#remaining-gaps)). +- **No authenticated channel is needed.** The earlier design filled an index + from the automerge worker over `fetch()` requests checked by `clientId`. + The spike behind that is kept below, in case a later design needs to know + who sent something. +- **Legacy `/automerge%3A…` URLs** stay keyed by URL and unchecked. Their + replies still ride `HANDOFF_CHANNEL`, which any context can answer first. + +## 6. Package-supplied path providers + +Approved in conversation 2026-10-28. Implemented in `patchwork-filesystem` +(`sub` children, the editing methods, `WalkOptions.providers`), +`patchwork-plugins` (`pathProviderFor`, `packagePathProviders`, +`pathProviderTypes`), the tab resolver and the router, and the base +sidebar. The built-in folder provider doesn't offer editing yet. In the +sidebar, a document dropped among an opened provider's children is added +(linked) and left where it was. The row itself still takes drops as a folder +entry. Files, new documents and sub-documents can't be dropped among a +provider's children. + +The sidebar can open up a doc that isn't a folder when a built-in provider +walks it: a directory doc, a branches doc, the account. This section proposes +letting a datatype say what its children are too: a tldraw doc's embedded +docs, say, or the docs a markdown doc links to. It lifts the non-goal +"Path providers supplied by packages". + +### The plugin + +A `patchwork:path-provider` plugin is a `PathProvider`: `matches`, `lookup` +and `list`, exported from `patchwork-filesystem` already. A datatype package +registers one beside its datatype. + +- **Built-ins come first and can't be overridden.** Account, directory, folder + and branches are checked first; package providers after, in registration + order. A package can't take over how folders resolve. + - One exception: the folder built-in matches anything with a `docs` array, + so a typed document that keeps one (a tldraw canvas holds its scripts + there) would never reach its own type's provider. Such a document is + walked by a provider declared for its type, if there is one, and as a + folder otherwise. An untyped or `folder`-typed folder is always the + built-in's. A provider for such a type should list the `docs` entries as + the folder provider does, so paths into them pin the same either way. +- **It's a pure function of the document.** `list` and `lookup` read only the + node they're given, at the heads they're given. That's what lets a file + list built through a provider be rebuilt to the same bytes. +- **It says which datatypes it walks** with `supportedDatatypes`, as a tool + does. That's how the sidebar knows a row of that type can open up, from its + link's `type`, without loading the doc. + +### Where it runs: tabs only + +The automerge worker never loads package code, so it never runs package +providers. A tab does: its sidebar, its walks, and its resolver. + +- **The service worker asks tabs first.** Live → pinned redirects and reads at + heads through a package provider are answered by a tab. A page made the + request, so there is one. +- **`/packages` is the worker's.** Tabs decline it (section 4), so package + providers don't apply under `/packages`. Packages are folders and + directories anyway. +- **A pinned URL needs a tab to rebuild its list.** If a pinned URL goes + through a package provider and its list is lost, only a tab with that + provider loaded can rebuild it. Without one, it fails as a network error. + +### Trust + +Nothing between a URL and its bytes trusts a provider (section 5). A file list +built through one is checked against the URL; bytes are checked against the +list. + +What a provider *can* do is the same as any package code in a tab: +- name the wrong children in the sidebar; +- resolve a live path to a document it shouldn't; +- by being pinned, choose the version a URL names. + +That's the "whoever builds a URL chooses its version" gap, applied to +providers. It adds no access a package doesn't already have. + +### Versions + +A new version of the providing package can change what a live path resolves +to. Pinned paths don't change. A package's provider applies only to a document a walk +goes on *through*. Whether a document a path ends at is a container or a +file, and what a file list counts as a file, are the built-ins' call. So a +path pins to the same URL with or without the provider, and a provider +installed later doesn't change a URL pinned before it. A list rebuilt through a changed provider won't +match its hash, so the URL fails rather than serving something else. + +### What a provider offers + +The sidebar shows what a provider offers and no more. Two things are optional. + +**Editing.** A provider may implement any of these: +- `add(handle, node, child, index?)`; +- `rename(handle, node, name, to)`; +- `remove(handle, node, name)`. + +Each takes the doc's handle and changes the doc. The sidebar turns on Rename, +Remove, drops and new-doc only where the provider offers them. A child of a +provider that offers none is read-only, as directory and branches children are +now. + +The built-in folder provider would offer all three. That would make the +sidebar's folder editing one case of this rather than its own code path, +though there's no need to move it yet. + +**Values as sub-documents.** A child that is a value inside the doc, not a +link, can be offered as a sub-document: +- `{ name, sub: [...segments] }` is opened as automerge-repo's sub-handle, + `handle.sub(...segments)`, at the URL `automerge:<docId>/<path>`; +- the tool gets that handle, so it edits just that part of the doc; +- the live path names it as an in-doc step (rule 3): a plain name after the + doc's own step, pinned by the doc's heads. + +Values a provider doesn't offer as sub-documents stay hidden, as now. + +## Remaining gaps + +These replace the deleted `package-load-gate.md` note. + +- Offline, `servePassthrough` falls back to the URL-keyed `patchwork` cache, + which any code on the origin can write. Site assets, the automerge worker's + script included, come from there. +- Packages can write the automerge worker's IndexedDB/OPFS storage. + Automerge verifies change hashes when it loads a document, so tampered + changes don't load as the heads they claim. That check is planned to become + optional; when it does, this gap reopens unless patchwork keeps it on. +- Whoever builds a pinned URL chooses its version. A live → pinned redirect + answered by a tab can name a wrong version, and so can a package that + builds import URLs for another. It can't poison the store, but it decides + which version gets loaded. +- A pinned URL can't be served if its file list is gone (evicted, cleared, + or held only in a service worker that stopped) and no resolver can + rebuild it. A rebuild reads each file at its current heads, so it matches + only if nothing changed since. It tries the one-file list a redirect hands + out first, then the whole container's, so a reloaded page rebuilds as + cheaply as it was pinned. The ModuleWatcher pins again on boot, so + imports recover; a hand-made share link doesn't. +- Legacy `/automerge%3A…` URLs stay keyed by URL and unchecked. +- Protected package code is stored in plaintext. Today keyhive's secrets sit + in the same storage as its ciphertext, so this exposes nothing new; + revisit when keyhive protects its keys at rest. +- Nothing cleans up the content store. Deduplication slows growth, and the + existing quota warning still fires. + +## Spikes before planning + +- (For the earlier, index-based section 5. Not needed now, but kept.) Can + the service worker `postMessage` to a SharedWorker client, and can that + worker reply with the service worker able to check its `Client.url`, in + Chromium, Firefox and WebKit? Fallback: the worker posts answers to the + service worker as `fetch()` requests, whose `clientId` the service worker + can look up. Spike: `packages/e2e/spikes/sw-sharedworker`, run in CI by + `.github/workflows/spike-sw-sharedworker.yml` (run 40640485949: Chrome 147, + Firefox 148, Safari 26.4). + + **Answered: only the fetch path works everywhere.** + + | | Chromium | Firefox | WebKit | + |---|---|---|---| + | Worker is a controlled `sharedworker` client, `url` includes the query | yes | yes | yes | + | `navigator.serviceWorker` in the worker | no | yes | yes | + | `navigator.serviceWorker.controller` in the worker | no | null | yes | + | Service worker `client.postMessage` reaches the worker | no | yes | yes | + | Worker replies with `controller.postMessage`, `event.source` checkable | no | no | yes | + | Worker replies on a port sent with the ping (`source` is null) | no | yes | yes | + | Worker `fetch()`, `clients.get(clientId)` gives `sharedworker` and its url | yes | yes | yes | + | Dedicated `Worker` from the same URL: same `url`, `type: "worker"` | yes | yes | yes | + + Chromium doesn't expose `ServiceWorkerContainer` in workers, so the service + worker can't message the automerge worker at all there. In Firefox the + worker's first `fetch()`, made while its module was still evaluating, went + to the network instead of the service worker. Its later fetches were + intercepted. So answers should be retried, or the worker should hold them + until it has seen one of its own fetches answered by the service worker. +- Does automerge verify change hashes when loading a document from storage? + **Answered: yes**, though it's planned to become optional (see + [Remaining gaps](#remaining-gaps)). + +## Testing + +- Unit: + - Path grammar: live and pinned round trips, heads sorting, segments that + look like doc steps but aren't. + - Providers: account namespace (union order, shadowing, fallback to + `rootFolderUrl`), directory, folder, branches. + - `pinPath` against a fixture folder that is renamed between versions. + - Router: `path=` parsing and serializing; `path=` alone; `doc=`/`path=` + disagreement. + - `ModuleWatcher` `folder` source: add, remove, rename without reload, only + the changed package reloading, union deduplication. + - Migration from a module-settings doc. + - File lists: canonical bytes (key order, no whitespace), tails for folder + and directory fixtures, no document ids, and a list that agrees with + `readPath` for every file it covers. + - The request flow with a fake `Cache` and an injected hash: + - a forged list or store entry is rejected; + - a resolver's mismatching bytes fail as a network error and are never + stored; + - a file read at its recorded heads after it has changed still serves + the old bytes; + - protected non-package content is never stored; + - a tail the list doesn't cover answers 404. + - The service worker itself, in node, against a fake global scope + (`core/bootloader/test/service-worker-harness.ts`). +- E2E, run in CI: + - Open a document with `&path=` alone. + - A page `cache.put`s forged bytes under a real content key, reloads, and + gets the real module. + +## Implementation phases + +Each phase can ship on its own and gets its own plan: + +1. **Grammar, providers, namespace** in `patchwork-filesystem`, plus the + account `namespace` field and its fallback. No behaviour change yet. +2. **Service worker:** `/@/` URLs, the content store, resolvers no longer + writing the cache. Done against the earlier index-based section 5; the + rework replaces the index, the manifest handler and the authenticated + channel with file lists. +3. **Routing:** `&path=`, opening at a path, rebased on `open-document-path`. +4. **Packages:** the `folder` source, `/packages`, migration, deprecating + `moduleSettingsUrl`. diff --git a/packages/e2e/spikes/sw-sharedworker/.gitignore b/packages/e2e/spikes/sw-sharedworker/.gitignore new file mode 100644 index 000000000..fbca22537 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs b/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs new file mode 100644 index 000000000..070c5efa8 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/playwright.config.mjs @@ -0,0 +1,23 @@ +import { defineConfig, devices } from "@playwright/test"; + +const PORT = 5199; + +export default defineConfig({ + testDir: ".", + testMatch: "*.spec.mjs", + timeout: 60_000, + retries: 0, + reporter: "list", + use: { baseURL: `http://localhost:${PORT}` }, + webServer: { + command: `node serve.mjs`, + env: { PORT: String(PORT) }, + url: `http://localhost:${PORT}/`, + reuseExistingServer: false, + }, + projects: [ + { name: "chromium", use: devices["Desktop Chrome"] }, + { name: "firefox", use: devices["Desktop Firefox"] }, + { name: "webkit", use: devices["Desktop Safari"] }, + ], +}); diff --git a/packages/e2e/spikes/sw-sharedworker/public/index.html b/packages/e2e/spikes/sw-sharedworker/public/index.html new file mode 100644 index 000000000..e8722eb8e --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/index.html @@ -0,0 +1,64 @@ +<!doctype html> +<meta charset="utf-8" /> +<title>spike: service worker ↔ shared worker +
running…
+ diff --git a/packages/e2e/spikes/sw-sharedworker/public/shared.js b/packages/e2e/spikes/sw-sharedworker/public/shared.js new file mode 100644 index 000000000..9363d16a3 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/shared.js @@ -0,0 +1,64 @@ +// Loaded twice by the page: as the module SharedWorker the spike is about, and +// as a dedicated Worker from the same URL, the impostor any page on the origin +// can start. The service worker should be able to tell them apart. + +const isShared = + typeof SharedWorkerGlobalScope !== "undefined" && + self instanceof SharedWorkerGlobalScope; +const kind = isShared ? "shared" : "dedicated"; +const sw = self.navigator?.serviceWorker; + +const env = { + kind, + hasNavigatorServiceWorker: !!sw, + hasController: !!sw?.controller, + pingsReceived: [], +}; + +function onPing(event, via) { + const data = event.data; + if (data?.type !== "ping") return; + env.pingsReceived.push(via); + sw?.controller?.postMessage({ type: "pong", nonce: data.nonce }); + event.ports?.[0]?.postMessage({ type: "pong", nonce: data.nonce }); + fetch(`/__spike/pong?nonce=${data.nonce}`).catch(() => {}); +} + +if (sw) { + sw.addEventListener("message", (event) => onPing(event, "navigator.serviceWorker")); + sw.startMessages?.(); +} +// Not expected to fire for a service worker's postMessage; recorded in case. +self.addEventListener("message", (event) => onPing(event, "self")); + +// Worker-initiated: hand the service worker a port to call us on. +const { port1, port2 } = new MessageChannel(); +port1.onmessage = (event) => { + if (event.data?.type === "ping-on-hello-port") { + port1.postMessage({ type: "pong-on-hello-port", nonce: event.data.nonce }); + } +}; +try { + sw?.controller?.postMessage({ type: "hello", kind }, [port2]); + env.helloPosted = !!sw?.controller; +} catch (error) { + env.helloPosted = `threw: ${error}`; +} + +const ready = (async () => { + env.probe = await fetch("/__spike/probe") + .then((response) => response.text()) + .catch((error) => `error: ${error}`); + env.fetchesReachServiceWorker = env.probe === "sw"; + await fetch(`/__spike/hello?kind=${kind}`).catch(() => {}); + return env; +})(); + +if (isShared) { + self.addEventListener("connect", (event) => { + const port = event.ports[0]; + ready.then((env) => port.postMessage({ type: "env", env })); + }); +} else { + ready.then((env) => self.postMessage({ type: "env", env })); +} diff --git a/packages/e2e/spikes/sw-sharedworker/public/sw.js b/packages/e2e/spikes/sw-sharedworker/public/sw.js new file mode 100644 index 000000000..7d645a3e7 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/public/sw.js @@ -0,0 +1,145 @@ +// Spike: can the service worker talk to a SharedWorker client and tell, from +// what the platform reports, that an answer came from that worker? +// +// Three reply paths are measured for a service-worker-initiated ping: +// controller: the worker posts to navigator.serviceWorker.controller, and +// we read event.source (a Client: url, type) +// port: the worker answers on a MessagePort we sent with the ping +// fetch: the worker fetch()es /__spike/pong, and we look its clientId up +// and one worker-initiated path: +// hello: the worker posts a MessagePort to the controller at startup; +// we check event.source, keep the port, and ping over it. + +const WORKER_PATH = "/shared.js?v=spike"; + +self.addEventListener("install", () => self.skipWaiting()); +self.addEventListener("activate", (event) => + event.waitUntil(self.clients.claim()) +); + +const describe = (source) => { + if (!source) return null; + if (typeof Client !== "undefined" && source instanceof Client) { + return { kind: "Client", url: source.url, type: source.type, id: source.id }; + } + if (typeof MessagePort !== "undefined" && source instanceof MessagePort) { + return { kind: "MessagePort" }; + } + return { kind: Object.prototype.toString.call(source) }; +}; + +const replies = new Map(); // nonce -> { via -> observation } +const record = (nonce, via, observation) => { + if (!replies.has(nonce)) replies.set(nonce, {}); + replies.get(nonce)[via] = { at: Date.now(), ...observation }; +}; +const collect = (nonce, ms) => + new Promise((resolve) => setTimeout(() => resolve(replies.get(nonce) ?? {}), ms)); + +const hellos = []; // { source, port } +const helloFetches = []; +const forged = []; + +self.addEventListener("message", (event) => { + const data = event.data ?? {}; + switch (data.type) { + case "run": + event.waitUntil(run(event.ports[0])); + break; + case "hello": + hellos.push({ + kind: data.kind, + source: describe(event.source), + hasPort: !!event.ports[0], + port: event.ports[0], + }); + break; + case "pong": + record(data.nonce, "controller", { source: describe(event.source) }); + break; + case "forged-pong": + forged.push({ source: describe(event.source) }); + break; + } +}); + +self.addEventListener("fetch", (event) => { + const url = new URL(event.request.url); + if (!url.pathname.startsWith("/__spike/")) return; + event.respondWith( + (async () => { + const client = event.clientId + ? await self.clients.get(event.clientId) + : undefined; + const observation = { + clientId: event.clientId, + client: describe(client), + }; + if (url.pathname === "/__spike/pong") { + record(url.searchParams.get("nonce"), "fetch", observation); + } else if (url.pathname === "/__spike/hello") { + helloFetches.push({ kind: url.searchParams.get("kind"), ...observation }); + } + // The worker compares this with what the server says, to tell whether + // its fetches reach us at all. + return new Response("sw", { headers: { "cache-control": "no-store" } }); + })() + ); +}); + +async function pingOverHelloPort(hello) { + const nonce = crypto.randomUUID(); + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 3000); + hello.port.onmessage = (event) => { + if (event.data?.nonce !== nonce) return; + clearTimeout(timer); + resolve(true); + }; + hello.port.postMessage({ type: "ping-on-hello-port", nonce }); + }); +} + +async function run(reply) { + const results = { workerHref: new URL(WORKER_PATH, self.location).href }; + + const all = await self.clients.matchAll({ type: "all", includeUncontrolled: true }); + results.allClients = all.map(describe); + results.controlledSharedWorkers = ( + await self.clients.matchAll({ type: "sharedworker" }) + ).map(describe); + results.allSharedWorkers = ( + await self.clients.matchAll({ type: "sharedworker", includeUncontrolled: true }) + ).map(describe); + + const target = all.find((client) => client.type === "sharedworker"); + results.sharedWorkerFound = !!target; + results.sharedWorkerUrlMatches = target?.url === results.workerHref; + + if (target) { + const nonce = crypto.randomUUID(); + const { port1, port2 } = new MessageChannel(); + port1.onmessage = (event) => { + if (event.data?.nonce === nonce) { + record(nonce, "port", { source: describe(event.source) }); + } + }; + try { + target.postMessage({ type: "ping", nonce }, [port2]); + results.swToWorkerPostMessage = "sent"; + } catch (error) { + results.swToWorkerPostMessage = `threw: ${error}`; + } + results.swToWorkerReplies = await collect(nonce, 3000); + } + + results.hellos = hellos.map(({ port, ...rest }) => rest); + results.helloFetches = helloFetches; + const sharedHello = hellos.find((hello) => hello.source?.type === "sharedworker"); + results.helloPortRoundTrip = sharedHello?.port + ? await pingOverHelloPort(sharedHello) + : null; + results.forgedFromWindow = forged; + + reply.postMessage(results); +} diff --git a/packages/e2e/spikes/sw-sharedworker/serve.mjs b/packages/e2e/spikes/sw-sharedworker/serve.mjs new file mode 100644 index 000000000..9d5e8b685 --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/serve.mjs @@ -0,0 +1,31 @@ +// A static server for the spike. /__spike/* answers "server", so a worker +// can tell whether its fetches went through the service worker. +import { createServer } from "node:http"; +import { readFile } from "node:fs/promises"; +import { extname, join, normalize } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("./public/", import.meta.url)); +const port = Number(process.env.PORT ?? 5199); +const types = { ".html": "text/html", ".js": "text/javascript" }; + +createServer(async (request, response) => { + const { pathname } = new URL(request.url, "http://localhost"); + const headers = { "cache-control": "no-store" }; + if (pathname.startsWith("/__spike/")) { + response.writeHead(200, headers).end("server"); + return; + } + const file = normalize(join(root, pathname === "/" ? "index.html" : pathname)); + if (!file.startsWith(root)) { + response.writeHead(403).end(); + return; + } + try { + const body = await readFile(file); + headers["content-type"] = types[extname(file)] ?? "application/octet-stream"; + response.writeHead(200, headers).end(body); + } catch { + response.writeHead(404).end(); + } +}).listen(port, () => console.log(`spike on http://localhost:${port}`)); diff --git a/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs b/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs new file mode 100644 index 000000000..c782a173b --- /dev/null +++ b/packages/e2e/spikes/sw-sharedworker/spike.spec.mjs @@ -0,0 +1,25 @@ +import { test, expect } from "@playwright/test"; +import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; + +// Records what each browser does; it only fails if the page never reports. +test("service worker ↔ shared worker messaging", async ({ page, browserName }) => { + page.on("console", (message) => console.log(`[${browserName}] ${message.text()}`)); + await page.goto("/"); + const handle = await page.waitForFunction(() => window.__results, null, { + timeout: 45_000, + }); + const results = await handle.jsonValue(); + const json = JSON.stringify(results, null, 2); + console.log(json); + + const dir = new URL("./results/", import.meta.url); + mkdirSync(dir, { recursive: true }); + writeFileSync(new URL(`${browserName}.json`, dir), json); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `### ${browserName}\n\n\`\`\`json\n${json}\n\`\`\`\n` + ); + } + expect(results.error).toBeUndefined(); +}); diff --git a/packages/e2e/tests/content-store.spec.ts b/packages/e2e/tests/content-store.spec.ts new file mode 100644 index 000000000..a63f47bd6 --- /dev/null +++ b/packages/e2e/tests/content-store.spec.ts @@ -0,0 +1,100 @@ +import { expect, test } from "@playwright/test"; +import { waitForRepoReady } from "./helpers.js"; + +/** + * The service worker's content store for pinned `/@/` URLs. Anything on the + * origin can write CacheStorage, so the store is untrusted: every read is + * hashed and checked against the file list the pinned URL names by hash. + * This plays the page that tries to get around that. + */ + +declare global { + interface Window { + patchwork?: { + account: { + documentId: string; + doc(): { rootFolderUrl?: string }; + }; + }; + } +} + +test("a forged store entry is never served", async ({ page, browserName }) => { + test.skip( + browserName !== "chromium", + "needs the full-UI boot for an account, which is only reliable on chromium (see install-tool.spec.ts)" + ); + test.setTimeout(300_000); + + await page.goto("/", { timeout: 120_000 }); + await page.waitForFunction(() => !!window.patchwork?.account, null, { + timeout: 180_000, + }); + + const { pinned, acctHash, sha256 } = await page.evaluate(async () => { + const encoder = new TextEncoder(); + const ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + const base58 = (bytes: Uint8Array) => { + let n = 0n; + for (const byte of bytes) n = n * 256n + BigInt(byte); + let out = ""; + while (n > 0n) { + out = ALPHABET[Number(n % 58n)] + out; + n /= 58n; + } + for (const byte of bytes) { + if (byte !== 0) break; + out = "1" + out; + } + return out; + }; + const sha = async (text: string) => + new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(text))); + + const account = window.patchwork!.account; + const acctHash = base58((await sha(account.documentId)).slice(0, 12)); + + const file = window.repo.create>(); + file.change((d) => { + d.content = "export default 42"; + d.mimeType = "text/javascript"; + }); + const root = await window.repo.find<{ docs: unknown[] }>( + account.doc().rootFolderUrl! + ); + root.change((d) => { + d.docs.push({ name: "forge-test.js", type: "file", url: file.url }); + }); + + // The live URL redirects to the pinned one; fetch follows it. + let pinned = ""; + for (let i = 0; i < 30 && !pinned; i++) { + const response = await fetch(`/@/${acctHash}/forge-test.js`); + if (response.ok && response.redirected) pinned = response.url; + else await new Promise((resolve) => setTimeout(resolve, 1000)); + } + const digest = await sha("export default 42"); + const sha256 = btoa(String.fromCharCode(...digest)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); + return { pinned, acctHash, sha256 }; + }); + expect(pinned).toContain("/@/"); + + await page.evaluate( + async ({ acctHash, sha256 }) => { + const cache = await caches.open(`patchwork-content-${acctHash}`); + await cache.put(`/.content/${sha256}`, new Response("export default 666")); + }, + { acctHash, sha256 } + ); + + await page.reload({ timeout: 120_000 }); + await waitForRepoReady(page, 180_000); + const value = await page.evaluate( + async (url) => (await import(/* @vite-ignore */ url)).default, + pinned + ); + expect(value).toBe(42); +}); diff --git a/packages/e2e/tests/path-routing.spec.ts b/packages/e2e/tests/path-routing.spec.ts new file mode 100644 index 000000000..1d84254b0 --- /dev/null +++ b/packages/e2e/tests/path-routing.spec.ts @@ -0,0 +1,72 @@ +import { expect, test } from "@playwright/test"; + +/** + * `&path=` in the hash: a live path rooted at the account. A hash with only a + * path opens what it names, and the router writes the document's url beside + * it as `doc=`. + */ + +declare global { + interface Window { + patchwork?: { + account: { + documentId: string; + doc(): { rootFolderUrl?: string }; + }; + }; + } +} + +test("a hash with only path= opens what it names", async ({ page, browserName }) => { + test.skip( + browserName !== "chromium", + "needs the full-UI boot for an account, which is only reliable on chromium (see install-tool.spec.ts)" + ); + test.setTimeout(300_000); + + await page.goto("/", { timeout: 120_000 }); + await page.waitForFunction(() => !!window.patchwork?.account, null, { + timeout: 180_000, + }); + + const { url, path } = await page.evaluate(async () => { + const ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + const base58 = (bytes: Uint8Array) => { + let n = 0n; + for (const byte of bytes) n = n * 256n + BigInt(byte); + let out = ""; + while (n > 0n) { + out = ALPHABET[Number(n % 58n)] + out; + n /= 58n; + } + for (const byte of bytes) { + if (byte !== 0) break; + out = "1" + out; + } + return out; + }; + const account = window.patchwork!.account; + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(account.documentId)) + ); + const acctHash = base58(digest.slice(0, 12)); + + const file = window.repo.create>(); + file.change((d) => { + d["@patchwork"] = { type: "file" }; + d.content = "opened by path"; + d.mimeType = "text/plain"; + }); + const root = await window.repo.find<{ docs: unknown[] }>(account.doc().rootFolderUrl!); + root.change((d) => { + d.docs.push({ name: "path test", type: "file", url: file.url }); + }); + return { url: file.url, path: `${acctHash}/${encodeURIComponent("path test")}` }; + }); + + await page.evaluate((path) => (location.hash = `path=${path}`), path); + await expect + .poll(() => page.evaluate(() => location.hash), { timeout: 60_000 }) + .toContain(`doc=${url}`); + expect(await page.evaluate(() => location.hash)).toContain(`path=${path}`); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f1593ba90..271648470 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -217,6 +217,9 @@ importers: '@automerge/automerge-repo': specifier: 2.6.0-subduction.48 version: 2.6.0-subduction.48(patch_hash=3560381ba62410bed0716e7ef08af4200bcf8c0af4aa0794c1d5d05d23e97d5e) + bs58: + specifier: ^6.0.0 + version: 6.0.0 debug: specifier: ^4.4.3 version: 4.4.3 diff --git a/sites/bench/README.md b/sites/bench/README.md index 730bbb565..4d73ba9bd 100644 --- a/sites/bench/README.md +++ b/sites/bench/README.md @@ -129,6 +129,9 @@ server holds our heads" means that peer and not a sibling. from a fresh tab, three tabs flushing concurrently, sync latency with the server; with the longest main-thread stall (a 5ms timer's lateness) and long task time recorded around the heavy parts. +- `content-hash.spec` — what the service worker pays on every hit of the `/@/` + content store, which it hashes before serving since anything on the origin + can write CacheStorage: SHA-256 of 10KB, 100KB and 1MB, median of 200. Cross-tab timings use epoch milliseconds, since `performance.now()` counts from each page's own navigation start. `find()` in the helpers retries on diff --git a/sites/bench/tests/content-hash.spec.ts b/sites/bench/tests/content-hash.spec.ts new file mode 100644 index 000000000..b0d23740b --- /dev/null +++ b/sites/bench/tests/content-hash.spec.ts @@ -0,0 +1,38 @@ +// What the service worker pays on every hit of the content store: it hashes +// what it read before serving it, since anything on the origin can write +// CacheStorage. SHA-256 over typical module sizes, in the page, which runs the +// same crypto.subtle the service worker does. +import { test } from "@playwright/test"; +import { median, openTab, record } from "./bench.js"; + +const SIZES = [ + ["10KB", 10 * 1024], + ["100KB", 100 * 1024], + ["1MB", 1024 * 1024], +] as const; +const RUNS = 200; + +test("patchwork: sha-256 of a content store hit", async ({ context }) => { + const page = await openTab(context, "patchwork", { server: "none" }); + for (const [label, bytes] of SIZES) { + const times = await page.evaluate( + async ({ bytes, runs }) => { + const data = new Uint8Array(bytes).map((_, i) => i % 251); + const times: number[] = []; + for (let i = 0; i < runs; i++) { + const start = performance.now(); + await crypto.subtle.digest("SHA-256", data); + times.push(performance.now() - start); + } + return times; + }, + { bytes, runs: RUNS } + ); + record({ + metric: `sha-256 of a ${label} hit`, + mode: "patchwork", + value: median(times), + unit: "ms", + }); + } +}); diff --git a/vitest.config.ts b/vitest.config.ts index baa96d0fa..53a7fe9dc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,8 +3,10 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { workspace: [ + "core/bootloader", "core/filesystem", "core/patchwork", + "core/plugins", "packages/edge-handles", ], },