From aa1c19af9effcd2381d6bdce78ac3c440c9b8b53 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:49:06 +0000 Subject: [PATCH 1/3] fix(service): Cap known values of runs without a token, key write-token runs on the main thread Address the review of #45: - Count `known` across all sessions of a run without the token (10,000 values, 256 Ki characters). A request past it fails with 400 and ends the run. No value is dropped, so no key changes. - Key and redact runs with the token on the main thread, as on main. The worker serves runs without the token only, with a 256 MiB old space and a 4 MiB semi-space. - Leave values over 32,767 characters out of `known` and templates. V8 cannot run a pattern with them, so those requests failed before. - Count matches in `extractValues()` as it finds them, so the cap of 100,000 values stops before memory grows. - Write the key text of a body without a copy of it. - Group IPv4-mapped IPv6 addresses in all forms as IPv4. - README: split deploy order for the PROXY header, the /48 gap. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- README.md | 116 ++++++++---- packages/roach/src/values.ts | 66 +++++-- packages/service/src/key-worker.ts | 42 ++-- packages/service/src/recorder.ts | 143 ++++++++++---- packages/service/src/request-key.ts | 48 +++-- packages/service/src/server.ts | 51 +++-- packages/service/src/service.ts | 22 ++- packages/service/tests/recording.test.ts | 28 ++- packages/service/tests/service.test.ts | 232 ++++++++++++++++++++++- 9 files changed, 584 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index e161522..3083276 100644 --- a/README.md +++ b/README.md @@ -361,8 +361,11 @@ as part of a word and does not match. finds it. For example, a pattern finds a short commit id only in the output of `git push`, but the model then quotes it alone in its reply. A pattern wins at the same place. A session remembers all values of its - requests, also after it ends. A limit there would change the keys of - later requests. + requests until it ends. It does not drop a value, because that would + change the keys of later requests. Only a value of over 32,767 + characters stays out: V8 cannot run a pattern with it, so it would fail + each later request of the session. A run without the token has a limit + (see below): past it, the request fails. - Values in thinking blocks of the request do not count for `n`, and the proxy does not look for remembered values in them. A replay keeps the recorded text of a thinking block, so when a later request sends it back, @@ -379,24 +382,32 @@ real way use the same recording. The service takes any pattern that compiles in 1 s, up to 2048 characters. A run without the token must use the same patterns as the run -that recorded, or its keys do not match. All runs share one process, so -the service makes each key in a worker process, one key at a time. So a -slow key does not stop the other work of the service. Each key has a time -limit: 1 s, and 0.5 s for each MiB of the body. The built-in patterns take -about 60 to 110 ms for each MiB. A request whose key takes longer fails -with HTTP 400. Some work, such as `JSON.parse()` of a large body, cannot -stop at the time limit. When a key goes 1 s past its time limit, the -service stops the worker and starts a new one. The worker has 384 MiB of -heap. A key that needs more stops the worker, and its request fails with -HTTP 400. The key of a 64 MiB body needs about 320 MiB. +that recorded, or its keys do not match. Each key has a time limit: 1 s, +and 0.5 s for each MiB of the body. The built-in patterns take about 60 +to 110 ms for each MiB. A request whose key takes longer fails with HTTP 400. + +All runs share one process. Anyone can start a run without the token, so +the service makes the keys of these runs in a worker process, one key at +a time. So a slow key does not stop the other work of the service. Some +work, such as `JSON.parse()` of a large body, cannot stop at the time +limit. When a key goes 1 s past its time limit, the service stops the +worker and starts a new one. The worker has 256 MiB of heap. A key that +needs more stops the worker, and its request fails with HTTP 400. The key +of a 16 MiB body of small text blocks needs less. A run with the token +makes its keys in the main process, so it has no memory limit of the +worker. The key of a 64 MiB body can need about 1 GiB. The limits never change a key: a request gets the key that it always -had, or it fails. The keys of runs with the token come before the keys of -runs without it. A run without the token has smaller limits: +had, or it fails. A run without the token has smaller limits: - A key has at most 5 s. - A request can have at most 100,000 changing values. A request with more fails with HTTP 400, and the run goes on. +- The sessions of the run remember at most 10,000 values, with 256 Ki + characters, together. A request that would add more fails with HTTP + 400, and the run ends. A session forgets its values when it ends. The + model requests of Junior have about 500 values with 10 K characters for + each session. - The keys of the run and the redaction of its misses (see [Misses](#misses)) have 30 s in total. In the 14 days to 2026-11-05, the keys of the largest run took 9.5 s (p99 9.1 s, about 1,000 keys). @@ -453,9 +464,10 @@ shows it. - A miss has no change when there is no sample, such as for a recording from an older version, or when the texts are the same in the part that the sample keeps. -- The proxy redacts the text of the miss before it compares it. This runs - in the worker process of the keys, with the time limit of a key (see - [Changing values](#changing-values)). Past it, the miss has no change. +- The proxy redacts the text of the miss before it compares it. This has + the time limit of a key, and for a run without the token it runs in the + worker process of the keys (see [Changing values](#changing-values)). + Past the limit, the miss has no change. - When the requests of a test run in parallel, their order can change. Then a sample can come from another request of the test. @@ -569,10 +581,10 @@ config is `RoachServiceConfig` in `packages/service/src/service.ts`. `deploy/gcp It checks the address of each connection after DNS, so a host name cannot point a run at the metadata server, at the VPC, or at the VM. Only tests and local development set `privateNetworks`. - - The service makes keys in a worker process, and a key has a time - limit and a memory limit (`key-worker.ts`, `request-key.ts`). So a - slow pattern or a large body fails only its own request, and it does - not stop the service. + - A key has a time limit. The service makes the keys of runs without + the token in a worker process with a memory limit (`key-worker.ts`, + `request-key.ts`). So a slow pattern or a large body fails only its + own request, and it does not stop the service. - **Sentry.** With `sentryDsn`, each read and write of a recording sends a span to Sentry. Each proxied request sends a span and metrics. Each unexpected error sends an error event. See [Sentry](#sentry). @@ -677,8 +689,9 @@ To see how well the cache works: 60 minutes without a request to the service fails. - The service can have 100 open runs without a token, for all tenants together, and 10 for each client address. All addresses of one IPv6 - /64 count as one client. Another one gets HTTP 429 until one ends. Runs - with the token have no cap. With `proxyProtocol`, the address of the + /64 count as one client, and an IPv4 address in IPv6 form, such as + `::ffff:192.0.2.1`, counts as its IPv4 address. Another one gets HTTP + 429 until one ends. Runs with the token have no cap. With `proxyProtocol`, the address of the client comes from the PROXY protocol v1 header of the load balancer. A connection without the header uses the address of the connection. - A proxied request body over 64 MiB gets HTTP 413, and over 16 MiB for a @@ -688,9 +701,12 @@ To see how well the cache works: [Changing values](#changing-values). - A `CONNECT` tunnel that sends no byte for 10 s after the `200` closes. A tunnel that sends and gets no byte for 10 minutes closes. -- The credentials that a run learns and the values that a session - remembers have limits. See [Config](#config) and +- The credentials that a run learns and the values that the sessions of + a run without the token remember have limits. See [Config](#config) and [Changing values](#changing-values). +- A gap: a client with an IPv6 /48 has 65,536 /64s, so it can open runs + until the service has 100 runs without a token. Then other clients get + HTTP 429 until its runs end. - GCS deletes each recording 30 days after it was written, also when runs still replay it. Then `auto` mode records it again, and `replay` mode fails the request with HTTP 412. @@ -782,24 +798,45 @@ the last green commit. The service reads its config only when it starts. With `proxyProtocol`, it reads a PROXY header when a connection has one, and it uses the address of the connection when it does not. A service without -`proxyProtocol` cannot read the header. To turn on the PROXY header: +`proxyProtocol` cannot read the header, so a load balancer that sends the +header before the service reads it fails each request with HTTP 400. To +turn on the PROXY header, apply in this order. Run the commands in +`deploy/gcp`: 1. Merge the change, and let the `Deploy` workflow deploy the image. -2. Apply Terraform: `terraform apply -var-file=roach.tfvars`. The load - balancer then sends the header, but the service still has its old - config, so proxied requests fail with HTTP 400 until step 3. -3. Restart the VM at once, so that it reads the new config: - `gcloud compute instances reset roach --project roach-511216 --zone us-east1-b`. - The apply does not restart the VM. +2. Apply only the config secret: + + ```sh + terraform apply -var-file=roach.tfvars \ + -target=google_secret_manager_secret_version.config + ``` + +3. Restart the VM, so that it reads the new config. It then reads a + header when a connection has one: + + ```sh + gcloud compute instances reset roach --project roach-511216 --zone us-east1-b + ``` + +4. When the service answers, apply the rest. The load balancer then + sends the header (`proxy_header` of `google_compute_target_ssl_proxy` + and of `google_compute_health_check`): + + ```sh + terraform apply -var-file=roach.tfvars + ``` + +No step fails requests. From step 3 to step 4, the service sees the +addresses of the load balancer, so all clients share the limit of runs +for each client. The load balancer writes its header before the first byte of the client, so a client cannot put its own header first. A client can send its own header only when the service has `proxyProtocol` and the load balancer -does not send the header, such as when the config has `proxyProtocol` -but the load balancer has no `proxy_header`. With the steps above, this -does not occur: until step 3 the service does not read headers. A false -header changes only the address for the limit of runs for each client, -and the firewall lets only the Google ranges connect to the VM. +does not send the header: from step 3 to step 4 above. So do step 4 at +once. A false header changes only the address for the limit of runs for +each client, and the firewall lets only the Google ranges connect to the +VM. To add a project, add its repository to the repositories of the `ROACH_TOKEN` organization secret, in the GitHub settings of the @@ -932,8 +969,9 @@ In `packages/service/src/`: - `addresses.ts`: the public addresses that the proxy can send requests to. - `recorder.ts`: modes, sessions, replay, and misses. - `request-key.ts`: the key of a request. -- `key-worker.ts`: the worker process that makes keys and redacts the text - of misses, with a time limit and a memory limit. +- `key-worker.ts`: the worker process that makes the keys of runs without + the token and redacts the text of their misses, with a time limit and a + memory limit. - `misses.ts`: request samples and the first change of a miss. - `store.ts`: the formats of recordings and request samples, and the store contract. diff --git a/packages/roach/src/values.ts b/packages/roach/src/values.ts index 3673f13..5171640 100644 --- a/packages/roach/src/values.ts +++ b/packages/roach/src/values.ts @@ -155,7 +155,8 @@ export type KnownValues = ReadonlyMap; * value at the same place. * * Past `maxValues` values, it stops and throws an error with HTTP status - * 400. So a text with millions of values cannot fill the memory. + * 400. It counts each value when it finds it, so a text with millions of + * values cannot fill the memory before the error. */ export function extractValues( text: string, @@ -174,28 +175,51 @@ export function extractValues( const search = hasKnown ? new RegExp(`${pattern.source}|(${literals([...known.keys()])})`, "g") : pattern; + // An `exec()` loop, not `replace()`: `replace()` finds all matches + // before it calls its function, so it cannot stop at `maxValues`. The + // loop finds the same matches, also the empty ones. + const parts: string[] = []; let count = 0; - const result = text.replace(search, (...match: unknown[]) => { - count += 1; - if (count > maxValues) { - throw Object.assign( - new Error("the request has too many changing values"), - { status: 400 }, - ); + let end = 0; + search.lastIndex = 0; + try { + for (let match = search.exec(text); match; match = search.exec(text)) { + count += 1; + if (count > maxValues) { + throw Object.assign( + new Error("the request has too many changing values"), + { status: 400 }, + ); + } + const value = match[0]; + const name = + hasKnown && match[knownGroup] !== undefined + ? known.get(value)! + : names[groups.findIndex((group) => match[group] !== undefined)]!; + const list = values.get(name) ?? []; + list.push(value); + values.set(name, list); + parts.push(text.slice(end, match.index), `<<${name}>>`); + end = match.index + value.length; + if (value.length === 0) search.lastIndex += 1; } - const value = match[0] as string; - const name = - hasKnown && match[knownGroup] !== undefined - ? known.get(value)! - : names[groups.findIndex((group) => match[group] !== undefined)]!; - const list = values.get(name) ?? []; - list.push(value); - values.set(name, list); - return `<<${name}>>`; - }); - return { text: result, values }; + } finally { + // `pattern` is shared, so it must not keep a position. + search.lastIndex = 0; + } + parts.push(text.slice(end)); + return { text: parts.join(""), values }; } +/** + * Whether `value` can be in `known` or in a template (`templateValues()`). + * V8 does not run a pattern with a literal of over 32,767 characters on a + * text that can hold it. Such a value in `known` would fail each later + * request of its session that could repeat it, so to leave it out changes + * no key. + */ +export const canBeKnown = (value: string): boolean => value.length <= 32_767; + const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); /** @@ -212,13 +236,13 @@ const literals = (values: string[]) => /** * Write each value of the request that a recorded response repeats as * `<>`. A value is replaced only where it is not part of a longer - * word or number. + * word or number. A value that `canBeKnown()` refuses stays as it is. */ export function templateValues(text: string, values: RequestValues): string { const placeholders = new Map(); for (const [name, list] of values) { list.forEach((value, index) => { - if (!placeholders.has(value)) { + if (!placeholders.has(value) && canBeKnown(value)) { placeholders.set(value, `<<${name}:${index + 1}>>`); } }); diff --git a/packages/service/src/key-worker.ts b/packages/service/src/key-worker.ts index 4805d33..38b9e09 100644 --- a/packages/service/src/key-worker.ts +++ b/packages/service/src/key-worker.ts @@ -1,13 +1,15 @@ /** - * The worker process that makes keys and redacts the text of misses. + * The worker process that makes the keys of runs without the token, and + * redacts the text of their misses. * - * The value patterns of a run come from its client, and redaction takes - * time for each credential that a run knows. Both run in this worker, so a - * slow request does not stop the event loop that answers all runs. The - * service has one worker. It does one job at a time, and each job has a - * time limit (the `timeout` of `vm`). The jobs of runs with the write token - * go before the jobs of runs without it. A job of an ended run does not - * start. + * Anyone can start a run without the token, and its value patterns and + * bodies come from its client. Redaction takes time for each credential + * that a run knows. Both run in this worker, so a slow request does not + * stop the event loop that answers all runs. The service has one worker. + * It does one job at a time, in order, and each job has a time limit (the + * `timeout` of `vm`). A job of an ended run does not start. A run with the + * token does this work on the main thread (`recorder.ts`), because the + * key of a 64 MiB body can need more memory than the worker has. * * The worker is a child process with a heap limit (`MAX_HEAP_MB`), so one * request cannot fill the memory of the service. It is not a worker @@ -68,8 +70,6 @@ export interface JobOptions { timeLimitMs: number; /** The most changing values that a key can have. */ maxValues: number; - /** Run the job before the jobs that do not have this. */ - first: boolean; /** Called before the job starts. When it returns `true`, it does not. */ skip(): boolean; } @@ -90,11 +90,13 @@ interface JobResult { const MARK = "roach-key-worker"; /** - * The old-generation heap of the worker, in MiB. The key of a 64 MiB body, - * the largest body of a run with the token, needs about 320 MiB. At 256 MiB, - * a 64 MiB body with a screenshot fails. + * The heap of the worker, in MiB: the old generation and each semi-space + * of the young generation. The worker makes the keys of runs without the + * token, with bodies of at most 16 MiB. The key of a 16 MiB body of small + * text blocks peaks at about 330 MiB of RSS. The worst body, such as 16 MiB + * of `{}`, stops the worker at about 500 MiB of RSS. */ -const MAX_HEAP_MB = 384; +const MAX_HEAP_MB = { old: 256, semi: 4 }; /** The time after the time limit of a job before the worker stops. */ const WATCHDOG_MARGIN_MS = 1000; @@ -163,7 +165,11 @@ function failWorker(stopped: ChildProcess, why: string): void { function startWorker(): ChildProcess { const started = fork(fileURLToPath(import.meta.url), [MARK], { - execArgv: [...process.execArgv, `--max-old-space-size=${MAX_HEAP_MB}`], + execArgv: [ + ...process.execArgv, + `--max-old-space-size=${MAX_HEAP_MB.old}`, + `--max-semi-space-size=${MAX_HEAP_MB.semi}`, + ], // `Map` values, such as `known`, need the advanced serialization. serialization: "advanced", stdio: ["ignore", "inherit", "inherit", "ipc"], @@ -205,11 +211,7 @@ function hold(child: ChildProcess | undefined, busy: boolean): void { /** Start the next job, when the worker has none. */ function next(): void { while (!running && queue.length > 0) { - const index = Math.max( - 0, - queue.findIndex((pending) => pending.first), - ); - const [pending] = queue.splice(index, 1) as [Pending]; + const pending = queue.shift()!; if (pending.skip()) { pending.reject(jobError("this run has ended", { status: 410 })); continue; diff --git a/packages/service/src/recorder.ts b/packages/service/src/recorder.ts index b9c67e5..fe51e9e 100644 --- a/packages/service/src/recorder.ts +++ b/packages/service/src/recorder.ts @@ -42,7 +42,7 @@ import { type JobError, type JobOptions, } from "./key-worker.ts"; -import { keyTimeLimit, type KeyedRequest } from "./request-key.ts"; +import { keyRequest, keyTimeLimit, type KeyedRequest } from "./request-key.ts"; import { createSecrets, redactUrl, type LearnLimit } from "./secrets.ts"; import { recordingKey, @@ -65,8 +65,10 @@ import type { RefusalReason, } from "@sentry/roach/types"; import { + canBeKnown, fillValues, templateValues, + withTimeLimit, type RequestValues, } from "@sentry/roach/values"; @@ -340,11 +342,20 @@ export interface WorkLimit { * with HTTP 400. Its key does not change. */ maxValues: number; + /** + * The most values in `known` of all sessions of the run, and their + * characters. A request that would add more fails with HTTP 400, and + * the run ends. The values stay, so no key changes. + */ + maxKnown: { values: number; chars: number }; /** The time of all jobs of the run, in ms. Past it, the run ends. */ totalMs: number; /** - * Set for a run without a token. A key that goes past its time limit - * ends the run. The jobs of the run go after the jobs of other runs. + * Set for a run without a token. Its keys and the redaction of its + * misses run in the key worker, and a key that goes past its time limit + * ends the run. A run with the token does them on the main thread, as + * before the worker, so the memory limit of the worker fails none of + * its requests. */ untrusted: boolean; } @@ -366,11 +377,54 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { const { workLimit } = config; let closed = false; let workMs = 0; + /** The values in `known` of all sessions, and their characters. */ + const knownTotal = { values: 0, chars: 0 }; + /** Clear `known` of a session. */ + const forget = (session: Session) => { + knownTotal.values -= session.known.size; + for (const value of session.known.keys()) { + knownTotal.chars -= value.length; + } + session.known.clear(); + }; + /** + * Add the values of a request to `known` of its session. A value that + * `canBeKnown()` refuses stays out: before, it failed each later request + * of the session, so no key changes. Past `workLimit.maxKnown`, it adds + * nothing, ends the run, and throws an error with HTTP status 400. + */ + const remember = (session: Session, values: RequestValues) => { + const added = new Map(); + let chars = 0; + for (const [name, list] of values) { + for (const value of list) { + if (session.known.has(value) || added.has(value)) continue; + if (!canBeKnown(value)) continue; + added.set(value, name); + chars += value.length; + } + } + const { maxKnown } = workLimit; + if ( + knownTotal.values + added.size > maxKnown.values || + knownTotal.chars + chars > maxKnown.chars + ) { + if (!closed) config.endRun("its sessions have too many changing values"); + throw Object.assign( + new Error( + `the sessions of this run have over ${maxKnown.values} changing values or ${maxKnown.chars} characters of them. This run has ended.`, + ), + { status: 400 }, + ); + } + for (const [value, name] of added) session.known.set(value, name); + knownTotal.values += added.size; + knownTotal.chars += chars; + }; /** The options of a job of `bytes` bytes in the key worker. */ const jobOptions = (bytes: number): JobOptions => ({ timeLimitMs: Math.min(workLimit.maxJobMs, keyTimeLimit(bytes)), maxValues: workLimit.maxValues, - first: !workLimit.untrusted, skip: () => closed, }); /** Count the time of a job. Past `workLimit.totalMs`, end the run. */ @@ -541,7 +595,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { ended.passed = passed; const recorded = [...ended.recorded]; ended.recorded.clear(); - ended.known.clear(); + forget(ended); if (passed) await write(recorded); else drop(recorded.map(([key]) => key)); return ended.missed; @@ -629,33 +683,52 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { const { keyed, key } = await telemetry.span( { op: "roach.key", attributes: { "roach.rule": rule.name } }, async () => { - const result = await keyInWorker( - rule, - { - method: request.method, - url: request.url.href, - headers: request.headers, - body: request.body.toString("utf8"), - }, - owner?.known, - jobOptions(request.body.length), - ).catch((error: JobError) => { - countWork(error.ms); - if (error.timedOut && workLimit.untrusted && !closed) { - config.endRun("a key went past its time limit"); - error.message += " This run has ended."; - } - throw error; - }); - countWork(result.ms); - const name = recordingKey(rule.name, result.value.key); + const toKey = { + method: request.method, + url: request.url.href, + headers: request.headers, + body: request.body.toString("utf8"), + }; + const options = jobOptions(request.body.length); + const value = workLimit.untrusted + ? await keyInWorker(rule, toKey, owner?.known, options).then( + (result) => { + countWork(result.ms); + return result.value; + }, + (error: JobError) => { + countWork(error.ms); + if (error.timedOut && !closed) { + config.endRun("a key went past its time limit"); + error.message += " This run has ended."; + } + throw error; + }, + ) + : keyRequest( + rule, + toKey, + owner?.known, + options.timeLimitMs, + options.maxValues, + ); + const name = recordingKey(rule.name, value.key); telemetry.annotate({ "roach.key": name }); - return { keyed: result.value, key: name }; + return { keyed: value, key: name }; }, ); /** The redacted text of the key, or `undefined` past the time limit. */ - const redactedText = () => - redactInWorker( + const redactedText = async (): Promise => { + if (!workLimit.untrusted) { + try { + return withTimeLimit(keyTimeLimit(keyed.text.length), () => + secrets.redactRequest(keyed.text, request), + ); + } catch { + return undefined; + } + } + return redactInWorker( secrets.list(), keyed.text, { @@ -674,15 +747,10 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { return undefined; }, ); - // A session remembers every value, also after it ends. A limit here - // would change the keys of later requests that repeat a value. - if (owner) { - for (const [name, list] of keyed.values) { - for (const value of list) { - if (!owner.known.has(value)) owner.known.set(value, name); - } - } - } + }; + // A session remembers every value of its requests. A value that it + // dropped would change the keys of later requests that repeat it. + if (owner) remember(owner, keyed.values); const report = (result: ResponseSource, response?: ProxyResponse) => { const attributes = { "roach.rule": rule.name, @@ -823,6 +891,7 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { ); const extra = ended.length - MAX_ENDED_SESSIONS + 1; for (const old of ended.slice(0, Math.max(0, extra))) { + forget(old); sessions.delete(old.id); } const id = randomBytes(8).toString("hex"); diff --git a/packages/service/src/request-key.ts b/packages/service/src/request-key.ts index 4f4d9c1..f970469 100644 --- a/packages/service/src/request-key.ts +++ b/packages/service/src/request-key.ts @@ -8,9 +8,9 @@ * `<>` in the key (`@sentry/roach/values`). * * The patterns of `values` come from the client, and all runs share one - * process. A slow pattern would block every run. So the service makes keys - * in a worker thread (`key-worker.ts`), and a key has a time limit - * (`keyTimeLimit()`). + * process. A slow pattern would block every run. So a key has a time + * limit (`keyTimeLimit()`), and the service makes the keys of runs without + * the token in a worker process (`key-worker.ts`). */ import { createHash } from "node:crypto"; import type { IncomingHttpHeaders } from "node:http"; @@ -65,19 +65,30 @@ export interface KeyedRequest { values: RequestValues; } -/** JSON with sorted object keys, so equal bodies give equal keys. */ -function stableStringify(value: unknown): string { +/** + * JSON with sorted object keys, so equal bodies give equal keys. A value + * in `replace` is written as its string instead. + */ +function stableStringify( + value: unknown, + replace?: ReadonlyMap, +): string { if (value === null || typeof value !== "object") { return JSON.stringify(value); } + const replaced = replace?.get(value); + if (replaced !== undefined) return JSON.stringify(replaced); if (Array.isArray(value)) { - return `[${value.map(stableStringify).join(",")}]`; + return `[${value.map((item) => stableStringify(item, replace)).join(",")}]`; } const entries = Object.entries(value as Record) .filter(([, item]) => item !== undefined) .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); return `{${entries - .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`) + .map( + ([key, item]) => + `${JSON.stringify(key)}:${stableStringify(item, replace)}`, + ) .join(",")}}`; } @@ -108,11 +119,18 @@ function normalizeJson( maxValues: number, ): { text: string; values: RequestValues } { const thinking: string[] = []; + // The mark of each thinking block, numbered in the order of the body. + // The text gets the mark in place of the block, without a copy of the + // body, because a copy of a large body needs hundreds of MiB. + const marks = new Map(); // The values of thinking blocks count toward `maxValues` too. let left = maxValues; - const mark = (value: unknown): unknown => { - if (Array.isArray(value)) return value.map(mark); - if (value === null || typeof value !== "object") return value; + const mark = (value: unknown): void => { + if (value === null || typeof value !== "object") return; + if (Array.isArray(value)) { + for (const item of value) mark(item); + return; + } const record = value as Record; if (THINKING_BLOCK_TYPES.has(String(record.type))) { const block = extractValues( @@ -123,14 +141,14 @@ function normalizeJson( ); for (const list of block.values.values()) left -= list.length; thinking.push(block.text); - return `\uE000${thinking.length - 1}`; + marks.set(record, `\uE000${thinking.length - 1}`); + return; } - return Object.fromEntries( - Object.entries(record).map(([name, item]) => [name, mark(item)]), - ); + for (const item of Object.values(record)) mark(item); }; + mark(json); const { text, values } = extractValues( - stableStringify(mark(json)), + stableStringify(json, marks), patterns, known, left, diff --git a/packages/service/src/server.ts b/packages/service/src/server.ts index 7bc5cbf..db61450 100644 --- a/packages/service/src/server.ts +++ b/packages/service/src/server.ts @@ -100,32 +100,41 @@ export function clientAddress(incoming: http.IncomingMessage): string { /** * The group of a client address for the limit of runs for each client. An * IPv6 client often has a whole /64, so the group of an IPv6 address is - * its /64, such as `2001:db8:0:1::/64`. An IPv4 address, also in IPv6 - * form (`::ffff:192.0.2.1`), is its own group. + * its /64, such as `2001:db8:0:1::/64`. An IPv4 address is its own group, + * also in each IPv6 form, such as `::ffff:192.0.2.1`, `::ffff:c000:201`, + * and `0:0:0:0:0:ffff:192.0.2.1`. */ export function clientGroup(address: string): string { - const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address); - if (mapped) return mapped[1]!; if (isIP(address) !== 6) return address; - // Write `::` as its zero groups. An address that ends in IPv4 form has - // two groups in it, but only the first four groups count. - const [head = "", tail] = address.toLowerCase().split("::"); + const groups = ipv6Groups(address); + if ( + groups.slice(0, 5).every((group) => group === 0) && + groups[5] === 0xffff + ) { + const [high = 0, low = 0] = groups.slice(6); + return [high >> 8, high & 0xff, low >> 8, low & 0xff].join("."); + } + return `${groups + .slice(0, 4) + .map((group) => group.toString(16)) + .join(":")}::/64`; +} + +/** The eight 16-bit groups of an IPv6 address. */ +function ipv6Groups(address: string): number[] { + // The URL parser writes each IPv6 form with hex groups only, such as + // `::ffff:c000:201`. A zone, such as `%eth0`, is not part of it. + const [head = "", tail] = new URL( + `http://[${address.split("%")[0]}]`, + ).hostname + .slice(1, -1) + .split("::"); const front = head ? head.split(":") : []; const back = tail ? tail.split(":") : []; - const groups = - tail === undefined - ? front - : [ - ...front, - ...Array(Math.max(0, 8 - front.length - back.length)).fill( - "0", - ), - ...back, - ]; - const prefix = groups - .slice(0, 4) - .map((group) => Number.parseInt(group, 16).toString(16)); - return `${prefix.join(":")}::/64`; + const zeros = Array(8 - front.length - back.length).fill("0"); + return [...front, ...zeros, ...back].map((group) => + Number.parseInt(group, 16), + ); } /** An error that answers a request with its own status. */ diff --git a/packages/service/src/service.ts b/packages/service/src/service.ts index 938ba8b..3a8f398 100644 --- a/packages/service/src/service.ts +++ b/packages/service/src/service.ts @@ -165,22 +165,32 @@ const MAX_PUBLIC_RUNS_PER_CLIENT = 10; /** A proxied request body of a run without a token over this gets 413. */ const PUBLIC_MAX_BODY_BYTES = 16 * 1024 * 1024; /** - * The time that a run can use in the key worker (`key-worker.ts`). A key - * has 1 s and 0.5 s for each MiB of its body. For a run without a token, - * one job has at most 5 s and all jobs 30 s. Past that, or when a key goes - * past its time limit, the run ends. In the 14 days to 2026-11-05, the key - * time of a run was at most 9.5 s (p99 9.1 s, about 1,000 keys). A request - * of a run without a token can have at most 100,000 changing values. + * The work of a run without a token (`WorkLimit`). A key has 1 s and 0.5 s + * for each MiB of its body. One job in the key worker (`key-worker.ts`) has + * at most 5 s, and all jobs 30 s. Past that, or when a key goes past its + * time limit, the run ends. In the 14 days to 2026-11-05, the key time of + * a run was at most 9.5 s (p99 9.1 s, about 1,000 keys). + * + * A request can have at most 100,000 changing values. The sessions of a + * run can remember (`known`) at most 10,000 values with 256 Ki characters. + * Past that, the run ends, so 100 runs keep at most about 200 MiB. The + * largest model request in the 14 days to 2026-11-05 had 162,707 input + * tokens, about 0.65 MB. In 24 model requests of Junior, the patterns of + * its evals found at most 770 values with 15 K characters for each MiB. So + * a session has about 500 values with 10 K characters, and the limits are + * about 20 times that. The evals run one session at a time. */ const PUBLIC_WORK_LIMIT: WorkLimit = { maxJobMs: 5000, maxValues: 100_000, + maxKnown: { values: 10_000, chars: 256 * 1024 }, totalMs: 30_000, untrusted: true, }; const WRITE_WORK_LIMIT: WorkLimit = { maxJobMs: Infinity, maxValues: Infinity, + maxKnown: { values: Infinity, chars: Infinity }, totalMs: Infinity, untrusted: false, }; diff --git a/packages/service/tests/recording.test.ts b/packages/service/tests/recording.test.ts index 7591ded..efdfe3f 100644 --- a/packages/service/tests/recording.test.ts +++ b/packages/service/tests/recording.test.ts @@ -31,7 +31,7 @@ import { import { startRoachService, type RoachService } from "../src/service.ts"; import { keyRequest } from "../src/request-key.ts"; import type { RecordingMode, RecordingRule } from "@sentry/roach/types"; -import { VALUE_PATTERNS } from "@sentry/roach/values"; +import { extractValues, VALUE_PATTERNS } from "@sentry/roach/values"; const TOKEN = "write-token"; const TENANT = "acme/alpha"; @@ -648,6 +648,32 @@ describe("recording", () => { ]); }); + it("finds the values that replace() finds", () => { + // `extractValues()` counts each value when it finds it. It must find + // the same values as `replace()`, also empty ones, and with known + // values. + const patterns = { x: "x*", d: String.raw`\d{0,2}`, w: "ab|a" }; + const known = new Map([["zz", "z"]]); + const search = new RegExp( + `(x*)|(\\d{0,2})|(ab|a)|((?(); + const expected = text.replace(search, (...match: string[]) => { + const name = + names[match.slice(1, 5).findIndex((group) => group !== undefined)]!; + values.set(name, [...(values.get(name) ?? []), match[0]!]); + return `<<${name}>>`; + }); + expect(extractValues(text, patterns, known)).toEqual({ + text: expected, + values, + }); + } + }); + it("keeps the keys of earlier versions", () => { // These keys come from an earlier version of the service. A change to // how the service makes keys must not change them, or recordings stop diff --git a/packages/service/tests/service.test.ts b/packages/service/tests/service.test.ts index f334db7..70a68a5 100644 --- a/packages/service/tests/service.test.ts +++ b/packages/service/tests/service.test.ts @@ -29,6 +29,8 @@ import { } from "@sentry/roach/client"; import { describeRecordingMisses } from "@sentry/roach/report"; import { keyInWorker } from "../src/key-worker.ts"; +import { keyRequest } from "../src/request-key.ts"; +import { clientGroup } from "../src/server.ts"; import { startRoachService, type RoachService } from "../src/service.ts"; import type { RemoteRun, RunConfig } from "@sentry/roach/types"; import { VALUE_PATTERNS } from "@sentry/roach/values"; @@ -44,6 +46,17 @@ const runs: RemoteRoach[] = []; const hash = (token: string) => createHash("sha256").update(token).digest("hex"); +/** The heap that this process uses, in bytes. */ +const heap = () => process.memoryUsage().heapUsed; + +/** `count` distinct values `v`, from `v`, with spaces between. */ +const ids = (from: number, count: number) => + Array.from({ length: count }, (_, n) => `v${from + n}`).join(" "); + +/** `count` runs of `z` of different lengths, from 30,000 + `from`. */ +const zRuns = (from: number, count: number) => + Array.from({ length: count }, (_, n) => "z".repeat(30_000 + from + n)); + /** Start a run, with one recorded rule. Without `token`, it has no token. */ async function startRun( token: string | undefined, @@ -742,6 +755,200 @@ describe("limits of the service", () => { expect((await send(writer.url, { prompt })).status).toBe(200); }); + it("fails a request with millions of values before it fills the memory", async () => { + // `replace()` finds all matches before it counts them, so these bodies + // stopped the worker at its heap limit. The count stops at the limit. + const rules = [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { v: "x" }, + }, + ]; + const reader = await startRun(undefined, "replay", { rules }); + for (const body of [ + { v: "x".repeat(8 * 1024 * 1024) }, + "x ".repeat(4 * 1024 * 1024), + ]) { + const started = performance.now(); + const tooMany = await send(reader.url, body); + expect(tooMany.status).toBe(400); + expect(tooMany.body).toMatch(/too many changing values/); + expect(performance.now() - started).toBeLessThan(5000); + } + // The worker did not stop, and the run goes on. + expect((await send(reader.url, { prompt: "x" })).status).toBe(412); + await expect(reader.stats()).resolves.toBeDefined(); + }); + + it("ends a run without a token whose sessions remember too many values", async () => { + // The probe of the review: 64 sessions, each with one 15 MB request of + // distinct values. Before the limit, the main heap grew past 1.2 GB. + const rules = [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { v: "[a-z]{150}" }, + }, + ]; + const before = heap(); + let peak = before; + const sampler = setInterval(() => (peak = Math.max(peak, heap())), 20); + onTestFinished(() => clearInterval(sampler)); + const reader = await startRun(undefined, "replay", { rules }); + const sessions = await Promise.all( + Array.from({ length: 64 }, (_, n) => reader.startSession(`s${n}`)), + ); + let next = 0; + // A distinct value of 150 letters: the number in base 25, with the + // digits `b` to `z`, after `a` as padding. + const value = () => + (next++) + .toString(25) + .replace(/./g, (digit) => + String.fromCharCode(98 + Number.parseInt(digit, 25)), + ) + .padStart(150, "a"); + const results: Array<{ status: number; body: string }> = []; + for (const opened of sessions) { + const words = Array.from({ length: 15_000_000 / 151 }, value); + const result = await send(opened.url, { prompt: words.join(" ") }).catch( + (error: unknown) => ({ status: 407, body: String(error) }), + ); + results.push(result); + if (result.status !== 412) break; + } + expect(results).toHaveLength(1); + expect(results[0]!.status).toBe(400); + expect(results[0]!.body).toMatch( + /over 10000 changing values or 262144 characters.*This run has ended/s, + ); + await expect(reader.stats()).rejects.toThrow(/404/); + expect(peak - before).toBeLessThan(300 * 1024 * 1024); + }); + + it("counts the remembered values of all sessions of a run without a token", async () => { + const rules = [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { v: "v\\d+", z: "z+" }, + }, + ]; + + // 6,000 and then 5,000 values of two open sessions are over 10,000. + const values = await startRun(undefined, "replay", { rules }); + const first = await values.startSession("first"); + const second = await values.startSession("second"); + expect((await send(first.url, { prompt: ids(0, 6000) })).status).toBe(412); + const over = await send(second.url, { prompt: ids(6000, 5000) }); + expect(over.status).toBe(400); + expect(over.body).toMatch(/over 10000 changing values/); + await expect(values.stats()).rejects.toThrow(/404/); + + // A session forgets its values when it ends. + const ended = await startRun(undefined, "replay", { rules }); + const early = await ended.startSession("early"); + expect((await send(early.url, { prompt: ids(0, 6000) })).status).toBe(412); + await early.end(true); + const late = await ended.startSession("late"); + expect((await send(late.url, { prompt: ids(6000, 6000) })).status).toBe( + 412, + ); + // Values that the session knows do not count again. + expect((await send(late.url, { prompt: ids(6000, 6000) })).status).toBe( + 412, + ); + + // Nine values of about 30,000 characters are over 256 Ki characters. + const chars = await startRun(undefined, "replay", { rules }); + const long = await chars.startSession("long"); + const eight = await send(long.url, { prompt: zRuns(0, 8).join(" ") }); + expect(eight.status).toBe(412); + const ninth = await send(long.url, { prompt: zRuns(8, 1).join(" ") }); + expect(ninth.status).toBe(400); + expect(ninth.body).toMatch(/or 262144 characters/); + + // A run with the token has no limit. + const writer = await startRun("write-token", "auto", { rules }); + const opened = await writer.startSession("writer"); + expect((await send(opened.url, { prompt: ids(0, 20_000) })).status).toBe( + 200, + ); + }); + + it("leaves a value of over 32,767 characters out of the memory of a session", async () => { + const rule = { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { v: "y+" }, + }; + const toKey = (prompt: string) => ({ + method: "POST", + url: `${origin}/v1/messages`, + headers: {}, + body: JSON.stringify({ prompt }), + }); + const long = "y".repeat(40_000); + // Before, the session remembered the value, and V8 does not run the + // pattern of a value this long. So each later request failed and had no + // key. To leave the value out changes no key. + expect(() => + keyRequest(rule, toKey("later y"), new Map([[long, "v"]])), + ).toThrow(/too large/); + + for (const [token, mode, status] of [ + [undefined, "replay", 412], + ["write-token", "auto", 200], + ] as const) { + const run = await startRun(token, mode, { rules: [rule] }); + const opened = await run.startSession("long"); + expect((await send(opened.url, { prompt: long })).status).toBe(status); + for (let n = 0; n < 3; n += 1) { + expect( + (await send(opened.url, { prompt: `later ${n} y` })).status, + ).toBe(status); + } + await opened.end(true); + await expect(run.stats()).resolves.toBeDefined(); + } + // Each recording has the key of its request without the long value. + const keys = [long, "later 0 y", "later 1 y", "later 2 y"].map( + (prompt) => `${keyRequest(rule, toKey(prompt)).key}.json`, + ); + expect((await recordings("acme/alpha")).toSorted()).toEqual( + keys.toSorted(), + ); + }); + + it("keys a 60 MiB body of small text blocks from a run with the token", async () => { + // The worker has a heap limit for runs without the token. A run with + // the token makes its keys on the main thread, so a body that keyed + // before the worker still keys. + const writer = await startRun("write-token", "auto", { + rules: [ + { + name: "model", + match: { method: "POST", url: `${origin}/v1/` }, + values: { uuid: VALUE_PATTERNS.uuid, time: VALUE_PATTERNS.isoTime }, + }, + ], + }); + const block = { type: "text", text: "hello there" }; + const count = Math.floor( + (60 * 1024 * 1024) / (JSON.stringify(block).length + 1), + ); + const body = { + model: "m", + messages: [ + { role: "user", content: Array.from({ length: count }, () => block) }, + ], + }; + const sent = await send(writer.url, body); + expect(sent).toMatchObject({ status: 200, source: "live" }); + expect(await writer.close()).toMatchObject({ written: 1 }); + }, 120_000); + it("ends a run without a token whose key fills the memory of the worker", async () => { // Each `{}` is an object for `JSON.parse()`, so the body needs more // memory than the worker has. @@ -759,8 +966,8 @@ describe("limits of the service", () => { expect(Math.max(...waits)).toBeLessThan(500); await expect(reader.stats()).rejects.toThrow(/404/); // The next key gets a new worker. - const writer = await startRun("write-token", "auto"); - expect((await send(writer.url, { prompt: "hi" })).status).toBe(200); + const next = await startRun(undefined, "replay"); + expect((await send(next.url, { prompt: "hi" })).status).toBe(412); }); it("stops a key that the time limit of vm cannot stop", async () => { @@ -770,10 +977,11 @@ describe("limits of the service", () => { const options = { timeLimitMs: 1, maxValues: Infinity, - first: true, skip: () => false, }; - const body = `[${"{},".repeat(16 * 1024 * 1024)}{}]`; + // An object with 2.1 M keys takes about 2 s to parse, and it fits in + // the heap of the worker. + const body = `{${Array.from({ length: 2_100_000 }, (_, n) => `"k${n}":0`).join(",")}}`; const started = performance.now(); const error = await keyInWorker( rule, @@ -1055,4 +1263,20 @@ describe("limits of the service", () => { expect(ended.status).toBe(200); expect((await startFrom("203.0.113.1")).status).toBe(201); }); + + it("groups an IPv4 address in each IPv6 form as the IPv4 address", () => { + for (const address of [ + "192.0.2.1", + "::ffff:192.0.2.1", + "::FFFF:c000:201", + "0:0:0:0:0:ffff:192.0.2.1", + "0000:0000:0000:0000:0000:ffff:c000:0201", + ]) { + expect(clientGroup(address)).toBe("192.0.2.1"); + } + expect(clientGroup("2001:db8:0:1:ffff::1")).toBe("2001:db8:0:1::/64"); + expect(clientGroup("2001:DB8:0:1::1%eth0")).toBe("2001:db8:0:1::/64"); + // An address that only ends in IPv4 form is IPv6. + expect(clientGroup("::ffff:0:192.0.2.1")).toBe("0:0:0:0::/64"); + }); }); From 25eda67c862cf59aad384ceeec30ca05ce81af28 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:45:20 +0000 Subject: [PATCH 2/3] fix(service): Limit the request bodies that runs without a token hold Address the review of #48: - Runs without the token share a budget of 128 MiB of request bodies. A request holds its bytes from when they come until it finishes, also while its key waits for the key worker. A request that does not fit gets 503 with `Retry-After: 1`, and the run goes on. A client that declares a large body and then stops holds only what it sent. 30 and 60 parallel 16 MiB bodies now peak at about 1 GB of RSS for the service and the worker, down from 1.9 GB and 3.4 GB. - README: the deploy order resets the VM twice, and a reset ends open runs. Document the body budget. - Fix the doc comment of `canBeKnown()`: V8 fails on any text. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- README.md | 14 ++++- packages/roach/src/values.ts | 8 ++- packages/service/src/server.ts | 86 +++++++++++++++++++++++--- packages/service/src/service.ts | 21 ++++++- packages/service/tests/service.test.ts | 54 ++++++++++++++++ 5 files changed, 169 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3083276..caf6fb6 100644 --- a/README.md +++ b/README.md @@ -697,6 +697,12 @@ To see how well the cache works: - A proxied request body over 64 MiB gets HTTP 413, and over 16 MiB for a run without the token. A control API body over 1 MiB gets HTTP 413 before the service parses it. +- The requests of all runs without the token can hold 128 MiB of bodies + at one time. A request holds its body until it finishes, also while its + key waits for the key worker. A request whose body does not fit gets + HTTP 503 with `Retry-After: 1`, and the run goes on. The clients of + model APIs send a 503 again. A client that does not gets the 503, and + its test fails. Runs with the token have no such limit. - Keys and the redaction of misses have time limits. See [Changing values](#changing-values). - A `CONNECT` tunnel that sends no byte for 10 s after the `200` closes. A @@ -826,9 +832,11 @@ turn on the PROXY header, apply in this order. Run the commands in terraform apply -var-file=roach.tfvars ``` -No step fails requests. From step 3 to step 4, the service sees the -addresses of the load balancer, so all clients share the limit of runs -for each client. +No step fails requests with HTTP 400. But step 1 and step 3 each reset +the VM, and a reset ends the open runs, so their CI jobs fail (see +[Deploy](#deploy)). Do step 3 when few runs are open. From step 3 to +step 4, the service sees the addresses of the load balancer, so all +clients share the limit of runs for each client. The load balancer writes its header before the first byte of the client, so a client cannot put its own header first. A client can send its own diff --git a/packages/roach/src/values.ts b/packages/roach/src/values.ts index 5171640..ac1b636 100644 --- a/packages/roach/src/values.ts +++ b/packages/roach/src/values.ts @@ -213,9 +213,11 @@ export function extractValues( /** * Whether `value` can be in `known` or in a template (`templateValues()`). - * V8 does not run a pattern with a literal of over 32,767 characters on a - * text that can hold it. Such a value in `known` would fail each later - * request of its session that could repeat it, so to leave it out changes + * V8 cannot run a pattern with a literal of over 32,767 characters on + * any text, also on a text of 5 characters. (A literal with a character + * over U+00FF fails only on a text with such a character, and only such a + * text can hold it.) Such a value in `known` failed each later request of + * its session, so those requests had no key, and to leave it out changes * no key. */ export const canBeKnown = (value: string): boolean => value.length <= 32_767; diff --git a/packages/service/src/server.ts b/packages/service/src/server.ts index db61450..4b12ed2 100644 --- a/packages/service/src/server.ts +++ b/packages/service/src/server.ts @@ -164,13 +164,53 @@ function authorityOf(origin: URL): string { } /** - * Read a whole body. A body over `maxBytes` fails with HTTP 413. Only + * The bytes of request bodies that the requests of many targets can hold + * at one time (`ProxyTarget.bodyBudget`). `used` counts the bodies of the + * requests that did not finish. + */ +export interface BodyBudget { + max: number; + used: number; +} + +/** The bytes that one request holds of its `BodyBudget`. */ +interface Held { + budget: BodyBudget | undefined; + bytes: number; +} + +/** Hold `bytes` more of the budget. Returns `false` when they do not fit. */ +function hold(held: Held, bytes: number): boolean { + const { budget } = held; + if (!budget) return true; + if (budget.used + bytes > budget.max) return false; + budget.used += bytes; + held.bytes += bytes; + return true; +} + +/** Give back all bytes that a request holds. */ +function release(held: Held): void { + if (held.budget) held.budget.used -= held.bytes; + held.bytes = 0; +} + +const busyError = () => + httpError( + 503, + "the service has too many large requests at this time. Send the request again later.", + ); + +/** + * Read a whole body. A body over `maxBytes` fails with HTTP 413. A body + * that does not fit in the budget of `held` fails with HTTP 503. Only * request bodies have a limit: the client sends them, and a large upstream * response is not the fault of the client. */ function readBody( stream: http.IncomingMessage, maxBytes = Infinity, + held: Held = { budget: undefined, bytes: 0 }, ): Promise { const declared = Number(stream.headers["content-length"]); if (declared > maxBytes) { @@ -178,15 +218,26 @@ function readBody( stream.resume(); return Promise.reject(httpError(413, `body is over ${maxBytes} bytes`)); } + // Hold the bytes when they come, not the declared length. So a client + // that declares a large body and then sends nothing holds nothing. return new Promise((resolve, reject) => { const chunks: Buffer[] = []; let size = 0; + let failed = false; stream.on("data", (chunk: Buffer) => { + if (failed) return; size += chunk.length; - if (size > maxBytes) { - // Read the rest without keeping it, so the client gets the 413. + const error = + size > maxBytes + ? httpError(413, `body is over ${maxBytes} bytes`) + : !hold(held, chunk.length) + ? busyError() + : undefined; + if (error) { + // Read the rest without keeping it, so the client gets the error. + failed = true; chunks.length = 0; - reject(httpError(413, `body is over ${maxBytes} bytes`)); + reject(error); return; } chunks.push(chunk); @@ -292,6 +343,7 @@ async function proxyRequest( incoming: http.IncomingMessage, outgoing: http.ServerResponse, tunnelOrigin: string | undefined, + held: Held, ): Promise { const url = new URL( tunnelOrigin ? `${tunnelOrigin}${incoming.url}` : (incoming.url ?? ""), @@ -308,7 +360,7 @@ async function proxyRequest( method: incoming.method ?? "GET", url, headers: incoming.headers, - body: await readBody(incoming, maxBodyBytes), + body: await readBody(incoming, maxBodyBytes, held), }; const rule = recorder.ruleFor(request); @@ -432,6 +484,12 @@ export interface ProxyTarget { closed?: boolean; /** A request body over this size gets HTTP 413. Default: 64 MiB. */ maxBodyBytes?: number; + /** + * The budget that the request bodies of this target take from, until + * each request finishes. A body that does not fit gets HTTP 503, and the + * run goes on. Targets can share one budget. + */ + bodyBudget?: BodyBudget; /** The time of the last proxied request, from `Date.now()`. */ lastUsed?: number; /** Gets a span for each proxied request. */ @@ -568,11 +626,21 @@ function serveProxied( attributes: { "roach.origin": origin, "http.request.method": method }, }, async () => { + const held: Held = { budget: target.bodyBudget, bytes: 0 }; try { - await proxyRequest(target, session, incoming, outgoing, tunnelOrigin); + await proxyRequest( + target, + session, + incoming, + outgoing, + tunnelOrigin, + held, + ); } catch (error) { // In the span, so that the Sentry event is in the trace. answerError(target, origin, error, outgoing); + } finally { + release(held); } telemetry.annotate({ "http.response.status_code": outgoing.statusCode, @@ -598,7 +666,11 @@ function answerError( ? reportInternal("Request", error, target.telemetry.capture(error)) : (error as Error).message; if (!outgoing.headersSent) { - outgoing.writeHead(status ?? 502, { "content-type": "text/plain" }); + outgoing.writeHead(status ?? 502, { + "content-type": "text/plain", + // Clients of model APIs send a 503 again, after this time in s. + ...(status === 503 ? { "retry-after": "1" } : {}), + }); } outgoing.end(`Roach error: ${text}\n`); } diff --git a/packages/service/src/service.ts b/packages/service/src/service.ts index 3a8f398..073d2fd 100644 --- a/packages/service/src/service.ts +++ b/packages/service/src/service.ts @@ -57,6 +57,7 @@ import { readJson, sameToken, sendJson, + type BodyBudget, type ProxyTarget, type TargetResult, } from "./server.ts"; @@ -164,6 +165,16 @@ const MAX_PUBLIC_RUNS = 100; const MAX_PUBLIC_RUNS_PER_CLIENT = 10; /** A proxied request body of a run without a token over this gets 413. */ const PUBLIC_MAX_BODY_BYTES = 16 * 1024 * 1024; +/** + * The bytes of request bodies that all runs without a token can hold at + * one time. A request holds its body until it finishes, also while its key + * waits for the key worker. A body that does not fit gets HTTP 503, and + * the run goes on. With 128 MiB, 30 or 60 parallel bodies of 16 MiB take + * at most about 1.1 GB of RSS for the service and the key worker. Without + * the budget, 60 took 3.4 GB. The largest model request in the 14 days + * to 2026-11-05 was about 0.65 MB. + */ +const PUBLIC_BODY_BUDGET_BYTES = 128 * 1024 * 1024; /** * The work of a run without a token (`WorkLimit`). A key has 1 s and 0.5 s * for each MiB of its body. One job in the key worker (`key-worker.ts`) has @@ -528,6 +539,8 @@ export async function startRoachService( }); } const runs = new Map(); + /** The request bodies of all runs without a token. */ + const publicBodies: BodyBudget = { max: PUBLIC_BODY_BUDGET_BYTES, used: 0 }; const authority = await createCertificateAuthority(config.ca); const storeOf = (tenant: string): RecordingStore => config.bucket @@ -573,7 +586,13 @@ export async function startRoachService( ), // A public run must not be an open proxy to the allowed origins. replayOnly: !canWrite, - ...(canWrite ? {} : { client, maxBodyBytes: PUBLIC_MAX_BODY_BYTES }), + ...(canWrite + ? {} + : { + client, + maxBodyBytes: PUBLIC_MAX_BODY_BYTES, + bodyBudget: publicBodies, + }), privateNetworks: config.privateNetworks === true, // Each run learns only the credentials of its own requests. recorder: createRecorder( diff --git a/packages/service/tests/service.test.ts b/packages/service/tests/service.test.ts index 70a68a5..96d2422 100644 --- a/packages/service/tests/service.test.ts +++ b/packages/service/tests/service.test.ts @@ -12,6 +12,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { ProxyAgent, request } from "undici"; import { afterEach, @@ -46,6 +47,13 @@ const runs: RemoteRoach[] = []; const hash = (token: string) => createHash("sha256").update(token).digest("hex"); +/** A body that sends one byte and then nothing. */ +function stalledBody(): Readable { + const body = new Readable({ read() {} }); + body.push("{"); + return body; +} + /** The heap that this process uses, in bytes. */ const heap = () => process.memoryUsage().heapUsed; @@ -98,6 +106,7 @@ async function send( status: response.statusCode, body: await response.body.text(), source: response.headers["x-roach"], + retryAfter: response.headers["retry-after"], }; } @@ -755,6 +764,51 @@ describe("limits of the service", () => { expect((await send(writer.url, { prompt })).status).toBe(200); }); + it("answers 503 when the bodies of runs without a token fill the budget", async () => { + // Each request holds its body until its key is done. Before the budget, + // 30 parallel bodies of 16 MiB took about 1.9 GB. + const reader = await startRun(undefined, "replay"); + const prompt = "a".repeat(16 * 1024 * 1024 - 64); + const results = await Promise.all( + Array.from({ length: 12 }, () => send(reader.url, { prompt })), + ); + // The budget holds 128 MiB, 8 of these bodies. The bytes count when + // they come, so parts of more bodies can fill it. + const busy = results.filter((result) => result.status === 503); + expect(busy.length).toBeGreaterThanOrEqual(4); + for (const result of results) expect([412, 503]).toContain(result.status); + for (const result of busy) { + expect(result.body).toMatch(/too many large requests/); + expect(result.retryAfter).toBe("1"); + } + // The run goes on, and the bodies gave their bytes back. + await expect(reader.stats()).resolves.toBeDefined(); + const again = await Promise.all( + Array.from({ length: 4 }, () => send(reader.url, { prompt })), + ); + expect(again.map((result) => result.status)).toEqual([412, 412, 412, 412]); + + // A client that declares large bodies and then stops holds only the + // bytes that it sent. + const stalled = new AbortController(); + const stalls = Array.from({ length: 9 }, () => { + const agent = new ProxyAgent({ uri: reader.url, proxyTunnel: true }); + agents.push(agent); + return request(`${origin}/v1/messages`, { + method: "POST", + dispatcher: agent, + headers: { "content-length": String(16 * 1024 * 1024 - 64) }, + body: stalledBody(), + signal: stalled.signal, + }).catch(() => undefined); + }); + // Give the stalled requests the time to reach the service. + await new Promise((resolve) => setTimeout(resolve, 300)); + expect((await send(reader.url, { prompt })).status).toBe(412); + stalled.abort(); + await Promise.all(stalls); + }); + it("fails a request with millions of values before it fills the memory", async () => { // `replace()` finds all matches before it counts them, so these bodies // stopped the worker at its heap limit. The count stops at the limit. From 8812e5c34b66f688079114a1c9a5b31d84df6896 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:29:30 +0000 Subject: [PATCH 3/3] fix(service): Give each client a part of the body budget, and time out slow bodies One client that started a run without a token could hold all 128 MiB of the body budget: 8 tunnels that each sent 16 MiB - 1 KiB and then one byte now and then. Node does not time out requests in a tunnel, because the tunnel server does not listen, so nothing gave the bytes back. Every other run without a token then got HTTP 503. - The runs of one client (`clientGroup()` of the client that started the run) can hold 32 MiB of the budget. - A body of a run without a token that does not come in 30 s gets HTTP 408, the connection closes, and the bytes go back. - Count a 503 as `roach.request` with `roach.result:busy`, and a 408 as `roach.result:timeout`, and add a TELEMETRY.md query. - README: the new limits, the memory bound for write bodies of a normal size, and the gap that Node does not time out tunneled requests. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- README.md | 38 +++++-- TELEMETRY.md | 19 +++- packages/service/src/server.ts | 109 ++++++++++++++----- packages/service/src/service.ts | 27 +++-- packages/service/tests/service.test.ts | 143 ++++++++++++++++++++++++- 5 files changed, 288 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index caf6fb6..3f40b02 100644 --- a/README.md +++ b/README.md @@ -606,11 +606,11 @@ attribute of Roach starts with `roach.`. Metrics have only attributes with few values: `roach.tenant`, `roach.mode`, `roach.rule`, `roach.origin`, `roach.result`, and `roach.reason`. -| Metric | Type | Value | -| ------------------------ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `roach.request` | counter | 1 for each proxied request. `roach.result` is `replayed`, `live`, `missed`, `passthrough`, or `refused`. A `passthrough` or `refused` request has no `roach.rule`. A `refused` request has `roach.reason`, and `other` for an origin not in `allow`. | -| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `roach.result`. | -| `roach.recording.write` | counter | Recordings by `roach.rule`. `roach.result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | +| Metric | Type | Value | +| ------------------------ | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `roach.request` | counter | 1 for each proxied request. `roach.result` is `replayed`, `live`, `missed`, `passthrough`, `refused`, `busy` (HTTP 503: the body did not fit in the budget), or `timeout` (HTTP 408: the body did not come in time). A `passthrough`, `refused`, `busy`, or `timeout` request has no `roach.rule`. A `refused` request has `roach.reason`, and `other` for an origin not in `allow`. | +| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `roach.result`. | +| `roach.recording.write` | counter | Recordings by `roach.rule`. `roach.result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | Each proxied request is one trace. Each span has `roach.tenant`, `roach.mode`, `roach.run.id` (the `name` of the run, else its id), and the @@ -698,11 +698,20 @@ To see how well the cache works: run without the token. A control API body over 1 MiB gets HTTP 413 before the service parses it. - The requests of all runs without the token can hold 128 MiB of bodies - at one time. A request holds its body until it finishes, also while its - key waits for the key worker. A request whose body does not fit gets - HTTP 503 with `Retry-After: 1`, and the run goes on. The clients of - model APIs send a 503 again. A client that does not gets the 503, and - its test fails. Runs with the token have no such limit. + at one time, and the runs of one client 32 MiB. The client of a run is + the client that started it, grouped as for the limit of runs. A request + holds its body until it finishes, also while its key waits for the key + worker. A request whose body does not fit gets HTTP 503 with + `Retry-After: 1`, and the run goes on. The clients of model APIs send a + 503 again. A client that does not gets the 503, and its test fails. + Runs with the token have no such limit. +- A body of a run without the token that does not come in 30 s gets + HTTP 408, and its connection closes. The run goes on. +- With these limits, the service and the key worker use at most about + 1.1 GB of the 2 GB of the VM. That is for request bodies of runs with + the token of a normal size. A body of 64 MiB from a run with the token, + at the same time as runs without the token fill the budget, took about + 2.0 GB. - Keys and the redaction of misses have time limits. See [Changing values](#changing-values). - A `CONNECT` tunnel that sends no byte for 10 s after the `200` closes. A @@ -712,7 +721,14 @@ To see how well the cache works: [Changing values](#changing-values). - A gap: a client with an IPv6 /48 has 65,536 /64s, so it can open runs until the service has 100 runs without a token. Then other clients get - HTTP 429 until its runs end. + HTTP 429 until its runs end. Its runs can also hold all of the budget of + bodies, for 30 s at a time. +- A gap: Node times out the headers and the body of a request only on a + server that listens. The servers of tunnels, and the HTTP server with + `proxyProtocol`, get their connections from another server. So a + request of a run with the token, and a control API call with + `proxyProtocol`, have no time limit for their headers and body. A + tunnel closes only after 10 minutes without a byte. - GCS deletes each recording 30 days after it was written, also when runs still replay it. Then `auto` mode records it again, and `replay` mode fails the request with HTTP 412. diff --git a/TELEMETRY.md b/TELEMETRY.md index ad45555..e5e8912 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -50,6 +50,11 @@ For metrics, use `-f dataset=tracemetrics`, `roach.reason` tells why: `notAllowed` (the origin is not in `allow`), `notPublic` (the address is not public), or `replayOnly` (a run without the token, and no rule records the request). +- **Busy**: the body of a run without the token did not fit in the + budget of bodies, and the proxy answered HTTP 503 with `Retry-After`. + The run goes on, and the client can send the request again. +- **Timeout**: the body of a run without the token did not come in 30 s, + and the proxy answered HTTP 408. - **Dropped**: a live response was recorded, but its session failed, so the recording was not written. The next run sends the request live again. @@ -96,8 +101,8 @@ The examples use the `sentry` CLI. The same fields work in Explore. The counts are estimates (see [Sampling](#sampling)). Cached and uncached requests by rule. The hit rate of a rule is -`replayed / (replayed + live + missed)`. Do not count `passthrough` or -`refused`. +`replayed / (replayed + live + missed)`. Do not count `passthrough`, +`refused`, `busy`, or `timeout`. ```bash sentry explore sentry/roach --dataset metrics --metric roach.request --agg sum \ @@ -121,6 +126,16 @@ sentry explore sentry/roach --dataset metrics --metric roach.request --agg sum \ -F roach.origin ``` +Requests of runs without the token that the budget of bodies stopped +(`busy`, HTTP 503) or that were too slow (`timeout`, HTTP 408), by tenant. +A steady count of `busy` from many tenants shows that the budget is too +small. A count from one tenant shows one client that fills its part. + +```bash +sentry explore sentry/roach --dataset metrics --metric roach.request --agg sum \ + --period 24h -q 'roach.result:[busy,timeout]' -F roach.tenant -F roach.result +``` + Time saved by a replay. Compare `live` and `replayed` for each rule. Use `--agg p95` for the slow tail. diff --git a/packages/service/src/server.ts b/packages/service/src/server.ts index 4b12ed2..38987ca 100644 --- a/packages/service/src/server.ts +++ b/packages/service/src/server.ts @@ -166,33 +166,49 @@ function authorityOf(origin: URL): string { /** * The bytes of request bodies that the requests of many targets can hold * at one time (`ProxyTarget.bodyBudget`). `used` counts the bodies of the - * requests that did not finish. + * requests that did not finish, and `clients` counts them for each client + * (`ProxyTarget.client`). One client can hold at most `perClient` bytes, + * so it cannot take all of `max`. A body that takes longer than `readMs` + * to come fails with HTTP 408 and gives its bytes back. */ export interface BodyBudget { max: number; + perClient: number; + readMs: number; used: number; + clients: Map; } /** The bytes that one request holds of its `BodyBudget`. */ interface Held { budget: BodyBudget | undefined; + client: string; bytes: number; } /** Hold `bytes` more of the budget. Returns `false` when they do not fit. */ function hold(held: Held, bytes: number): boolean { - const { budget } = held; + const { budget, client } = held; if (!budget) return true; - if (budget.used + bytes > budget.max) return false; + const ofClient = budget.clients.get(client) ?? 0; + if (budget.used + bytes > budget.max || ofClient + bytes > budget.perClient) { + return false; + } budget.used += bytes; + budget.clients.set(client, ofClient + bytes); held.bytes += bytes; return true; } /** Give back all bytes that a request holds. */ function release(held: Held): void { - if (held.budget) held.budget.used -= held.bytes; + const { budget, client, bytes } = held; held.bytes = 0; + if (!budget || bytes === 0) return; + budget.used -= bytes; + const left = (budget.clients.get(client) ?? 0) - bytes; + if (left > 0) budget.clients.set(client, left); + else budget.clients.delete(client); } const busyError = () => @@ -203,14 +219,15 @@ const busyError = () => /** * Read a whole body. A body over `maxBytes` fails with HTTP 413. A body - * that does not fit in the budget of `held` fails with HTTP 503. Only - * request bodies have a limit: the client sends them, and a large upstream - * response is not the fault of the client. + * that does not fit in the budget of `held` fails with HTTP 503, and one + * that does not come in the `readMs` of the budget fails with HTTP 408. + * Only request bodies have a limit: the client sends them, and a large + * upstream response is not the fault of the client. */ function readBody( stream: http.IncomingMessage, maxBytes = Infinity, - held: Held = { budget: undefined, bytes: 0 }, + held: Held = { budget: undefined, client: "", bytes: 0 }, ): Promise { const declared = Number(stream.headers["content-length"]); if (declared > maxBytes) { @@ -224,26 +241,46 @@ function readBody( const chunks: Buffer[] = []; let size = 0; let failed = false; + const fail = (error: Error) => { + // Read the rest without keeping it, so the client gets the error. + failed = true; + chunks.length = 0; + clearTimeout(deadline); + reject(error); + }; + // Node does not time out a request in a tunnel (`listenProxy()`), so + // a client that sends a byte now and then could hold its bytes for + // as long as the tunnel stays open. + const readMs = held.budget?.readMs; + const deadline = + readMs === undefined + ? undefined + : setTimeout( + () => + fail( + httpError(408, `the body did not come in ${readMs / 1000} s`), + ), + readMs, + ); stream.on("data", (chunk: Buffer) => { if (failed) return; size += chunk.length; - const error = - size > maxBytes - ? httpError(413, `body is over ${maxBytes} bytes`) - : !hold(held, chunk.length) - ? busyError() - : undefined; - if (error) { - // Read the rest without keeping it, so the client gets the error. - failed = true; - chunks.length = 0; - reject(error); - return; + if (size > maxBytes) { + fail(httpError(413, `body is over ${maxBytes} bytes`)); + } else if (!hold(held, chunk.length)) { + fail(busyError()); + } else { + chunks.push(chunk); } - chunks.push(chunk); }); - stream.on("end", () => resolve(Buffer.concat(chunks))); - stream.on("error", reject); + stream.on("end", () => { + clearTimeout(deadline); + resolve(Buffer.concat(chunks)); + }); + stream.on("error", (error) => { + clearTimeout(deadline); + reject(error); + }); }); } @@ -490,6 +527,12 @@ export interface ProxyTarget { * run goes on. Targets can share one budget. */ bodyBudget?: BodyBudget; + /** + * The client whose part of `bodyBudget` the requests take. The service + * sets it for a run without the token: the address group + * (`clientGroup()`) of the client that started the run. + */ + client?: string; /** The time of the last proxied request, from `Date.now()`. */ lastUsed?: number; /** Gets a span for each proxied request. */ @@ -626,7 +669,11 @@ function serveProxied( attributes: { "roach.origin": origin, "http.request.method": method }, }, async () => { - const held: Held = { budget: target.bodyBudget, bytes: 0 }; + const held: Held = { + budget: target.bodyBudget, + client: target.client ?? "", + bytes: 0, + }; try { await proxyRequest( target, @@ -665,11 +712,25 @@ function answerError( status === undefined ? reportInternal("Request", error, target.telemetry.capture(error)) : (error as Error).message; + // A body that did not fit in the budget, or that did not come in time, + // is not a miss and not an error. Count it, so that Sentry shows it. + const result = + status === 503 ? "busy" : status === 408 ? "timeout" : undefined; + if (result) { + target.telemetry.annotate({ "roach.result": result }); + target.telemetry.count("roach.request", 1, { + "roach.origin": origin, + "roach.result": result, + }); + } if (!outgoing.headersSent) { outgoing.writeHead(status ?? 502, { "content-type": "text/plain", // Clients of model APIs send a 503 again, after this time in s. ...(status === 503 ? { "retry-after": "1" } : {}), + // The rest of a slow body does not come in time, so close the + // connection and do not wait for it. + ...(status === 408 ? { connection: "close" } : {}), }); } outgoing.end(`Roach error: ${text}\n`); diff --git a/packages/service/src/service.ts b/packages/service/src/service.ts index 073d2fd..8a47b8b 100644 --- a/packages/service/src/service.ts +++ b/packages/service/src/service.ts @@ -132,11 +132,6 @@ interface Run extends ProxyTarget { token: string; started: number; lastUsed: number; - /** - * The address group (`clientGroup()`) of the client that started a run - * without a token. - */ - client?: string; } const MODES = new Set(["auto", "off", "record", "replay"]); @@ -175,6 +170,20 @@ const PUBLIC_MAX_BODY_BYTES = 16 * 1024 * 1024; * to 2026-11-05 was about 0.65 MB. */ const PUBLIC_BODY_BUDGET_BYTES = 128 * 1024 * 1024; +/** + * The part of `PUBLIC_BODY_BUDGET_BYTES` that the runs of one client + * (`clientGroup()` of the client that started the run) can hold, so that + * one client cannot take all of it. It is 2 bodies of 16 MiB, or about 50 + * of the largest model requests. + */ +const PUBLIC_BODY_CLIENT_BYTES = 32 * 1024 * 1024; +/** + * A body of a run without a token that does not come in this time gets + * HTTP 408, and gives its bytes back. So a client that sends a byte now + * and then cannot hold its part of the budget. 16 MiB in 30 s is about + * 4.5 Mbit/s. + */ +const PUBLIC_BODY_READ_MS = 30_000; /** * The work of a run without a token (`WorkLimit`). A key has 1 s and 0.5 s * for each MiB of its body. One job in the key worker (`key-worker.ts`) has @@ -540,7 +549,13 @@ export async function startRoachService( } const runs = new Map(); /** The request bodies of all runs without a token. */ - const publicBodies: BodyBudget = { max: PUBLIC_BODY_BUDGET_BYTES, used: 0 }; + const publicBodies: BodyBudget = { + max: PUBLIC_BODY_BUDGET_BYTES, + perClient: PUBLIC_BODY_CLIENT_BYTES, + readMs: PUBLIC_BODY_READ_MS, + used: 0, + clients: new Map(), + }; const authority = await createCertificateAuthority(config.ca); const storeOf = (tenant: string): RecordingStore => config.bucket diff --git a/packages/service/tests/service.test.ts b/packages/service/tests/service.test.ts index 96d2422..16d7e61 100644 --- a/packages/service/tests/service.test.ts +++ b/packages/service/tests/service.test.ts @@ -54,6 +54,75 @@ function stalledBody(): Readable { return body; } +/** Wait `ms` of real time, also when `setTimeout` is fake. */ +async function realWait(ms: number) { + const end = performance.now() + ms; + while (performance.now() < end) { + await new Promise((resolve) => setImmediate(resolve)); + } +} + +/** Wait until `check` is true, for at most 10 s of real time. */ +async function until(check: () => boolean) { + const end = performance.now() + 10_000; + while (!check()) { + if (performance.now() > end) throw new Error("the wait timed out"); + await new Promise((resolve) => setImmediate(resolve)); + } +} + +/** + * Open a `CONNECT` tunnel of `run` to the upstream on a raw socket, and + * start a request that declares a body of `declared` bytes. It sends + * `sent` bytes of it, and then nothing. `reply` is all that the tunnel + * gets until it closes, and `status()` is the status that it got so far. + */ +async function startBody( + proxy: string, + run: RemoteRun, + declared: number, + sent: number, +) { + const { hostname, port } = new URL(proxy); + const { username, password } = new URL(run.url); + const authority = new URL(origin).host; + const socket = connect(Number(port), hostname); + onTestFinished(() => { + socket.destroy(); + }); + await once(socket, "connect"); + const credentials = Buffer.from( + `${decodeURIComponent(username)}:${decodeURIComponent(password)}`, + ).toString("base64"); + socket.write( + `CONNECT ${authority} HTTP/1.1\r\nHost: ${authority}\r\nProxy-Authorization: Basic ${credentials}\r\n\r\n`, + ); + const [established] = (await once(socket, "data")) as [Buffer]; + expect(established.toString()).toMatch(/^HTTP\/1.1 200/); + const chunks: Buffer[] = []; + socket.on("data", (chunk: Buffer) => chunks.push(chunk)); + socket.on("error", () => socket.destroy()); + const reply = once(socket, "close").then(() => + Buffer.concat(chunks).toString(), + ); + socket.write( + `POST /v1/messages HTTP/1.1\r\nhost: ${authority}\r\ncontent-length: ${declared}\r\n\r\n`, + ); + socket.write(Buffer.alloc(sent, "a")); + const status = () => + Number( + /^HTTP\/1.1 (\d{3})/.exec(Buffer.concat(chunks).toString())?.[1] ?? 0, + ); + return { socket, reply, status }; +} + +/** The status of a whole body of `bytes` through a raw tunnel of `run`. */ +async function answer(proxy: string, run: RemoteRun, bytes: number) { + const { status } = await startBody(proxy, run, bytes, bytes); + await until(() => status() > 0); + return status(); +} + /** The heap that this process uses, in bytes. */ const heap = () => process.memoryUsage().heapUsed; @@ -719,9 +788,11 @@ describe("limits of the service", () => { }, ], }); + // 7 bodies of 4 MiB fit in the 32 MiB that the runs of one client can + // hold. const prompt = "a".repeat(4 * 1024 * 1024); const sent = Promise.allSettled( - Array.from({ length: 20 }, () => send(run.url, { prompt })), + Array.from({ length: 7 }, () => send(run.url, { prompt })), ); const waits = await caWaitsUntil(sent); // The keys are made in a worker, so the service answers at once. @@ -772,8 +843,8 @@ describe("limits of the service", () => { const results = await Promise.all( Array.from({ length: 12 }, () => send(reader.url, { prompt })), ); - // The budget holds 128 MiB, 8 of these bodies. The bytes count when - // they come, so parts of more bodies can fill it. + // The runs of one client can hold 32 MiB, 2 of these bodies. The bytes + // count when they come, so parts of more bodies can fill it. const busy = results.filter((result) => result.status === 503); expect(busy.length).toBeGreaterThanOrEqual(4); for (const result of results) expect([412, 503]).toContain(result.status); @@ -784,9 +855,9 @@ describe("limits of the service", () => { // The run goes on, and the bodies gave their bytes back. await expect(reader.stats()).resolves.toBeDefined(); const again = await Promise.all( - Array.from({ length: 4 }, () => send(reader.url, { prompt })), + Array.from({ length: 2 }, () => send(reader.url, { prompt })), ); - expect(again.map((result) => result.status)).toEqual([412, 412, 412, 412]); + expect(again.map((result) => result.status)).toEqual([412, 412]); // A client that declares large bodies and then stops holds only the // bytes that it sent. @@ -809,6 +880,68 @@ describe("limits of the service", () => { await Promise.all(stalls); }); + it("keeps the budget for other clients while one client stalls its bodies", async () => { + const local = await startRoachService({ + directory, + privateNetworks: true, + writeTokenHash: hash("write-token"), + proxyProtocol: true, + }); + onTestFinished(() => local.close()); + const config = { + tenant: "acme/alpha", + mode: "replay", + allow: [origin], + rules: [ + { name: "model", match: { method: "POST", url: `${origin}/v1/` } }, + ], + }; + const startFrom = async (client: string) => + JSON.parse( + (await rawRequest(local.url, client, "POST", "/__roach/runs", config)) + .body, + ) as RemoteRun; + const attacker = await startFrom("203.0.113.1"); + const other = await startFrom("203.0.113.2"); + onTestFinished(() => { + vi.useRealTimers(); + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + + // Each body declares 16 MiB, sends all but 1 KiB, and then stops. Before + // the part for each client, 8 of them held the whole budget. + const size = 16 * 1024 * 1024; + const stalls = await Promise.all( + Array.from({ length: 8 }, () => + startBody(local.url, attacker, size, size - 1024), + ), + ); + await until(() => + stalls.every( + ({ socket, status }) => socket.writableLength === 0 || status() > 0, + ), + ); + await realWait(500); + const small = 700 * 1024; + expect(await answer(local.url, other, small)).toBe(412); + // The runs of the attacker can hold only 32 MiB, 2 of the bodies. + await until( + () => stalls.filter(({ status }) => status() === 503).length >= 6, + ); + const holders = stalls.filter(({ status }) => status() === 0); + expect(holders.length).toBeLessThanOrEqual(2); + + // A body that does not come in 30 s gets HTTP 408, and its tunnel + // closes. Then the attacker has its part again. + await vi.advanceTimersByTimeAsync(30_000); + for (const holder of holders) { + expect(await holder.reply).toMatch( + /^HTTP\/1.1 408 [^]*the body did not come in 30 s/, + ); + } + expect(await answer(local.url, attacker, small)).toBe(412); + }); + it("fails a request with millions of values before it fills the memory", async () => { // `replace()` finds all matches before it counts them, so these bodies // stopped the worker at its heap limit. The count stops at the limit.