From 8a5574439fb0823c0a3c6471e04d878b5747ca76 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:15:10 +0000 Subject: [PATCH 1/2] fix(service): Time out slow headers on servers that do not listen, and do not report aborted requests Node checks `headersTimeout` and `requestTimeout` only after the `listening` event. The tunnel server never listens, and with `proxyProtocol` the HTTP server does not either, so their requests had no time limit. Send the event to them, and give headers 10 s. Node checks each second. A client that closes its connection before its body came gave an unexpected ECONNRESET, so it went to Sentry. It now gets status 400 and counts as `roach.request` with `roach.result:aborted`. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- README.md | 13 ++-- TELEMETRY.md | 7 +- packages/service/src/server.ts | 44 +++++++++++- packages/service/tests/service.test.ts | 93 ++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e161522..5725369 100644 --- a/README.md +++ b/README.md @@ -594,11 +594,11 @@ attribute of Roach starts with `roach.`. Metrics have only attributes with few values: `roach.tenant`, `roach.mode`, `roach.rule`, `roach.origin`, `roach.result`, and `roach.reason`. -| Metric | Type | Value | -| ------------------------ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `roach.request` | counter | 1 for each proxied request. `roach.result` is `replayed`, `live`, `missed`, `passthrough`, or `refused`. A `passthrough` or `refused` request has no `roach.rule`. A `refused` request has `roach.reason`, and `other` for an origin not in `allow`. | -| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `roach.result`. | -| `roach.recording.write` | counter | Recordings by `roach.rule`. `roach.result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | +| Metric | Type | Value | +| ------------------------ | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `roach.request` | counter | 1 for each proxied request. `roach.result` is `replayed`, `live`, `missed`, `passthrough`, `refused`, or `aborted` (the client closed the request before its body came). A `passthrough`, `refused`, or `aborted` request has no `roach.rule`. A `refused` request has `roach.reason`, and `other` for an origin not in `allow`. | +| `roach.request.duration` | distribution (ms) | The time to answer a request that a rule records, by `roach.result`. | +| `roach.recording.write` | counter | Recordings by `roach.rule`. `roach.result` is `written` (the proxy sent them to the store) or `dropped` (their session failed). | Each proxied request is one trace. Each span has `roach.tenant`, `roach.mode`, `roach.run.id` (the `name` of the run, else its id), and the @@ -686,6 +686,9 @@ To see how well the cache works: before the service parses it. - Keys and the redaction of misses have time limits. See [Changing values](#changing-values). +- Request headers that do not come in 10 s get HTTP 408, and a request + whose body does not come in 300 s too. This is also true in a tunnel + and with `proxyProtocol`. - A `CONNECT` tunnel that sends no byte for 10 s after the `200` closes. A tunnel that sends and gets no byte for 10 minutes closes. - The credentials that a run learns and the values that a session diff --git a/TELEMETRY.md b/TELEMETRY.md index ad45555..841dffb 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -50,6 +50,9 @@ For metrics, use `-f dataset=tracemetrics`, `roach.reason` tells why: `notAllowed` (the origin is not in `allow`), `notPublic` (the address is not public), or `replayOnly` (a run without the token, and no rule records the request). +- **Aborted**: the client closed its connection before the body of its + request came. The proxy sent the request nowhere. It is not an error, + so Sentry gets no event. - **Dropped**: a live response was recorded, but its session failed, so the recording was not written. The next run sends the request live again. @@ -96,8 +99,8 @@ The examples use the `sentry` CLI. The same fields work in Explore. The counts are estimates (see [Sampling](#sampling)). Cached and uncached requests by rule. The hit rate of a rule is -`replayed / (replayed + live + missed)`. Do not count `passthrough` or -`refused`. +`replayed / (replayed + live + missed)`. Do not count `passthrough`, +`refused`, or `aborted`. ```bash sentry explore sentry/roach --dataset metrics --metric roach.request --agg sum \ diff --git a/packages/service/src/server.ts b/packages/service/src/server.ts index 7bc5cbf..07b7f21 100644 --- a/packages/service/src/server.ts +++ b/packages/service/src/server.ts @@ -72,6 +72,15 @@ const TUNNEL_FIRST_BYTE_MS = 10_000; * call can take minutes before its first byte, so this is long. */ const TUNNEL_IDLE_MS = 10 * 60 * 1000; +/** + * The HTTP servers of the proxy. Headers that do not come in 10 s get + * HTTP 408 from Node. A whole request, with its body, has Node's default + * of 300 s. Node checks these times each second. + */ +const HTTP_OPTIONS: http.ServerOptions = { + headersTimeout: 10_000, + connectionsCheckingInterval: 1000, +}; /** * A PROXY protocol v1 header, such as * `PROXY TCP4 203.0.113.7 10.0.0.2 51234 443`. It is at most 107 bytes. @@ -183,10 +192,23 @@ function readBody( chunks.push(chunk); }); stream.on("end", () => resolve(Buffer.concat(chunks))); - stream.on("error", reject); + stream.on("error", (error: Error & { code?: string }) => { + // A client that closes its connection before its body came is not a + // failure of the proxy, so it does not go to Sentry. `method` is set + // only on a request, not on an upstream response. + const aborted = stream.method !== null && error.code === "ECONNRESET"; + reject(aborted ? abortedError() : error); + }); }); } +/** The error of a request whose client closed it before its body came. */ +const abortedError = () => + Object.assign( + httpError(400, "the client closed the request before its body came"), + { aborted: true }, + ); + /** * Read a JSON object body of a control API call. A body over * `MAX_CONTROL_BODY_BYTES` fails with HTTP 413 before it is parsed. Any @@ -588,6 +610,14 @@ function answerError( status === undefined ? reportInternal("Request", error, target.telemetry.capture(error)) : (error as Error).message; + if ((error as { aborted?: boolean }).aborted) { + const attributes = { "roach.result": "aborted" }; + target.telemetry.annotate(attributes); + target.telemetry.count("roach.request", 1, { + ...attributes, + "roach.origin": origin, + }); + } if (!outgoing.headersSent) { outgoing.writeHead(status ?? 502, { "content-type": "text/plain" }); } @@ -612,7 +642,7 @@ export async function listenProxy( const sockets = new Set(); // Requests inside a CONNECT tunnel. The tunnel gives the origin. - const tunnelServer = http.createServer((incoming, outgoing) => { + const tunnelServer = http.createServer(HTTP_OPTIONS, (incoming, outgoing) => { const tunnel = tunnels.get(incoming.socket); if (!tunnel) { outgoing.writeHead(400).end(); @@ -620,7 +650,7 @@ export async function listenProxy( } serveProxied(tunnel, incoming, outgoing, tunnel.origin); }); - const server = http.createServer((incoming, outgoing) => { + const server = http.createServer(HTTP_OPTIONS, (incoming, outgoing) => { // A request to the proxy itself has a path. A proxied plain HTTP // request has an absolute URL. if (!incoming.url?.startsWith("/")) { @@ -736,6 +766,14 @@ export async function listenProxy( }, ); + // Node checks `headersTimeout` and `requestTimeout` only after the + // `listening` event of a server. The tunnel server, and `server` with + // `proxyProtocol`, get their connections from another server and never + // listen. So send the event to them. This relies on Node internals + // (`setupConnectionsTracking` in `lib/_http_server.js`), and the test + // "times out slow headers" fails when Node changes them. + tunnelServer.emit("listening"); + if (listener !== server) server.emit("listening"); await new Promise((resolve, reject) => { listener.once("error", reject); listener.listen(options.port, options.host, () => resolve()); diff --git a/packages/service/tests/service.test.ts b/packages/service/tests/service.test.ts index f334db7..b5308a9 100644 --- a/packages/service/tests/service.test.ts +++ b/packages/service/tests/service.test.ts @@ -44,6 +44,36 @@ const runs: RemoteRoach[] = []; const hash = (token: string) => createHash("sha256").update(token).digest("hex"); +/** + * Open a `CONNECT` tunnel to the upstream through the service at + * `serviceUrl`, with the credentials of the proxy URL of a run. Returns + * the socket after the `200`. + */ +async function openTunnel(serviceUrl: string, runUrl: string) { + const { username, password } = new URL(runUrl); + const authority = new URL(origin).host; + const socket = connect(Number(new URL(serviceUrl).port), "127.0.0.1"); + onTestFinished(() => { + socket.destroy(); + }); + await once(socket, "connect"); + const credentials = Buffer.from(`${username}:${password}`).toString("base64"); + socket.write( + `CONNECT ${authority} HTTP/1.1\r\nHost: ${authority}\r\nProxy-Authorization: Basic ${credentials}\r\n\r\n`, + ); + const [reply] = (await once(socket, "data")) as [Buffer]; + expect(reply.toString()).toMatch(/^HTTP\/1.1 200/); + return socket; +} + +/** All that a socket gets until it closes. */ +async function readUntilClose(socket: ReturnType) { + const chunks: Buffer[] = []; + socket.on("data", (chunk: Buffer) => chunks.push(chunk)); + await once(socket, "close"); + return Buffer.concat(chunks).toString(); +} + /** Start a run, with one recorded rule. Without `token`, it has no token. */ async function startRun( token: string | undefined, @@ -984,6 +1014,69 @@ describe("limits of the service", () => { ]), ).resolves.toBe("closed"); }); + it("times out slow headers, also with proxyProtocol and in a tunnel", async () => { + // With `proxyProtocol`, the HTTP servers get their connections from a + // TCP server, and Node did not check their time limits. + const local = await startRoachService({ + directory, + privateNetworks: true, + writeTokenHash: hash("write-token"), + proxyProtocol: true, + }); + onTestFinished(() => local.close()); + const started = await rawRequest( + local.url, + "203.0.113.1", + "POST", + "/__roach/runs", + { + tenant: "acme/alpha", + mode: "replay", + allow: [origin], + rules: [{ name: "model", match: {} }], + }, + ); + const run = JSON.parse(started.body) as RemoteRun; + const direct = connect(Number(new URL(local.url).port), "127.0.0.1"); + await once(direct, "connect"); + const tunnel = await openTunnel(local.url, run.url); + const begin = performance.now(); + direct.write( + "PROXY TCP4 203.0.113.1 10.0.0.2 51234 443\r\nGET / HTTP/1.1\r\nhost: roach\r\n", + ); + tunnel.write("POST /v1/messages HTTP/1.1\r\nhost: upstream\r\n"); + // The headers never end. Node answers 408 after 10 s. + const replies = await Promise.all( + [direct, tunnel].map((socket) => + Promise.race([ + readUntilClose(socket), + new Promise((resolve) => setTimeout(() => resolve("open"), 15_000)), + ]), + ), + ); + for (const reply of replies) expect(reply).toMatch(/^HTTP\/1.1 408 /); + expect(performance.now() - begin).toBeGreaterThan(9000); + }); + + it("does not send a request to Sentry when its client closes it", async () => { + const run = await startRun(undefined, "replay"); + const log = vi.spyOn(process.stderr, "write"); + onTestFinished(() => log.mockRestore()); + const tunnel = await openTunnel(service.url, run.url); + tunnel.write( + "POST /v1/messages HTTP/1.1\r\nhost: upstream\r\ncontent-length: 1000\r\n\r\n{", + ); + // Let the proxy start to read the body, then close the connection. + await new Promise((resolve) => setTimeout(resolve, 200)); + tunnel.destroy(); + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(log.mock.calls.map(([text]) => String(text)).join("")).not.toMatch( + /Request failed/, + ); + // The run goes on. + expect((await send(run.url, { prompt: "hi" })).status).toBe(412); + }); + it("limits the runs without a token of each client", async () => { // Behind the load balancer, each connection starts with a PROXY // header. It gives the address of the client. From 3f11cb0b210e089261f38cd70f95a3dde18b401d Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:40:16 +0000 Subject: [PATCH 2/2] fix(service): Lower the body share of one client to 16 MiB With 32 MiB, 4 IPv4 addresses or 4 /64s could fill the 128 MiB budget of runs without a token. With 16 MiB it takes 8. 16 MiB is still one body of the largest size, or about 25 of the largest model requests. The README said that a client with a /48 holds the budget "for 30 s at a time". A body gets HTTP 408 after 30 s, but the client can start the next one at once, so the hold does not stop. --- README.md | 5 +++-- packages/service/src/service.ts | 6 +++--- packages/service/tests/service.test.ts | 17 +++++++---------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7ebe2ae..cb29413 100644 --- a/README.md +++ b/README.md @@ -698,7 +698,7 @@ To see how well the cache works: run without the token. A control API body over 1 MiB gets HTTP 413 before the service parses it. - The requests of all runs without the token can hold 128 MiB of bodies - at one time, and the runs of one client 32 MiB. The client of a run is + at one time, and the runs of one client 16 MiB. The client of a run is the client that started it, grouped as for the limit of runs. A request holds its body until it finishes, also while its key waits for the key worker. A request whose body does not fit gets HTTP 503 with @@ -725,7 +725,8 @@ To see how well the cache works: - A gap: a client with an IPv6 /48 has 65,536 /64s, so it can open runs until the service has 100 runs without a token. Then other clients get HTTP 429 until its runs end. Its runs can also hold all of the budget of - bodies, for 30 s at a time. + bodies. A body gets HTTP 408 after 30 s, but the client can start the + next one at once, so it holds the budget for as long as it sends. - GCS deletes each recording 30 days after it was written, also when runs still replay it. Then `auto` mode records it again, and `replay` mode fails the request with HTTP 412. diff --git a/packages/service/src/service.ts b/packages/service/src/service.ts index 8a47b8b..798217c 100644 --- a/packages/service/src/service.ts +++ b/packages/service/src/service.ts @@ -173,10 +173,10 @@ const PUBLIC_BODY_BUDGET_BYTES = 128 * 1024 * 1024; /** * The part of `PUBLIC_BODY_BUDGET_BYTES` that the runs of one client * (`clientGroup()` of the client that started the run) can hold, so that - * one client cannot take all of it. It is 2 bodies of 16 MiB, or about 50 - * of the largest model requests. + * one client cannot take all of it. It takes 8 clients to fill the budget. + * It is 1 body of 16 MiB, or about 25 of the largest model requests. */ -const PUBLIC_BODY_CLIENT_BYTES = 32 * 1024 * 1024; +const PUBLIC_BODY_CLIENT_BYTES = 16 * 1024 * 1024; /** * A body of a run without a token that does not come in this time gets * HTTP 408, and gives its bytes back. So a client that sends a byte now diff --git a/packages/service/tests/service.test.ts b/packages/service/tests/service.test.ts index f0df55c..a49d40e 100644 --- a/packages/service/tests/service.test.ts +++ b/packages/service/tests/service.test.ts @@ -806,11 +806,11 @@ describe("limits of the service", () => { }, ], }); - // 7 bodies of 4 MiB fit in the 32 MiB that the runs of one client can + // 3 bodies of 4 MiB fit in the 16 MiB that the runs of one client can // hold. const prompt = "a".repeat(4 * 1024 * 1024); const sent = Promise.allSettled( - Array.from({ length: 7 }, () => send(run.url, { prompt })), + Array.from({ length: 3 }, () => send(run.url, { prompt })), ); const waits = await caWaitsUntil(sent); // The keys are made in a worker, so the service answers at once. @@ -861,7 +861,7 @@ describe("limits of the service", () => { const results = await Promise.all( Array.from({ length: 12 }, () => send(reader.url, { prompt })), ); - // The runs of one client can hold 32 MiB, 2 of these bodies. The bytes + // The runs of one client can hold 16 MiB, 1 of these bodies. The bytes // count when they come, so parts of more bodies can fill it. const busy = results.filter((result) => result.status === 503); expect(busy.length).toBeGreaterThanOrEqual(4); @@ -872,10 +872,7 @@ describe("limits of the service", () => { } // The run goes on, and the bodies gave their bytes back. await expect(reader.stats()).resolves.toBeDefined(); - const again = await Promise.all( - Array.from({ length: 2 }, () => send(reader.url, { prompt })), - ); - expect(again.map((result) => result.status)).toEqual([412, 412]); + expect((await send(reader.url, { prompt })).status).toBe(412); // A client that declares large bodies and then stops holds only the // bytes that it sent. @@ -942,12 +939,12 @@ describe("limits of the service", () => { await realWait(500); const small = 700 * 1024; expect(await answer(local.url, other, small)).toBe(412); - // The runs of the attacker can hold only 32 MiB, 2 of the bodies. + // The runs of the attacker can hold only 16 MiB, 1 of the bodies. await until( - () => stalls.filter(({ status }) => status() === 503).length >= 6, + () => stalls.filter(({ status }) => status() === 503).length >= 7, ); const holders = stalls.filter(({ status }) => status() === 0); - expect(holders.length).toBeLessThanOrEqual(2); + expect(holders.length).toBeLessThanOrEqual(1); // A body that does not come in 30 s gets HTTP 408, and its tunnel // closes. Then the attacker has its part again.