fix(evals): Fail an item when its MCP server is not connected at init

evals/README.md

@@ -243,7 +243,7 @@ rationale in `runner/run.ts`). `requestedTrials = requestedIds.length * iteratio
 - A calibrated suite gates on an aggregate rate instead of one flaky item blocking every PR; `--pass-threshold 1.0` restores strict all-pass
 - The trial count matters as much as the scores: the Langfuse SDK drops an item whose task throws, so gating on the results it returns would report `7/7 passed` on a run where three trials never executed
 
-Harness failures (MCP spawn, OpenRouter, judge) are therefore left to throw rather than being converted into a `FAIL` verdict. A broken harness shows up as a shortfall, not as a failing eval.
+Harness failures (MCP spawn, an MCP server not connected at init, OpenRouter, judge) are therefore left to throw rather than being converted into a `FAIL` verdict. A broken harness shows up as a shortfall, not as a failing eval.
 
 **Location:** `runner/experiment.ts` (`buildRunSummary`, `resolveExitCode`)
 
@@ -421,13 +421,10 @@ above). The snapshot holds the same fields flattened, one object per case, in th
 **Symptom:** Test 2 fails after Test 1, passes alone.<br>
 **Solution:** ✅ Isolated agent + MCP instance per test.
 
-### Agent claims the Apify MCP server is "still connecting" (remote/sandboxed environments)
-**Symptom:** A transcript shows the agent reading a system notice that MCP servers are still
-connecting, concluding the Apify tools are unavailable, and falling back to a built-in tool.<br>
-**Cause:** In sandboxed remote Claude Code environments the MCP handshake can race the first
-turn; a fast model sometimes doesn't wait or retry. On a local machine the CLI completes the
-handshake before the first turn, so this doesn't reproduce there.<br>
-**Solution:** Retry the item once; a repeat failure on a local machine is real.
+### Apify MCP server not connected at init
+**Symptom:** `MCP server "apify" not connected at init (status: pending)`, then `retrying once`; a failed retry shows `🔥` and counts as a failed trial.<br>
+**Cause:** The CLI waits 5 s (`MCP_CONNECT_TIMEOUT_MS`) for the server, then emits `init` with it `pending` and runs the first turn without its tools. Common on a loaded machine (high `--concurrency`, few CPUs).<br>
+**Solution:** Automatic: the run aborts at `init`; `pending`/`failed` retry once, other statuses fail at once. If it recurs, lower `--concurrency`.
 
 ### Agent answers from memory or shells out instead of using our tools
 **Symptom:** The judge reports the agent used `Bash`, `WebSearch`, or its own knowledge.<br>
@@ -459,7 +456,7 @@ Fork PRs cannot run evals because GitHub withholds repository secrets. Push the
 
 Both tiers need `ANTHROPIC_API_KEY`, the three `LANGFUSE_*` keys, and `APIFY_TOKEN` (mapped from the `APIFY_TEST_USER_API_TOKEN` repository secret). `OPENROUTER_API_KEY` is required only by the merge judge. The three publish cases also need write access to `apify/normal-mode-test-actor`.
 
-The pr tier has run on a hosted runner in 4m31s, inside its ten-minute target; the merge tier's threshold and 90-minute timeout are still provisional. Transient network and provider failures retry once; the handshake race above does not, because it produces a wrong answer rather than an error.
+The pr tier has run on a hosted runner in 4m31s, inside its ten-minute target; the merge tier's threshold and 90-minute timeout are still provisional. Transient network and provider failures retry once, as does an MCP server `pending` or `failed` at init (see Common issues).
 
 ## Writing tool descriptions
 

evals/agent/claude_agent.ts

@@ -172,6 +172,14 @@ export async function runAgentConversation(options: AgentRunOptions): Promise<Ag
     const receivedAt: number[] = [];
     try {
         for await (const message of query({ prompt, options: sdkOptions })) {
+            // The CLI emits init before a slow server is up (`pending`); the first turn would then run without its tools.
+            if (message.type === 'system' && message.subtype === 'init') {
+                const status =
+                    message.mcp_servers.find((server) => server.name === MCP_SERVER_NAME)?.status ?? 'missing';
+                if (status !== 'connected') {
+                    throw new Error(`MCP server "${MCP_SERVER_NAME}" not connected at init (status: ${status})`);
+                }
+            }
             messages.push(message);
             receivedAt.push(Date.now());
         }

evals/runner/experiment.ts

@@ -351,13 +351,14 @@ export function formatTeardownHint(runId: string): string[] {
 }
 
 /**
- * Messages of failures worth replaying the agent run for: the network or the provider
- * dropped the request, not the agent's own doing.
+ * Messages of failures worth replaying the agent run for: the network, the provider or the
+ * MCP handshake failed, not the agent's own doing.
  *
  * Matched on the message because the SDK surfaces them as plain `Error`s with no code or
  * status. Deliberately narrow: retrying a deterministic failure (a bad prompt, a missing
  * binary, a tool timeout) only doubles the spend and, for a case with fixed-name fixtures,
  * turns the replay itself into a name collision that fails the zero-tool-error gate.
+ * An MCP server `pending` or `failed` at init is replayed once; other statuses are configuration errors.
  */
 const TRANSIENT_AGENT_ERROR_PATTERNS = [
     /connection error/i,
@@ -368,6 +369,7 @@ const TRANSIENT_AGENT_ERROR_PATTERNS = [
     /\b(408|429|500|502|503|504)\b/,
     /overloaded/i,
     /rate.?limit/i,
+    /MCP server ".+" not connected at init \(status: (pending|failed)\)/,
 ];
 
 /** Whether an agent-run failure is transient, so replaying the prompt could recover it. */

tests/unit/evals.claude_agent.test.ts

@@ -2,6 +2,7 @@ import type * as ClaudeAgentSdk from '@anthropic-ai/claude-agent-sdk';
 import type { Options, PreToolUseHookInput, SDKMessage } from '@anthropic-ai/claude-agent-sdk';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
+import { isTransientAgentError } from '../../evals/runner/experiment.js';
 import { TOOL_CALL_DENY_REASON, TOOL_CALL_MAX_TURNS } from '../../evals/runner/tool_call_mode.js';
 import { REPORT_PROBLEM_NUDGE } from '../../src/tools/dev/report_problem.js';
 
@@ -40,6 +41,24 @@ function resultMessage(overrides: Record<string, unknown> = {}): SDKMessage {
     } as unknown as SDKMessage;
 }
 
+/** The SDK `init` message; only the fields the run and the adapter read. */
+function initMessage(mcpServers: { name: string; status: string }[]): SDKMessage {
+    return {
+        type: 'system',
+        subtype: 'init',
+        claude_code_version: '2.1.206',
+        mcp_servers: mcpServers,
+    } as unknown as SDKMessage;
+}
+
+function assistantMessage(text: string): SDKMessage {
+    return {
+        type: 'assistant',
+        message: { id: 'msg-1', content: [{ type: 'text', text }] },
+        parent_tool_use_id: null,
+    } as unknown as SDKMessage;
+}
+
 /** A minimal SDK stream a mocked `query()` can yield to let the adapter succeed. */
 async function* fakeStream(messages: SDKMessage[]): AsyncGenerator<SDKMessage> {
     for (const message of messages) yield message;
@@ -195,4 +214,69 @@ describe('runAgentConversation()', () => {
 
         await expect(runAgentConversation(baseOptions())).rejects.toBe(originalError);
     });
+
+    describe('with an init message', () => {
+        let isPastInit: boolean;
+
+        /** Yields init with the given servers, then records whether the run read any further. */
+        function mockInitStream(mcpServers: { name: string; status: string }[]) {
+            isPastInit = false;
+            mocks.query.mockImplementation(({ options }: { options: Options }) => {
+                capturedOptions = options;
+                return (async function* () {
+                    yield initMessage(mcpServers);
+                    isPastInit = true;
+                    yield assistantMessage('done');
+                    yield resultMessage();
+                })();
+            });
+        }
+
+        it('throws when the MCP server is not connected at init', async () => {
+            mockInitStream([{ name: 'apify', status: 'pending' }]);
+
+            await expect(runAgentConversation(baseOptions())).rejects.toThrow(
+                'MCP server "apify" not connected at init (status: pending)',
+            );
+            expect(isPastInit).toBe(false);
+            expect(capturedOptions?.abortController?.signal.aborted).toBe(true);
+        });
+
+        it('throws when the MCP server is missing from init', async () => {
+            mockInitStream([]);
+
+            await expect(runAgentConversation(baseOptions())).rejects.toThrow(
+                'MCP server "apify" not connected at init (status: missing)',
+            );
+            expect(isPastInit).toBe(false);
+        });
+
+        it('runs to completion when the MCP server is connected at init', async () => {
+            mockInitStream([{ name: 'apify', status: 'connected' }]);
+
+            const run = await runAgentConversation(baseOptions());
+
+            expect(isPastInit).toBe(true);
+            expect(run.claudeCodeVersion).toBe('2.1.206');
+            expect(run.conversation.turns.at(-1)?.finalResponse).toBe('done');
+        });
+
+        it('keeps the not-connected error transient when stderr is appended', async () => {
+            const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+            mocks.query.mockImplementation(({ options }: { options: Options }) => {
+                return (async function* () {
+                    options.stderr?.('some line\n');
+                    yield initMessage([{ name: 'apify', status: 'pending' }]);
+                })();
+            });
+
+            const error = await runAgentConversation(baseOptions()).catch((caught: unknown) => caught);
+
+            const message = error instanceof Error ? error.message : String(error);
+            expect(message).toContain('MCP server "apify" not connected at init (status: pending)');
+            expect(message).toContain('\n  [claude-stderr] some line');
+            expect(isTransientAgentError(error)).toBe(true);
+            errorSpy.mockRestore();
+        });
+    });
 });

tests/unit/evals.experiment.test.ts

@@ -181,16 +181,21 @@ describe('isTransientAgentError()', () => {
         'read ECONNRESET',
         'API error: 529 overloaded_error',
         'Request failed with status 503',
+        'MCP server "apify" not connected at init (status: pending)',
+        'MCP server "apify" not connected at init (status: failed)',
     ])('treats "%s" as transient', (message) => {
         expect(isTransientAgentError(new Error(message))).toBe(true);
     });
 
-    it.each(['spawn ENOENT', 'Prompt is too long', 'stdio binary not found', 'Invalid model name'])(
-        'treats "%s" as permanent',
-        (message) => {
-            expect(isTransientAgentError(new Error(message))).toBe(false);
-        },
-    );
+    it.each([
+        'spawn ENOENT',
+        'Prompt is too long',
+        'stdio binary not found',
+        'Invalid model name',
+        'MCP server "apify" not connected at init (status: disabled)',
+    ])('treats "%s" as permanent', (message) => {
+        expect(isTransientAgentError(new Error(message))).toBe(false);
+    });
 });
 
 describe('createExperimentTask()', () => {
@@ -250,6 +255,19 @@ describe('createExperimentTask()', () => {
         expect(mocks.runAgentConversation).toHaveBeenCalledTimes(2);
     });
 
+    it('retries once and never judges when the MCP server stays unconnected', async () => {
+        mocks.runAgentConversation.mockRejectedValue(
+            new Error('MCP server "apify" not connected at init (status: pending)'),
+        );
+
+        await expect(makeMcpAgentTask()(makeItem())).rejects.toThrow(
+            'Item "search-001": MCP server "apify" not connected at init (status: pending)',
+        );
+        expect(mocks.runAgentConversation).toHaveBeenCalledTimes(2);
+        expect(mocks.evaluateConversation).not.toHaveBeenCalled();
+        expect(mocks.emitObservations).not.toHaveBeenCalled();
+    });
+
     it('still scores the item when emitting the agent trace throws', async () => {
         mocks.runAgentConversation.mockResolvedValue(makeAgentRun());
         mocks.emitObservations.mockImplementation(() => {