From ec4f7f8cef697b0b7d398fd9ee65f79032477f2c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:22:00 +0000 Subject: [PATCH 1/5] Initial plan From f83316c23d4419083cf3f72e5dded6a02f242bcc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:33:59 +0000 Subject: [PATCH 2/5] Retry MCP gateway startup once and diagnose pre-agent checkout failures Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/handle_agent_failure.cjs | 47 +- .../setup/js/handle_agent_failure.test.cjs | 70 +++ actions/setup/js/start_mcp_gateway.cjs | 491 ++++++++++-------- actions/setup/js/start_mcp_gateway.test.cjs | 54 +- 4 files changed, 418 insertions(+), 244 deletions(-) diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index f8fef8f3534..24d5018ef0f 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -1951,7 +1951,7 @@ async function getSecretRedactionError(jobId, step) { /** * Find the failed agent step using Actions metadata only, never agent-generated log text. * @param {{startupFailure?: boolean}} [options] - Select the first failed step only for a validated collector startup failure. - * @returns {Promise<{failingStep: string, agentConclusion?: string, attributionUnavailable?: string, hasNoJobs?: boolean, secretRedactionFailed?: boolean, stepError?: string, cancelledWithoutTimeout?: boolean, isTimedOut?: boolean}>} + * @returns {Promise<{failingStep: string, agentConclusion?: string, attributionUnavailable?: string, hasNoJobs?: boolean, secretRedactionFailed?: boolean, stepError?: string, cancelledWithoutTimeout?: boolean, isTimedOut?: boolean, preAgentFailure?: boolean, checkoutRef?: string}>} */ async function getFailedAgentDiagnostics({ startupFailure = false } = {}) { let jobs; @@ -2012,11 +2012,22 @@ async function getFailedAgentDiagnostics({ startupFailure = false } = {}) { if (agentConclusion !== "failure" && agentConclusion !== "timed_out" && !isCancelledTimeout) { return { failingStep: "", agentConclusion }; } - const steps = agentJob.steps?.slice(); + const steps = agentJob.steps?.slice() || []; + const setupBoundary = steps.findIndex(step => step.name === "Start MCP Gateway" || (typeof step.name === "string" && step.name.startsWith("Execute "))); + const setupFailure = + setupBoundary >= 0 && + ["skipped", "failure", "timed_out"].includes(steps[setupBoundary].conclusion) && + (startupFailure || steps[setupBoundary].conclusion === "skipped" || steps.some(step => typeof step.name === "string" && step.name.startsWith("Execute ") && step.conclusion === "skipped")) && + !steps.some(step => typeof step.name === "string" && step.name.startsWith("Execute ") && step.conclusion !== "skipped") + ? steps + .slice(0, setupBoundary + (steps[setupBoundary].name === "Start MCP Gateway" ? 1 : 0)) + .find(step => ["failure", "timed_out"].includes(step.conclusion) && (step.name === "Start MCP Gateway" || (typeof step.name === "string" && step.name.startsWith("Checkout ")))) + : undefined; if (!startupFailure) steps?.reverse(); const redactionStep = steps?.find(step => step.name === "Redact secrets in logs" && ["failure", "timed_out"].includes(step.conclusion)); // Failed redaction blocks safe diagnostics even after an agent crash; startup keeps priority. - const failedStep = (!startupFailure && redactionStep) || steps?.find(step => (["failure", "timed_out"].includes(step.conclusion) || (isCancelledTimeout && step.conclusion === "cancelled")) && typeof step.name === "string"); + const failedStep = + (!startupFailure && redactionStep) || setupFailure || steps?.find(step => (["failure", "timed_out"].includes(step.conclusion) || (isCancelledTimeout && step.conclusion === "cancelled")) && typeof step.name === "string"); if (!failedStep) { return { failingStep: "", agentConclusion, ...(isCancelledTimeout ? { isTimedOut: true } : {}), attributionUnavailable: "The failed agent job had no failed step metadata." }; } @@ -2028,6 +2039,10 @@ async function getFailedAgentDiagnostics({ startupFailure = false } = {}) { return { failingStep, agentConclusion, + ...(failedStep === setupFailure ? { preAgentFailure: true } : {}), + ...(failedStep === setupFailure && failedStep.name === "Checkout repository (gh-aw default)" && process.env.GH_AW_DEFAULT_CHECKOUT_USES_TRIGGER_REF === "true" && /^refs\/pull\/[1-9]\d*\/merge$/.test(context.ref || "") + ? { checkoutRef: context.ref } + : {}), ...(isCancelledTimeout ? { isTimedOut: true } : {}), ...(failedStep === redactionStep ? { secretRedactionFailed: true, stepError: await getSecretRedactionError(agentJob.id, redactionStep) } : {}), }; @@ -2035,10 +2050,21 @@ async function getFailedAgentDiagnostics({ startupFailure = false } = {}) { /** * Add step-level context to generic agent and infrastructure failures. - * @param {{failureCategories: string[], failingStep: string, agentConclusion?: string, attributionUnavailable?: string, stepError?: string, engineFailureContext: ReportBlock[], checkoutError?: string, items?: Array}} options + * @param {{failureCategories: string[], failingStep: string, agentConclusion?: string, attributionUnavailable?: string, stepError?: string, engineFailureContext: ReportBlock[], checkoutError?: string, items?: Array, preAgentFailure?: boolean, checkoutRef?: string}} options * @returns {ReportBlock[]} */ -function buildFailureDiagnosticsContext({ failureCategories, failingStep, agentConclusion = "", attributionUnavailable = "", stepError = "", engineFailureContext, checkoutError = "", items = [] }) { +function buildFailureDiagnosticsContext({ + failureCategories, + failingStep, + agentConclusion = "", + attributionUnavailable = "", + stepError = "", + engineFailureContext, + checkoutError = "", + items = [], + preAgentFailure = false, + checkoutRef = "", +}) { const checkoutContext = checkoutError ? [ nodes.heading("Runtime Checkout Rejected"), @@ -2051,7 +2077,8 @@ function buildFailureDiagnosticsContext({ failureCategories, failingStep, agentC ] : []; const infrastructureMessages = items.filter(item => item?.type === "report_incomplete" && item.reason === "infrastructure_error"); - const isGenericFailure = Boolean(stepError) || failureCategories.includes("agent_failure") || failureCategories.includes("engine_driver_failure") || failureCategories.includes("agent_startup_failure") || infrastructureMessages.length > 0; + const isGenericFailure = + preAgentFailure || Boolean(stepError) || failureCategories.includes("agent_failure") || failureCategories.includes("engine_driver_failure") || failureCategories.includes("agent_startup_failure") || infrastructureMessages.length > 0; const hasModelEndpointMismatch = failureCategories.includes("model_endpoint_mismatch"); if (!isGenericFailure && !hasModelEndpointMismatch) { return checkoutContext; @@ -2074,6 +2101,14 @@ function buildFailureDiagnosticsContext({ failureCategories, failingStep, agentC metadata.push(nodes.strong([nodes.text("Failing step:")]), nodes.text(" Unavailable")); } context.push(nodes.paragraph(metadata)); + if (preAgentFailure) { + context.push(nodes.paragraph([nodes.strong([nodes.text("Pre-agent setup failure:")]), nodes.text(" The agent did not start. Inspect the failing setup step in the linked run.")])); + if (checkoutRef) { + context.push( + nodes.paragraph([nodes.text("Requested pull-request merge ref: "), nodes.code(checkoutRef), nodes.text(". Verify that this ref is still available and the pull request is mergeable; checkout did not fall back to the head branch.")]) + ); + } + } if (stepError) { context.push(...renderErrorDetails(stepError)); } else if (engineFailureContext.length) { diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index 19ebea52de8..bc8335133a9 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -578,6 +578,76 @@ describe("handle_agent_failure", () => { }); describe("failure diagnostics", () => { + it("reports a failed merge-ref checkout as pre-agent with the step and ref", async () => { + process.env.GH_AW_DEFAULT_CHECKOUT_USES_TRIGGER_REF = "true"; + global.context.ref = "refs/pull/42/merge"; + global.github = { + paginate: vi.fn().mockResolvedValue([ + { + name: "agent", + conclusion: "failure", + steps: [ + { name: "Checkout repository (gh-aw default)", conclusion: "failure" }, + { name: "Start MCP Gateway", conclusion: "skipped" }, + { name: "Execute GitHub Copilot CLI", conclusion: "skipped" }, + { name: "Upload logs", conclusion: "failure" }, + ], + }, + ]), + rest: { actions: { listJobsForWorkflowRun: vi.fn() } }, + }; + + const diagnostics = await getFailedAgentDiagnostics(); + expect(diagnostics).toEqual({ + failingStep: "Checkout repository (gh-aw default)", + agentConclusion: "failure", + preAgentFailure: true, + checkoutRef: "refs/pull/42/merge", + }); + const rendered = buildFailureDiagnosticsContext({ failureCategories: ["agent_failure"], engineFailureContext: "", ...diagnostics }); + expect(rendered).toContain("Pre-agent setup failure:"); + expect(rendered).toContain("The agent did not start."); + expect(rendered).toContain("`Checkout repository (gh-aw default)`"); + expect(rendered).toContain("`refs/pull/42/merge`"); + expect(rendered).toContain("Verify that this ref is still available"); + }); + + it.each(["Start MCP Gateway", "Checkout org/other"])("attributes pre-agent setup failure to %s without assuming the checkout ref", async failingStep => { + global.context.ref = "refs/pull/42/merge"; + global.github = { + paginate: vi.fn().mockResolvedValue([ + { + name: "agent", + conclusion: "failure", + steps: [ + { name: failingStep, conclusion: "failure" }, + { name: "Execute Codex CLI", conclusion: "skipped" }, + ], + }, + ]), + rest: { actions: { listJobsForWorkflowRun: vi.fn() } }, + }; + await expect(getFailedAgentDiagnostics()).resolves.toEqual({ failingStep, agentConclusion: "failure", preAgentFailure: true }); + }); + + it("does not classify a checkout after agent execution as pre-agent", async () => { + global.github = { + paginate: vi.fn().mockResolvedValue([ + { + name: "agent", + conclusion: "failure", + steps: [ + { name: "Start MCP Gateway", conclusion: "success" }, + { name: "Execute Claude Code CLI", conclusion: "success" }, + { name: "Checkout org/other", conclusion: "failure" }, + ], + }, + ]), + rest: { actions: { listJobsForWorkflowRun: vi.fn() } }, + }; + await expect(getFailedAgentDiagnostics()).resolves.toEqual({ failingStep: "Checkout org/other", agentConclusion: "failure" }); + }); + it("includes the runtime checkout rejection and remediation in failure diagnostics", () => { const rendered = buildFailureDiagnosticsContext({ failureCategories: ["agent_failure"], diff --git a/actions/setup/js/start_mcp_gateway.cjs b/actions/setup/js/start_mcp_gateway.cjs index 0b307b015f1..cd07475b15d 100644 --- a/actions/setup/js/start_mcp_gateway.cjs +++ b/actions/setup/js/start_mcp_gateway.cjs @@ -726,6 +726,31 @@ function stopGatewayProcess(pid) { } } +/** + * Retry a failed container startup once, without retrying configuration errors. + * @template T + * @param {() => Promise} start + * @returns {Promise} + */ +async function startGatewayWithRetry(start) { + for (let attempt = 1; ; attempt += 1) { + try { + return await start(); + } catch (err) { + stopGatewayProcess(activeGatewayPid); + activeGatewayPid = null; + clearGatewayStartupMarker(getGatewayStartupMarkerPath()); + const message = redactGatewayDiagnostics(getErrorMessage(err)); + if (message.startsWith(`${ERR_CONFIG}:`)) throw err; + if (attempt === 2) { + throw codedError(`Pre-agent failure in step "Start MCP Gateway" after 2 startup attempts: ${message}`, ERR_SYSTEM, { cause: err }); + } + core.warning(`Start MCP Gateway: startup attempt 1 failed; retrying once. ${message}`); + await sleep(1000); + } + } +} + /** * HTTP GET helper – returns { statusCode, body }. * @param {string} url @@ -1036,263 +1061,266 @@ async function main() { // ----------------------------------------------------------------------- const outputPath = path.join(configDir, "gateway-output.json"); const gatewayStartupMarker = getGatewayStartupMarkerPath(); - clearGatewayStartupMarker(gatewayStartupMarker); - - // Clean up any stale gateway container from a previous run on this runner. - // On persistent self-hosted runners a prior job's gateway container may still - // be running and holding the host port, causing "bind: address already in use" - // when we try to start the new one. Force-removing by the well-known container - // name is idempotent: docker rm -f exits non-zero when the container doesn't - // exist, but the trailing || true (and 2>/dev/null) make the command succeed. - core.info("Cleaning up any stale awmg-mcpg container from a previous run..."); - try { - execSync("docker rm -f awmg-mcpg 2>/dev/null && echo 'Removed stale awmg-mcpg container' || true", { stdio: "inherit", timeout: DOCKER_CLEANUP_TIMEOUT_MS }); - } catch { - // Non-fatal: proceed even if the cleanup command itself fails - core.info("Could not remove stale awmg-mcpg container (may not exist)"); - } - core.info(""); - - core.info("Starting gateway container..."); - core.info(""); - - const gatewayStartTime = nowMs(); - - // Split docker command into args, respecting simple quoting - let args = Array.from(dockerCommand.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || []); - const cmd = args.shift(); - if (!cmd) { - setFailed(`${ERR_CONFIG}: MCP_GATEWAY_DOCKER_COMMAND did not contain an executable command`); - return; - } - const gatewayEnv = { ...process.env }; - // Insert before all existing flags and the image, never into the command string. - args.splice(1, 0, ...caMountArgs); - args = injectCustomGatewayEnvArgs(args, gatewayEnv); - args = injectProxyEnvArgs(args, gatewayEnv); - - const outputFd = fs.openSync(outputPath, "w", 0o600); - const stderrPath = createGatewayStderrPath(); - const stderrFd = fs.openSync(stderrPath, "w", 0o600); - - const child = spawn(cmd, args, { - stdio: ["pipe", outputFd, stderrFd], - env: { ...gatewayEnv, MCP_GATEWAY_LOG_DIR: logDir }, - detached: true, - }); - child.on("error", err => { - activeGatewayPid = null; - core.error(`ERROR: Failed to launch MCP gateway process: ${getErrorMessage(err)}`); - }); - activeGatewayPid = child.pid || null; - - // Write configuration to stdin then close - if (!child.stdin) { - stopGatewayProcess(activeGatewayPid); - setFailed(`${ERR_SYSTEM}: Gateway process stdin is not available`); - return; - } - child.stdin.write(JSON.stringify(configObj)); - child.stdin.end(); + const { gatewayPid, stderrPath } = await startGatewayWithRetry(async () => { + clearGatewayStartupMarker(gatewayStartupMarker); + + // Clean up any stale gateway container from a previous run on this runner. + // On persistent self-hosted runners a prior job's gateway container may still + // be running and holding the host port, causing "bind: address already in use" + // when we try to start the new one. Force-removing by the well-known container + // name is idempotent: docker rm -f exits non-zero when the container doesn't + // exist, but the trailing || true (and 2>/dev/null) make the command succeed. + core.info("Cleaning up any stale awmg-mcpg container from a previous run..."); + try { + execSync("docker rm -f awmg-mcpg 2>/dev/null && echo 'Removed stale awmg-mcpg container' || true", { stdio: "inherit", timeout: DOCKER_CLEANUP_TIMEOUT_MS }); + } catch { + // Non-fatal: proceed even if the cleanup command itself fails + core.info("Could not remove stale awmg-mcpg container (may not exist)"); + } + core.info(""); - // Allow the child to run independently - child.unref(); + core.info("Starting gateway container..."); + core.info(""); - const gatewayPid = child.pid; - if (!gatewayPid) { - setFailed(`${ERR_SYSTEM}: Failed to start gateway container`); - return; - } - - core.info(`Gateway started with PID: ${gatewayPid}`); - printTiming(gatewayStartTime, "Gateway container launch"); - core.info("Verifying gateway process is running..."); + const gatewayStartTime = nowMs(); - if (isProcessAlive(gatewayPid)) { - core.info(`Gateway process confirmed running (PID: ${gatewayPid})`); - } else { - core.error("ERROR: Gateway process exited immediately after start"); - core.error(""); - printGatewayStartupDiagnostics(stderrPath, outputPath); - setFailed(`${ERR_SYSTEM}: Gateway process exited immediately after start`); - return; - } - core.info(""); - - // ----------------------------------------------------------------------- - // Wait for gateway to initialise - // ----------------------------------------------------------------------- - core.info("Waiting for gateway to initialize..."); - await sleep(5000); - core.info("Checking if gateway process is still alive after initialization..."); - - if (!isProcessAlive(gatewayPid)) { - core.error(`ERROR: Gateway process (PID: ${gatewayPid}) exited during initialization`); - core.error(""); - printGatewayStartupDiagnostics(stderrPath, outputPath); - setFailed(`${ERR_SYSTEM}: Gateway process (PID: ${gatewayPid}) exited during initialization`); - return; - } - core.info(`Gateway process is still running (PID: ${gatewayPid})`); - core.info(""); + // Split docker command into args, respecting simple quoting + let args = Array.from(dockerCommand.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || []); + const cmd = args.shift(); + if (!cmd) { + throw codedError("MCP_GATEWAY_DOCKER_COMMAND did not contain an executable command", ERR_CONFIG); + } + const gatewayEnv = { ...process.env }; + // Insert before all existing flags and the image, never into the command string. + args.splice(1, 0, ...caMountArgs); + args = injectCustomGatewayEnvArgs(args, gatewayEnv); + args = injectProxyEnvArgs(args, gatewayEnv); - // ----------------------------------------------------------------------- - // Health check loop - // ----------------------------------------------------------------------- - core.info("Waiting for gateway to be ready..."); - const healthCheckStart = nowMs(); - const healthHost = "localhost"; - const healthUrl = `http://${healthHost}:${gatewayPort}/health`; - - const healthRetryConfig = resolveGatewayHealthRetryConfig(); - const retryBudgetSeconds = Math.round(healthRetryConfig.retryBudgetMs / 1000); - core.info(`Health endpoint: ${healthUrl}`); - core.info(`(Note: MCP_GATEWAY_DOMAIN is '${gatewayDomain}' for container access)`); - core.info( - `Retrying up to ${healthRetryConfig.maxTotalAttempts} times with exponential backoff (${healthRetryConfig.initialDelayMs}ms initial delay, capped at ${healthRetryConfig.maxDelayMs}ms, ~${retryBudgetSeconds}s cumulative retry delay excluding request time)` - ); - core.info(""); + const outputFd = fs.openSync(outputPath, "w", 0o600); + const stderrPath = createGatewayStderrPath(); + const stderrFd = fs.openSync(stderrPath, "w", 0o600); - const maxTotalAttempts = healthRetryConfig.maxTotalAttempts; - const startupDeadline = gatewayStartTime + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS; - let httpCode = 0; - let healthBody = ""; - let succeeded = false; - let attemptsMade = 0; + let child; + try { + child = spawn(cmd, args, { + stdio: ["pipe", outputFd, stderrFd], + env: { ...gatewayEnv, MCP_GATEWAY_LOG_DIR: logDir }, + detached: true, + }); + } finally { + fs.closeSync(outputFd); + fs.closeSync(stderrFd); + } + child.on("error", err => { + activeGatewayPid = null; + core.error(`ERROR: Failed to launch MCP gateway process: ${getErrorMessage(err)}`); + }); + activeGatewayPid = child.pid || null; - core.info("=== Health Check Progress ==="); - try { - await retryGatewayStartup( - async remainingMs => { - // Counts total health-check attempts, including the final successful attempt. - attemptsMade += 1; - const elapsedSec = Math.floor((nowMs() - healthCheckStart) / 1000); - if (attemptsMade % 10 === 1 || attemptsMade === 1) { - core.info(`Attempt ${attemptsMade}/${maxTotalAttempts} (${elapsedSec}s elapsed)...`); - } + // Write configuration to stdin then close + if (!child.stdin) { + stopGatewayProcess(activeGatewayPid); + throw codedError("Gateway process stdin is not available"); + } + child.stdin.on("error", err => core.warning(`Gateway stdin: ${redactGatewayDiagnostics(getErrorMessage(err))}`)); + child.stdin.write(JSON.stringify(configObj)); + child.stdin.end(); - const res = await httpGet(healthUrl, Math.min(2000, remainingMs)); - httpCode = res.statusCode; - healthBody = res.body; - if (httpCode === 200 && healthBody) { - core.info(`✓ Health check succeeded on attempt ${attemptsMade} (${elapsedSec}s elapsed)`); - return; - } - throw codedError(`Health endpoint not ready (HTTP ${httpCode || 0})`); - }, - healthRetryConfig, - startupDeadline, - "MCP gateway health check", - () => isProcessAlive(gatewayPid) - ); - succeeded = true; - } catch { - // Retry exhaustion is ignored here; handled below using existing diagnostics. - } + // Allow the child to run independently + child.unref(); - core.info("=== End Health Check Progress ==="); - core.info(""); + const gatewayPid = child.pid; + if (!gatewayPid) { + throw codedError("Failed to start gateway container"); + } - core.info(`Final HTTP code: ${httpCode}`); - core.info(`Total attempts: ${attemptsMade}`); - if (healthBody) { - core.info(`Health response body: ${healthBody}`); - } else { - core.info("Health response body: (empty)"); - } + core.info(`Gateway started with PID: ${gatewayPid}`); + printTiming(gatewayStartTime, "Gateway container launch"); + core.info("Verifying gateway process is running..."); - if (succeeded) { - printTiming(healthCheckStart, "Health check wait"); - const safeOutputsReadinessStart = nowMs(); - try { - await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartTime); - } finally { - printTiming(safeOutputsReadinessStart, "Safe outputs readiness wait"); - } - core.info("Gateway is ready!"); - writeGatewayStartupMarker(gatewayStartupMarker); - } else { - core.error(""); - core.error("ERROR: Gateway failed to become ready"); - core.error(`Last HTTP code: ${httpCode}`); - core.error(`Last health response: ${healthBody || "(empty)"}`); - core.error(""); - core.error("Checking if gateway process is still alive..."); if (isProcessAlive(gatewayPid)) { - core.error(`Gateway process (PID: ${gatewayPid}) is still running`); + core.info(`Gateway process confirmed running (PID: ${gatewayPid})`); } else { - core.error(`Gateway process (PID: ${gatewayPid}) has exited`); + core.error("ERROR: Gateway process exited immediately after start"); + core.error(""); + printGatewayStartupDiagnostics(stderrPath, outputPath); + throw codedError("Gateway process exited immediately after start"); } - core.error(""); - core.error("Docker container status:"); - try { - execSync("docker ps -a 2>/dev/null | head -20", { stdio: "inherit", timeout: CONTAINER_STATUS_TIMEOUT_MS }); - } catch { - core.error("Could not list docker containers"); + core.info(""); + + // ----------------------------------------------------------------------- + // Wait for gateway to initialise + // ----------------------------------------------------------------------- + core.info("Waiting for gateway to initialize..."); + await sleep(5000); + core.info("Checking if gateway process is still alive after initialization..."); + + if (!isProcessAlive(gatewayPid)) { + core.error(`ERROR: Gateway process (PID: ${gatewayPid}) exited during initialization`); + core.error(""); + printGatewayStartupDiagnostics(stderrPath, outputPath); + throw codedError(`Gateway process (PID: ${gatewayPid}) exited during initialization`); } - core.error(""); - printGatewayStartupDiagnostics(stderrPath, outputPath); - core.error(""); - core.error("Checking network connectivity to gateway port..."); + core.info(`Gateway process is still running (PID: ${gatewayPid})`); + core.info(""); + + // ----------------------------------------------------------------------- + // Health check loop + // ----------------------------------------------------------------------- + core.info("Waiting for gateway to be ready..."); + const healthCheckStart = nowMs(); + const healthHost = "localhost"; + const healthUrl = `http://${healthHost}:${gatewayPort}/health`; + + const healthRetryConfig = resolveGatewayHealthRetryConfig(); + const retryBudgetSeconds = Math.round(healthRetryConfig.retryBudgetMs / 1000); + core.info(`Health endpoint: ${healthUrl}`); + core.info(`(Note: MCP_GATEWAY_DOMAIN is '${gatewayDomain}' for container access)`); + core.info( + `Retrying up to ${healthRetryConfig.maxTotalAttempts} times with exponential backoff (${healthRetryConfig.initialDelayMs}ms initial delay, capped at ${healthRetryConfig.maxDelayMs}ms, ~${retryBudgetSeconds}s cumulative retry delay excluding request time)` + ); + core.info(""); + + const maxTotalAttempts = healthRetryConfig.maxTotalAttempts; + const startupDeadline = gatewayStartTime + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS; + let httpCode = 0; + let healthBody = ""; + let succeeded = false; + let attemptsMade = 0; + + core.info("=== Health Check Progress ==="); try { - // Validate gatewayPort is numeric to prevent shell injection - const safePort = String(gatewayPort).replace(/[^0-9]/g, ""); - execSync(`netstat -tlnp 2>/dev/null | grep ":${safePort}" || ss -tlnp 2>/dev/null | grep ":${safePort}" || echo "Port ${safePort} does not appear to be listening"`, { stdio: "inherit", timeout: CONTAINER_STATUS_TIMEOUT_MS }); + await retryGatewayStartup( + async remainingMs => { + // Counts total health-check attempts, including the final successful attempt. + attemptsMade += 1; + const elapsedSec = Math.floor((nowMs() - healthCheckStart) / 1000); + if (attemptsMade % 10 === 1 || attemptsMade === 1) { + core.info(`Attempt ${attemptsMade}/${maxTotalAttempts} (${elapsedSec}s elapsed)...`); + } + + const res = await httpGet(healthUrl, Math.min(2000, remainingMs)); + httpCode = res.statusCode; + healthBody = res.body; + if (httpCode === 200 && healthBody) { + core.info(`✓ Health check succeeded on attempt ${attemptsMade} (${elapsedSec}s elapsed)`); + return; + } + throw codedError(`Health endpoint not ready (HTTP ${httpCode || 0})`); + }, + healthRetryConfig, + startupDeadline, + "MCP gateway health check", + () => isProcessAlive(gatewayPid) + ); + succeeded = true; } catch { - // ignore + // Retry exhaustion is ignored here; handled below using existing diagnostics. } - try { - process.kill(gatewayPid); - } catch { - // ignore + + core.info("=== End Health Check Progress ==="); + core.info(""); + + core.info(`Final HTTP code: ${httpCode}`); + core.info(`Total attempts: ${attemptsMade}`); + if (healthBody) { + core.info(`Health response body: ${healthBody}`); + } else { + core.info("Health response body: (empty)"); } - setFailed(`${ERR_SYSTEM}: Gateway failed to become ready`); - return; - } - core.info(""); - // ----------------------------------------------------------------------- - // Wait for gateway output (rewritten configuration) - // ----------------------------------------------------------------------- - core.info("Reading gateway output configuration..."); - const outputWaitStart = nowMs(); - const waitAttempts = 10; - for (let i = 0; i < waitAttempts; i++) { - try { - const stat = fs.statSync(outputPath); - if (stat.size > 0) { - core.info("Gateway output received!"); - break; + if (succeeded) { + printTiming(healthCheckStart, "Health check wait"); + const safeOutputsReadinessStart = nowMs(); + try { + await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartTime); + } finally { + printTiming(safeOutputsReadinessStart, "Safe outputs readiness wait"); } - } catch { - // Output file not ready yet — stat failure is ignored. + core.info("Gateway is ready!"); + writeGatewayStartupMarker(gatewayStartupMarker); + } else { + core.error(""); + core.error("ERROR: Gateway failed to become ready"); + core.error(`Last HTTP code: ${httpCode}`); + core.error(`Last health response: ${healthBody || "(empty)"}`); + core.error(""); + core.error("Checking if gateway process is still alive..."); + if (isProcessAlive(gatewayPid)) { + core.error(`Gateway process (PID: ${gatewayPid}) is still running`); + } else { + core.error(`Gateway process (PID: ${gatewayPid}) has exited`); + } + core.error(""); + core.error("Docker container status:"); + try { + execSync("docker ps -a 2>/dev/null | head -20", { stdio: "inherit", timeout: CONTAINER_STATUS_TIMEOUT_MS }); + } catch { + core.error("Could not list docker containers"); + } + core.error(""); + printGatewayStartupDiagnostics(stderrPath, outputPath); + core.error(""); + core.error("Checking network connectivity to gateway port..."); + try { + // Validate gatewayPort is numeric to prevent shell injection + const safePort = String(gatewayPort).replace(/[^0-9]/g, ""); + execSync(`netstat -tlnp 2>/dev/null | grep ":${safePort}" || ss -tlnp 2>/dev/null | grep ":${safePort}" || echo "Port ${safePort} does not appear to be listening"`, { stdio: "inherit", timeout: CONTAINER_STATUS_TIMEOUT_MS }); + } catch { + // ignore + } + try { + process.kill(gatewayPid); + } catch { + // ignore + } + throw codedError("Gateway failed to become ready"); } - if (i < waitAttempts - 1) { - await sleep(1000); + core.info(""); + + // ----------------------------------------------------------------------- + // Wait for gateway output (rewritten configuration) + // ----------------------------------------------------------------------- + core.info("Reading gateway output configuration..."); + const outputWaitStart = nowMs(); + const waitAttempts = 10; + for (let i = 0; i < waitAttempts; i++) { + try { + const stat = fs.statSync(outputPath); + if (stat.size > 0) { + core.info("Gateway output received!"); + break; + } + } catch { + // Output file not ready yet — stat failure is ignored. + } + if (i < waitAttempts - 1) { + await sleep(1000); + } } - } - printTiming(outputWaitStart, "Gateway output wait"); - core.info(""); + printTiming(outputWaitStart, "Gateway output wait"); + core.info(""); - // Verify output was written - let outputSize = 0; - try { - outputSize = fs.statSync(outputPath).size; - } catch { - // File doesn't exist — ignored, treated as empty output. - } - if (outputSize === 0) { - core.error("ERROR: Gateway did not write output configuration"); - core.error(""); - printGatewayStartupDiagnostics(stderrPath, outputPath); + // Verify output was written + let outputSize = 0; try { - process.kill(gatewayPid); + outputSize = fs.statSync(outputPath).size; } catch { - // ignore + // File doesn't exist — ignored, treated as empty output. } - setFailed(`${ERR_SYSTEM}: Gateway did not write output configuration`); - return; - } + if (outputSize === 0) { + core.error("ERROR: Gateway did not write output configuration"); + core.error(""); + printGatewayStartupDiagnostics(stderrPath, outputPath); + try { + process.kill(gatewayPid); + } catch { + // ignore + } + throw codedError("Gateway did not write output configuration"); + } + return { gatewayPid, stderrPath }; + }); // Restrict permissions try { @@ -1613,6 +1641,7 @@ module.exports = { validateGatewayAgentIdentifiers, waitForSafeOutputsReadiness, retryGatewayStartup, + startGatewayWithRetry, httpGet, writeGatewayStartupMarker, }; diff --git a/actions/setup/js/start_mcp_gateway.test.cjs b/actions/setup/js/start_mcp_gateway.test.cjs index 2c568fc974d..efcba708f54 100644 --- a/actions/setup/js/start_mcp_gateway.test.cjs +++ b/actions/setup/js/start_mcp_gateway.test.cjs @@ -35,11 +35,48 @@ import { validateGatewayAgentIdentifiers, waitForSafeOutputsReadiness, retryGatewayStartup, + startGatewayWithRetry, httpGet, } from "./start_mcp_gateway.cjs"; import { writeMcpConfiguration } from "./mcp_configuration.cjs"; import { applyGatewayRuntimeTimeouts } from "./tool_timeouts.cjs"; +describe("MCP gateway container startup retry", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("does not restart a successful gateway", async () => { + const start = vi.fn().mockResolvedValue({ gatewayPid: 123 }); + await expect(startGatewayWithRetry(start)).resolves.toEqual({ gatewayPid: 123 }); + expect(start).toHaveBeenCalledOnce(); + }); + + it("recovers on the single retry", async () => { + vi.useFakeTimers(); + const start = vi.fn().mockRejectedValueOnce(new Error("ERR_SYSTEM: container exited")).mockResolvedValue({ gatewayPid: 456 }); + const result = startGatewayWithRetry(start); + await vi.runAllTimersAsync(); + await expect(result).resolves.toEqual({ gatewayPid: 456 }); + expect(start).toHaveBeenCalledTimes(2); + }); + + it("reports the pre-agent step after exactly two failed attempts", async () => { + vi.useFakeTimers(); + const start = vi.fn().mockRejectedValue(new Error("ERR_SYSTEM: Gateway failed to become ready")); + const result = expect(startGatewayWithRetry(start)).rejects.toThrow('Pre-agent failure in step "Start MCP Gateway" after 2 startup attempts: ERR_SYSTEM: Gateway failed to become ready'); + await vi.runAllTimersAsync(); + await result; + expect(start).toHaveBeenCalledTimes(2); + }); + + it("does not retry invalid configuration", async () => { + const start = vi.fn().mockRejectedValue(new Error("ERR_CONFIG: invalid agent ID")); + await expect(startGatewayWithRetry(start)).rejects.toThrow("ERR_CONFIG: invalid agent ID"); + expect(start).toHaveBeenCalledOnce(); + }); +}); + describe("MCP gateway runner proxy environment", () => { it.each([ ["HTTP_PROXY", ["HTTP_PROXY", "NO_PROXY"]], @@ -121,7 +158,7 @@ describe("MCP gateway runner proxy environment", () => { it("adds runner proxy flags before launching the gateway", () => { const source = fs.readFileSync(new URL("./start_mcp_gateway.cjs", import.meta.url), "utf8"); - expect(source.indexOf("args = injectProxyEnvArgs(args, gatewayEnv)")).toBeLessThan(source.indexOf("const child = spawn(cmd, args")); + expect(source.indexOf("args = injectProxyEnvArgs(args, gatewayEnv)")).toBeLessThan(source.indexOf("child = spawn(cmd, args")); }); it.each(["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"])("redacts %s values in gateway diagnostics", name => { @@ -137,7 +174,7 @@ describe("MCP gateway CA launcher arguments", () => { const require = createRequire(import.meta.url); const spawn = vi.fn(() => ({ on: vi.fn(), - stdin: { write: vi.fn(), end: vi.fn() }, + stdin: { write: vi.fn(), end: vi.fn(), on: vi.fn() }, unref: vi.fn(), // Stop before health polling; this test covers the launch boundary. pid: undefined, @@ -164,6 +201,7 @@ describe("MCP gateway CA launcher arguments", () => { chmodSync: vi.fn(), rmSync: vi.fn(), openSync: vi.fn(() => 42), + closeSync: vi.fn(), mkdtempSync: vi.fn(() => directory), }, os: { tmpdir: () => directory }, @@ -179,14 +217,16 @@ describe("MCP gateway CA launcher arguments", () => { try { fs.writeFileSync(caPath, "certificate validated by gateway", { mode: 0o600 }); const source = fs.readFileSync(new URL("./start_mcp_gateway.cjs", import.meta.url), "utf8"); - await vm.runInNewContext(`${source}\nmain();`, { + await vm.runInNewContext(`${source}\nsleep = async () => {}; main().catch(err => core.setFailed(err.message));`, { require: id => mocks[id] || require(id), module: { exports: {} }, process: { env, umask: vi.fn(), on: vi.fn() }, core, }); - expect(spawn).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledTimes(2); + expect(mocks.fs.closeSync).toHaveBeenCalledTimes(4); + expect(mocks.child_process.execSync).toHaveBeenCalledTimes(2); const [command, args, options] = spawn.mock.calls[0]; const launchedConfig = JSON.parse(spawn.mock.results[0].value.stdin.write.mock.calls[0][0]); const target = launchedConfig.mcpServers.private.caCertFile; @@ -208,7 +248,7 @@ describe("MCP gateway CA launcher arguments", () => { ]); expect(options.env.NO_PROXY).toBe(inheritedProxy ? "host.docker.internal,awf-enclave-mcp" : undefined); expect(launchedConfig.mcpServers.public).not.toHaveProperty("caCertFile"); - expect(core.setFailed).toHaveBeenCalledExactlyOnceWith("ERR_SYSTEM: Failed to start gateway container"); + expect(core.setFailed).toHaveBeenCalledExactlyOnceWith('ERR_SYSTEM: Pre-agent failure in step "Start MCP Gateway" after 2 startup attempts: ERR_SYSTEM: Failed to start gateway container'); } finally { fs.rmSync(directory, { recursive: true, force: true }); } @@ -335,8 +375,8 @@ describe("start_mcp_gateway logging", () => { it.each([false, true])("times health and safe-output readiness separately when readiness fails: %s", async fails => { const source = fs.readFileSync(new URL("./start_mcp_gateway.cjs", import.meta.url), "utf8"); - const start = source.indexOf(" if (succeeded) {"); - const end = source.indexOf("\n } else {", start); + const start = source.indexOf(" if (succeeded) {"); + const end = source.indexOf("\n } else {", start); const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; const runSuccessBlock = new AsyncFunction( "{ nowMs, printTiming, waitForSafeOutputsReadiness, core, writeGatewayStartupMarker }", From 394b9fd12c650a5bffd3e9caa6b44cd7dd6af203 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:38:48 +0000 Subject: [PATCH 3/5] Apply remaining changes Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- schemas/audit.schema.json | 82 ++++++++++++++++++++++++++++++++++ schemas/logs-jsonl.schema.json | 82 ++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/schemas/audit.schema.json b/schemas/audit.schema.json index 3cb26d86827..94781137dee 100644 --- a/schemas/audit.schema.json +++ b/schemas/audit.schema.json @@ -5887,6 +5887,88 @@ "matches" ], "additionalProperties": false + }, + "model_catalog": { + "type": [ + "null", + "object" + ], + "properties": { + "source": { + "type": "string" + }, + "models_fetch_complete": { + "type": "boolean" + }, + "endpoints": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "models": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "supported_endpoints": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + }, + "warning": { + "type": "string" + } + }, + "required": [ + "provider", + "configured", + "models" + ], + "additionalProperties": false + } + }, + "warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "source", + "models_fetch_complete", + "endpoints" + ], + "additionalProperties": false } }, "required": [ diff --git a/schemas/logs-jsonl.schema.json b/schemas/logs-jsonl.schema.json index 84c671cbc4e..3b73d7fb132 100644 --- a/schemas/logs-jsonl.schema.json +++ b/schemas/logs-jsonl.schema.json @@ -9878,6 +9878,88 @@ "matches" ], "additionalProperties": false + }, + "model_catalog": { + "type": [ + "null", + "object" + ], + "properties": { + "source": { + "type": "string" + }, + "models_fetch_complete": { + "type": "boolean" + }, + "endpoints": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "models": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "supported_endpoints": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + }, + "warning": { + "type": "string" + } + }, + "required": [ + "provider", + "configured", + "models" + ], + "additionalProperties": false + } + }, + "warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "source", + "models_fetch_complete", + "endpoints" + ], + "additionalProperties": false } }, "required": [ From 0858968e48eba2b5ccef9c6328aef6c5e3013443 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:46:29 +0000 Subject: [PATCH 4/5] Fix MCP gateway retry and diagnostics review Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/constants.cjs | 7 ++ actions/setup/js/handle_agent_failure.cjs | 51 ++++++++++---- .../setup/js/handle_agent_failure.test.cjs | 24 +++++++ actions/setup/js/start_mcp_gateway.cjs | 54 ++++++++------- actions/setup/js/start_mcp_gateway.test.cjs | 66 +++++++++++++++++-- 5 files changed, 158 insertions(+), 44 deletions(-) diff --git a/actions/setup/js/constants.cjs b/actions/setup/js/constants.cjs index c7c2470d004..dfbefb82e51 100644 --- a/actions/setup/js/constants.cjs +++ b/actions/setup/js/constants.cjs @@ -204,6 +204,12 @@ const SCAN_WINDOW_MS = 24 * 60 * 60 * 1000; */ const PROXY_ENV_NAMES = Object.freeze(["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]); +/** + * GitHub Actions step name for starting the MCP gateway. + * @type {string} + */ +const STEP_NAME_START_MCP_GATEWAY = "Start MCP Gateway"; + module.exports = { AGENT_OUTPUT_FILENAME, TMP_GH_AW_PATH, @@ -231,4 +237,5 @@ module.exports = { MAX_RESTORE_PAGES, SCAN_WINDOW_MS, PROXY_ENV_NAMES, + STEP_NAME_START_MCP_GATEWAY, }; diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 24d5018ef0f..1ce8fe3ba69 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -40,7 +40,7 @@ const { parseTokenUsageJsonl, generateTokenUsageSummary } = require("./parse_mcp const { readDedupedTokenUsage, getTokenUsagePaths } = require("./parse_token_usage.cjs"); const { extractShellCommandDetailsFromToolData } = require("./tool_call_details.cjs"); const { resolveFailureIssueRepo } = require("./repo_helpers.cjs"); -const { GITHUB_API_VERSION } = require("./constants.cjs"); +const { GITHUB_API_VERSION, STEP_NAME_START_MCP_GATEWAY } = require("./constants.cjs"); const { buildEmptyOutputOutcome, buildCompletionEvidence, EMPTY_OUTPUT_CAUSES, EMPTY_OUTPUT_FAILURE_CAUSES } = require("./empty_output_outcome.cjs"); const { isAgentExecutionEvent } = require("./agent_execution.cjs"); const { collectUnifiedSession, isModelEndpointMismatchData } = require("./unified_session.cjs"); @@ -53,6 +53,27 @@ const path = require("path"); const DEFAULT_ACTION_FAILURE_ISSUE_EXPIRES_HOURS = 24 * 7; const ALLOWED_AGENT_CONCLUSIONS = new Set(["success", "failure", "cancelled", "skipped", "neutral", "timed_out", "action_required", "stale"]); +const EXECUTE_AGENT_STEP_NAME_RE = /^Execute .+ CLI$/; + +function isExecuteAgentStep(step) { + return typeof step.name === "string" && EXECUTE_AGENT_STEP_NAME_RE.test(step.name); +} + +function isGatewayStartupStep(step) { + return step.name === STEP_NAME_START_MCP_GATEWAY; +} + +function isCheckoutStep(step) { + return typeof step.name === "string" && step.name.startsWith("Checkout "); +} + +function isSetupFailureStep(step) { + return isGatewayStartupStep(step) || isCheckoutStep(step); +} + +function isFailedStep(step) { + return ["failure", "timed_out"].includes(step.conclusion); +} /** Claude Code error emitted when a `--continue` resume finds no deferred tool marker. */ const NO_DEFERRED_MARKER_LINE_RE = /No deferred tool marker found/i; /** Claude harness line reporting that the no-deferred-marker error was recovered by a fresh retry. */ @@ -2013,21 +2034,23 @@ async function getFailedAgentDiagnostics({ startupFailure = false } = {}) { return { failingStep: "", agentConclusion }; } const steps = agentJob.steps?.slice() || []; - const setupBoundary = steps.findIndex(step => step.name === "Start MCP Gateway" || (typeof step.name === "string" && step.name.startsWith("Execute "))); - const setupFailure = - setupBoundary >= 0 && - ["skipped", "failure", "timed_out"].includes(steps[setupBoundary].conclusion) && - (startupFailure || steps[setupBoundary].conclusion === "skipped" || steps.some(step => typeof step.name === "string" && step.name.startsWith("Execute ") && step.conclusion === "skipped")) && - !steps.some(step => typeof step.name === "string" && step.name.startsWith("Execute ") && step.conclusion !== "skipped") - ? steps - .slice(0, setupBoundary + (steps[setupBoundary].name === "Start MCP Gateway" ? 1 : 0)) - .find(step => ["failure", "timed_out"].includes(step.conclusion) && (step.name === "Start MCP Gateway" || (typeof step.name === "string" && step.name.startsWith("Checkout ")))) - : undefined; + const setupBoundary = steps.findIndex(step => isGatewayStartupStep(step) || isExecuteAgentStep(step)); + let setupFailure; + if (setupBoundary >= 0) { + const boundaryStep = steps[setupBoundary]; + const executionSteps = steps.filter(isExecuteAgentStep); + const boundaryFailed = ["skipped", "failure", "timed_out"].includes(boundaryStep.conclusion); + const agentNeverRan = executionSteps.every(step => step.conclusion === "skipped"); + const boundaryIndicatesPreAgentFailure = startupFailure || boundaryStep.conclusion === "skipped" || executionSteps.some(step => step.conclusion === "skipped"); + if (boundaryFailed && boundaryIndicatesPreAgentFailure && agentNeverRan) { + const setupStepsEnd = isGatewayStartupStep(boundaryStep) ? setupBoundary + 1 : setupBoundary; + setupFailure = steps.slice(0, setupStepsEnd).find(step => isFailedStep(step) && isSetupFailureStep(step)); + } + } if (!startupFailure) steps?.reverse(); - const redactionStep = steps?.find(step => step.name === "Redact secrets in logs" && ["failure", "timed_out"].includes(step.conclusion)); + const redactionStep = steps?.find(step => step.name === "Redact secrets in logs" && isFailedStep(step)); // Failed redaction blocks safe diagnostics even after an agent crash; startup keeps priority. - const failedStep = - (!startupFailure && redactionStep) || setupFailure || steps?.find(step => (["failure", "timed_out"].includes(step.conclusion) || (isCancelledTimeout && step.conclusion === "cancelled")) && typeof step.name === "string"); + const failedStep = (!startupFailure && redactionStep) || setupFailure || steps?.find(step => (isFailedStep(step) || (isCancelledTimeout && step.conclusion === "cancelled")) && typeof step.name === "string"); if (!failedStep) { return { failingStep: "", agentConclusion, ...(isCancelledTimeout ? { isTimedOut: true } : {}), attributionUnavailable: "The failed agent job had no failed step metadata." }; } diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index bc8335133a9..3780b749c7a 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -630,6 +630,30 @@ describe("handle_agent_failure", () => { await expect(getFailedAgentDiagnostics()).resolves.toEqual({ failingStep, agentConclusion: "failure", preAgentFailure: true }); }); + it("ignores an unrelated Execute step before the generated checkout and execution steps", async () => { + global.github = { + paginate: vi.fn().mockResolvedValue([ + { + name: "agent", + conclusion: "failure", + steps: [ + { name: "Execute migrations", conclusion: "success" }, + { name: "Checkout repository (gh-aw default)", conclusion: "failure" }, + { name: "Start MCP Gateway", conclusion: "skipped" }, + { name: "Execute GitHub Copilot CLI", conclusion: "skipped" }, + ], + }, + ]), + rest: { actions: { listJobsForWorkflowRun: vi.fn() } }, + }; + + await expect(getFailedAgentDiagnostics()).resolves.toEqual({ + failingStep: "Checkout repository (gh-aw default)", + agentConclusion: "failure", + preAgentFailure: true, + }); + }); + it("does not classify a checkout after agent execution as pre-agent", async () => { global.github = { paginate: vi.fn().mockResolvedValue([ diff --git a/actions/setup/js/start_mcp_gateway.cjs b/actions/setup/js/start_mcp_gateway.cjs index cd07475b15d..356a0956987 100644 --- a/actions/setup/js/start_mcp_gateway.cjs +++ b/actions/setup/js/start_mcp_gateway.cjs @@ -43,7 +43,7 @@ const { getSetupTimeoutMs } = require("./child_process_timeouts.cjs"); const { applyGatewayRuntimeTimeouts } = require("./tool_timeouts.cjs"); const { fetchMCPToolsResult } = require("./mount_mcp_as_cli.cjs"); const { writeMcpConfiguration } = require("./mcp_configuration.cjs"); -const { PROXY_ENV_NAMES } = require("./constants.cjs"); +const { PROXY_ENV_NAMES, STEP_NAME_START_MCP_GATEWAY } = require("./constants.cjs"); const { redactCliProxyDiagnostics } = require("./redact_cli_proxy_diagnostics.cjs"); const { prepareMCPServerCAs } = require("./mcp_server_ca.cjs"); const { ERR_CONFIG, ERR_SYSTEM } = require("./error_codes.cjs"); @@ -318,15 +318,19 @@ async function waitForSafeOutputsReadiness(config, gatewayPort, agentId, retryCo () => !permanentFailure && !insufficientBudget ); } catch (err) { - throw codedError( - `MCP gateway safeoutputs readiness failed after ${attempts} attempt(s): ` + - (permanentFailure - ? "permanent authentication or MCP protocol failure. The agent will not start. Check gateway credentials and MCP backend configuration." - : "no tools registered or backend unavailable. The agent will not start. Check MCP gateway startup logs for connection resets (ECONNRESET) or delayed backend registration.") + - ` Last probe: ${lastProbe}.`, - ERR_SYSTEM, - { cause: err } + const readinessError = Object.assign( + codedError( + `MCP gateway safeoutputs readiness failed after ${attempts} attempt(s): ` + + (permanentFailure + ? "permanent authentication or MCP protocol failure. The agent will not start. Check gateway credentials and MCP backend configuration." + : "no tools registered or backend unavailable. The agent will not start. Check MCP gateway startup logs for connection resets (ECONNRESET) or delayed backend registration.") + + ` Last probe: ${lastProbe}.`, + ERR_SYSTEM, + { cause: err } + ), + { nonRetryable: permanentFailure } ); + throw readinessError; } } @@ -729,24 +733,27 @@ function stopGatewayProcess(pid) { /** * Retry a failed container startup once, without retrying configuration errors. * @template T - * @param {() => Promise} start + * @param {(startupDeadline: number) => Promise} start + * @param {number} [startupDeadline] * @returns {Promise} */ -async function startGatewayWithRetry(start) { +async function startGatewayWithRetry(start, startupDeadline = Infinity) { for (let attempt = 1; ; attempt += 1) { try { - return await start(); + return await start(startupDeadline); } catch (err) { stopGatewayProcess(activeGatewayPid); activeGatewayPid = null; clearGatewayStartupMarker(getGatewayStartupMarkerPath()); const message = redactGatewayDiagnostics(getErrorMessage(err)); - if (message.startsWith(`${ERR_CONFIG}:`)) throw err; - if (attempt === 2) { - throw codedError(`Pre-agent failure in step "Start MCP Gateway" after 2 startup attempts: ${message}`, ERR_SYSTEM, { cause: err }); + const isNonRetryable = err instanceof Error && "nonRetryable" in err && err.nonRetryable === true; + if (message.startsWith(`${ERR_CONFIG}:`) || message.startsWith(`${ERR_SAFE_OUTPUTS_GATEWAY_CONFIG}:`) || isNonRetryable) throw err; + if (attempt === 2 || nowMs() >= startupDeadline) { + const attemptDescription = attempt === 1 ? "1 startup attempt" : `${attempt} startup attempts`; + throw codedError(`Pre-agent failure in step "${STEP_NAME_START_MCP_GATEWAY}" after ${attemptDescription}: ${message}`, ERR_SYSTEM, { cause: err }); } - core.warning(`Start MCP Gateway: startup attempt 1 failed; retrying once. ${message}`); - await sleep(1000); + core.warning(`${STEP_NAME_START_MCP_GATEWAY}: startup attempt ${attempt} failed; retrying once. ${message}`); + await sleep(Math.min(1000, Math.max(0, startupDeadline - nowMs()))); } } } @@ -1061,7 +1068,10 @@ async function main() { // ----------------------------------------------------------------------- const outputPath = path.join(configDir, "gateway-output.json"); const gatewayStartupMarker = getGatewayStartupMarkerPath(); - const { gatewayPid, stderrPath } = await startGatewayWithRetry(async () => { + const healthRetryConfig = resolveGatewayHealthRetryConfig(); + const gatewayStartupStartedAt = nowMs(); + const startupDeadline = gatewayStartupStartedAt + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS; + const { gatewayPid, stderrPath } = await startGatewayWithRetry(async sharedStartupDeadline => { clearGatewayStartupMarker(gatewayStartupMarker); // Clean up any stale gateway container from a previous run on this runner. @@ -1152,7 +1162,7 @@ async function main() { // Wait for gateway to initialise // ----------------------------------------------------------------------- core.info("Waiting for gateway to initialize..."); - await sleep(5000); + await sleep(Math.min(5000, Math.max(0, sharedStartupDeadline - nowMs()))); core.info("Checking if gateway process is still alive after initialization..."); if (!isProcessAlive(gatewayPid)) { @@ -1172,7 +1182,6 @@ async function main() { const healthHost = "localhost"; const healthUrl = `http://${healthHost}:${gatewayPort}/health`; - const healthRetryConfig = resolveGatewayHealthRetryConfig(); const retryBudgetSeconds = Math.round(healthRetryConfig.retryBudgetMs / 1000); core.info(`Health endpoint: ${healthUrl}`); core.info(`(Note: MCP_GATEWAY_DOMAIN is '${gatewayDomain}' for container access)`); @@ -1182,7 +1191,6 @@ async function main() { core.info(""); const maxTotalAttempts = healthRetryConfig.maxTotalAttempts; - const startupDeadline = gatewayStartTime + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS; let httpCode = 0; let healthBody = ""; let succeeded = false; @@ -1209,7 +1217,7 @@ async function main() { throw codedError(`Health endpoint not ready (HTTP ${httpCode || 0})`); }, healthRetryConfig, - startupDeadline, + sharedStartupDeadline, "MCP gateway health check", () => isProcessAlive(gatewayPid) ); @@ -1233,7 +1241,7 @@ async function main() { printTiming(healthCheckStart, "Health check wait"); const safeOutputsReadinessStart = nowMs(); try { - await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartTime); + await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartupStartedAt); } finally { printTiming(safeOutputsReadinessStart, "Safe outputs readiness wait"); } diff --git a/actions/setup/js/start_mcp_gateway.test.cjs b/actions/setup/js/start_mcp_gateway.test.cjs index efcba708f54..a85fa76c89a 100644 --- a/actions/setup/js/start_mcp_gateway.test.cjs +++ b/actions/setup/js/start_mcp_gateway.test.cjs @@ -75,6 +75,49 @@ describe("MCP gateway container startup retry", () => { await expect(startGatewayWithRetry(start)).rejects.toThrow("ERR_CONFIG: invalid agent ID"); expect(start).toHaveBeenCalledOnce(); }); + + it("does not retry safe-output configuration errors", async () => { + const start = vi.fn().mockRejectedValue(new Error("ERR_SAFE_OUTPUTS_GATEWAY_CONFIG: missing gateway port")); + await expect(startGatewayWithRetry(start)).rejects.toThrow("ERR_SAFE_OUTPUTS_GATEWAY_CONFIG: missing gateway port"); + expect(start).toHaveBeenCalledOnce(); + }); + + it("preserves one startup deadline across attempts", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1000); + const deadline = 5000; + const attemptDeadlines = []; + const start = vi + .fn() + .mockImplementationOnce(async sharedDeadline => { + attemptDeadlines.push(sharedDeadline); + throw new Error("ERR_SYSTEM: transient launch failure"); + }) + .mockImplementationOnce(async sharedDeadline => { + attemptDeadlines.push(sharedDeadline); + return { gatewayPid: 456 }; + }); + const result = startGatewayWithRetry(start, deadline); + await vi.runAllTimersAsync(); + + await expect(result).resolves.toEqual({ gatewayPid: 456 }); + expect(attemptDeadlines).toEqual([deadline, deadline]); + }); + + it("does not restart after the shared startup deadline expires", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1000); + const start = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 100)); + throw new Error("ERR_SYSTEM: gateway startup timed out"); + }); + const result = startGatewayWithRetry(start, 1100); + const rejection = expect(result).rejects.toThrow("after 1 startup attempt"); + await vi.advanceTimersByTimeAsync(100); + + await rejection; + expect(start).toHaveBeenCalledOnce(); + }); }); describe("MCP gateway runner proxy environment", () => { @@ -380,7 +423,7 @@ describe("start_mcp_gateway logging", () => { const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; const runSuccessBlock = new AsyncFunction( "{ nowMs, printTiming, waitForSafeOutputsReadiness, core, writeGatewayStartupMarker }", - `const succeeded = true, healthCheckStart = 10, gatewayStartTime = 0; + `const succeeded = true, healthCheckStart = 10, gatewayStartupStartedAt = 0; const configObj = {}, gatewayPort = 8080, agentId = "test", healthRetryConfig = {}, gatewayStartupMarker = "marker"; ${source.slice(start, end)} }` @@ -401,7 +444,8 @@ describe("start_mcp_gateway logging", () => { writeGatewayStartupMarker: marker, }); if (fails) { - await expect(result).rejects.toBe(failure); + const rejection = expect(result).rejects.toBe(failure); + await rejection; expect(marker).not.toHaveBeenCalled(); } else { await result; @@ -748,7 +792,9 @@ describe("start_mcp_gateway startup diagnostics", () => { [undefined, "readiness-test-key", "a numeric MCP_GATEWAY_PORT"], ["1234", undefined, "MCP_GATEWAY_AGENT_ID"], ])("requires safe-output gateway configuration with a stable error code", async (port, agentId, message) => { - await expect(waitForSafeOutputsReadiness(config, port, agentId, retryConfig)).rejects.toThrow(`ERR_SAFE_OUTPUTS_GATEWAY_CONFIG: MCP gateway safeoutputs readiness requires ${message}`); + const start = vi.fn(() => waitForSafeOutputsReadiness(config, port, agentId, retryConfig)); + await expect(startGatewayWithRetry(start)).rejects.toThrow(`ERR_SAFE_OUTPUTS_GATEWAY_CONFIG: MCP gateway safeoutputs readiness requires ${message}`); + expect(start).toHaveBeenCalledOnce(); }); it.each(["safeoutputs", "safe-outputs"])("probes the exact configured %s backend endpoint", async serverName => { @@ -792,12 +838,15 @@ describe("start_mcp_gateway startup diagnostics", () => { it("fails immediately without accepting tools after an authentication failure", async () => { const port = await startBackend(() => [{ name: "noop" }], { initStatus: 401 }); - const error = await waitForSafeOutputsReadiness(config, port, "readiness-test-key", retryConfig).catch(err => err); + const start = vi.fn(() => waitForSafeOutputsReadiness(config, port, "readiness-test-key", retryConfig)); + const error = await startGatewayWithRetry(start).catch(err => err); expect(error.message).toContain("permanent authentication or MCP protocol failure"); expect(error.message).toContain("Last probe: initialize failed (HTTP 401)."); expect(error.cause).toBeInstanceOf(Error); + expect(error.nonRetryable).toBe(true); expect(error.stack).not.toContain("readiness-test-key"); + expect(start).toHaveBeenCalledOnce(); expect(requests).toEqual(["initialize"]); expect(JSON.stringify(global.core.warning.mock.calls)).not.toContain("readiness-test-key"); }); @@ -805,12 +854,15 @@ describe("start_mcp_gateway startup diagnostics", () => { it.each(['{"jsonrpc":"2.0","id":2,"error":{"code":-32602,"message":"readiness-test-key"}}', "malformed readiness-test-key"])("fails immediately on permanent tools/list protocol failures", async listResponse => { const port = await startBackend(() => [], { listResponse }); - const error = await waitForSafeOutputsReadiness(config, port, "readiness-test-key", retryConfig).catch(err => err); + const start = vi.fn(() => waitForSafeOutputsReadiness(config, port, "readiness-test-key", retryConfig)); + const error = await startGatewayWithRetry(start).catch(err => err); expect(error.message).toContain("permanent authentication or MCP protocol failure"); expect(error.message).toContain("Last probe: tools/list failed (HTTP 200)."); expect(error.cause).toBeInstanceOf(Error); + expect(error.nonRetryable).toBe(true); expect(error.stack).not.toContain("readiness-test-key"); + expect(start).toHaveBeenCalledOnce(); expect(requests.filter(method => method === "initialize")).toHaveLength(1); expect(JSON.stringify(global.core.warning.mock.calls)).not.toContain("readiness-test-key"); }); @@ -933,7 +985,7 @@ describe("start_mcp_gateway startup diagnostics", () => { it("gates startup completion and outputs on safeoutputs readiness", () => { const source = fs.readFileSync(new URL("./start_mcp_gateway.cjs", import.meta.url), "utf8"); - const gate = source.indexOf("await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartTime)"); + const gate = source.indexOf("await waitForSafeOutputsReadiness(configObj, gatewayPort, agentId, healthRetryConfig, gatewayStartupStartedAt)"); expect(gate).toBeGreaterThan(0); expect(gate).toBeLessThan(source.indexOf("writeGatewayStartupMarker(gatewayStartupMarker)")); expect(gate).toBeLessThan(source.indexOf("fs.appendFileSync(githubOutput, outputs")); @@ -942,7 +994,7 @@ describe("start_mcp_gateway startup diagnostics", () => { expect(source).toContain("core.setFailed(/^ERR_[A-Z]+:/.test(text) ? text : `${ERR_SYSTEM}: ${text}`)"); expect(source).toContain("setFailed(getErrorMessage(err));"); expect(source).toContain("setFailed(message);"); - expect(source).toContain("const startupDeadline = gatewayStartTime + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS"); + expect(source).toContain("const startupDeadline = gatewayStartupStartedAt + healthRetryConfig.backendStartupTimeoutMs + MCP_GATEWAY_HEALTH_REGISTRATION_MARGIN_MS"); expect(source).toContain("httpGet(healthUrl, Math.min(2000, remainingMs))"); }); }); From 26726ecdcb0d1281b407709a30dd8997457db1ff Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:58:04 +0000 Subject: [PATCH 5/5] Add changeset --- .changeset/patch-retry-gateway-startup.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/patch-retry-gateway-startup.md diff --git a/.changeset/patch-retry-gateway-startup.md b/.changeset/patch-retry-gateway-startup.md new file mode 100644 index 00000000000..5dbfc6001c9 --- /dev/null +++ b/.changeset/patch-retry-gateway-startup.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Retry MCP gateway startup failures once and improve diagnostics for pre-agent checkout failures.