diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index a38a0f252e0..5faa07a089b 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"904043e6f7d1d1121005edda7e346e977d286e134537e7bd802a9966571ef72d","body_hash":"0daed06ad9ab37f712e54a71683b2f3c690ab0cce61735ab8ecf312216a2f4ec","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.90","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ff0cbc82b1f5518390c58553431e35b62580466e05b3edcbfac040f2ba86ebdb","body_hash":"0daed06ad9ab37f712e54a71683b2f3c690ab0cce61735ab8ecf312216a2f4ec","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.90","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.50","digest":"sha256:91fe4af8373c723d4d2e0706a0c35d310c2eeadaea5e0738c725b3861042f620","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.50@sha256:91fe4af8373c723d4d2e0706a0c35d310c2eeadaea5e0738c725b3861042f620"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.50","digest":"sha256:11f12bcd3dde377ba822df7c670d6b937e9e0cebc46218946d4edc972d7fd965","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.50@sha256:11f12bcd3dde377ba822df7c670d6b937e9e0cebc46218946d4edc972d7fd965"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.50","digest":"sha256:cc9b6b979edf4aafbb3811c2bdc4891039d5346473afa88e304aa692167004f9","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.50@sha256:cc9b6b979edf4aafbb3811c2bdc4891039d5346473afa88e304aa692167004f9"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.30","digest":"sha256:ab5a436a1490438db473e4e3d4c973cb1d75e3cb233fb08b73d31b42d7d18fba","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.30@sha256:ab5a436a1490438db473e4e3d4c973cb1d75e3cb233fb08b73d31b42d7d18fba"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v1.12.2","digest":"sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6","pinned_image":"ghcr.io/github/github-mcp-server:v1.12.2@sha256:508a0857ec762b1ab1cece29193345b501fab1dd9d1228a7b617062954cecac6"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -159,7 +159,7 @@ jobs: GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential","single_agent_control"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test.","hypothesis":"H0: no change in pass_rate between inline_strict and delegated_sequential. H1: delegated_sequential improves pass_rate by \u003e= 0.15 absolute versus inline_strict. Note: single_agent_control is a synthetic negative baseline (always FAIL by design) excluded from H1 comparisons.","metric":"pass_rate","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[34,33,33],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 904043e6f7d1d1121005edda7e346e977d286e134537e7bd802a9966571ef72d:0daed06ad9ab37f712e54a71683b2f3c690ab0cce61735ab8ecf312216a2f4ec + GH_AW_HARNESS_VERSION: ff0cbc82b1f5518390c58553431e35b62580466e05b3edcbfac040f2ba86ebdb:0daed06ad9ab37f712e54a71683b2f3c690ab0cce61735ab8ecf312216a2f4ec with: script: | const path = require('path'); @@ -1167,6 +1167,15 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + - env: + SMOKE_EXECUTION: ${{ steps.agentic_execution.outcome }} + SMOKE_VARIANT: ${{ needs.activation.outputs.sub_agent_strategy }} + if: always() + name: Assert sub-agent model and wire API evidence + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\nif (process.env.SMOKE_VARIANT === \"single_agent_control\") {\n assert.equal(started.length, 0, \"Control variant must not invoke sub-agents\");\n return;\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"/chat/completions\"],\n [\"mini-whoami\", \"gpt-5-mini\", \"/responses\"],\n [\"nano-whoami\", \"gpt-5-nano\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\nfor (const [agentName, model, endpoint] of expected) {\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" + # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact - name: Upload agent output fallback artifact diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index 0b4dae0cdad..a70c52a6753 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -54,6 +54,68 @@ evals: sandbox: agent: id: awf +post-steps: + - name: Assert sub-agent model and wire API evidence + if: always() + uses: actions/github-script@v9.0.0 + env: + SMOKE_VARIANT: ${{ needs.activation.outputs.sub_agent_strategy }} + SMOKE_EXECUTION: ${{ steps.agentic_execution.outcome }} + with: + script: | + const assert = require("node:assert/strict"); + const fs = require("node:fs"); + const path = require("node:path"); + const root = "/tmp/gh-aw"; + const { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/unified_session.cjs")); + writeUnifiedSession({ rootDir: root, engine: "copilot" }); + const readJsonl = file => fs.readFileSync(file, "utf8") + .split("\n").filter(line => line.trim()).map(line => JSON.parse(line)); + const events = readJsonl(path.join(root, "usage/aw_session.jsonl")); + const started = events.filter(event => event.type === "subagent.started"); + if (process.env.SMOKE_VARIANT === "single_agent_control") { + assert.equal(started.length, 0, "Control variant must not invoke sub-agents"); + return; + } + assert.equal(process.env.SMOKE_EXECUTION, "success", "Copilot SDK execution failed"); + const proxyPaths = [ + "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", + "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl", + "sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl", + ]; + const proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file)); + assert.ok(proxyFile, "Missing api-proxy token usage evidence"); + const requests = readJsonl(proxyFile); + const expected = [ + ["haiku-whoami", "claude-haiku-4.5", "/chat/completions"], + ["mini-whoami", "gpt-5-mini", "/responses"], + ["nano-whoami", "gpt-5-nano", "/responses"], + ]; + const servedModel = model => String(model || "") + .replace(/^copilot(?:-(?:responses|completions))?(?:-\d+)?\//, "") + .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, "") + .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, "$1-$2.$3"); + for (const [agentName, model, endpoint] of expected) { + const invocations = started.filter(event => event.data.agentName === agentName); + assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`); + const invocation = invocations[0]; + const invocationId = invocation.data.invocationId || invocation.agentId; + assert.ok(invocationId, `${agentName} is missing an invocation ID`); + const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId); + assert.ok(activity.some(event => event.type === "subagent.completed"), `${agentName} did not complete`); + assert.ok(!activity.some(event => event.type === "subagent.failed"), `${agentName} failed`); + const models = activity.flatMap(event => [ + event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel, + ]).filter(Boolean); + assert.ok(models.length > 0, `${agentName} has no recorded model`); + assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`); + assert.ok(requests.some(request => + servedModel(request.model) === model && + (request.path || request.endpoint) === endpoint && + Number(request.status) === 200 && + request.purpose !== "routing_classification"), + `${agentName} has no successful ${model} request on ${endpoint}`); + } --- # Smoke Test: Copilot SDK Inline Sub-Agents diff --git a/actions/setup/js/awf_reflect.cjs b/actions/setup/js/awf_reflect.cjs index b9c677ad73a..b06aae1efdc 100644 --- a/actions/setup/js/awf_reflect.cjs +++ b/actions/setup/js/awf_reflect.cjs @@ -24,6 +24,7 @@ const tls = require("tls"); const { withRetry, sleep } = require("./error_recovery.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { AWF_REFLECT_OUTPUT_PATH } = require("./awf_reflect_paths.cjs"); +const { qualifyModelForMultiProvider, isCopilotProviderName } = require("./copilot_sdk_multi_provider.cjs"); function parseReflectTimeoutMs(value) { const rawValue = String(value || "").trim(); @@ -919,15 +920,13 @@ function resolveOpenAICompatibleEndpointFromReflect(options) { * Returns `null` when no configured endpoints are present or the data is * unavailable. * - * Each endpoint becomes a `NamedProviderConfig` (using the endpoint's `provider` - * field as the stable name) and every model advertised by that endpoint becomes a - * `ProviderModelConfig` tuple `{ id, provider }` referencing it. Callers can - * derive provider-qualified selection ids as `"/"` if needed. + * Copilot endpoints become separate responses and completions providers. Other + * endpoints retain a single provider. Models reference their inferred wire API. * * The primary model is the first model that matches `options.model` (if set), * otherwise the first model across all providers. - * A valid `options.wireApi` overrides inference for the primary provider only, - * except for Anthropic providers, which do not use an OpenAI wire API. + * A valid `options.wireApi` routes only the configured Copilot model; it never + * changes the wire API used by the other models. The primary model is qualified. * * @param {{ * model?: string, @@ -970,6 +969,11 @@ function resolveMultiProviderFromReflect(options) { // provider label (e.g. two "copilot" entries at different ports). /** @type {Map} */ const providerNameCount = new Map(); + const configuredWireApi = String(options?.wireApi || "") + .toLowerCase() + .trim(); + let configuredModelMapped = false; + let mappedConfiguredModel = ""; for (const endpoint of endpoints) { const baseUrl = endpointBaseUrl(endpoint); @@ -985,18 +989,62 @@ function resolveMultiProviderFromReflect(options) { } // Ensure unique provider names by appending a suffix when the same name appears twice. - const existing = providerNameCount.get(rawProviderName) ?? 0; - providerNameCount.set(rawProviderName, existing + 1); - const providerName = existing === 0 ? rawProviderName : `${rawProviderName}-${existing}`; + const isCopilot = isCopilotProviderName(rawProviderName); + const stableProviderName = isCopilot ? "copilot" : rawProviderName; + const existing = providerNameCount.get(stableProviderName) ?? 0; + providerNameCount.set(stableProviderName, existing + 1); + const suffix = existing === 0 ? "" : `-${existing}`; + const providerName = `${stableProviderName}${suffix}`; const endpointModels = Array.isArray(endpoint.models) ? endpoint.models.filter(m => typeof m === "string" && m.trim().length > 0) : []; - // Infer provider type and wire API using the configured model if available, - // otherwise fall back to the first model. - // For multi-model providers (e.g. Copilot), different models may have different wire APIs, - // so we prefer the configured model to ensure the correct wireApi is selected. + if (isCopilot) { + /** @type {Array<"responses" | "completions">} */ + const wireApis = ["responses", "completions"]; + for (const wireApi of wireApis) { + providers.push({ name: `copilot-${wireApi}${suffix}`, type: "openai", baseUrl, wireApi }); + } + // Routing metadata uses model_id (not id); HTTP paths exclude ws:/responses. + const routingModels = Array.isArray(endpoint.routing_models) ? endpoint.routing_models : Array.isArray(rd.routing_models) ? rd.routing_models : []; + for (const modelId of endpointModels) { + const catalogEntry = getCatalogModelEntry(options?.modelsJson ?? null, modelId, "github-copilot"); + let wireApi = inferWireApiForModel("openai", modelId, catalogEntry); + const metadata = routingModels.find(model => model?.model_id === modelId); + const supported = metadata?.supported_endpoints; + if (Array.isArray(supported)) { + const responses = supported.includes("/responses"); + const completions = supported.includes("/chat/completions"); + if (!responses && !completions) { + logger(`sdk-mode(multi): skipping Copilot model="${modelId}" with no supported SDK wire API`); + continue; + } + if (responses !== completions) wireApi = responses ? "responses" : "completions"; + } + const matchesConfigured = configuredModel === modelId || configuredModel === `copilot/${modelId}` || ["responses", "completions"].some(api => configuredModel === `copilot-${api}${suffix}/${modelId}`); + if (!configuredModelMapped && matchesConfigured) { + configuredModelMapped = true; + const qualifiedWireApi = wireApis.find(api => configuredModel === `copilot-${api}${suffix}/${modelId}`); + if (qualifiedWireApi && (!Array.isArray(supported) || supported.includes(qualifiedWireApi === "responses" ? "/responses" : "/chat/completions"))) { + wireApi = qualifiedWireApi; + } + if (configuredWireApi === "responses" || configuredWireApi === "completions") { + const requestedEndpoint = configuredWireApi === "responses" ? "/responses" : "/chat/completions"; + if (Array.isArray(supported) && !supported.includes(requestedEndpoint)) { + throw new Error(`Model endpoint mismatch: model "${modelId}" with COPILOT_PROVIDER_WIRE_API=${configuredWireApi} requires ${requestedEndpoint}; supported endpoints: ${supported.join(", ")}`); + } + wireApi = configuredWireApi; + } + mappedConfiguredModel = `copilot-${wireApi}${suffix}/${modelId}`; + } + models.push({ id: modelId, provider: `copilot-${wireApi}${suffix}` }); + } + logger(`sdk-mode(multi): split provider="${providerName}" into responses/completions modelCount=${endpointModels.length} baseUrl="${baseUrl}"`); + continue; + } + + // Other endpoints retain their provider-wide inference. const firstModel = endpointModels.length > 0 ? endpointModels[0] : ""; - const modelForInference = configuredModel && endpointModels.includes(configuredModel) ? configuredModel : firstModel; + const modelForInference = endpointModels.find(modelId => configuredModel === modelId || configuredModel === `${providerName}/${modelId}`) || firstModel; const catalogProviderName = rawProviderName.toLowerCase() === "copilot" ? "github-copilot" : rawProviderName; const catalogEntry = modelForInference ? getCatalogModelEntry(options?.modelsJson ?? null, modelForInference, catalogProviderName) : null; const providerType = inferProviderTypeForModel(rawProviderName, modelForInference, catalogEntry); @@ -1026,13 +1074,9 @@ function resolveMultiProviderFromReflect(options) { // Determine the primary model: prefer the configured model if it appears in the model list; // otherwise fall back to the first model across all providers. - let primaryModel = ""; - if (configuredModel) { - const match = models.find(m => m.id === configuredModel); - if (match) primaryModel = match.id; - } + let primaryModel = qualifyModelForMultiProvider(configuredModel, { providers, models }) || mappedConfiguredModel; if (!primaryModel && models.length > 0) { - primaryModel = models[0].id; + primaryModel = `${models[0].provider}/${models[0].id}`; } if (!primaryModel) { @@ -1040,12 +1084,12 @@ function resolveMultiProviderFromReflect(options) { return null; } - const primaryProviderName = models.find(m => m.id === primaryModel)?.provider; + const primaryProviderName = models.find(m => `${m.provider}/${m.id}` === primaryModel)?.provider; const primaryProvider = providers.find(p => p.name === primaryProviderName); const wireApi = String(options?.wireApi || "") .toLowerCase() .trim(); - if (primaryProvider && primaryProvider.type !== "anthropic" && (wireApi === "responses" || wireApi === "completions")) { + if (primaryProvider && !primaryProvider.name.startsWith("copilot-") && primaryProvider.type !== "anthropic" && (wireApi === "responses" || wireApi === "completions")) { primaryProvider.wireApi = wireApi; logger(`sdk-mode(multi): primary provider="${primaryProvider.name}" wireApi="${wireApi}" selected from configured wire API`); } diff --git a/actions/setup/js/awf_reflect.test.cjs b/actions/setup/js/awf_reflect.test.cjs index 4ee05e6c2e8..42fee967ec7 100644 --- a/actions/setup/js/awf_reflect.test.cjs +++ b/actions/setup/js/awf_reflect.test.cjs @@ -959,6 +959,73 @@ describe("awf_reflect.cjs", () => { }); describe("resolveMultiProviderFromReflect", () => { + const splitEndpoint = { + provider: "copilot", + configured: true, + port: 10002, + models: ["claude-haiku-4.5", "gpt-5.6-luna", "gpt-5-mini", "gpt-4o", "unsupported"], + routing_models: [ + { model_id: "claude-haiku-4.5", supported_endpoints: ["/chat/completions", "/v1/messages"] }, + { model_id: "gpt-5.6-luna", supported_endpoints: ["/responses", "ws:/responses"] }, + { model_id: "gpt-5-mini", supported_endpoints: ["/chat/completions", "/responses", "ws:/responses"] }, + { model_id: "gpt-4o", supported_endpoints: ["/chat/completions", "/responses"] }, + { model_id: "unsupported", supported_endpoints: ["/v1/messages"] }, + ], + }; + + it("splits Copilot models using HTTP endpoint support and per-model inference", () => { + const result = resolveMultiProviderFromReflect({ reflectData: { endpoints: [splitEndpoint] }, model: "gpt-5.6-luna" }); + expect(result.providers).toEqual([ + { name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "responses" }, + { name: "copilot-completions", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "completions" }, + ]); + expect(result.models).toEqual([ + { id: "claude-haiku-4.5", provider: "copilot-completions" }, + { id: "gpt-5.6-luna", provider: "copilot-responses" }, + { id: "gpt-5-mini", provider: "copilot-responses" }, + { id: "gpt-4o", provider: "copilot-completions" }, + ]); + expect(result.model).toBe("copilot-responses/gpt-5.6-luna"); + }); + + it.each(["gpt-5.6-luna", "copilot/gpt-5.6-luna", "copilot-responses/gpt-5.6-luna"])("rejects an unsupported endpoint override for %s", model => { + expect(() => resolveMultiProviderFromReflect({ reflectData: { endpoints: [splitEndpoint] }, model, wireApi: "completions" })).toThrow( + 'Model endpoint mismatch: model "gpt-5.6-luna" with COPILOT_PROVIDER_WIRE_API=completions requires /chat/completions; supported endpoints: /responses, ws:/responses' + ); + }); + + it("routes a compatible override only for the selected model, not its siblings", () => { + const model = "gpt-5-mini"; + const result = resolveMultiProviderFromReflect({ reflectData: { endpoints: [splitEndpoint] }, model, wireApi: "completions" }); + expect(result.model).toBe("copilot-completions/gpt-5-mini"); + expect(result.models.find(entry => entry.id === "gpt-5.6-luna").provider).toBe("copilot-responses"); + expect(result.models.find(entry => entry.id === "claude-haiku-4.5").provider).toBe("copilot-completions"); + expect(result.providers[0].wireApi).toBe("responses"); + }); + + it("uses top-level routing model metadata when an endpoint does not have it", () => { + const result = resolveMultiProviderFromReflect({ + reflectData: { + endpoints: [{ provider: "github", configured: true, port: 10002, models: ["gpt-5.6-luna"] }], + routing_models: [{ model_id: "gpt-5.6-luna", supported_endpoints: ["/chat/completions"] }], + }, + }); + expect(result.model).toBe("copilot-completions/gpt-5.6-luna"); + }); + + it("preserves a qualified API preference for a both-supported model instead of selecting the first model", () => { + const reflectData = { endpoints: [splitEndpoint] }; + expect(resolveMultiProviderFromReflect({ reflectData, model: "copilot-responses/gpt-4o" }).model).toBe("copilot-responses/gpt-4o"); + expect(resolveMultiProviderFromReflect({ reflectData, model: "copilot-completions/gpt-5-mini" }).model).toBe("copilot-completions/gpt-5-mini"); + expect(resolveMultiProviderFromReflect({ reflectData, model: "copilot-responses/gpt-4o", wireApi: "completions" }).model).toBe("copilot-completions/gpt-4o"); + }); + + it("qualifies a fallback and preserves valid endpoint-qualified selections", () => { + const reflectData = { endpoints: [splitEndpoint, { ...splitEndpoint, port: 10003 }] }; + expect(resolveMultiProviderFromReflect({ reflectData, model: "unknown" }).model).toBe("copilot-completions/claude-haiku-4.5"); + expect(resolveMultiProviderFromReflect({ reflectData, model: "copilot-responses-1/gpt-5-mini" }).model).toBe("copilot-responses-1/gpt-5-mini"); + }); + it("returns null when reflectData is null", () => { const logs = []; const result = resolveMultiProviderFromReflect({ reflectData: null, logger: msg => logs.push(msg) }); @@ -971,8 +1038,8 @@ describe("awf_reflect.cjs", () => { reflectData: { endpoints: [{ provider: "copilot", port: 10002, configured: true, models: ["gpt-5.4"] }] }, }); expect(result).not.toBeNull(); - expect(result.providers).toHaveLength(1); - expect(result.model).toBe("gpt-5.4"); + expect(result.providers).toHaveLength(2); + expect(result.model).toBe("copilot-responses/gpt-5.4"); }); it("rewrites provider baseUrl to the host bridge in sbx HOSTALIASES mode", () => { @@ -1033,7 +1100,7 @@ describe("awf_reflect.cjs", () => { ], }; const result = resolveMultiProviderFromReflect({ reflectData }); - expect(result.model).toBe("gpt-5.4"); + expect(result.model).toBe("openai/gpt-5.4"); }); it("prefers the configured model when it appears in model list", () => { @@ -1044,7 +1111,7 @@ describe("awf_reflect.cjs", () => { ], }; const result = resolveMultiProviderFromReflect({ reflectData, model: "claude-sonnet-4.6" }); - expect(result.model).toBe("claude-sonnet-4.6"); + expect(result.model).toBe("anthropic/claude-sonnet-4.6"); }); it("falls back to first model when configured model is not found in list", () => { @@ -1055,7 +1122,7 @@ describe("awf_reflect.cjs", () => { ], }; const result = resolveMultiProviderFromReflect({ reflectData, model: "nonexistent-model" }); - expect(result.model).toBe("gpt-5.4"); + expect(result.model).toBe("openai/gpt-5.4"); }); it("derives provider baseUrl from models_url origin when available", () => { @@ -1083,9 +1150,9 @@ describe("awf_reflect.cjs", () => { }, }; const result = resolveMultiProviderFromReflect({ reflectData, modelsJson }); - expect(result.providers[0]).toMatchObject({ name: "copilot", type: "openai", wireApi: "responses" }); - expect(result.providers[1]).toMatchObject({ name: "anthropic", type: "anthropic" }); - expect(result.providers[1]).not.toHaveProperty("wireApi"); + expect(result.providers[0]).toMatchObject({ name: "copilot-responses", type: "openai", wireApi: "responses" }); + expect(result.providers[2]).toMatchObject({ name: "anthropic", type: "anthropic" }); + expect(result.providers[2]).not.toHaveProperty("wireApi"); }); it("infers wireApi from the base catalog model for a Copilot utility variant", () => { @@ -1102,7 +1169,7 @@ describe("awf_reflect.cjs", () => { reflectData, modelsJson, }); - expect(result.providers[0]).toMatchObject({ name: "copilot", type: "openai", wireApi: "responses" }); + expect(result.providers[0]).toMatchObject({ name: "copilot-responses", type: "openai", wireApi: "responses" }); }); it.each([null, { providers: { "github-copilot": { models: { "gpt-6-luna": { wire_api: "completions" } } } } }])("honors an explicit Responses API for gpt-6-luna with a missing or stale catalog", modelsJson => { @@ -1112,8 +1179,8 @@ describe("awf_reflect.cjs", () => { reflectData: { endpoints: [{ provider: "copilot", port: 10002, configured: true, models: ["gpt-6-luna"] }] }, modelsJson, }); - expect(result.model).toBe("gpt-6-luna"); - expect(result.providers[0]).toMatchObject({ name: "copilot", type: "openai", wireApi: "responses" }); + expect(result.model).toBe("copilot-responses/gpt-6-luna"); + expect(result.providers[0]).toMatchObject({ name: "copilot-responses", type: "openai", wireApi: "responses" }); }); it("applies the wire API override only to the selected provider", () => { @@ -1127,8 +1194,8 @@ describe("awf_reflect.cjs", () => { ], }, }); - expect(result.providers[0].wireApi).toBe("completions"); - expect(result.providers[1]).toMatchObject({ name: "copilot-1", wireApi: "responses" }); + expect(result.models[0].provider).toBe("copilot-completions"); + expect(result.models[1].provider).toBe("copilot-responses-1"); }); it("honors an explicit Completions API over catalog inference", () => { @@ -1138,7 +1205,7 @@ describe("awf_reflect.cjs", () => { reflectData: { endpoints: [{ provider: "copilot", port: 10002, configured: true, models: ["gpt-6-luna"] }] }, modelsJson: { providers: { "github-copilot": { models: { "gpt-6-luna": { wire_api: "responses" } } } } }, }); - expect(result.providers[0].wireApi).toBe("completions"); + expect(result.model).toBe("copilot-completions/gpt-6-luna"); }); it.each(["Responses", " responses "])("normalizes explicit wire API overrides (%s)", wireApi => { @@ -1150,14 +1217,13 @@ describe("awf_reflect.cjs", () => { expect(result.providers[0].wireApi).toBe("responses"); }); - it("applies an explicit wire API override to the fallback primary model", () => { + it("does not apply an unmatched model's wire API override to the fallback primary model", () => { const result = resolveMultiProviderFromReflect({ model: "nonexistent-model", wireApi: "responses", - reflectData: { endpoints: [{ provider: "copilot", port: 10002, configured: true, models: ["gpt-6-luna"] }] }, + reflectData: { endpoints: [{ provider: "copilot", port: 10002, configured: true, models: ["gpt-4o"] }] }, }); - expect(result.model).toBe("gpt-6-luna"); - expect(result.providers[0].wireApi).toBe("responses"); + expect(result.model).toBe("copilot-completions/gpt-4o"); }); it("infers the fallback primary model wire API instead of inheriting an automatic GPT default", () => { @@ -1166,7 +1232,7 @@ describe("awf_reflect.cjs", () => { reflectData: { endpoints: [{ provider: "openai", port: 10002, configured: true, models: ["gpt-4.1"] }] }, modelsJson: { providers: { openai: { models: { "gpt-4.1": { wire_api: "completions" } } } } }, }); - expect(result.model).toBe("gpt-4.1"); + expect(result.model).toBe("openai/gpt-4.1"); expect(result.providers[0].wireApi).toBe("completions"); }); @@ -1197,10 +1263,9 @@ describe("awf_reflect.cjs", () => { ], }; const result = resolveMultiProviderFromReflect({ reflectData }); - expect(result.providers[0].name).toBe("copilot"); - expect(result.providers[1].name).toBe("copilot-1"); - expect(result.models[0]).toEqual({ id: "gpt-5.4", provider: "copilot" }); - expect(result.models[1]).toEqual({ id: "gpt-5.5", provider: "copilot-1" }); + expect(result.providers.map(provider => provider.name)).toEqual(["copilot-responses", "copilot-completions", "copilot-responses-1", "copilot-completions-1"]); + expect(result.models[0]).toEqual({ id: "gpt-5.4", provider: "copilot-responses" }); + expect(result.models[1]).toEqual({ id: "gpt-5.5", provider: "copilot-responses-1" }); }); it("skips endpoints with no resolvable baseUrl", () => { diff --git a/actions/setup/js/copilot_dynamic_workflow.test.cjs b/actions/setup/js/copilot_dynamic_workflow.test.cjs index cf247d0ac80..bb4da9dd113 100644 --- a/actions/setup/js/copilot_dynamic_workflow.test.cjs +++ b/actions/setup/js/copilot_dynamic_workflow.test.cjs @@ -216,7 +216,7 @@ describe("Copilot dynamic workflow sessions", () => { .split("\n") .map(JSON.parse); expect(events.filter(event => event.type.startsWith("workflow.run_"))).toEqual(lifecycle.filter(event => event.type.startsWith("workflow.run_"))); - expect(events.find(event => event.type === "subagent.started")).toEqual(lifecycle[3]); + expect(events.find(event => event.type === "subagent.started")).toEqual({ ...lifecycle[3], data: { ...lifecycle[3].data, agentDisplayName: lifecycle[3].data.agentName } }); expect(events.find(event => event.type === "assistant.message")).toEqual(lifecycle[5]); expect(events.filter(event => event.type === "assistant.message_delta")).toEqual([delta]); expect(setTimeoutSpy.mock.calls.filter(([, timeout]) => timeout === 1234)).toHaveLength(0); diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 6d9fab6727f..c3a541002cd 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -91,6 +91,7 @@ const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSi const { isCrashSignalExitCode, crashSignalNameForExitCode } = require("./harness_crash_signals.cjs"); const { isCAPIQuotaExceededError, isCAPIServerError } = require("./detect_agent_errors.cjs"); const { applyModelFallback } = require("./model_fallback.cjs"); +const { parseMultiProviderJson, qualifyModelForMultiProvider } = require("./copilot_sdk_multi_provider.cjs"); const { isRoutingReasoningEffort } = require("./copilot_routing_effort.cjs"); const { resolveAWFModelRoutingSelection, isModelAvailableInReflectData, getAWFModelRoutingPolicy, getAWFModelRoutingFailureCode } = require("./awf_model_routing.cjs"); const { recordAWFModelRoutingOutcome } = require("./awf_model_routing.cjs"); @@ -928,11 +929,16 @@ function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToke if (!copilotSDKMode) { return sdkEnv; } + const multiProvider = parseMultiProviderJson(multiProviderJson); + const qualifiedModel = qualifyModelForMultiProvider(resolvedModel || multiProvider?.model, multiProvider); + if (multiProvider && !qualifiedModel) { + throw new Error(`Copilot SDK model '${resolvedModel || multiProvider.model}' is not available in the configured BYOK provider catalog`); + } return { ...sdkEnv, COPILOT_CONNECTION_TOKEN: copilotConnectionToken, ...(multiProviderJson ? { GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: multiProviderJson } : {}), - COPILOT_MODEL: resolvedModel, + COPILOT_MODEL: qualifiedModel || resolvedModel, ...(routingEffort ? { COPILOT_REASONING_EFFORT: routingEffort } : {}), // Native Copilot CLI BYOK env vars — consumed by the headless sidecar for all sessions. COPILOT_PROVIDER_BASE_URL: providerBaseUrl, @@ -1267,7 +1273,7 @@ async function main() { let multiProviderJson = ""; let primaryProviderName = ""; if (copilotSDKMode) { - const configuredModel = process.env.COPILOT_MODEL || ""; + const configuredModel = modelRoutingSelection?.wire_model || process.env.COPILOT_MODEL || ""; const modelsJson = loadModelsJson(); const multiProvider = resolveMultiProviderFromReflect({ model: configuredModel, wireApi: process.env.COPILOT_PROVIDER_WIRE_API, reflectData: awfReflectData, modelsJson, logger: log }); @@ -1278,25 +1284,13 @@ async function main() { resolvedModel = multiProvider.model; // Set the primary provider's details as COPILOT_PROVIDER_* env vars for the headless sidecar // (which still reads those to configure its own sub-agent sessions). - primaryProviderName = multiProvider.models.find(m => m.id === resolvedModel)?.provider ?? multiProvider.providers[0]?.name ?? ""; + primaryProviderName = multiProvider.models.find(m => `${m.provider}/${m.id}` === resolvedModel)?.provider ?? multiProvider.providers[0]?.name ?? ""; const primaryProvider = multiProvider.providers.find(p => p.name === primaryProviderName) ?? multiProvider.providers[0]; - if (modelRoutingSelection) { - providerWireApi = modelRoutingSelection.endpoint === "/responses" ? "responses" : "completions"; - if (primaryProvider) primaryProvider.wireApi = providerWireApi; - } providerBaseUrl = primaryProvider?.baseUrl ?? ""; providerType = primaryProvider?.type ?? "openai"; providerWireApi = providerWireApi || primaryProvider?.wireApi || ""; multiProviderJson = JSON.stringify({ model: multiProvider.model, providers: multiProvider.providers, models: multiProvider.models }); - // For BYOK copilot providers, prefix the model with "copilot/" so subagents treat it as BYOK. - // The headless sidecar reads COPILOT_MODEL to configure sub-agent sessions spawned via the task tool, - // and the "copilot/" prefix signals to use the custom provider config from COPILOT_PROVIDER_* env vars. - const isCopilotProvider = primaryProviderName && (primaryProviderName.toLowerCase().includes("copilot") || primaryProviderName.toLowerCase().includes("github-copilot")); - if (!modelRoutingSelection && isCopilotProvider && resolvedModel && !resolvedModel.includes("/")) { - resolvedModel = `copilot/${resolvedModel}`; - } - log(`copilot-sdk driver mode: multi-provider config resolved (${multiProvider.providers.length} providers, ${multiProvider.models.length} models, model=${resolvedModel})`); logCopilotInferenceConfiguration({ copilotSDKMode, @@ -1402,6 +1396,7 @@ async function main() { lastExitCode = 1; } else { let driverServerArgs = parseCopilotSDKServerArgsFromEnv(process.env.GH_AW_COPILOT_SDK_SERVER_ARGS, { logger: log }); + if (driverServerArgs.length === 0) driverServerArgs = buildCopilotSDKServerArgs(childEnv ?? process.env); if (process.env.GITHUB_WORKSPACE) { driverServerArgs = [...driverServerArgs, "--add-dir", process.env.GITHUB_WORKSPACE]; log(`copilot-sdk driver mode: appended workspace --add-dir ${process.env.GITHUB_WORKSPACE}`); diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 08e23fbfc8e..9ad462c9ab7 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -387,6 +387,53 @@ describe("copilot_harness.cjs", () => { }); describe("buildCopilotSDKChildEnv", () => { + it("rejects an unknown model instead of passing it bare with a valid BYOK catalog", () => { + expect(() => + buildCopilotSDKChildEnv({ + sdkEnv: {}, + copilotSDKMode: true, + copilotConnectionToken: "", + providerBaseUrl: "http://api-proxy:10002", + providerType: "openai", + providerWireApi: "responses", + resolvedModel: "unknown", + multiProviderJson: JSON.stringify({ + model: "copilot-responses/gpt-5.6-luna", + providers: [{ name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002" }], + models: [{ id: "gpt-5.6-luna", provider: "copilot-responses" }], + }), + }) + ).toThrow("not available in the configured BYOK provider catalog"); + }); + + it.each(["gpt-5.6-luna", "copilot/gpt-5.6-luna", "copilot-responses/gpt-5.6-luna"])("passes a qualified routed model (%s) with isolated model routes", resolvedModel => { + const config = { + model: "copilot-responses/gpt-5.6-luna", + providers: [ + { name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "responses" }, + { name: "copilot-completions", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "completions" }, + ], + models: [ + { id: "gpt-5.6-luna", provider: "copilot-responses" }, + { id: "claude-haiku-4.5", provider: "copilot-completions" }, + ], + }; + const env = buildCopilotSDKChildEnv({ + sdkEnv: {}, + copilotSDKMode: true, + copilotConnectionToken: "", + providerBaseUrl: "http://api-proxy:10002", + providerType: "openai", + providerWireApi: "responses", + resolvedModel, + routingEffort: "high", + multiProviderJson: JSON.stringify(config), + }); + expect(env.COPILOT_MODEL).toBe("copilot-responses/gpt-5.6-luna"); + expect(env.COPILOT_REASONING_EFFORT).toBe("high"); + expect(JSON.parse(env.GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON)).toEqual(config); + }); + it("includes native sidecar provider vars and multiProviderJson when wireApi is configured", () => { const multiProviderJson = JSON.stringify({ model: "gpt-5.4", providers: [{ name: "copilot", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "completions" }], models: [{ id: "gpt-5.4", provider: "copilot" }] }); const env = buildCopilotSDKChildEnv({ @@ -405,7 +452,7 @@ describe("copilot_harness.cjs", () => { COPILOT_SDK_URI: "http://127.0.0.1:4000", COPILOT_CONNECTION_TOKEN: "token-123", GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: multiProviderJson, - COPILOT_MODEL: "gpt-5.4", + COPILOT_MODEL: "copilot/gpt-5.4", COPILOT_REASONING_EFFORT: "high", COPILOT_PROVIDER_BASE_URL: "http://api-proxy:10002", COPILOT_PROVIDER_TYPE: "openai", @@ -1234,6 +1281,23 @@ describe("copilot_harness.cjs", () => { expect(spawnImpl).toHaveBeenCalledWith("copilot", engineGeneratedArgs, expect.objectContaining({ stdio: ["ignore", "pipe", "pipe"] })); }); + it.each(["not-json", '{"port":3002}'])("retains headless defaults and workspace access for invalid SDK args %s", async value => { + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.exitCode = null; + child.signalCode = null; + const spawnImpl = vi.fn(() => child); + const logger = vi.fn(); + const env = { COPILOT_SDK_URI: "http://127.0.0.1:3002" }; + let args = parseCopilotSDKServerArgsFromEnv(value, { logger }); + if (args.length === 0) args = buildCopilotSDKServerArgs(env); + args = [...args, "--add-dir", "/w"]; + await startCopilotSDKServer({ command: "copilot", env, serverArgs: args, logger, spawnImpl, waitForReady: async () => {} }); + expect(spawnImpl).toHaveBeenCalledWith("copilot", ["--headless", "--no-auto-update", "--port", "3002", "--add-dir", "/w"], expect.anything()); + expect(logger.mock.calls.flat().join("\n")).toMatch(/GH_AW_COPILOT_SDK_SERVER_ARGS/); + }); + it("uses only base headless args when extraArgs is empty or omitted", async () => { const child = new EventEmitter(); child.stdout = new PassThrough(); diff --git a/actions/setup/js/copilot_sdk_driver.cjs b/actions/setup/js/copilot_sdk_driver.cjs index 879050d6ea4..cf786458185 100644 --- a/actions/setup/js/copilot_sdk_driver.cjs +++ b/actions/setup/js/copilot_sdk_driver.cjs @@ -27,10 +27,10 @@ "use strict"; const fs = require("fs"); -const { runWithCopilotSDK, extractPromptFromArgs } = require("./copilot_sdk_session.cjs"); +const { runWithCopilotSDK, extractPromptFromArgs, isCopilotSDKBareMode, resolveCopilotSDKWorkingDirectory } = require("./copilot_sdk_session.cjs"); const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); const { parseCopilotSDKToolConfig } = require("./copilot_sdk_tool_config.cjs"); -const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); +const { parseMultiProviderJson, qualifyModelForMultiProvider } = require("./copilot_sdk_multi_provider.cjs"); const { applyModelFallback } = require("./model_fallback.cjs"); const { resolveRoutingReasoningEffort } = require("./copilot_routing_effort.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); @@ -109,6 +109,9 @@ async function main() { const sdkModels = multiProviderConfig.models; const routingEnabled = process.env.GH_AW_MODEL_ROUTING === "1"; let model = (routingEnabled ? process.env.COPILOT_MODEL : applyModelFallback(process.env, "COPILOT_MODEL", log)) || multiProviderConfig.model || undefined; + const qualifiedModel = qualifyModelForMultiProvider(model, multiProviderConfig); + if (!qualifiedModel) throw new Error(`Cannot qualify session model "${model || "(none)"}" against the configured provider models`); + model = qualifiedModel; const reasoningEffort = resolveRoutingReasoningEffort(process.env); log(`multi-provider mode: ${providers.length} providers, ${sdkModels.length} models, model=${model ?? "(env)"}`); for (const p of providers) { @@ -135,6 +138,8 @@ async function main() { models: sdkModels, permissionConfig, toolConfig, + workingDirectory: resolveCopilotSDKWorkingDirectory(), + bare: isCopilotSDKBareMode(process.env.GH_AW_COPILOT_SDK_SERVER_ARGS, log), }); process.exit(result.exitCode); diff --git a/actions/setup/js/copilot_sdk_driver.test.cjs b/actions/setup/js/copilot_sdk_driver.test.cjs index e53ead6bd85..eb5430310da 100644 --- a/actions/setup/js/copilot_sdk_driver.test.cjs +++ b/actions/setup/js/copilot_sdk_driver.test.cjs @@ -1,7 +1,6 @@ import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; import { createRequire } from "module"; import * as fs from "fs"; -import * as os from "os"; import * as path from "path"; const require = createRequire(import.meta.url); @@ -10,18 +9,129 @@ const { runWithCopilotSDK, parsePermissionConfigFromServerArgs } = require("./co describe("copilot_sdk_driver.cjs", () => { let testSessionStateDir; let prevSessionStateDir; + let agentDirectorySpy; beforeAll(() => { prevSessionStateDir = process.env.GH_AW_SESSION_STATE_BASE_DIR; - testSessionStateDir = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-test-session-state-")); + testSessionStateDir = fs.mkdtempSync(path.join(process.cwd(), ".gh-aw-test-session-state-")); process.env.GH_AW_SESSION_STATE_BASE_DIR = testSessionStateDir; + const runtimeFs = require("fs"); + const readdirSync = runtimeFs.readdirSync; + agentDirectorySpy = vi.spyOn(runtimeFs, "readdirSync").mockImplementation((directory, options) => { + // Never inspect the developer checkout's agents when exercising the SDK. + if (!String(directory).startsWith(testSessionStateDir) && String(directory).endsWith("/.github/agents")) return []; + return readdirSync(directory, options); + }); }); afterAll(() => { if (prevSessionStateDir === undefined) delete process.env.GH_AW_SESSION_STATE_BASE_DIR; else process.env.GH_AW_SESSION_STATE_BASE_DIR = prevSessionStateDir; if (testSessionStateDir) fs.rmSync(testSessionStateDir, { recursive: true, force: true }); + agentDirectorySpy.mockRestore(); }); describe("runWithCopilotSDK", () => { + it.each([ + ["gpt-5.4", "claude-sonnet-4.6", "copilot-responses", "copilot-completions"], + ["claude-sonnet-4.6", "gpt-5.4", "copilot-completions", "copilot-responses"], + ])("qualifies %s session and cross-provider %s custom agents at runtime", async (sessionModel, agentModel, sessionProvider, agentProvider) => { + const workspace = path.join(testSessionStateDir, `${sessionProvider}-workspace`); + const workflow = path.join(testSessionStateDir, `${sessionProvider}-workflow`); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + fs.mkdirSync(path.join(workflow, ".github/agents"), { recursive: true }); + fs.writeFileSync( + path.join(workspace, ".github/agents/researcher.agent.md"), + `---\nname: researcher\ndescription: >-\n Research across\n providers\ntools:\n - read\n - grep\nmodel: copilot/${agentModel}\n---\nResearch carefully.\n` + ); + fs.writeFileSync(path.join(workflow, ".github/agents/workflow.agent.md"), "---\nname: workflow-agent\nmodel: medium\ntools: read, grep\n---\nWorkflow prompt."); + for (const alias of ["small", "large"]) fs.writeFileSync(path.join(workflow, `.github/agents/${alias}.agent.md`), `---\nmodel: ${alias}\n---\nAlias prompt.`); + fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), "---\nname: unknown-agent\nmodel: unavailable-model\n---\nKeep this agent."); + const createSession = vi.fn().mockResolvedValue({ + sessionId: `${sessionProvider}-agent-session`, + on: () => {}, + sendAndWait: vi.fn().mockResolvedValue({ data: { content: "done" } }), + disconnect: vi.fn().mockResolvedValue(undefined), + }); + const constructor = vi.fn(); + class FakeCopilotClient { + constructor(options) { + constructor(options); + } + start = vi.fn().mockResolvedValue(undefined); + createSession = createSession; + stop = vi.fn().mockResolvedValue(undefined); + } + const logger = vi.fn(); + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test prompt", + model: `copilot/${sessionModel}`, + models: [ + { id: sessionModel, provider: sessionProvider }, + { id: agentModel, provider: agentProvider }, + ], + providers: [ + { name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "responses" }, + { name: "copilot-completions", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "completions" }, + ], + workingDirectory: workspace, + agentsBaseDir: workflow, + bare: true, + logger, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: vi.fn(() => ({})) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + expect(constructor).toHaveBeenCalledWith(expect.objectContaining({ workingDirectory: workspace })); + expect(createSession).toHaveBeenCalledWith( + expect.objectContaining({ + model: `${sessionProvider}/${sessionModel}`, + workingDirectory: workspace, + skipCustomInstructions: true, + customAgents: expect.arrayContaining([ + { name: "researcher", displayName: "researcher", description: "Research across providers", tools: ["read", "grep"], prompt: "Research carefully.", model: `${agentProvider}/${agentModel}` }, + { name: "workflow-agent", displayName: "workflow-agent", tools: ["read", "grep"], prompt: "Workflow prompt.", model: `${sessionProvider}/${sessionModel}` }, + { name: "unknown-agent", displayName: "unknown-agent", prompt: "Keep this agent.", model: `${sessionProvider}/${sessionModel}` }, + { name: "small", displayName: "small", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, + { name: "large", displayName: "large", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, + ]), + }) + ); + expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning:.*unknown-agent.*unavailable-model.*session model/i)); + const captured = fs + .readFileSync(path.join(testSessionStateDir, `${sessionProvider}-agent-session/events.jsonl`), "utf8") + .trim() + .split("\n") + .map(JSON.parse); + expect(captured.filter(event => event.type === "subagent.model_unavailable")).toEqual([ + expect.objectContaining({ data: { agentName: "unknown-agent", declaredModel: "unavailable-model", model: `${sessionProvider}/${sessionModel}` } }), + ]); + }); + + it("captures ephemeral selection and final model results including BYOK and subagent context", async () => { + const event = { type: "model.call_final_result", ephemeral: true, timestamp: "2026-10-10T00:00:00Z", data: { isByok: true, model: "anthropic/claude-sonnet-4.6", agentId: "researcher", parentToolCallId: "delegate-1" } }; + const selected = { type: "subagent.selected", ephemeral: true, timestamp: event.timestamp, data: { agentName: "researcher", model: "anthropic/claude-sonnet-4.6", resolvedModel: "anthropic/claude-sonnet-4.6" } }; + let onEvent = () => {}; + class FakeCopilotClient { + start = vi.fn().mockResolvedValue(undefined); + stop = vi.fn().mockResolvedValue(undefined); + createSession = vi.fn().mockResolvedValue({ + sessionId: "final-model-result", + on: handler => { + onEvent = handler; + }, + sendAndWait: async () => { + onEvent(selected); + onEvent(event); + return { data: { content: "done" } }; + }, + disconnect: vi.fn().mockResolvedValue(undefined), + }); + } + await runWithCopilotSDK({ sdkUri: "http://127.0.0.1:3002", prompt: "test", logger: () => {}, sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: vi.fn(() => ({})) }, approveAll: () => "allow" } }); + const events = fs.readFileSync(path.join(testSessionStateDir, "final-model-result/events.jsonl"), "utf8").trim().split("\n").map(JSON.parse); + expect(events).toContainEqual(event); + expect(events).toContainEqual(selected); + }); + it("disconnects session and stops client on success", async () => { const disconnect = vi.fn().mockResolvedValue(undefined); const stop = vi.fn().mockResolvedValue(undefined); @@ -990,7 +1100,7 @@ describe("copilot_sdk_driver.cjs", () => { sdkUri: "http://127.0.0.1:3002", prompt: "test prompt", logger: () => {}, - model: "gpt-5.4", + model: "copilot/gpt-5.4", providers, models, sdkModule: { @@ -1003,7 +1113,7 @@ describe("copilot_sdk_driver.cjs", () => { expect(result.exitCode).toBe(0); expect(createSession).toHaveBeenCalledWith( expect.objectContaining({ - model: "gpt-5.4", + model: "copilot/gpt-5.4", providers, models, }) diff --git a/actions/setup/js/copilot_sdk_multi_provider.cjs b/actions/setup/js/copilot_sdk_multi_provider.cjs index f5eda347657..b6d0974ec75 100644 --- a/actions/setup/js/copilot_sdk_multi_provider.cjs +++ b/actions/setup/js/copilot_sdk_multi_provider.cjs @@ -58,8 +58,38 @@ function parseMultiProviderJson(value) { } } +/** + * Resolve a catalog model to its BYOK selection id without guessing unknown models. + * + * @param {string | undefined} model + * @param {{ providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null | undefined} config + * @returns {string | null} + */ +function qualifyModelForMultiProvider(model, config) { + const value = typeof model === "string" ? model.trim() : ""; + if (!value || !config) return null; + const models = config.models.filter(entry => config.providers.some(provider => provider.name === entry.provider)); + const qualified = models.find(entry => `${entry.provider}/${entry.id}` === value); + if (qualified) return value; + const legacy = value.startsWith("copilot/"); + const id = legacy ? value.slice("copilot/".length) : value; + const matches = models.filter(entry => entry.id === id && (!legacy || isCopilotProviderName(entry.provider))); + if (new Set(matches.map(entry => entry.provider)).size > 1) { + throw new Error(`Ambiguous model "${value}"; specify an explicit /: ${matches.map(entry => `${entry.provider}/${entry.id}`).join(", ")}`); + } + const match = matches[0]; + return match ? `${match.provider}/${match.id}` : null; +} + +/** @param {string} name @returns {boolean} */ +function isCopilotProviderName(name) { + return /^(?:copilot(?:-(?:responses|completions))?|github-copilot|github)(?:-\d+)?$/i.test(name); +} + module.exports = { isValidProviderConfig, isValidModelConfig, parseMultiProviderJson, + qualifyModelForMultiProvider, + isCopilotProviderName, }; diff --git a/actions/setup/js/copilot_sdk_multi_provider.test.cjs b/actions/setup/js/copilot_sdk_multi_provider.test.cjs new file mode 100644 index 00000000000..b05f4b98d79 --- /dev/null +++ b/actions/setup/js/copilot_sdk_multi_provider.test.cjs @@ -0,0 +1,56 @@ +const { qualifyModelForMultiProvider, parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); + +describe("qualifyModelForMultiProvider", () => { + const config = { + providers: [{ name: "copilot-responses" }, { name: "copilot-completions-1" }, { name: "openai" }], + models: [ + { id: "gpt-5.6-luna", provider: "copilot-responses" }, + { id: "claude-haiku-4.5", provider: "copilot-completions-1" }, + { id: "org/model", provider: "openai" }, + { id: "orphan", provider: "missing" }, + ], + }; + + it.each([ + ["gpt-5.6-luna", "copilot-responses/gpt-5.6-luna"], + ["copilot/gpt-5.6-luna", "copilot-responses/gpt-5.6-luna"], + ["claude-haiku-4.5", "copilot-completions-1/claude-haiku-4.5"], + ["copilot/claude-haiku-4.5", "copilot-completions-1/claude-haiku-4.5"], + ["copilot-completions-1/claude-haiku-4.5", "copilot-completions-1/claude-haiku-4.5"], + ["org/model", "openai/org/model"], + ["openai/org/model", "openai/org/model"], + ])("qualifies %s", (model, expected) => { + expect(qualifyModelForMultiProvider(model, config)).toBe(expected); + }); + + it.each(["small", "medium", "large", "", "unknown", "orphan", "other/gpt-5.6-luna", "copilot/org/model", undefined])("does not guess %s", model => { + expect(qualifyModelForMultiProvider(model, config)).toBeNull(); + }); + + it("returns null without a catalog", () => { + expect(qualifyModelForMultiProvider("gpt-5.6-luna", null)).toBeNull(); + }); + + it.each(["shared", "copilot/shared"])("rejects ambiguous model %s and accepts explicit qualification", model => { + const ambiguous = { + providers: [{ name: "copilot-responses" }, { name: "copilot-completions" }], + models: [ + { id: "shared", provider: "copilot-responses" }, + { id: "shared", provider: "copilot-completions" }, + ], + }; + expect(() => qualifyModelForMultiProvider(model, ambiguous)).toThrow(/Ambiguous model.*explicit \//); + expect(qualifyModelForMultiProvider("copilot-completions/shared", ambiguous)).toBe("copilot-completions/shared"); + }); + + it("uses the same per-model mapping after JSON handoff", () => { + const parsed = parseMultiProviderJson( + JSON.stringify({ + model: "copilot-completions/claude-haiku-4.5", + providers: [{ name: "copilot-completions", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "completions" }], + models: [{ id: "claude-haiku-4.5", provider: "copilot-completions" }], + }) + ); + expect(qualifyModelForMultiProvider("claude-haiku-4.5", parsed)).toBe(parsed.model); + }); +}); diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 39bac3e8ce8..c609a51b9fb 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -35,8 +35,10 @@ const { buildCopilotSDKToolCallBudget } = require("./copilot_sdk_tool_budget.cjs const { resolveModelWithFallback } = require("./model_fallback.cjs"); const { COPILOT_WORKFLOW_EVENT_TYPES } = require("./copilot_workflow_events.cjs"); const { copilotSessionContext } = require("./copilot_session.cjs"); +const { loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); +const { qualifyModelForMultiProvider, parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); -const CAPTURED_EPHEMERAL_EVENTS = new Set([...COPILOT_WORKFLOW_EVENT_TYPES, "assistant.usage", "assistant.message_delta", "assistant.reasoning_delta", "model.call_failure"]); +const CAPTURED_EPHEMERAL_EVENTS = new Set([...COPILOT_WORKFLOW_EVENT_TYPES, "assistant.usage", "assistant.message_delta", "assistant.reasoning_delta", "model.call_failure", "model.call_final_result"]); // Default timeout for a single sendAndWait call: 10 minutes. // This is intentionally generous — the headless Copilot CLI has its own internal @@ -88,6 +90,23 @@ function extractPromptFromArgs(args) { return null; } +/** @param {string | undefined} serverArgs @param {(message: string) => void} [logger] @returns {boolean} */ +function isCopilotSDKBareMode(serverArgs, logger = message => process.stderr.write(`[copilot-sdk] ${message}\n`)) { + try { + const args = JSON.parse(serverArgs || "[]"); + if (!Array.isArray(args) || args.some(arg => typeof arg !== "string")) throw new Error("expected a string array"); + return args.includes("--no-custom-instructions"); + } catch (error) { + logger(`warning: Invalid GH_AW_COPILOT_SDK_SERVER_ARGS: ${getErrorMessage(error)}; using sidecar default args`); + return false; + } +} + +/** @returns {string} */ +function resolveCopilotSDKWorkingDirectory() { + return path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); +} + /** * Run a Copilot agentic session using the @github/copilot-sdk. * @@ -126,6 +145,9 @@ function extractPromptFromArgs(args) { * defineTool?: typeof import("@github/copilot-sdk").defineTool, * }, * sessionStateBaseDir?: string, + * workingDirectory?: string, + * agentsBaseDir?: string, + * bare?: boolean, * }} options * @returns {Promise<{exitCode: number, output: string, hasOutput: boolean, durationMs: number}>} */ @@ -146,6 +168,9 @@ async function runWithCopilotSDK({ coreLogger, sdkModule, sessionStateBaseDir, + workingDirectory = resolveCopilotSDKWorkingDirectory(), + agentsBaseDir = path.join(os.tmpdir(), "gh-aw"), + bare = isCopilotSDKBareMode(process.env.GH_AW_COPILOT_SDK_SERVER_ARGS, logger), }) { // Lazy-require to avoid loading the SDK when it is not needed. // The SDK is large and has side-effects on import (worker threads, etc.). @@ -192,7 +217,7 @@ async function runWithCopilotSDK({ }); const client = new CopilotClient({ connection, - workingDirectory: process.env.GITHUB_WORKSPACE || process.cwd(), + workingDirectory, logLevel, }); /** @type {any} */ @@ -321,7 +346,7 @@ async function runWithCopilotSDK({ coreLogger, logger: log, onDenied: requestSummary => recordToolDenial(`permission denied: ${requestSummary}`), - workspaceRoot: process.env.GITHUB_WORKSPACE, + workspaceRoot: workingDirectory, }); const toolCallBudget = buildCopilotSDKToolCallBudget(toolConfig?.maxToolCalls, event => { writeDriverEvent(event.exhausted ? "guard.tool_call_budget_exceeded" : "guard.tool_call_budget_debit", event); @@ -330,12 +355,34 @@ async function runWithCopilotSDK({ // Build session config using the multi-provider surface. /** @type {any} */ const sdkReasoningEffort = reasoningEffort; + const reflectedConfig = parseMultiProviderJson(process.env.GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON); + providers ??= reflectedConfig?.providers; + providerModels ??= reflectedConfig?.models; + const envModel = process.env.GH_AW_MODEL_ROUTING === "1" ? process.env.COPILOT_MODEL : resolveModelWithFallback(process.env, "COPILOT_MODEL"); + const requestedModel = model || envModel || reflectedConfig?.model || undefined; + const providerConfig = providers && providerModels ? { providers, models: providerModels } : null; + const sessionModel = providerConfig ? qualifyModelForMultiProvider(requestedModel, providerConfig) : requestedModel; + if (providerConfig && !sessionModel) throw new Error(`Cannot qualify session model "${requestedModel || "(none)"}" against the configured provider models`); + const unavailableModels = []; + const customAgents = loadCopilotSDKCustomAgents(workingDirectory, agentsBaseDir, log).map(agent => { + const requested = agent.model; + const alias = !requested || ["small", "medium", "large"].includes(requested.toLowerCase()); + const qualified = alias ? sessionModel : providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; + if (!alias && !qualified) { + log(`warning: custom agent "${agent.name}" model "${requested}" is not in the configured provider models; using session model "${sessionModel || "(none)"}"`); + unavailableModels.push({ agentName: agent.name, declaredModel: requested, model: sessionModel || undefined }); + } + return { ...agent, model: qualified || sessionModel || undefined }; + }); /** @type {import("@github/copilot-sdk").SessionConfig} */ const sessionConfig = { - model: model || resolveModelWithFallback(process.env, "COPILOT_MODEL") || undefined, + model: sessionModel || undefined, ...(sdkReasoningEffort ? { reasoningEffort: sdkReasoningEffort } : {}), providers, models: providerModels, + workingDirectory, + skipCustomInstructions: bare, + customAgents, onPermissionRequest, ...(toolCallBudget ? { hooks: { onPreToolUse: toolCallBudget.onPreToolUse } } : {}), ...buildCopilotSDKSessionToolConfig(toolConfig, sdk, webFetchOptions), @@ -349,6 +396,7 @@ async function runWithCopilotSDK({ fs.mkdirSync(sessionDir, { recursive: true }); const eventsPath = path.join(sessionDir, "events.jsonl"); eventsStream = fs.createWriteStream(eventsPath, { flags: "a" }); + for (const unavailable of unavailableModels) writeDriverEvent("subagent.model_unavailable", unavailable); // Snapshot to a non-null local for closure-safe writes (JSDoc nullability narrowing). const stream = eventsStream; const assistantEvents = []; @@ -373,6 +421,10 @@ async function runWithCopilotSDK({ session.on(event => { if (event.ephemeral && !CAPTURED_EPHEMERAL_EVENTS.has(event.type)) return; const data = { ...event.data }; + if (event.type.startsWith("subagent.") && data.agentName && data.agentDisplayName) data.agentDisplayName = data.agentName; + if (data.agentMetrics) { + data.agentMetrics = Object.fromEntries(Object.entries(data.agentMetrics).map(([id, metric]) => [id, metric.agentName ? { ...metric, agentDisplayName: metric.agentName } : metric])); + } const context = copilotSessionContext(event); const scope = JSON.stringify([context.agentId, context.parentToolUseId]); const toolKey = JSON.stringify([scope, data.toolCallId]); @@ -594,4 +646,4 @@ async function runWithCopilotSDK({ } } -module.exports = { SDK_SEND_TIMEOUT_MS_DEFAULT, SDK_POST_COMPLETION_IDLE_MS_DEFAULT, SDK_IDLE_TIMEOUT_PATTERN, extractPromptFromArgs, runWithCopilotSDK }; +module.exports = { SDK_SEND_TIMEOUT_MS_DEFAULT, SDK_POST_COMPLETION_IDLE_MS_DEFAULT, SDK_IDLE_TIMEOUT_PATTERN, extractPromptFromArgs, runWithCopilotSDK, isCopilotSDKBareMode, resolveCopilotSDKWorkingDirectory }; diff --git a/actions/setup/js/copilot_sdk_session.test.cjs b/actions/setup/js/copilot_sdk_session.test.cjs new file mode 100644 index 00000000000..56b658b1b79 --- /dev/null +++ b/actions/setup/js/copilot_sdk_session.test.cjs @@ -0,0 +1,226 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { createRequire } from "module"; +import * as fs from "fs"; +import * as path from "path"; +import os from "node:os"; + +const require = createRequire(import.meta.url); +const { runWithCopilotSDK, resolveCopilotSDKWorkingDirectory } = require("./copilot_sdk_session.cjs"); +const models = [ + { id: "gpt-5.4", provider: "openai" }, + { id: "claude-sonnet-4.6", provider: "anthropic" }, +]; +const providers = [ + { name: "openai", type: "openai", baseUrl: "http://api-proxy:10002" }, + { name: "anthropic", type: "anthropic", baseUrl: "http://api-proxy:10002" }, +]; + +describe("copilot_sdk_session runtime configuration", () => { + let base; + beforeEach(() => { + base = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-session-test-")); + vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); + vi.stubEnv("GH_AW_ENGINE_CWD", base); + vi.stubEnv("COPILOT_MODEL", ""); + vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); + vi.stubEnv("GH_AW_MODEL_ROUTING", ""); + vi.stubEnv("GH_AW_COPILOT_SDK_SERVER_ARGS", "[]"); + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", JSON.stringify({ model: "claude-sonnet-4.6", models, providers })); + }); + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(base, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it("resolves engine.cwd against the workspace rather than the driver process directory", async () => { + vi.stubEnv("GITHUB_WORKSPACE", "/w"); + vi.stubEnv("GH_AW_ENGINE_CWD", "packages/app"); + const cwd = vi.spyOn(process, "cwd").mockReturnValue("/w/packages/app"); + expect(resolveCopilotSDKWorkingDirectory()).toBe("/w/packages/app"); + cwd.mockRestore(); + + const app = path.join(base, "packages/app"); + fs.mkdirSync(path.join(app, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(app, ".github/agents/local.agent.md"), "---\nmodel: small\n---\nApp agent."); + vi.stubEnv("GITHUB_WORKSPACE", base); + vi.spyOn(process, "cwd").mockReturnValue(app); + const createSession = vi.fn().mockResolvedValue({ + sessionId: "relative-cwd", + on: () => {}, + sendAndWait: async () => ({ data: { content: "done" } }), + disconnect: async () => {}, + }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: () => {}, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + expect(createSession).toHaveBeenCalledWith( + expect.objectContaining({ + workingDirectory: app, + customAgents: [{ name: "local", displayName: "local", prompt: "App agent.", model: "anthropic/claude-sonnet-4.6" }], + }) + ); + }); + + it.each(["{bad json", '{"args":["--no-custom-instructions"]}'])("warns and starts a session for invalid server args %s", serverArgs => { + vi.stubEnv("GH_AW_COPILOT_SDK_SERVER_ARGS", serverArgs); + const createSession = vi.fn().mockResolvedValue({ + sessionId: "invalid-args", + on: () => {}, + sendAndWait: async () => ({ data: { content: "done" } }), + disconnect: async () => {}, + }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const logger = vi.fn(); + return runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }).then(result => { + expect(result.exitCode).toBe(0); + expect(createSession).toHaveBeenCalledWith(expect.objectContaining({ skipCustomInstructions: false })); + expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning: Invalid GH_AW_COPILOT_SDK_SERVER_ARGS.*using sidecar default args/)); + }); + }); + + it("preserves explicit agent models without a BYOK catalog or misleading warnings", async () => { + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", ""); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(base, ".github/agents/researcher.agent.md"), "---\nmodel: claude-sonnet-4.6\n---\nResearch carefully."); + const createSession = vi.fn().mockResolvedValue({ + sessionId: "no-catalog", + on: () => {}, + sendAndWait: async () => ({ data: { content: "done" } }), + disconnect: async () => {}, + }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const logger = vi.fn(); + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + model: "gpt-5.4", + logger, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + expect(createSession).toHaveBeenCalledWith(expect.objectContaining({ model: "gpt-5.4", customAgents: [{ name: "researcher", displayName: "researcher", prompt: "Research carefully.", model: "claude-sonnet-4.6" }] })); + expect(logger.mock.calls.flat().some(message => message.includes("warning: custom agent"))).toBe(false); + }); + + it("fails an unknown session model explicitly rather than warning and continuing", async () => { + const createSession = vi.fn(); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const logger = vi.fn(); + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + model: "unavailable-session-model", + logger, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toMatch(/Cannot qualify session model "unavailable-session-model"/); + expect(logger).toHaveBeenCalledWith(expect.stringMatching(/error: Cannot qualify session model/)); + expect(createSession).not.toHaveBeenCalled(); + }); + + it("attributes SDK lifecycle and metrics to declared names instead of task display names", async () => { + let handler; + const events = [ + { type: "subagent.started", data: { invocationId: "child", agentName: "file-summarizer", agentDisplayName: "readme-summarizer" } }, + { type: "subagent.completed", data: { invocationId: "child", agentName: "file-summarizer", agentDisplayName: "summarize-readme" } }, + { type: "session.shutdown", data: { agentMetrics: { child: { agentName: "file-summarizer", agentDisplayName: "readme-summarizer" }, main: { totalNanoAiu: 1 } } } }, + ]; + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = async () => ({ + sessionId: "declared-names", + on: callback => { + handler = callback; + }, + sendAndWait: async () => { + events.forEach(event => handler(event)); + return { data: { content: "done" } }; + }, + disconnect: async () => {}, + }); + } + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: () => {}, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + const captured = fs + .readFileSync(path.join(base, "declared-names/events.jsonl"), "utf8") + .trim() + .split("\n") + .map(line => JSON.parse(line)); + expect(captured[0].data.agentDisplayName).toBe("file-summarizer"); + expect(captured[1].data.agentDisplayName).toBe("file-summarizer"); + expect(captured[2].data.agentMetrics.child.agentDisplayName).toBe("file-summarizer"); + expect(captured[2].data.agentMetrics.main).toEqual({ totalNanoAiu: 1 }); + expect(events[0].data.agentDisplayName).toBe("readme-summarizer"); + expect(events[2].data.agentMetrics.child.agentDisplayName).toBe("readme-summarizer"); + }); + + it.each(["routing", "environment", "fallback", "reflect"])("qualifies the %s model before createSession", async source => { + const expected = source === "reflect" ? "anthropic/claude-sonnet-4.6" : "openai/gpt-5.4"; + if (source === "routing" || source === "environment") vi.stubEnv("COPILOT_MODEL", "gpt-5.4"); + if (source === "routing") { + vi.stubEnv("GH_AW_MODEL_ROUTING", "1"); + vi.stubEnv("GH_AW_MODEL_FALLBACK", "claude-sonnet-4.6"); + } + if (source === "fallback") vi.stubEnv("GH_AW_MODEL_FALLBACK", "gpt-5.4"); + vi.stubEnv("GH_AW_COPILOT_SDK_SERVER_ARGS", '["--no-custom-instructions"]'); + const createSession = vi.fn().mockResolvedValue({ + sessionId: source, + on: () => {}, + sendAndWait: async () => ({ data: { content: "done" } }), + disconnect: async () => {}, + }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: () => {}, + agentsBaseDir: base, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + expect(createSession).toHaveBeenCalledWith(expect.objectContaining({ model: expected, providers, models, workingDirectory: base, skipCustomInstructions: true, customAgents: [] })); + }); +}); diff --git a/actions/setup/js/copilot_workflow_events.cjs b/actions/setup/js/copilot_workflow_events.cjs index 65659f645c4..f0ff173c2f9 100644 --- a/actions/setup/js/copilot_workflow_events.cjs +++ b/actions/setup/js/copilot_workflow_events.cjs @@ -27,6 +27,17 @@ const COPILOT_WORKFLOW_EVENT_FIELDS = { spawnDepth: ["spawnDepth", "spawn_depth"], }, "subagent.configured": { invocationId: ["invocationId"], model: ["model"], contextTier: ["contextTier"], reasoningEffort: ["reasoningEffort"], multiTurn: ["multiTurn"] }, + "subagent.selected": { + invocationId: ["invocationId"], + toolCallId: ["toolCallId"], + agentName: ["agentName"], + agentDisplayName: ["agentDisplayName"], + tools: ["tools"], + model: ["model"], + selectedModel: ["selectedModel"], + resolvedModel: ["resolvedModel"], + modelSelectionSource: ["modelSelectionSource"], + }, "subagent.request": { invocationId: ["invocationId"], agentName: ["agentName"], diff --git a/actions/setup/js/copilot_workflow_events.test.cjs b/actions/setup/js/copilot_workflow_events.test.cjs new file mode 100644 index 00000000000..dad76f72f02 --- /dev/null +++ b/actions/setup/js/copilot_workflow_events.test.cjs @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { COPILOT_WORKFLOW_EVENT_TYPES } = require("./copilot_workflow_events.cjs"); +const { normalizeCopilotSession } = require("./copilot_session.cjs"); +const { normalizeUnifiedSessionEvent } = require("./unified_session_payload.cjs"); + +describe("Copilot subagent selection evidence", () => { + it("captures selected events and preserves model evidence through unified projection", () => { + const event = { + type: "subagent.selected", + ephemeral: true, + agentId: "child", + timestamp: "2026-10-10T00:00:00Z", + data: { + agentName: "researcher", + agentDisplayName: "Researcher", + tools: ["read", "grep"], + invocationId: "invoke-1", + toolCallId: "delegate-1", + parentToolCallId: "delegate-1", + model: "copilot/claude-sonnet-4.6", + selectedModel: "anthropic/claude-sonnet-4.6", + resolvedModel: "anthropic/claude-sonnet-4.6", + modelSelectionSource: "custom-agent", + }, + }; + expect(COPILOT_WORKFLOW_EVENT_TYPES.has(event.type)).toBe(true); + const normalized = normalizeCopilotSession([event]).find(entry => entry.type === event.type); + expect(normalizeUnifiedSessionEvent(normalized)).toMatchObject({ + type: event.type, + agentId: "child", + data: event.data, + }); + }); +}); diff --git a/actions/setup/js/extract_inline_sub_agents.cjs b/actions/setup/js/extract_inline_sub_agents.cjs index 31ac7aa6b17..226269cecc8 100644 --- a/actions/setup/js/extract_inline_sub_agents.cjs +++ b/actions/setup/js/extract_inline_sub_agents.cjs @@ -330,4 +330,62 @@ function writeInlineSubAgents(content, workspaceDir, agentsBaseDir, engineId) { return mainContent; } -module.exports = { extractInlineSubAgents, writeInlineSubAgents, getEngineSubAgentTarget, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers }; +/** + * Read repository and workflow-written Copilot agents when the SDK session starts. + * Workflow definitions override repository definitions with the same name. + * + * @param {string} workspaceDir + * @param {string} agentsBaseDir + * @param {(message: string) => void} logger + * @returns {import("@github/copilot-sdk").CustomAgentConfig[]} + */ +function loadCopilotSDKCustomAgents(workspaceDir, agentsBaseDir, logger) { + const { extractFrontmatterAndBody } = require("./frontmatter_hash_pure.cjs"); + const { parseDocument } = require("./work_queue_yaml.cjs"); + const target = inlineArtifactTarget("SUB_AGENT", getEngineSubAgentTarget("copilot")); + const directories = new Set([path.join(workspaceDir, ".github/agents"), path.join(agentsBaseDir, target.dir)]); + const agents = new Map(); + for (const directory of directories) { + let entries; + try { + entries = fs.readdirSync(directory, { withFileTypes: true }); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) logger(`warning: cannot read custom agents in ${directory}: ${String(error)}`); + continue; + } + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isFile() || !entry.name.endsWith(".agent.md")) continue; + const agentPath = path.join(directory, entry.name); + try { + const { frontmatterText, markdown } = extractFrontmatterAndBody(fs.readFileSync(agentPath, "utf8")); + /** @type {any} */ + const document = parseDocument(frontmatterText); + if (!document) throw new Error("cannot parse agent frontmatter"); + if (document.errors.length) throw new Error(document.errors.map(error => error.message).join("; ")); + const frontmatter = document.toJS() || {}; + if (typeof frontmatter !== "object" || Array.isArray(frontmatter)) throw new Error("agent frontmatter must be a mapping"); + const name = typeof frontmatter.name === "string" && frontmatter.name.trim() ? frontmatter.name.trim() : entry.name.slice(0, -".agent.md".length); + let tools = frontmatter.tools; + if (typeof tools === "string") + tools = tools + .split(",") + .map(tool => tool.trim()) + .filter(Boolean); + if (tools !== undefined && tools !== null && (!Array.isArray(tools) || !tools.every(tool => typeof tool === "string"))) throw new Error("agent tools must be a list of names or a comma-separated string"); + agents.set(name, { + name, + displayName: name, + prompt: markdown.trim(), + ...(typeof frontmatter.description === "string" ? { description: frontmatter.description } : {}), + ...(typeof frontmatter.model === "string" ? { model: frontmatter.model.trim() } : {}), + ...(tools !== undefined ? { tools } : {}), + }); + } catch (error) { + logger(`warning: cannot load custom agent ${agentPath}: ${String(error)}`); + } + } + } + return [...agents.values()]; +} + +module.exports = { extractInlineSubAgents, writeInlineSubAgents, getEngineSubAgentTarget, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents }; diff --git a/actions/setup/js/extract_inline_sub_agents.test.cjs b/actions/setup/js/extract_inline_sub_agents.test.cjs index 588b9e58536..da252ae39ee 100644 --- a/actions/setup/js/extract_inline_sub_agents.test.cjs +++ b/actions/setup/js/extract_inline_sub_agents.test.cjs @@ -4,7 +4,6 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; const fs = require("fs"); const path = require("path"); -const os = require("os"); // Provide a minimal core mock so the module loads correctly. global.core = { @@ -14,11 +13,36 @@ global.core = { setFailed: () => {}, }; -const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers } = require("./extract_inline_sub_agents.cjs"); +const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); // Helper: returns a ## agent: `name` start marker line. const agentMarker = name => `## agent: \`${name}\``; +describe("loadCopilotSDKCustomAgents", () => { + it("loads runtime files, preserving YAML descriptions and workflow precedence", () => { + const base = fs.mkdtempSync(path.join(process.cwd(), ".sdk-agents-test-")); + try { + const workspace = path.join(base, "workspace"); + const workflow = path.join(base, "workflow"); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + fs.mkdirSync(path.join(workflow, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(workspace, ".github/agents/helper.agent.md"), "---\nname: helper\n---\nRepository version."); + fs.writeFileSync(path.join(workflow, ".github/agents/helper.agent.md"), "---\nname: helper\ndescription: |\n First line\n Second line\ntools: []\nmodel: small\n---\nWorkflow version."); + fs.writeFileSync(path.join(workspace, ".github/agents/plain.agent.md"), "Prompt without frontmatter."); + fs.writeFileSync(path.join(workspace, ".github/agents/ignored.md"), "Ignored file."); + fs.writeFileSync(path.join(workspace, ".github/agents/broken.agent.md"), "---\ntools: {invalid: true}\n---\nInvalid tools."); + const warnings = []; + expect(loadCopilotSDKCustomAgents(workspace, workflow, message => warnings.push(message))).toEqual([ + { name: "helper", displayName: "helper", description: "First line\nSecond line\n", tools: [], model: "small", prompt: "Workflow version." }, + { name: "plain", displayName: "plain", prompt: "Prompt without frontmatter." }, + ]); + expect(warnings).toEqual([expect.stringContaining("agent tools must be a list")]); + } finally { + fs.rmSync(base, { recursive: true, force: true }); + } + }); +}); + // ───────────────────────────────────────────────────────────────────────────── // extractInlineSubAgents — unit tests // ───────────────────────────────────────────────────────────────────────────── @@ -230,7 +254,7 @@ describe("writeInlineSubAgents", () => { let tmpDir; beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "inline-agents-test-")); + tmpDir = fs.mkdtempSync(path.join(process.cwd(), ".inline-agents-test-")); }); afterEach(() => { diff --git a/actions/setup/js/smoke_copilot_sub_agents.test.cjs b/actions/setup/js/smoke_copilot_sub_agents.test.cjs new file mode 100644 index 00000000000..18f9bd8e485 --- /dev/null +++ b/actions/setup/js/smoke_copilot_sub_agents.test.cjs @@ -0,0 +1,87 @@ +import fs from "node:fs"; +import { createRequire } from "node:module"; +import { describe, test, expect } from "vitest"; + +const require = createRequire(import.meta.url); +const workflow = fs.readFileSync(new URL("../../../.github/workflows/smoke-copilot-sub-agents.md", import.meta.url), "utf8"); +const script = workflow + .split(" script: |\n")[1] + .split("\n---")[0] + .replace(/^ /gm, ""); +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + +function evidence() { + const agents = [ + ["haiku-whoami", "copilot-completions/claude-haiku-4.5", "/chat/completions"], + ["mini-whoami", "copilot-responses/gpt-5-mini", "/responses"], + ["nano-whoami", "copilot-responses/gpt-5-nano", "/responses"], + ]; + return { + events: agents.flatMap(([agentName, model]) => [ + { type: "subagent.started", data: { agentName, model, invocationId: agentName } }, + { type: "subagent.completed", data: { model, invocationId: agentName } }, + ]), + requests: agents.map(([, model, endpoint]) => ({ model: model.split("/")[1], path: endpoint, status: 200 })), + }; +} + +async function assertEvidence({ events, requests }, variant = "inline_strict") { + const mockRequire = name => { + if (name.endsWith("unified_session.cjs")) return { writeUnifiedSession() {} }; + if (name === "node:fs") { + return { + existsSync: () => true, + readFileSync: file => (file.endsWith("aw_session.jsonl") ? events : requests).map(record => JSON.stringify(record)).join("\n"), + }; + } + return require(name); + }; + await new AsyncFunction("require", "process", script)(mockRequire, { + env: { RUNNER_TEMP: "/tmp", SMOKE_VARIANT: variant, SMOKE_EXECUTION: "success" }, + }); +} + +describe("Copilot sub-agent smoke evidence", () => { + test("accepts completed delegation with qualified models and matching proxy requests", async () => { + await expect(assertEvidence(evidence())).resolves.toBeUndefined(); + }); + + test("accepts the served dated Claude model identity", async () => { + const data = evidence(); + data.events[1].data.model = "copilot-completions/claude-haiku-4-5-20251001"; + await expect(assertEvidence(data)).resolves.toBeUndefined(); + }); + + test("rejects success without delegation", async () => { + await expect(assertEvidence({ events: [], requests: evidence().requests })).rejects.toThrow("must actually run"); + }); + + test("rejects a sub-agent that falls back to the main model", async () => { + const data = evidence(); + data.events[1].data.model = "copilot-responses/gpt-5.3-codex"; + await expect(assertEvidence(data)).rejects.toThrow("changed or fell back"); + }); + + test("rejects selection of a different model", async () => { + const data = evidence(); + data.events.push({ type: "subagent.selected", data: { invocationId: "haiku-whoami", selectedModel: "copilot-responses/gpt-5-mini" } }); + await expect(assertEvidence(data)).rejects.toThrow("changed or fell back"); + }); + + test("rejects requests using the wrong wire API", async () => { + const data = evidence(); + data.requests[0].path = "/responses"; + await expect(assertEvidence(data)).rejects.toThrow("no successful"); + }); + + test("rejects failed invocations even with successful model requests", async () => { + const data = evidence(); + data.events.push({ type: "subagent.failed", data: { invocationId: "haiku-whoami" } }); + await expect(assertEvidence(data)).rejects.toThrow("failed"); + }); + + test("requires no delegation in the negative control", async () => { + await expect(assertEvidence({ events: [], requests: [] }, "single_agent_control")).resolves.toBeUndefined(); + await expect(assertEvidence(evidence(), "single_agent_control")).rejects.toThrow("must not invoke"); + }); +}); diff --git a/actions/setup/js/unified_session_payload.cjs b/actions/setup/js/unified_session_payload.cjs index 2621d04e55c..225beed8bb7 100644 --- a/actions/setup/js/unified_session_payload.cjs +++ b/actions/setup/js/unified_session_payload.cjs @@ -23,6 +23,7 @@ const SCOPED_AGENT_TYPES = new Set([ "tool.execution_complete", "usage.report", "session.result", + "model.call_final_result", ]); /** @type {Fields} */ diff --git a/actions/setup/js/unified_session_payload.test.cjs b/actions/setup/js/unified_session_payload.test.cjs index 15cb37a93d9..059b5ba4aec 100644 --- a/actions/setup/js/unified_session_payload.test.cjs +++ b/actions/setup/js/unified_session_payload.test.cjs @@ -14,6 +14,32 @@ import { dynamicWorkflow } from "./fixtures/claude_dynamic_workflow.cjs"; import { DYNAMIC_WORKFLOW_EVENT_TYPES } from "./dynamic_workflow_session.cjs"; describe("essential unified session payloads", () => { + it.each([true, false])("retains final model result and BYOK=%s through persisted unified projection", isByok => { + const source = { + type: "model.call_final_result", + ephemeral: true, + agentId: "researcher", + parentToolCallId: "delegate-1", + data: { + model: "anthropic/claude-sonnet-4.6", + isByok, + result: { status: "success", usage: { inputTokens: 0, outputTokens: 42 }, stopReason: "end_turn" }, + extensionEvidence: { retained: true }, + }, + }; + const original = structuredClone(source); + const projected = normalizeUnifiedSessionEvent(source); + expect(projected.data).toEqual({ ...source.data, agentId: "researcher", parentToolCallId: "delegate-1" }); + const [unified] = mergeSessionSources([{ component: "agent", phase: "agent", path: "sdk.jsonl", events: normalizeCopilotSession([source]) }]); + const persisted = JSON.parse(JSON.stringify(unified)); + expect(persisted.data).toMatchObject(source.data); + expect(persisted.data.parentToolCallId).toBe("delegate-1"); + expect(normalizeUnifiedSessionEvent(persisted).data).toEqual(persisted.data); + const validate = createSessionValidator("unified").event; + expect(validate(persisted), JSON.stringify(validate.errors)).toBe(true); + expect(source).toEqual(original); + }); + it.each([401, 503, 0, null])("retains observed session-error status codes through projection and schema validation (%s)", statusCode => { const source = { type: "session.error", data: { errorType: "provider", message: "Observed failure.", statusCode, opaque: "PRIVATE_ERROR_CONTEXT" } }; const original = structuredClone(source); diff --git a/docs/adr/67509-split-copilot-sdk-providers-by-wire-api.md b/docs/adr/67509-split-copilot-sdk-providers-by-wire-api.md new file mode 100644 index 00000000000..cf6f7b8376d --- /dev/null +++ b/docs/adr/67509-split-copilot-sdk-providers-by-wire-api.md @@ -0,0 +1,102 @@ +# ADR-67509: Split Copilot SDK Providers by Wire API and Qualify Every Routed Model + +**Date**: 2026-10-10 +**Status**: Proposed +**Deciders**: gh-aw maintainers + +--- + +### Context + +The Copilot coding agent opened PR #67509 for issue #67507. + +The Copilot SDK engine configured a single provider per Copilot endpoint and passed bare model names +(for example `gpt-5.6-luna`) to routed SDK sessions. Two problems followed. First, bare model names +bypassed BYOK inference routing under organization billing, so spend attribution and proxy routing +were wrong. Second, one provider carries one wire API, but Copilot exposes both the Responses API +(`/responses`) and Chat Completions (`/chat/completions`) and different model families support +different ones — so a GPT main agent and a Claude sub-agent could not coexist in the same session. +Separately, declared sub-agents (inline and repository agent files) were parsed but never handed to +the SDK, so delegation silently ran on the main agent's model. Smoke coverage asserted on model-name +replies, which a model can hallucinate, so none of this was reliably caught. + +### Decision + +We will split each Copilot endpoint into two sibling SDK providers, `copilot-responses` and +`copilot-completions`, derived from `/reflect` endpoint capabilities, and route each model to the +provider matching its wire API (`inferWireApiForModel`, with explicit `wire_api`/`wireApi` from the +model catalog winning over the GPT-5+ `responses` heuristic). Every model identifier that reaches the +SDK — routed, fallback, reflected, and sub-agent — is qualified as `/` so BYOK +inference is used; explicitly configured API selections are preserved when supported. +Each Copilot model's provider must match its `/reflect` `supported_endpoints`. An incompatible +`COPILOT_PROVIDER_WIRE_API` override fails at startup before the sidecar starts with +`Model endpoint mismatch: …`, naming the model, override, and supported endpoints. +Declared agents are loaded as SDK `customAgents` carrying name, description, prompt, and tools, with +concrete models qualified and aliases inheriting the session model. Unknown concrete agent models +inherit without failing the run, but emit `subagent.model_unavailable` with the declared agent name, +declared model, and inherited model; audit surfaces this as a warning finding. Aliases inherit quietly. +Non-Copilot endpoints and CLI mode are deliberately left unchanged. Observability captures +`subagent.selected` and `model.call_final_result` including BYOK status, and `pkg/cli` normalizes +qualified identities (`copilot-(responses|completions)` stripped) so audit spend attribution still +groups by real model. Smoke tests now require evidence of completed delegation on the expected model +and proxy endpoint instead of model-name replies. + +### Alternatives Considered + +#### Alternative 1: Keep one provider per endpoint and pick a single wire API + +Continue with a provider-wide wire API, choosing `responses` or `completions` per run. This is the +smallest change and keeps the provider list short. Rejected because it structurally prevents +cross-family sessions: a Claude sub-agent delegated from a GPT main agent needs a different wire API +in the same session, which is the primary defect this PR fixes. + +#### Alternative 2: Qualify models only at the top-level session, not for sub-agents and fallbacks + +Qualify the main session model and leave sub-agent, fallback, and reflected models bare, relying on +server-side defaults. Tempting because it touches far fewer call sites. Rejected because the bare +paths are exactly where BYOK routing was being bypassed; partial qualification would leave spend +attribution wrong for delegated work and make failures intermittent and hard to diagnose. + +#### Alternative 3: Resolve wire API dynamically per request instead of per provider + +Let the driver choose the HTTP path at call time rather than modelling two providers. Rejected +because the SDK's provider abstraction owns `wireApi`; per-request override would fork the SDK's +transport handling and lose the SDK-native `customAgents` routing we rely on. + +### Consequences + +#### Positive + +- Cross-family sessions work: a GPT main agent on `/responses` can delegate to a Claude sub-agent on + `/chat/completions` within one session. +- Routed, fallback, reflected, and sub-agent calls all use BYOK inference, so org-billed runs route + through the intended proxy and attribute spend correctly. +- Declared inline and repository sub-agents are actually honored by the SDK, with their prompts and + tool sets intact. +- Smoke and audit assertions are grounded in delegation/proxy evidence rather than model self-reports, + which a model can fake. + +#### Negative + +- The provider list roughly doubles for every Copilot endpoint, making reflected configuration and + debug output noisier and harder to read. +- Wire-API inference adds a heuristic (`GPT-5+ → responses`) that will need maintenance as new model + families ship; a wrong inference produces a routing failure rather than a graceful degradation. +- Qualified identifiers (`copilot-responses/gpt-5.6-luna`) leak into logs, session records, and + fixtures, requiring normalization in `pkg/cli` and in any downstream consumer that matched on bare + model names. +- Behaviour now diverges between SDK mode and CLI mode, which stays on the old path — two code paths + to reason about until CLI mode is migrated. + +#### Neutral + +- Non-Copilot endpoints (including Anthropic-typed providers, where `wireApi` is ignored) are + unaffected by this change. +- Unknown sub-agent models inherit the session model without failing the run; structured events and + audit findings make this compatibility fallback visible. +- `docs/src/content/docs/reference/engines.md` documents the new provider naming, so the qualified + form is now part of the user-facing contract. + +--- + +*Proposed for review by the gh-aw maintainers.* diff --git a/docs/src/content/docs/reference/engines.md b/docs/src/content/docs/reference/engines.md index cf424a24ea9..7fea6510c85 100644 --- a/docs/src/content/docs/reference/engines.md +++ b/docs/src/content/docs/reference/engines.md @@ -561,6 +561,16 @@ Enable `engine.copilot-sdk: true` to run Copilot in SDK mode. In this mode, the harness starts a local sidecar and runs the SDK driver process instead of the default CLI-only flow. +The built-in SDK driver loads inline sub-agents and repository +`.github/agents/*.agent.md` files as custom agents. It qualifies session and +sub-agent models against the AWF model catalog. Copilot models use separate +Responses and Chat Completions providers according to their supported endpoints, +so GPT and Claude agents can use different wire APIs in the same session. +The aliases `small`, `medium`, and `large` inherit the session model; an unknown +sub-agent model logs a warning and inherits it. Sub-agent activity and final model +results, including BYOK status, are retained in `usage/aw_session.jsonl`. +This behavior applies to SDK mode, not the default CLI mode. + Use top-level `max-tool-denials` to stop SDK inference when tool requests are repeatedly denied. The default is `5`. This field is only supported when `engine.id: copilot` and diff --git a/pkg/cli/audit_finding_codes.go b/pkg/cli/audit_finding_codes.go index 0af796fadcd..00e620b17c9 100644 --- a/pkg/cli/audit_finding_codes.go +++ b/pkg/cli/audit_finding_codes.go @@ -15,6 +15,7 @@ const ( AuditFindingBlockedNetworkRequests AuditFindingCode = "blocked_network_requests" AuditFindingWorkflowSucceeded AuditFindingCode = "workflow_succeeded" AuditFindingSubagentModelNotObserved AuditFindingCode = "subagent_model_not_observed" + AuditFindingSubagentModelUnavailable AuditFindingCode = "subagent_model_unavailable" AuditFindingSubagentFailed AuditFindingCode = "subagent_failed" AuditFindingDetectionJobFailed AuditFindingCode = "threat_detection_job_failed" AuditFindingThreatDetected AuditFindingCode = "threat_detected" diff --git a/pkg/cli/audit_report_analysis.go b/pkg/cli/audit_report_analysis.go index 4b8c8bdaea9..e8043bc5a70 100644 --- a/pkg/cli/audit_report_analysis.go +++ b/pkg/cli/audit_report_analysis.go @@ -35,6 +35,7 @@ func generateFindings(processedRun ProcessedRun, metrics MetricsData, errors []V findings = append(findings, generateToolingFindings(processedRun)...) findings = append(findings, generateFirewallFindings(processedRun)...) findings = append(findings, generateSubagentModelFindings(processedRun.TokenUsage)...) + findings = append(findings, generateSubagentModelUnavailableFindings(processedRun.Run.LogsPath)...) findings = append(findings, generateSuccessFindings(processedRun.Run, metrics, errors)...) return findings } diff --git a/pkg/cli/audit_subagent_model.go b/pkg/cli/audit_subagent_model.go new file mode 100644 index 00000000000..e6a8eb3cc6e --- /dev/null +++ b/pkg/cli/audit_subagent_model.go @@ -0,0 +1,31 @@ +package cli + +import "fmt" + +func generateSubagentModelUnavailableFindings(runDir string) []AuditFinding { + attribution, _, err := readSessionModelRouting(runDir) + if err != nil { + auditReportLog.Printf("failed to read sub-agent model availability: %v", err) + return nil + } + if attribution == nil { + return nil + } + var findings []AuditFinding + seen := make(map[sessionSubagentModelUnavailable]struct{}) + for _, model := range attribution.UnavailableSubagentModels { + if _, duplicate := seen[model]; duplicate { + continue + } + seen[model] = struct{}{} + findings = append(findings, AuditFinding{ + Code: AuditFindingSubagentModelUnavailable, + Category: "tooling", + Severity: "medium", + Title: "Sub-agent Model Unavailable", + Description: fmt.Sprintf("Sub-agent %s declared model %s was unavailable; used inherited session model %s", model.AgentName, model.DeclaredModel, model.Model), + Impact: "The sub-agent used the session model instead of its declared model. This fallback is a warning, not a delegation failure.", + }) + } + return findings +} diff --git a/pkg/cli/audit_subagent_model_test.go b/pkg/cli/audit_subagent_model_test.go new file mode 100644 index 00000000000..2788f2afd2a --- /dev/null +++ b/pkg/cli/audit_subagent_model_test.go @@ -0,0 +1,78 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAuditSubagentModelUnavailable(t *testing.T) { + runDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(runDir, "usage"), 0755)) + event := `{"type":"subagent.model_unavailable","data":{"agentName":"research","declaredModel":"claude-haiku-4.5","model":"anthropic/claude-opus-4.8"},"provenance":{"component":"agent","phase":"agent"}}` + content := subagentSessionHeader + event + "\n" + event + "\n" + require.NoError(t, os.WriteFile(filepath.Join(runDir, "usage", "aw_session.jsonl"), []byte(content), 0600)) + + attribution, found, err := readSessionModelRouting(runDir) + require.NoError(t, err) + assert.False(t, found, "availability warnings do not imply workflow.info was found") + require.Len(t, attribution.UnavailableSubagentModels, 2) + assert.Nil(t, attribution.modelRouting()) + + run := ProcessedRun{Run: WorkflowRun{Conclusion: "success", LogsPath: runDir}} + findings := generateFindings(run, MetricsData{}, nil) + require.Len(t, findings, 2, "one deduplicated warning and the existing success finding") + warning := findings[0] + assert.Equal(t, AuditFindingSubagentModelUnavailable, warning.Code) + assert.Equal(t, "tooling", warning.Category) + assert.Equal(t, "medium", string(warning.Severity)) + assert.Contains(t, warning.Description, "research") + assert.Contains(t, warning.Description, "claude-haiku-4.5") + assert.Contains(t, warning.Description, "anthropic/claude-opus-4.8") + assert.Equal(t, AuditFindingWorkflowSucceeded, findings[1].Code) + assert.Empty(t, generateSubagentModelFindings(run.TokenUsage)) + + requests, actuals, _, err := parseSessionSubagentModels(strings.NewReader(content), true) + require.NoError(t, err) + assert.Empty(t, requests, "availability warnings must not create invocations or failures") + assert.Empty(t, actuals, "fallback metadata is not observed model traffic") +} + +func TestAuditSubagentModelUnavailableSources(t *testing.T) { + event := `{"type":"subagent.model_unavailable","data":{"agentName":"research","declaredModel":"haiku","model":"anthropic/opus"},"provenance":{"component":"agent","phase":"agent"}}` + for _, relative := range []string{"usage/aw_session.jsonl", "aw_session.jsonl"} { + t.Run(relative, func(t *testing.T) { + runDir := t.TempDir() + path := filepath.Join(runDir, filepath.FromSlash(relative)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(event), 0600)) + require.Len(t, generateSubagentModelUnavailableFindings(runDir), 1) + if relative == "usage/aw_session.jsonl" { + require.NoError(t, os.WriteFile(filepath.Join(runDir, "aw_session.jsonl"), []byte("invalid"), 0600)) + require.Len(t, generateSubagentModelUnavailableFindings(runDir), 1, "usage artifact takes precedence") + } + }) + } + assert.Empty(t, generateSubagentModelUnavailableFindings("")) + assert.Empty(t, generateSubagentModelUnavailableFindings(t.TempDir())) +} + +func TestDecodeSubagentModelUnavailableIgnoresInvalidEvidence(t *testing.T) { + for _, event := range []string{ + `{"type":"subagent.model_unavailable","data":{"agentName":"research","declaredModel":"haiku","model":"anthropic/opus"},"provenance":{"component":"detection","phase":"agent"}}`, + `{"type":"subagent.model_unavailable","data":{"agentName":"research","declaredModel":"haiku","model":"anthropic/opus"},"provenance":{"component":"agent","phase":"detection"}}`, + `{"type":"subagent.model_unavailable","data":{"agentName":"research","declaredModel":"haiku"},"provenance":{"component":"agent","phase":"agent"}}`, + `{"type":"subagent.model_unavailable","data":{"declaredModel":"haiku","model":"anthropic/opus"},"provenance":{"component":"agent","phase":"agent"}}`, + `{"type":"subagent.model_unavailable","data":{"agentName":"research","model":"anthropic/opus"},"provenance":{"component":"agent","phase":"agent"}}`, + `{"type":"subagent.model_unavailable","data":[],"provenance":{"component":"agent","phase":"agent"}}`, + `{"type":"subagent.model_unavailable","data":{"model":42},"provenance":{"component":"agent","phase":"agent"}}`, + } { + attribution := &sessionModelRoutingAttribution{} + require.NoError(t, decodeSessionModelRoutingEvent([]byte(event), 1, attribution)) + assert.Empty(t, attribution.UnavailableSubagentModels) + } +} diff --git a/pkg/cli/model_identity.go b/pkg/cli/model_identity.go index 3ab21da24c2..4749116b03b 100644 --- a/pkg/cli/model_identity.go +++ b/pkg/cli/model_identity.go @@ -13,6 +13,7 @@ import ( var ( modelDateSuffix = regexp.MustCompile(`-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$`) claudeModelVersionSpelling = regexp.MustCompile(`^(claude-.+)-([0-9]+)-([0-9]+)$`) + copilotSDKModelProvider = regexp.MustCompile(`^copilot-(?:responses|completions)(?:-[0-9]+)?$`) ) type modelIdentityResolver struct { @@ -45,6 +46,7 @@ func newModelIdentityResolver(runDir string) *modelIdentityResolver { func normalizeModelIdentity(model string) string { model, _, _ = strings.Cut(strings.ToLower(strings.TrimSpace(model)), "?") + model = modelNameWithoutProvider(model) model = modelDateSuffix.ReplaceAllString(model, "") return claudeModelVersionSpelling.ReplaceAllString(model, "$1-$2.$3") } @@ -106,7 +108,7 @@ func modelPatternMatches(pattern, observed, provider string) bool { observed, _, _ = strings.Cut(strings.ToLower(observed), "?") originalObserved := observed patternProvider, patternModel, patternQualified := strings.Cut(pattern, "/") - observedProvider, observedModel, observedQualified := strings.Cut(observed, "/") + observedProvider, _, observedQualified := strings.Cut(observed, "/") if patternQualified { if provider == "" && observedQualified { provider = observedProvider @@ -120,7 +122,7 @@ func modelPatternMatches(pattern, observed, provider string) bool { if provider != "" && normalizeModelProvider(observedProvider) != normalizeModelProvider(provider) { return modelPatternMatches(patternModel, originalObserved, "") } - observed = observedModel + observed = modelNameWithoutProvider(originalObserved) } patterns := []string{pattern, normalizeModelIdentity(pattern)} observedModels := []string{observed, normalizeModelIdentity(observed)} @@ -135,6 +137,10 @@ func modelPatternMatches(pattern, observed, provider string) bool { } func normalizeModelProvider(provider string) string { + provider = strings.ToLower(strings.TrimSpace(provider)) + if copilotSDKModelProvider.MatchString(provider) { + return "github-copilot" + } switch provider { case "github", "copilot", "github-copilot": return "github-copilot" @@ -148,8 +154,10 @@ func normalizeModelProvider(provider string) string { } func modelNameWithoutProvider(model string) string { - _, name, qualified := strings.Cut(model, "/") + provider, name, qualified := strings.Cut(model, "/") if qualified { + // SDK metric keys can repeat the provider after the qualified model. + name = strings.TrimSuffix(name, ":"+provider) return name } return model diff --git a/pkg/cli/model_identity_test.go b/pkg/cli/model_identity_test.go new file mode 100644 index 00000000000..dbec4e79ba9 --- /dev/null +++ b/pkg/cli/model_identity_test.go @@ -0,0 +1,90 @@ +package cli + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeSDKModelIdentity(t *testing.T) { + t.Parallel() + for _, test := range []struct { + model, want string + }{ + {"copilot-responses/gpt-5.6-luna", "gpt-5.6-luna"}, + {"copilot-responses/gpt-5.4-mini-2026-03-17:copilot-responses", "gpt-5.4-mini"}, + {"copilot-completions/claude-haiku-4.5", "claude-haiku-4.5"}, + {"copilot-completions/claude-haiku-4-5-20251001:copilot-completions", "claude-haiku-4.5"}, + {" COPILOT-COMPLETIONS/CLAUDE-HAIKU-4-5-20251001?effort=low ", "claude-haiku-4.5"}, + {"openai/gpt-5.4-mini", "gpt-5.4-mini"}, + {"copilot-responses/gpt-5.4-mini:other", "gpt-5.4-mini:other"}, + {"copilot-responses-1/gpt-5.4-mini:copilot-responses-1", "gpt-5.4-mini"}, + {"copilot-completions-2/claude-haiku-4-5-20251001:copilot-completions-2", "claude-haiku-4.5"}, + } { + t.Run(test.model, func(t *testing.T) { + require.Equal(t, test.want, normalizeModelIdentity(test.model)) + require.Equal(t, test.want, newModelIdentityResolver("").resolve(test.model, "", nil)) + }) + } +} + +func TestModelIdentityMatchesSDKProviders(t *testing.T) { + t.Parallel() + resolver := newModelIdentityResolver("") + for _, test := range []struct { + pattern, observed, provider string + want bool + }{ + {"copilot/gpt-*", "copilot-responses/gpt-5.6-luna:copilot-responses", "github-copilot", true}, + {"copilot/claude-haiku-4.5", "copilot-completions/claude-haiku-4-5-20251001:copilot-completions", "github-copilot", true}, + {"claude-haiku-4.5", "copilot-completions/claude-haiku-4-5-20251001:copilot-completions", "", true}, + {"copilot/claude-*", "copilot-completions/claude-haiku-4.5", "", true}, + {"anthropic/claude-*", "copilot-completions/claude-haiku-4.5", "", false}, + {"copilot/gpt-*", "copilot-completions/claude-haiku-4.5", "", false}, + {"copilot/claude-*", "claude-haiku-4.5", "anthropic", false}, + {"copilot/gpt-*", "copilot-responses-1/gpt-5.6-luna:copilot-responses-1", "github-copilot", true}, + {"copilot/claude-*", "copilot-completions-2/claude-haiku-4.5:copilot-completions-2", "", true}, + {"copilot/gpt-*", "copilot-arbitrary/gpt-5.6-luna", "", false}, + {"copilot/gpt-*", "copilot-responses-extra/gpt-5.6-luna", "", false}, + {"copilot/claude-*", "copilot-completions-1-extra/claude-haiku-4.5", "", false}, + } { + t.Run(test.pattern+"_"+test.observed+"_"+test.provider, func(t *testing.T) { + require.Equal(t, test.want, resolver.matches(test.pattern, test.observed, test.provider)) + }) + } + + require.Equal(t, "claude-haiku-4.5", resolver.resolve("haiku", "github-copilot", []string{ + "copilot-completions/claude-haiku-4-5-20251001:copilot-completions", + })) +} + +func TestNormalizeSDKModelProvider(t *testing.T) { + t.Parallel() + for _, provider := range []string{"copilot-responses", "copilot-completions", "copilot-responses-1", "copilot-completions-12", " COPILOT-RESPONSES-2 "} { + require.Equal(t, "github-copilot", normalizeModelProvider(provider)) + } + for _, provider := range []string{"copilot-arbitrary", "copilot-responses-extra", "copilot-completions-1-extra", "copilot-responses-", "copilot-responses-1.2", "other-copilot-responses-1"} { + require.Equal(t, provider, normalizeModelProvider(provider)) + } +} + +func TestSDKQualifiedModelsMergeAttributionWithoutMixingFamilies(t *testing.T) { + t.Parallel() + resolver := newModelIdentityResolver("") + actuals := resolveSubagentActualModels([]SubagentModelActual{ + {Model: "copilot-responses/gpt-5.4-mini:copilot-responses", Provider: "github-copilot", Requests: 2, TokenCoreMetrics: TokenCoreMetrics{InputTokens: 20}, AIC: 0.2}, + {Model: "gpt-5.4-mini", Provider: "github-copilot", Requests: 1, TokenCoreMetrics: TokenCoreMetrics{InputTokens: 10}, AIC: 0.1}, + {Model: "copilot-completions/claude-haiku-4-5-20251001:copilot-completions", Provider: "github-copilot", Requests: 1, TokenCoreMetrics: TokenCoreMetrics{InputTokens: 40}, AIC: 0.4}, + }, resolver, "gpt-5.4-mini", "claude-haiku-4.5") + require.Len(t, actuals, 2) + require.Equal(t, "gpt-5.4-mini", actuals[0].ResolvedModel) + require.Equal(t, 3, actuals[0].Requests) + require.Equal(t, 30, actuals[0].InputTokens) + require.InDelta(t, 0.3, actuals[0].AIC, 0.000001) + require.NotContains(t, actuals[0].ServedModels, "claude-haiku-4.5") + require.Equal(t, "claude-haiku-4.5", actuals[1].ResolvedModel) + require.Equal(t, 1, actuals[1].Requests) + require.Equal(t, 40, actuals[1].InputTokens) + require.InDelta(t, 0.4, actuals[1].AIC, 0.000001) + require.NotContains(t, actuals[1].ServedModels, "gpt-5.4-mini") +} diff --git a/pkg/cli/model_routing_session.go b/pkg/cli/model_routing_session.go index a5f3a496e04..db371eb60b9 100644 --- a/pkg/cli/model_routing_session.go +++ b/pkg/cli/model_routing_session.go @@ -64,8 +64,15 @@ type sessionModelRoutingEvent struct { } type sessionModelRoutingAttribution struct { - WorkflowInfo *sessionWorkflowInfo - Outcome *sessionModelRoutingOutcome + WorkflowInfo *sessionWorkflowInfo + Outcome *sessionModelRoutingOutcome + UnavailableSubagentModels []sessionSubagentModelUnavailable +} + +type sessionSubagentModelUnavailable struct { + AgentName string `json:"agentName"` + DeclaredModel string `json:"declaredModel"` + Model string `json:"model"` } func readSessionModelRouting(runDir string) (*sessionModelRoutingAttribution, bool, error) { @@ -140,6 +147,12 @@ func decodeSessionModelRoutingEvent(line []byte, lineNumber int, attribution *se return fmt.Errorf("invalid model_routing.outcome on line %d: %w", lineNumber, err) } attribution.Outcome = &outcome + case event.Type == "subagent.model_unavailable" && event.Provenance.Component == "agent" && event.Provenance.Phase == "agent": + var unavailable sessionSubagentModelUnavailable + if json.Unmarshal(event.Data, &unavailable) == nil && + unavailable.AgentName != "" && unavailable.DeclaredModel != "" && unavailable.Model != "" { + attribution.UnavailableSubagentModels = append(attribution.UnavailableSubagentModels, unavailable) + } } return nil } diff --git a/pkg/cli/testdata/subagent_attribution/copilot-qualified-cross-family/aw_session.jsonl b/pkg/cli/testdata/subagent_attribution/copilot-qualified-cross-family/aw_session.jsonl new file mode 100644 index 00000000000..ae512f004a5 --- /dev/null +++ b/pkg/cli/testdata/subagent_attribution/copilot-qualified-cross-family/aw_session.jsonl @@ -0,0 +1,9 @@ +{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}} +{"type":"session.init","data":{"sourceEngine":"copilot","sessionId":"copilot-qualified-parent"},"timestamp":"2026-10-08T10:00:00Z","provenance":{"component":"agent","phase":"agent","path":"agent-session.jsonl","index":0}} +{"type":"subagent.started","agentId":"child","data":{"invocationId":"child","agentName":"file-summarizer","agentDisplayName":"File-summarizer","model":"copilot-completions/claude-haiku-4.5","resolvedModel":"copilot-completions/claude-haiku-4-5-20251001"},"timestamp":"2026-10-08T10:00:01Z","provenance":{"component":"agent","phase":"agent","path":"agent-session.jsonl","index":1}} +{"type":"subagent.configured","agentId":"child","data":{"invocationId":"child","model":"copilot-completions/claude-haiku-4.5"},"timestamp":"2026-10-08T10:00:02Z","provenance":{"component":"agent","phase":"agent","path":"agent-session.jsonl","index":2}} +{"type":"subagent.completed","agentId":"child","data":{"invocationId":"child","agentName":"file-summarizer","outcome":"completed"},"timestamp":"2026-10-08T10:00:03Z","provenance":{"component":"agent","phase":"agent","path":"agent-session.jsonl","index":3}} +{"type":"session.result","data":{"agentMetrics":{"main":{"totalNanoAiu":250000000,"modelMetrics":{"copilot-responses/gpt-5.6-luna:copilot-responses":{"requests":{"count":1},"usage":{"inputTokens":100,"outputTokens":10},"totalNanoAiu":250000000}}},"child":{"agentName":"file-summarizer","totalNanoAiu":750000000,"modelMetrics":{"copilot-completions/claude-haiku-4-5-20251001:copilot-completions":{"requests":{"count":2},"usage":{"inputTokens":200,"outputTokens":20,"cacheReadTokens":30,"cacheWriteTokens":40},"totalNanoAiu":750000000}}}}},"timestamp":"2026-10-08T10:00:04Z","provenance":{"component":"agent","phase":"agent","path":"agent-session.jsonl","index":4}} +{"type":"firewall.token_usage","data":{"provider":"github-copilot","model":"claude-haiku-4.5","path":"/chat/completions","status":200,"requestId":"child-1","purpose":"subagent","inputTokens":100,"outputTokens":10,"aic":0.375},"timestamp":"2026-10-08T10:00:02Z","provenance":{"component":"firewall","phase":"agent","path":"token-usage.jsonl","index":0}} +{"type":"firewall.token_usage","data":{"provider":"github-copilot","model":"claude-haiku-4.5","path":"/chat/completions","status":200,"requestId":"child-2","purpose":"subagent","inputTokens":100,"outputTokens":10,"aic":0.375},"timestamp":"2026-10-08T10:00:03Z","provenance":{"component":"firewall","phase":"agent","path":"token-usage.jsonl","index":1}} +{"type":"firewall.token_usage","data":{"provider":"github-copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"requestId":"main-1","inputTokens":100,"outputTokens":10,"aic":0.25},"timestamp":"2026-10-08T10:00:04Z","provenance":{"component":"firewall","phase":"agent","path":"token-usage.jsonl","index":2}} diff --git a/pkg/cli/token_usage_subagent_session.go b/pkg/cli/token_usage_subagent_session.go index 87533690a36..441e7e16d28 100644 --- a/pkg/cli/token_usage_subagent_session.go +++ b/pkg/cli/token_usage_subagent_session.go @@ -41,6 +41,7 @@ type sessionSubagentData struct { RequestedModel string `json:"requestedModel"` ResolvedModel string `json:"resolvedModel"` Outcome string `json:"outcome"` + Cancelled bool `json:"cancelled"` ErrorCode string `json:"errorCode"` ErrorMessage string `json:"errorMessage"` ReasoningEffort string `json:"reasoningEffort"` @@ -906,6 +907,9 @@ func (models *subagentSessionModels) observeLifecycle(event sessionSubagentEvent } func observeLifecycleOutcome(row *SubagentModelRequest, agentUsage *AgentUsageBreakdown, event sessionSubagentEvent, data sessionSubagentData) { + if event.Type == "subagent.completed" && data.Cancelled && (row.CompletedCount > 0 || row.FailedCount > 0) { + return + } if event.Type == "subagent.completed" { row.CompletedCount++ row.IncompleteCount = max(0, row.IncompleteCount-1) diff --git a/pkg/cli/token_usage_subagent_session_test.go b/pkg/cli/token_usage_subagent_session_test.go index 027c58f8208..552add4cc13 100644 --- a/pkg/cli/token_usage_subagent_session_test.go +++ b/pkg/cli/token_usage_subagent_session_test.go @@ -100,6 +100,41 @@ func TestSubagentSessionStart(t *testing.T) { } } +func TestSessionSubagentSDKCancelledCompletion(t *testing.T) { + t.Parallel() + for _, terminal := range []string{"subagent.completed", "subagent.failed", ""} { + t.Run(terminal, func(t *testing.T) { + t.Parallel() + content := subagentSessionHeader + + subagentSessionRecord(`{"type":"subagent.started","data":{"invocationId":"child","agentName":"file-summarizer","model":"copilot-completions/claude-haiku-4.5"}}`) + if terminal != "" { + content += subagentSessionRecord(`{"type":"` + terminal + `","data":{"invocationId":"child","agentName":"file-summarizer"}}`) + } + content += subagentSessionRecord(`{"type":"subagent.completed","data":{"invocationId":"child","agentName":"file-summarizer","cancelled":true}}`) + requests, _, usage, found, err := parseSessionSubagentModelsDetailed(strings.NewReader(content), true) + require.NoError(t, err) + require.True(t, found) + require.Len(t, requests, 1) + require.Len(t, usage, 1) + assert.Equal(t, 1, requests[0].InvocationCount) + assert.Equal(t, 1, usage[0].InstanceCount) + assert.Zero(t, requests[0].IncompleteCount) + assert.Zero(t, usage[0].IncompleteCount) + if terminal == "subagent.failed" { + assert.Equal(t, 1, requests[0].FailedCount) + assert.Equal(t, 1, usage[0].FailedCount) + assert.Zero(t, requests[0].CompletedCount) + assert.Zero(t, usage[0].CompletedCount) + } else { + assert.Equal(t, 1, requests[0].CompletedCount) + assert.Equal(t, 1, usage[0].CompletedCount) + assert.Zero(t, requests[0].FailedCount) + assert.Zero(t, usage[0].FailedCount) + } + }) + } +} + func TestSessionSubagentModels(t *testing.T) { start := `{"type":"subagent.started","agentId":"research","data":{"agentName":"research","agentDisplayName":"routing-research","model":"opus","executionMode":"background"}}` config := `{"type":"subagent.configured","agentId":"research","data":{"model":"sonnet","reasoningEffort":"low"}}` @@ -475,6 +510,59 @@ func TestCopilotUnifiedFixtureRetainsFailedCrossFamilySubagent(t *testing.T) { require.Zero(t, summary.AgentUsage[1].AIC) } +func TestCopilotUnifiedFixtureAttributesQualifiedCrossFamilyModels(t *testing.T) { + t.Parallel() + fixture := filepath.Join("testdata", "subagent_attribution", "copilot-qualified-cross-family") + entries, err := readUnifiedTokenUsageEntries(fixture) + require.NoError(t, err) + require.Len(t, entries, 3) + require.Equal(t, "/chat/completions", entries[0].Path) + require.Equal(t, "/responses", entries[2].Path) + summary := &TokenUsageSummary{AICFound: true, TotalAIC: 1, ByModel: map[string]*ModelTokenUsage{ + "claude-haiku-4.5": {Provider: "github-copilot", Requests: 2}, + "gpt-5.6-luna": {Provider: "github-copilot", Requests: 1}, + }} + augmentSubagentModelAttribution(fixture, summary) + + require.Empty(t, summary.Warnings) + require.Zero(t, summary.MismatchCount) + require.Len(t, summary.SubagentModelRequests, 1) + request := summary.SubagentModelRequests[0] + require.Equal(t, "File-summarizer", request.AgentName) + require.Equal(t, "copilot-completions/claude-haiku-4.5", request.RequestedModel) + require.Equal(t, "claude-haiku-4.5", request.ResolvedModel) + require.Equal(t, "claude-haiku-4.5", request.EffectiveModel) + require.Equal(t, 1, request.CompletedCount) + require.Zero(t, request.FailedCount) + require.Contains(t, request.ServedModels, "claude-haiku-4.5") + require.NotContains(t, request.ServedModels, "gpt-5.6-luna") + require.Empty(t, request.ReasonCode) + + require.Len(t, summary.AgentUsage, 2) + main, child := summary.AgentUsage[0], summary.AgentUsage[1] + require.Equal(t, "main", main.AgentType) + require.Equal(t, 1, main.Requests) + require.InDelta(t, 0.25, main.AIC, 0.000001) + require.Equal(t, "gpt-5.6-luna", main.Models[0].ResolvedModel) + require.Contains(t, main.ServedModels, "gpt-5.6-luna") + require.NotContains(t, main.ServedModels, "claude-haiku-4.5") + require.Equal(t, "file-summarizer", child.AgentName) + require.Equal(t, "subagent", child.AgentType) + require.Equal(t, 2, child.Requests) + require.InDelta(t, 0.75, child.AIC, 0.000001) + require.Contains(t, child.ServedModels, "claude-haiku-4.5") + require.NotContains(t, child.ServedModels, "gpt-5.6-luna") + require.Len(t, child.Models, 1) + require.Equal(t, "claude-haiku-4.5", child.Models[0].ResolvedModel) + require.InDelta(t, 0.75, child.Models[0].AIC, 0.000001) + require.Len(t, summary.SubagentModelActuals, 1) + actual := summary.SubagentModelActuals[0] + require.Equal(t, "claude-haiku-4.5", actual.Model) + require.Equal(t, 2, actual.Requests) + require.Equal(t, TokenCoreMetrics{InputTokens: 200, OutputTokens: 20, CacheReadTokens: 30, CacheWriteTokens: 40}, actual.TokenCoreMetrics) + require.InDelta(t, 0.75, actual.AIC, 0.000001) +} + func TestCopilotAgentMetricsWithoutLifecycleRetainAttribution(t *testing.T) { root := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(root, "usage"), 0o755))