diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index b0f38796..ccfc754d 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2335,7 +2335,8 @@ jobs: - name: Run OpenCode PR Review model pool id: opencode_review_model_pool if: needs.coverage-evidence.result == 'success' - timeout-minutes: 350 + timeout-minutes: 330 + continue-on-error: true env: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} GITHUB_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} @@ -2343,27 +2344,23 @@ jobs: SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" - # Ordered contract-reliability first, then quota, with the rate-starved - # flagships last. The mini reasoning models (o4-mini, o3-mini, gpt-5- - # mini/nano/chat) reliably emit the strict review contract — every - # required label and only source-backed findings — so they lead. The - # high-quota non-reasoning models (deepseek-v3, mistral, llama-4) emit - # bare or hallucinated reviews the publish/approve gates reject, so - # they are fallbacks only. gpt-5/o3 ("Reasoning" tier, 8-12 req/day) - # stay last: first-placing them stalled every review until timeout - # because a rate-limited/hung flagship never fell back. - OPENCODE_MODEL_CANDIDATES: "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o3 github-models/openai/gpt-5" + # Ordered by observed successful review completion first, then + # contract-reliability and quota. DeepSeek V3 currently has the best + # success rate in the org queue, so it leads with enough time to finish + # a deep review. Compact reasoning models remain early fallbacks. The + # rate-starved flagships stay last because first-placing them stalled + # previous reviews until timeout instead of reaching healthier models. + OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o3 github-models/openai/gpt-5" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. OPENCODE_MODEL_ATTEMPTS: "1" - # 90 min per model — generous for a deep tool-using review, but bounded - # so a rate-limited model yields to the next one instead of eating the - # 350-min step. (20400s = 340min gave one model the entire budget with - # no fallback; 600s was too short for a proper review.) + # 90 min per model gives deep reviews enough room. The pool budget, + # not a short per-model cutoff, prevents stale provider calls from + # consuming the whole 360-min job before the publish gate logs why. OPENCODE_RUN_TIMEOUT_SECONDS: "5400" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "18000" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" OPENCODE_FIRST_ATTEMPT_AGENT: ci-review @@ -2720,12 +2717,7 @@ jobs: fi - name: Approve PR if OpenCode review passed - if: >- - always() - && ( - needs.coverage-evidence.result != 'success' - || steps.opencode_review_model_pool.outcome == 'success' - ) + if: always() timeout-minutes: 75 env: GH_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || steps.opencode_app_token.outputs.token || github.token }} @@ -3611,7 +3603,7 @@ jobs: "scripts/ci/test_strix_quick_gate.sh" emit_known_missing_string_finding \ "MODEL: github-models/deepseek/deepseek-r1-0528" \ - "OpenCode review must start with DeepSeek R1" \ + "OpenCode review config must keep the DeepSeek R1 session default" \ ".github/workflows/opencode-review.yml" \ "scripts/ci/test_strix_quick_gate.sh" diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index a21f339d..7a884ee9 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -33,7 +33,7 @@ on: review_dispatch_limit: description: Maximum OpenCode/Strix review dispatch actions per scheduler run required: false - default: "1" + default: "-1" type: string enable_auto_merge: description: Enable auto-merge for current-head approved PRs @@ -95,7 +95,7 @@ on: review_dispatch_limit: description: Maximum OpenCode/Strix review dispatch actions per scheduler run required: false - default: "1" + default: "-1" enable_auto_merge: description: Enable auto-merge for current-head approved PRs required: false diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index f51cfff5..4e35b8a9 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -521,16 +521,16 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout" - assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review has a bounded per-model timeout before trying fallback models" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' "opencode model pool has no script-level retry budget" + assert_file_contains "$workflow_file" 'timeout-minutes: 330' "opencode model pool keeps retrying with bounded queue impact while leaving approval headroom" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review has a deep per-model timeout before trying fallback models" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "18000"' "opencode model pool exits before the job timeout so the approval gate can publish a reason" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" assert_file_contains "$workflow_file" "always()" "opencode fallback chain uses always() so failed model steps cannot skip every fallback" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode fallback tries the catalog promptly instead of spending the entire review on one model" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review includes a broad catalog fallback pool" - assert_file_contains "$workflow_file" "steps.opencode_review_model_pool.outcome == 'success'" "opencode model step must succeed before review publication" - assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-chat github-models/openai/o3 github-models/mistral-ai/mistral-medium-2505 github-models/openai/gpt-5-nano github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct" "opencode review tries high-effort reasoning fallbacks before broader catalog models" + assert_file_not_contains "$workflow_file" "steps.opencode_review_model_pool.outcome == 'success'" "opencode approval gate still runs after model pool failure to publish a reason" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini" "opencode review starts with the observed high-success DeepSeek V3 path before compact reasoning fallbacks" assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against the PR-head worktree" assert_file_contains "$workflow_file" '"openai/o3"' "opencode config declares OpenAI o3 fallback" assert_file_contains "$workflow_file" '"openai/o4-mini"' "opencode config declares OpenAI o4-mini fallback" @@ -632,11 +632,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback has a bounded model review timeout before step timeout" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode catalog fallback has a bounded deep-review timeout before step timeout" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" - assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review tries compact OpenAI reasoning model fallbacks early" - assert_file_contains "$workflow_file" "github-models/openai/gpt-5-chat github-models/openai/o3 github-models/mistral-ai" "opencode review keeps full OpenAI and non-OpenAI catalog fallbacks after compact reasoning attempts" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review tries DeepSeek V3 before compact OpenAI reasoning model fallbacks" + assert_file_contains "$workflow_file" "github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" "opencode review keeps full reasoning-capable catalog fallbacks after compact attempts" assert_file_contains "$workflow_file" "coverage-evidence:" "opencode workflow measures coverage before review" assert_file_contains "$workflow_file" "github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target'" "manual and required OpenCode reviews measure coverage instead of approving skipped coverage evidence" assert_file_contains "$workflow_file" "Exchange OpenCode app token for target repository coverage reads" "coverage evidence can read private target repositories through the OpenCode app token" @@ -824,7 +824,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'GH_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || steps.review_read_app_token.outputs.token || github.token }}' "opencode manual dispatch uses the cross-repo approval token for target PR evidence lookups with app-token fallback" assert_file_contains "$workflow_file" 'repos/${GH_REPOSITORY}' "opencode review workflow uses env-backed repository context in shell commands" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review starts the central model pool" - assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini" "opencode review starts with faster reasoning-capable GitHub Models" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review starts with high-success DeepSeek V3 before faster reasoning-capable GitHub Models" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-r1-0528" "opencode review keeps a reachable DeepSeek R1 reasoning fallback model" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324" "opencode review has a reachable DeepSeek V3 fallback model" assert_file_contains "$workflow_file" "github-models/openai/gpt-5" "opencode review still has a bounded GPT-5 fallback model" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index ece0a6bc..01fbe801 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -79,9 +79,9 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_models assert set(candidate_models).issubset(set(models)) assert candidate_models[:3] == [ + "deepseek/deepseek-v3-0324", "openai/o4-mini", "openai/o3-mini", - "openai/gpt-5-mini", ] assert { "openai/gpt-5-chat", @@ -280,20 +280,21 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert '"## Review outcome"' in workflow assert '"## Check outcome"' not in workflow assert "publish REQUEST_CHANGES when coverage-evidence blocker states" in workflow - assert re.search(r"opencode-review-target:[\s\S]{0,240}timeout-minutes: 360", workflow) + assert re.search(r"opencode-review-target:[\s\S]{0,400}timeout-minutes: 360", workflow) assert 'timeout-minutes: 75' in workflow - assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 350", workflow) + assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,260}timeout-minutes: 330", workflow) + assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,320}continue-on-error: true", workflow) assert 'APPROVAL_CHECK_WAIT_ATTEMPTS: "81"' in workflow assert 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "30"' in workflow assert ( - 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/o4-mini ' + 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' + "github-models/openai/o4-mini " "github-models/openai/o3-mini " "github-models/openai/gpt-5-mini " "github-models/openai/gpt-5-nano " 'github-models/openai/gpt-5-chat ' "github-models/deepseek/deepseek-r1-0528 " "github-models/deepseek/deepseek-r1 " - "github-models/deepseek/deepseek-v3-0324 " "github-models/mistral-ai/mistral-medium-2505 " "github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 " "github-models/meta/llama-4-scout-17b-16e-instruct " @@ -303,7 +304,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "18000"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow assert "while :" in model_pool_runner assert "OpenCode model pool has no configured model candidates." in model_pool_runner