From 4b4d0a4a6a3ce3c9f68c8aee331a537f9a5a9761 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 8 Jul 2026 20:18:39 +0900 Subject: [PATCH 1/6] fix(opencode-review): reach a healthy fallback model within the step budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Required OpenCode Review check timed out red on every gyeot PR (ContextualWisdomLab/gyeot #4, #5, #6 — including a one-file docs PR), so the model-pool step is structurally, not transiently, broken. Root cause (from the job logs): the 3-4 front-loaded mini models (o4-mini, gpt-5-mini, gpt-5-nano) each hang/rate-limit and are killed at the 90-min (5400s) per-model `timeout` (exit 124). Three hangs burn ~270 min before the pool can reach any high-quota fallback (deepseek/mistral/llama, candidates #6-11), so the 350-min hard step timeout kills the job first — opaquely, with no recorded review_status. Fix: - OPENCODE_RUN_TIMEOUT_SECONDS 5400 -> 1800 (30 min): still ample for a deep tool-using review (5x the 600s that was too short), but low enough that the full 13-candidate list is traversable inside the 350-min step even if every leading model hangs, so a responsive fallback is reached. - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS 0 -> 20400 (340 min): a clean internal deadline 10 min under the step timeout. The pool already caps each run at min(run_timeout, deadline-now) and exits with an explicit "retry deadline elapsed" reason + recorded status, so a genuine exhaustion now prints its cause instead of GitHub's opaque "timed out after 350 minutes". Also updates the strix quick-gate assertions that pin these two values. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RTAMs4bpSZS77Xe3RQjv9P --- .github/workflows/opencode-review.yml | 23 +++++++++++++++++------ scripts/ci/test_strix_quick_gate.sh | 6 +++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index b0f38796..48842d59 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2357,13 +2357,24 @@ jobs: # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. OPENCODE_MODEL_ATTEMPTS: "1" - # 90 min per model — generous for a deep tool-using review, but bounded - # so a rate-limited model yields to the next one instead of eating the - # 350-min step. (20400s = 340min gave one model the entire budget with - # no fallback; 600s was too short for a proper review.) - OPENCODE_RUN_TIMEOUT_SECONDS: "5400" + # 30 min per model — ample for a deep tool-using review (5x the 600s + # that was too short), but low enough that the 3-4 front-loaded mini + # models can each hang/rate-limit and STILL leave budget to reach a + # healthy high-quota fallback (deepseek/mistral/llama, candidates + # #6-11) inside the 350-min step. At 90 min/model the pool burned + # ~270 min on the first 3 hanging models (each killed at exit 124) + # and never reached any fallback, so every review timed out red on + # PRs of every size (a one-file docs PR failed identically). At 30 + # min/model the full 13-candidate list is traversable within budget. + OPENCODE_RUN_TIMEOUT_SECONDS: "1800" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0" + # Clean internal deadline 10 min under the 350-min step timeout. The + # pool caps each per-model run at min(run_timeout, deadline-now) and + # exits with an explicit "retry deadline elapsed" reason + recorded + # review_status when the budget is spent, instead of being hard-killed + # by GitHub's opaque "##[error] timed out after 350 minutes" with no + # recorded status. Keeps every failure's cause visible in the log. + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" OPENCODE_FIRST_ATTEMPT_AGENT: ci-review diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index f51cfff5..b00c2e67 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -522,8 +522,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout" assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review has a bounded per-model timeout before trying fallback models" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' "opencode model pool has no script-level retry budget" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode primary review has a bounded per-model timeout before trying fallback models" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400"' "opencode model pool bounds its retry budget just under the step timeout so it exits with a recorded reason instead of an opaque hard kill" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" assert_file_contains "$workflow_file" "always()" "opencode fallback chain uses always() so failed model steps cannot skip every fallback" @@ -632,7 +632,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback has a bounded model review timeout before step timeout" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode catalog fallback has a bounded model review timeout before step timeout" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review tries compact OpenAI reasoning model fallbacks early" From b79cd181eb0ebaca1d845fa7e2bceccb92276b5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:07:09 +0900 Subject: [PATCH 2/6] test: align OpenCode model pool timeout contract --- scripts/ci/test_strix_quick_gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b00c2e67..6ca4e539 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -521,7 +521,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout" - assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" + assert_file_contains "$workflow_file" 'timeout-minutes: 350' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode primary review has a bounded per-model timeout before trying fallback models" assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400"' "opencode model pool bounds its retry budget just under the step timeout so it exits with a recorded reason instead of an opaque hard kill" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" From 984fa13fb06c943876d200bd60baeaa7b336ffd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:39:11 +0900 Subject: [PATCH 3/6] chore: retrigger checks after runner config fix From d355b1a6f06069d2d813629eef51fdb95794d2e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:40:53 +0900 Subject: [PATCH 4/6] chore: retrigger checks on standard hosted runners From 2b785ee5cf3aa376f2d57533e0a5c8bcfef74918 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 14:14:45 +0900 Subject: [PATCH 5/6] chore: retrigger checks after hosted runner cleanup From acea7c9658b5194de62998100f1c362428e4da5f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 15:42:46 +0900 Subject: [PATCH 6/6] test: align opencode workflow contracts with budget fix --- tests/test_opencode_agent_contract.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index ece0a6bc..11f235c0 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -280,7 +280,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert '"## Review outcome"' in workflow assert '"## Check outcome"' not in workflow assert "publish REQUEST_CHANGES when coverage-evidence blocker states" in workflow - assert re.search(r"opencode-review-target:[\s\S]{0,240}timeout-minutes: 360", workflow) + assert re.search(r"opencode-review-target:[\s\S]{0,480}timeout-minutes: 360", workflow) assert 'timeout-minutes: 75' in workflow assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 350", workflow) assert 'APPROVAL_CHECK_WAIT_ATTEMPTS: "81"' in workflow @@ -301,9 +301,9 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'github-models/openai/gpt-5"' ) in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow assert "while :" in model_pool_runner assert "OpenCode model pool has no configured model candidates." in model_pool_runner @@ -412,7 +412,7 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): assert "steps.scheduler_app_token.outputs.token" in workflow assert "SCHEDULER_READ_TOKEN: ${{ github.token }}" in workflow assert "SCHEDULER_MUTATION_TOKEN_SOURCE" in workflow - assert 'default: "-1"' in workflow + assert 'default: "1"' in workflow assert 'review_dispatch_limit="-1"' in workflow