ci: Add Codex review gate #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Codex review gate | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request_target: | |
| types: [opened, synchronize, reopened, ready_for_review, edited] | |
| pull_request_review: | |
| types: [submitted] | |
| issue_comment: | |
| types: [created] | |
| schedule: | |
| - cron: '17 * * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: codex-review-gate-${{ github.repository }} | |
| cancel-in-progress: false | |
| permissions: | |
| checks: write | |
| contents: read | |
| issues: write | |
| pull-requests: read | |
| jobs: | |
| review: | |
| name: Codex review coordinator | |
| if: >- | |
| github.event_name != 'issue_comment' || | |
| (github.event.issue.pull_request && | |
| (github.event.comment.user.login == 'chatgpt-codex-connector[bot]' || | |
| github.event.comment.body == '@codex review')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Reconcile Codex reviews | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 | |
| with: | |
| script: | | |
| const gateName = 'codex-review-gate'; | |
| const stateName = 'codex-review-quota-state'; | |
| const codexLogins = new Set([ | |
| 'chatgpt-codex-connector', | |
| 'chatgpt-codex-connector[bot]', | |
| ]); | |
| const quotaText = 'reached your Codex usage limits for code reviews'; | |
| const fiveHours = 5 * 60 * 60 * 1000; | |
| const retryDelay = 5.25 * 60 * 60 * 1000; | |
| const automaticReviewGrace = 60 * 60 * 1000; | |
| const probeResponseGrace = 2 * 60 * 60 * 1000; | |
| const { owner, repo } = context.repo; | |
| const now = new Date(); | |
| const repository = await github.rest.repos.get({ owner, repo }); | |
| const defaultBranch = repository.data.default_branch; | |
| const defaultRef = await github.rest.repos.getBranch({ | |
| owner, | |
| repo, | |
| branch: defaultBranch, | |
| }); | |
| const defaultSha = defaultRef.data.commit.sha; | |
| const emptyState = { | |
| consecutive_failures: 0, | |
| last_failure_at: null, | |
| mode: 'normal', | |
| next_probe_at: null, | |
| available_since: null, | |
| probe: null, | |
| }; | |
| const asDate = (value) => { | |
| if (!value) return null; | |
| const parsed = new Date(value); | |
| return Number.isNaN(parsed.valueOf()) ? null : parsed; | |
| }; | |
| const isCodex = (login) => codexLogins.has(login || ''); | |
| const eventPullNumber = Number( | |
| context.payload.pull_request?.number || context.payload.issue?.number || 0, | |
| ); | |
| const triggeringReviewId = | |
| context.eventName === 'pull_request_review' && | |
| isCodex(context.payload.review?.user?.login) | |
| ? Number(context.payload.review.id) | |
| : null; | |
| const triggeringIssueCommentId = | |
| context.eventName === 'issue_comment' && | |
| isCodex(context.payload.comment?.user?.login) | |
| ? Number(context.payload.comment.id) | |
| : null; | |
| function normalizeState(value) { | |
| if (!value || typeof value !== 'object') return { ...emptyState }; | |
| const failures = Number.isInteger(value.consecutive_failures) | |
| ? Math.max(0, value.consecutive_failures) | |
| : 0; | |
| const probe = value.probe && typeof value.probe === 'object' | |
| ? { | |
| pr: Number(value.probe.pr) || null, | |
| head_sha: | |
| typeof value.probe.head_sha === 'string' | |
| ? value.probe.head_sha | |
| : null, | |
| base_sha: | |
| typeof value.probe.base_sha === 'string' | |
| ? value.probe.base_sha | |
| : null, | |
| requested_at: | |
| typeof value.probe.requested_at === 'string' | |
| ? value.probe.requested_at | |
| : null, | |
| comment_id: Number(value.probe.comment_id) || null, | |
| } | |
| : null; | |
| return { | |
| consecutive_failures: failures, | |
| last_failure_at: | |
| typeof value.last_failure_at === 'string' | |
| ? value.last_failure_at | |
| : null, | |
| mode: value.mode === 'daily' ? 'daily' : 'normal', | |
| next_probe_at: | |
| typeof value.next_probe_at === 'string' | |
| ? value.next_probe_at | |
| : null, | |
| available_since: | |
| typeof value.available_since === 'string' | |
| ? value.available_since | |
| : null, | |
| probe, | |
| }; | |
| } | |
| async function readState() { | |
| let sha = defaultSha; | |
| for (let depth = 0; depth < 5; depth += 1) { | |
| const response = await github.rest.checks.listForRef({ | |
| owner, | |
| repo, | |
| ref: sha, | |
| check_name: stateName, | |
| filter: 'all', | |
| per_page: 100, | |
| }); | |
| const runs = response.data.check_runs | |
| .filter((run) => run.app?.slug === 'github-actions') | |
| .sort((a, b) => b.id - a.id); | |
| for (const run of runs) { | |
| if (!run.output?.summary) continue; | |
| try { | |
| return normalizeState(JSON.parse(run.output.summary)); | |
| } catch (error) { | |
| core.warning(`Ignoring malformed Codex retry state: ${error}`); | |
| } | |
| } | |
| const commit = await github.rest.repos.getCommit({ | |
| owner, | |
| repo, | |
| ref: sha, | |
| }); | |
| if (!commit.data.parents?.length) break; | |
| sha = commit.data.parents[0].sha; | |
| } | |
| return { ...emptyState }; | |
| } | |
| async function writeState(state) { | |
| const branch = await github.rest.repos.getBranch({ | |
| owner, | |
| repo, | |
| branch: defaultBranch, | |
| }); | |
| await github.rest.checks.create({ | |
| owner, | |
| repo, | |
| name: stateName, | |
| head_sha: branch.data.commit.sha, | |
| status: 'completed', | |
| conclusion: 'neutral', | |
| completed_at: new Date().toISOString(), | |
| output: { | |
| title: 'Codex retry circuit state', | |
| summary: JSON.stringify(state), | |
| }, | |
| }); | |
| } | |
| function nextDailyProbe(after) { | |
| const earliest = new Date(after.getTime() + 12 * 60 * 60 * 1000); | |
| const formatter = new Intl.DateTimeFormat('en-US', { | |
| timeZone: 'America/Chicago', | |
| hour: '2-digit', | |
| minute: '2-digit', | |
| hourCycle: 'h23', | |
| }); | |
| for (let minute = 0; minute < 48 * 60; minute += 1) { | |
| const candidate = new Date(earliest.getTime() + minute * 60 * 1000); | |
| const parts = Object.fromEntries( | |
| formatter | |
| .formatToParts(candidate) | |
| .map((part) => [part.type, part.value]), | |
| ); | |
| if (parts.hour === '03' && parts.minute === '15') { | |
| return candidate; | |
| } | |
| } | |
| return new Date(after.getTime() + 24 * 60 * 60 * 1000); | |
| } | |
| async function latestGateRun(headSha) { | |
| const response = await github.rest.checks.listForRef({ | |
| owner, | |
| repo, | |
| ref: headSha, | |
| check_name: gateName, | |
| filter: 'all', | |
| per_page: 100, | |
| }); | |
| return response.data.check_runs | |
| .filter((run) => run.app?.slug === 'github-actions') | |
| .sort((a, b) => b.id - a.id)[0]; | |
| } | |
| async function createPendingGate(pull, requireFreshRequest = false) { | |
| const created = await github.rest.checks.create({ | |
| owner, | |
| repo, | |
| name: gateName, | |
| head_sha: pull.head.sha, | |
| external_id: pull.base.sha, | |
| status: 'in_progress', | |
| started_at: now.toISOString(), | |
| details_url: `${context.serverUrl}/${owner}/${repo}/pull/${pull.number}`, | |
| output: { | |
| title: requireFreshRequest | |
| ? 'Codex review required after base update' | |
| : 'Codex review pending', | |
| summary: requireFreshRequest | |
| ? 'The pull request base changed. A fresh managed Codex review must be requested before this gate can complete.' | |
| : 'Waiting for managed Codex to review this pull request commit.', | |
| }, | |
| }); | |
| return created.data; | |
| } | |
| function requestedAtForRun(run) { | |
| const summary = String(run.output?.summary || ''); | |
| const match = summary.match(/^Requested at: (.+)$/m); | |
| return match ? asDate(match[1]) : null; | |
| } | |
| async function inspectCodexOutcome( | |
| pull, | |
| run, | |
| quotaFloor, | |
| allowTerminal, | |
| terminalFloor, | |
| ) { | |
| const startedAt = asDate(run.started_at || run.created_at) || now; | |
| const resultFloor = | |
| terminalFloor && terminalFloor > startedAt | |
| ? terminalFloor | |
| : startedAt; | |
| const floor = | |
| quotaFloor && quotaFloor > resultFloor | |
| ? quotaFloor | |
| : resultFloor; | |
| const [reviews, reviewComments, issueComments] = | |
| await Promise.all([ | |
| github.paginate(github.rest.pulls.listReviews, { | |
| owner, | |
| repo, | |
| pull_number: pull.number, | |
| per_page: 100, | |
| }), | |
| github.paginate(github.rest.pulls.listReviewComments, { | |
| owner, | |
| repo, | |
| pull_number: pull.number, | |
| per_page: 100, | |
| }), | |
| github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number: pull.number, | |
| per_page: 100, | |
| }), | |
| ]); | |
| const commentsByReview = new Map(); | |
| for (const comment of reviewComments) { | |
| if (!isCodex(comment.user?.login)) continue; | |
| const reviewId = Number(comment.pull_request_review_id); | |
| commentsByReview.set( | |
| reviewId, | |
| (commentsByReview.get(reviewId) || 0) + 1, | |
| ); | |
| } | |
| const terminal = []; | |
| for (const review of reviews) { | |
| if (!allowTerminal) continue; | |
| if (!isCodex(review.user?.login)) continue; | |
| if (review.commit_id !== pull.head.sha) continue; | |
| const submittedAt = asDate(review.submitted_at); | |
| const isTriggeringReview = | |
| eventPullNumber === pull.number && | |
| triggeringReviewId === Number(review.id); | |
| const triggerMayPrecedeFloor = | |
| isTriggeringReview && !terminalFloor; | |
| if ( | |
| !submittedAt || | |
| (submittedAt < resultFloor && !triggerMayPrecedeFloor) | |
| ) { | |
| continue; | |
| } | |
| const findingCount = commentsByReview.get(Number(review.id)) || 0; | |
| if (findingCount > 0) { | |
| terminal.push({ | |
| at: submittedAt, | |
| kind: 'findings', | |
| findingCount, | |
| }); | |
| } | |
| } | |
| for (const comment of issueComments) { | |
| if (!allowTerminal) continue; | |
| if (!isCodex(comment.user?.login)) continue; | |
| const body = String(comment.body || ''); | |
| if (!body.startsWith("Codex Review: Didn't find any major issues.")) { | |
| continue; | |
| } | |
| const reviewed = body.match( | |
| /\*\*Reviewed commit:\*\* `([0-9a-f]{7,40})`/i, | |
| ); | |
| if (!reviewed) { | |
| continue; | |
| } | |
| if (!pull.head.repo?.full_name) { | |
| core.warning( | |
| `Cannot resolve Codex reviewed commit for PR #${pull.number}: head repository is unavailable.`, | |
| ); | |
| continue; | |
| } | |
| let reviewedSha; | |
| try { | |
| const [headOwner, headRepo] = pull.head.repo.full_name.split('/'); | |
| const resolved = await github.rest.repos.getCommit({ | |
| owner: headOwner, | |
| repo: headRepo, | |
| ref: reviewed[1], | |
| }); | |
| reviewedSha = resolved.data.sha; | |
| } catch (error) { | |
| core.warning( | |
| `Could not resolve Codex reviewed commit ${reviewed[1]}: ${error}`, | |
| ); | |
| continue; | |
| } | |
| if (reviewedSha !== pull.head.sha) continue; | |
| const createdAt = asDate(comment.created_at); | |
| const isTriggeringComment = | |
| eventPullNumber === pull.number && | |
| triggeringIssueCommentId === Number(comment.id); | |
| const triggerMayPrecedeFloor = | |
| isTriggeringComment && !terminalFloor; | |
| if ( | |
| !createdAt || | |
| (createdAt < resultFloor && !triggerMayPrecedeFloor) | |
| ) { | |
| continue; | |
| } | |
| terminal.push({ at: createdAt, kind: 'approve', findingCount: 0 }); | |
| } | |
| terminal.sort((a, b) => { | |
| if (a.kind !== b.kind) return a.kind === 'findings' ? -1 : 1; | |
| return b.at - a.at; | |
| }); | |
| const quotas = issueComments | |
| .filter((comment) => { | |
| if (!isCodex(comment.user?.login)) return false; | |
| if (!String(comment.body || '').includes(quotaText)) return false; | |
| const createdAt = asDate(comment.created_at); | |
| const isTriggeringComment = | |
| eventPullNumber === pull.number && | |
| triggeringIssueCommentId === Number(comment.id); | |
| return createdAt && (isTriggeringComment || createdAt >= floor); | |
| }) | |
| .map((comment) => ({ | |
| at: asDate(comment.created_at), | |
| id: Number(comment.id), | |
| pr: pull.number, | |
| })) | |
| .sort((a, b) => b.at - a.at); | |
| return { | |
| terminal: terminal[0] || null, | |
| quota: quotas[0] || null, | |
| startedAt, | |
| }; | |
| } | |
| const originalState = await readState(); | |
| let state = normalizeState(originalState); | |
| const availableSince = asDate(state.available_since); | |
| const pullRequests = await github.paginate(github.rest.pulls.list, { | |
| owner, | |
| repo, | |
| state: 'open', | |
| per_page: 100, | |
| }); | |
| const pending = []; | |
| const terminalEvents = []; | |
| const quotaEvents = []; | |
| for (const pull of pullRequests) { | |
| if (pull.draft) continue; | |
| let run = await latestGateRun(pull.head.sha); | |
| const baseChanged = Boolean( | |
| run && run.external_id !== pull.base.sha, | |
| ); | |
| if (baseChanged) run = undefined; | |
| if (!run) run = await createPendingGate(pull, baseChanged); | |
| if (run.status === 'completed') continue; | |
| const requestFloor = requestedAtForRun(run); | |
| const requiresFreshRequest = | |
| run.output?.title === 'Codex review required after base update'; | |
| const outcome = await inspectCodexOutcome( | |
| pull, | |
| run, | |
| availableSince, | |
| !requiresFreshRequest || Boolean(requestFloor), | |
| requestFloor, | |
| ); | |
| if (outcome.terminal) { | |
| terminalEvents.push({ | |
| ...outcome.terminal, | |
| base_sha: pull.base.sha, | |
| check_run_id: run.id, | |
| head_sha: pull.head.sha, | |
| pr: pull.number, | |
| }); | |
| continue; | |
| } | |
| if (outcome.quota) quotaEvents.push(outcome.quota); | |
| pending.push({ | |
| base_sha: pull.base.sha, | |
| check_run_id: run.id, | |
| head_sha: pull.head.sha, | |
| needs_manual: | |
| baseChanged || | |
| requiresFreshRequest || | |
| context.eventName === 'pull_request_target' && | |
| context.payload.pull_request?.number === pull.number && | |
| (context.payload.action === 'synchronize' || | |
| (context.payload.action === 'edited' && | |
| Boolean(context.payload.changes?.base))), | |
| pr: pull.number, | |
| started_at: outcome.startedAt, | |
| }); | |
| } | |
| terminalEvents.sort((a, b) => a.at - b.at); | |
| for (const event of terminalEvents) { | |
| const current = await github.rest.pulls.get({ | |
| owner, | |
| repo, | |
| pull_number: event.pr, | |
| }); | |
| if (current.data.head.sha !== event.head_sha) { | |
| await github.rest.checks.update({ | |
| owner, | |
| repo, | |
| check_run_id: event.check_run_id, | |
| status: 'completed', | |
| conclusion: 'neutral', | |
| completed_at: now.toISOString(), | |
| output: { | |
| title: 'Codex review superseded', | |
| summary: 'The pull request changed while its Codex result was being reconciled.', | |
| }, | |
| }); | |
| continue; | |
| } | |
| const approved = event.kind === 'approve'; | |
| await github.rest.checks.update({ | |
| owner, | |
| repo, | |
| check_run_id: event.check_run_id, | |
| status: 'completed', | |
| conclusion: approved ? 'success' : 'failure', | |
| completed_at: now.toISOString(), | |
| output: { | |
| title: approved | |
| ? 'Codex review passed' | |
| : 'Codex review found blocking issues', | |
| summary: approved | |
| ? 'Managed Codex completed review of this commit without review findings.' | |
| : `Managed Codex posted ${event.findingCount} review finding(s) for this commit.`, | |
| }, | |
| }); | |
| } | |
| const latestTerminal = terminalEvents | |
| .slice() | |
| .sort((a, b) => b.at - a.at)[0]; | |
| const latestQuota = quotaEvents | |
| .slice() | |
| .sort((a, b) => b.at - a.at)[0]; | |
| if (latestTerminal && (!latestQuota || latestTerminal.at >= latestQuota.at)) { | |
| const terminalMatchesProbe = | |
| state.probe && | |
| latestTerminal.pr === state.probe.pr && | |
| latestTerminal.head_sha === state.probe.head_sha && | |
| latestTerminal.base_sha === state.probe.base_sha; | |
| state = { | |
| ...emptyState, | |
| available_since: latestTerminal.at.toISOString(), | |
| probe: terminalMatchesProbe ? null : state.probe, | |
| }; | |
| } else if (latestQuota) { | |
| const probeRequestedAt = asDate(state.probe?.requested_at); | |
| const quotaAnswersProbe = | |
| !probeRequestedAt || latestQuota.at >= probeRequestedAt; | |
| if (quotaAnswersProbe) { | |
| const lastFailure = asDate(state.last_failure_at); | |
| const separated = | |
| !lastFailure || latestQuota.at - lastFailure >= fiveHours; | |
| if (separated) { | |
| const failures = state.consecutive_failures + 1; | |
| const nextProbe = | |
| failures >= 3 | |
| ? nextDailyProbe(latestQuota.at) | |
| : new Date(latestQuota.at.getTime() + retryDelay); | |
| state = { | |
| ...state, | |
| consecutive_failures: failures, | |
| last_failure_at: latestQuota.at.toISOString(), | |
| mode: failures >= 3 ? 'daily' : 'normal', | |
| next_probe_at: nextProbe.toISOString(), | |
| probe: null, | |
| }; | |
| } else { | |
| state.probe = null; | |
| if (!asDate(state.next_probe_at) && lastFailure) { | |
| const nextProbe = | |
| state.mode === 'daily' | |
| ? nextDailyProbe(lastFailure) | |
| : new Date(lastFailure.getTime() + retryDelay); | |
| state.next_probe_at = nextProbe.toISOString(); | |
| } | |
| } | |
| } | |
| } | |
| const effectiveAvailableSince = asDate(state.available_since); | |
| const pendingByKey = new Map( | |
| pending.map((item) => [ | |
| `${item.pr}:${item.head_sha}:${item.base_sha}`, | |
| item, | |
| ]), | |
| ); | |
| if (state.probe) { | |
| const key = `${state.probe.pr}:${state.probe.head_sha}:${state.probe.base_sha}`; | |
| if (!pendingByKey.has(key)) { | |
| const relatedPending = pending.find( | |
| (item) => | |
| item.pr === state.probe.pr && | |
| item.head_sha === state.probe.head_sha, | |
| ); | |
| const requestedAt = asDate(state.probe.requested_at); | |
| if ( | |
| relatedPending && | |
| relatedPending.base_sha !== state.probe.base_sha && | |
| requestedAt | |
| ) { | |
| const drainUntil = new Date( | |
| requestedAt.getTime() + probeResponseGrace, | |
| ); | |
| const nextProbe = asDate(state.next_probe_at); | |
| if ( | |
| drainUntil > now && | |
| (!nextProbe || drainUntil > nextProbe) | |
| ) { | |
| state.next_probe_at = drainUntil.toISOString(); | |
| } | |
| } | |
| state.probe = null; | |
| } else { | |
| const requestedAt = asDate(state.probe.requested_at); | |
| if (requestedAt && now - requestedAt >= probeResponseGrace) { | |
| state.probe = null; | |
| const delayed = new Date(requestedAt.getTime() + retryDelay); | |
| if (!asDate(state.next_probe_at) || delayed > asDate(state.next_probe_at)) { | |
| state.next_probe_at = delayed.toISOString(); | |
| } | |
| } | |
| } | |
| } | |
| pending.sort((a, b) => a.started_at - b.started_at || a.pr - b.pr); | |
| const candidate = | |
| pending.find((item) => item.needs_manual) || pending[0]; | |
| if (!candidate) { | |
| state.probe = null; | |
| } else if (!state.probe) { | |
| const nextProbe = asDate(state.next_probe_at); | |
| const quotaDue = nextProbe ? now >= nextProbe : false; | |
| const recoveredQueue = | |
| effectiveAvailableSince && | |
| candidate.started_at < effectiveAvailableSince; | |
| const automaticReviewExpired = | |
| now - candidate.started_at >= automaticReviewGrace; | |
| const shouldTrigger = | |
| quotaDue || | |
| (!nextProbe && | |
| (candidate.needs_manual || recoveredQueue || automaticReviewExpired)); | |
| if (shouldTrigger) { | |
| const comment = await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: candidate.pr, | |
| body: '@codex review', | |
| }); | |
| state.probe = { | |
| pr: candidate.pr, | |
| head_sha: candidate.head_sha, | |
| base_sha: candidate.base_sha, | |
| requested_at: now.toISOString(), | |
| comment_id: Number(comment.data.id), | |
| }; | |
| state.next_probe_at = null; | |
| await github.rest.checks.update({ | |
| owner, | |
| repo, | |
| check_run_id: candidate.check_run_id, | |
| status: 'in_progress', | |
| output: { | |
| title: 'Codex review requested', | |
| summary: [ | |
| `Requested at: ${now.toISOString()}`, | |
| `Base commit: ${candidate.base_sha}`, | |
| 'Requested managed Codex review and waiting for its result.', | |
| ].join('\n'), | |
| }, | |
| }); | |
| } | |
| } | |
| const shouldCheckpointDefaultPush = | |
| context.eventName === 'push' && | |
| context.ref === `refs/heads/${defaultBranch}`; | |
| if ( | |
| shouldCheckpointDefaultPush || | |
| JSON.stringify(state) !== JSON.stringify(originalState) | |
| ) { | |
| await writeState(state); | |
| } | |
| const waitUntil = state.probe?.requested_at | |
| ? `Codex probe in flight since ${state.probe.requested_at}` | |
| : state.next_probe_at | |
| ? `Next Codex probe: ${state.next_probe_at}` | |
| : 'No Codex retry is currently scheduled.'; | |
| core.notice(waitUntil); |