Skip to content

fix(ai-review): only request changes for critical findings #27

fix(ai-review): only request changes for critical findings

fix(ai-review): only request changes for critical findings #27

Workflow file for this run

name: AI Review
on:
workflow_call:
inputs:
review_instructions:
description: "Custom review focus (optional). Overrides the default instructions."
required: false
type: string
pr_number:
description: "PR number to review (passed through from caller workflow_dispatch)."
required: false
type: string
commit_sha:
description: "Commit SHA to review (passed through from caller workflow_dispatch)."
required: false
type: string
secrets:
OPENAI_API_KEY:
required: true
AI_CODE_REVIEW_APP_ID:
required: false
AI_CODE_REVIEW_PRIVATE_KEY:
required: false
pull_request:
types:
- opened
- reopened
- synchronize
- edited
- ready_for_review
push:
branches: [main]
workflow_dispatch:
inputs:
pr_number:
description: PR number to review (leave blank to review a commit)
required: false
commit_sha:
description: Commit SHA to review (leave blank when using PR number)
required: false
env:
REVIEW_INSTRUCTIONS: >-
${{ inputs.review_instructions ||
'Review for bugs, security issues, and correctness problems. Keep feedback actionable.
Skip style/formatting nitpicks. If the change looks good, say so briefly.' }}
jobs:
ai-review:
if: |
github.event_name != 'pull_request' ||
github.event.pull_request.draft == false
runs-on: ubuntu-latest
timeout-minutes: 25
# Concurrency key uses the actual subject being reviewed (PR number or commit SHA),
# not github.sha (which is the same for all dispatch runs on the same branch).
# cancel-in-progress: false means a second dispatch for the same subject queues
# rather than cancelling the running review.
concurrency:
group: ai-review-${{ github.event.pull_request.number || inputs.pr_number || inputs.commit_sha || github.sha }}
cancel-in-progress: false
permissions:
contents: write
pull-requests: read
steps:
- name: "🧭 Resolve review target"
id: resolve-target
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
run: |
set -euo pipefail
echo "workflow_ref=${{ github.ref }}" >> "$GITHUB_OUTPUT"
if [ "${{ github.event_name }}" = "pull_request" ]; then
PR_NUMBER="${{ github.event.pull_request.number }}"
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
BASE_SHA="${{ github.event.pull_request.base.sha }}"
SUBJECT_URL="https://github.com/$REPO/pull/$PR_NUMBER"
COMPARE_BASE="$BASE_SHA"
COMPARE_HEAD="$HEAD_SHA"
echo "review_mode=pr" >> "$GITHUB_OUTPUT"
echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
echo "compare_base=$BASE_SHA" >> "$GITHUB_OUTPUT"
echo "compare_head=$HEAD_SHA" >> "$GITHUB_OUTPUT"
echo "subject_url=$SUBJECT_URL" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.pr_number }}" ]; then
PR_NUMBER="${{ inputs.pr_number }}"
PR_JSON=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json state,url,headRefOid,baseRefOid)
STATE=$(echo "$PR_JSON" | jq -r '.state')
URL=$(echo "$PR_JSON" | jq -r '.url')
HEAD_SHA=$(echo "$PR_JSON" | jq -r '.headRefOid')
BASE_SHA=$(echo "$PR_JSON" | jq -r '.baseRefOid')
if [ "$STATE" = "OPEN" ]; then
SUBJECT_URL="$URL"
COMPARE_BASE="$BASE_SHA"
COMPARE_HEAD="$HEAD_SHA"
echo "review_mode=pr" >> "$GITHUB_OUTPUT"
echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
echo "compare_base=$BASE_SHA" >> "$GITHUB_OUTPUT"
echo "compare_head=$HEAD_SHA" >> "$GITHUB_OUTPUT"
echo "subject_url=$URL" >> "$GITHUB_OUTPUT"
else
echo "Open PR review objects are not supported for PR state $STATE without a live PR diff."
echo "Use commit_sha to review the merged commit directly." >&2
exit 1
fi
else
SHA="${{ github.sha }}"
[ "${{ github.event_name }}" = "workflow_dispatch" ] \
&& [ -n "${{ inputs.commit_sha }}" ] \
&& SHA="${{ inputs.commit_sha }}"
COMMIT_JSON=$(gh api "repos/$REPO/commits/$SHA" --jq '{sha: .sha, parent: (.parents[0].sha // ""), url: .html_url}')
RESOLVED_SHA=$(echo "$COMMIT_JSON" | jq -r '.sha')
PARENT_SHA=$(echo "$COMMIT_JSON" | jq -r '.parent')
URL=$(echo "$COMMIT_JSON" | jq -r '.url')
COMMIT_SHA="$RESOLVED_SHA"
SUBJECT_URL="$URL"
COMPARE_BASE="$PARENT_SHA"
if [ -z "$COMPARE_BASE" ] || [ "$COMPARE_BASE" = "null" ]; then
COMPARE_BASE="$RESOLVED_SHA^"
fi
COMPARE_HEAD="$RESOLVED_SHA"
if [ "${{ github.event_name }}" = "push" ]; then
PARENT_COUNT=$(gh api "repos/$REPO/commits/$RESOLVED_SHA" --jq '.parents | length' 2>/dev/null || echo "1")
if [ "$PARENT_COUNT" -gt 1 ]; then
echo "review_mode=skip" >> "$GITHUB_OUTPUT"
echo "skip_reason=merge commit (${PARENT_COUNT} parents)" >> "$GITHUB_OUTPUT"
echo "subject_url=$URL" >> "$GITHUB_OUTPUT"
echo "target mode=skip subject=$URL range=$COMPARE_BASE..$COMPARE_HEAD skip_reason='merge commit'"
exit 0
fi
PR_COUNT=$(gh api "repos/$REPO/commits/$RESOLVED_SHA/pulls" \
-H "Accept: application/vnd.github.groot-preview+json" \
--jq 'length' 2>/dev/null || echo "0")
if [ "$PR_COUNT" -gt 0 ]; then
echo "review_mode=skip" >> "$GITHUB_OUTPUT"
echo "skip_reason=commit already belongs to $PR_COUNT pull request(s)" >> "$GITHUB_OUTPUT"
echo "subject_url=$URL" >> "$GITHUB_OUTPUT"
echo "target mode=skip subject=$URL range=$COMPARE_BASE..$COMPARE_HEAD skip_reason='commit already belongs to $PR_COUNT pull request(s)'"
exit 0
fi
fi
echo "review_mode=commit" >> "$GITHUB_OUTPUT"
echo "commit_sha=$RESOLVED_SHA" >> "$GITHUB_OUTPUT"
echo "compare_base=$COMPARE_BASE" >> "$GITHUB_OUTPUT"
echo "compare_head=$RESOLVED_SHA" >> "$GITHUB_OUTPUT"
echo "subject_url=$URL" >> "$GITHUB_OUTPUT"
fi
REVIEW_MODE=$(awk -F= '/^review_mode=/{print $2}' "$GITHUB_OUTPUT" | tail -n1)
PR_NUMBER_OUT=$(awk -F= '/^pr_number=/{print $2}' "$GITHUB_OUTPUT" | tail -n1)
COMMIT_SHA_OUT=$(awk -F= '/^commit_sha=/{print $2}' "$GITHUB_OUTPUT" | tail -n1)
echo "target mode=$REVIEW_MODE subject=$SUBJECT_URL range=$COMPARE_BASE..$COMPARE_HEAD pr=${PR_NUMBER_OUT:-} commit=${COMMIT_SHA_OUT:-}"
- name: "📥 Checkout review target"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
uses: actions/checkout@v5.0.0
with:
fetch-depth: 0
ref: ${{ steps.resolve-target.outputs.workflow_ref }}
- name: "🧾 Prepare review context"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
env:
REVIEW_MODE: ${{ steps.resolve-target.outputs.review_mode }}
PR_NUMBER: ${{ steps.resolve-target.outputs.pr_number }}
COMMIT_SHA: ${{ steps.resolve-target.outputs.commit_sha }}
COMPARE_BASE: ${{ steps.resolve-target.outputs.compare_base }}
COMPARE_HEAD: ${{ steps.resolve-target.outputs.compare_head }}
SUBJECT_URL: ${{ steps.resolve-target.outputs.subject_url }}
run: |
set -euo pipefail
mkdir -p .github/codex/context .github/codex/output
git fetch --no-tags origin "$COMPARE_BASE" "$COMPARE_HEAD"
git diff --stat "$COMPARE_BASE" "$COMPARE_HEAD" | tee .github/codex/output/diff-stat.txt
git diff --name-only "$COMPARE_BASE" "$COMPARE_HEAD" | tee .github/codex/output/diff-files.txt
{
echo "review_mode: $REVIEW_MODE"
[ -n "$PR_NUMBER" ] && echo "pr_number: $PR_NUMBER"
[ -n "$COMMIT_SHA" ] && echo "commit_sha: $COMMIT_SHA"
echo "compare_base: $COMPARE_BASE"
echo "compare_head: $COMPARE_HEAD"
echo "subject_url: $SUBJECT_URL"
echo "instructions: ${REVIEW_INSTRUCTIONS//$'\n'/ }"
} > .github/codex/context/review-target.txt
cat .github/codex/context/review-target.txt
- name: "📄 Write Codex prompt"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
run: |
mkdir -p .github/workflows
cat > .github/workflows/ai-review.prompt.md << 'PROMPT_EOF'
You are reviewing a GitHub change set for bugs, security issues, correctness problems, and user-visible regressions.
Read `.github/codex/context/review-target.txt` first. It tells you whether the subject is a pull request or a commit and which git range to inspect.
Execution mode:
- If `review_mode` is `pr`, post a native PR review directly to GitHub.
- If `review_mode` is `commit`, do not post to GitHub; write one markdown file for the workflow to post as a consolidated commit comment.
GitHub communication:
- Authentication is pre-configured with `GH_TOKEN`.
- You may use GitHub CLI (`gh`) or direct HTTPS requests to GitHub API.
- If using direct HTTPS requests, use `Authorization: Bearer $GH_TOKEN`.
- Choose the method that is most reliable for completing the task.
Review process:
1. Inspect the git diff for the provided range.
2. Open only the files needed to validate the diff.
3. Focus on actionable issues. Skip style, formatting, and low-value nits.
4. Only report findings you can defend from the code in this repository.
5. Keep the review concise.
6. Never call this review "WIP", "draft", or "preliminary".
If `review_mode` is `pr`:
1. Before deciding the verdict, fetch the PR's existing review threads via the GitHub API and inspect each thread's `isResolved` state and comments. Treat resolved threads as previously-handled: do not re-raise the same finding even if the underlying code still looks the same to you — the author has already given a reason and the maintainer chose to close it.
2. Classify every finding by severity before picking a verdict:
- **Critical** — the diff introduces a real bug that breaks correctness, a security vulnerability, data loss, or a user-visible regression. You must be able to point to the exact failing code path and state the concrete impact. If you cannot, or you are not confident the issue is real, it is NOT critical.
- **Non-critical** — everything else: edge-case concerns, possible improvements, defensive suggestions, questions, anything you are unsure about, and anything style-adjacent.
3. Choose the GitHub review event. Default to the least-blocking event that fits:
- `event=REQUEST_CHANGES` — use ONLY when there is at least one *new* *critical* issue the author has not addressed. Never request changes for non-critical findings, uncertain findings, or suggestions, no matter how many there are.
- `event=COMMENT` — use when you have findings but none are critical. Post the findings as inline comments so the author sees them; this does not gate the merge.
- `event=APPROVE` — use when no actionable issues remain (no unresolved threads with valid concerns and no new actionable issues at HEAD). Also use APPROVE — not COMMENT — when a previous review from this app requested changes, those critical issues are now resolved, and only non-critical findings remain; APPROVE is what clears the block on a re-review. Attach any remaining non-critical findings as inline comments.
4. When you are unsure whether a finding is critical, or unsure whether it is a real issue at all, treat it as non-critical. Blocking a merge is reserved for issues you are certain are real and critical.
5. Post the PR review directly to GitHub (do not delegate posting to workflow wrapper logic). Include a summary and inline comments when appropriate. In the summary, state the severity of what you found so a reader understands why the merge is or is not blocked.
If `review_mode` is `commit`:
1. Write `.github/codex/output/review.md`.
2. Use valid, readable markdown.
3. Keep it concise and actionable.
Never use or mention "WIP".
PROMPT_EOF
# PR mode requires a GitHub App with repo access and Pull requests: Read & write.
# Create app: https://github.com/organizations/jitsucom/settings/apps/new
# Install app to org/repo: https://github.com/settings/installations
# App settings page (replace APP_SLUG): https://github.com/settings/apps/APP_SLUG
# Generate private key in app settings, then store these repo/org secrets:
# - AI_CODE_REVIEW_APP_ID (App ID from app settings)
# - AI_CODE_REVIEW_PRIVATE_KEY (contents of downloaded .pem private key)
# Example CLI (org secrets, grant access to selected repos):
# gh secret set AI_CODE_REVIEW_APP_ID --org jitsucom --repos jitsu --body "<app-id>"
# gh secret set AI_CODE_REVIEW_PRIVATE_KEY --org jitsucom --repos jitsu < app-private-key.pem
- name: "🔐 Check required secrets"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
env:
REVIEW_MODE: ${{ steps.resolve-target.outputs.review_mode }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
AI_CODE_REVIEW_APP_ID: ${{ secrets.AI_CODE_REVIEW_APP_ID }}
AI_CODE_REVIEW_PRIVATE_KEY: ${{ secrets.AI_CODE_REVIEW_PRIVATE_KEY }}
run: |
set -euo pipefail
if [ -z "${OPENAI_API_KEY:-}" ]; then
echo "OPENAI_API_KEY secret is empty or unavailable to this workflow run." >&2
exit 1
fi
if [ "$REVIEW_MODE" = "pr" ]; then
if [ -z "${AI_CODE_REVIEW_APP_ID:-}" ] || [ -z "${AI_CODE_REVIEW_PRIVATE_KEY:-}" ]; then
echo "PR mode requires AI_CODE_REVIEW_APP_ID and AI_CODE_REVIEW_PRIVATE_KEY secrets." >&2
exit 1
fi
fi
echo "Required secrets are present for mode: $REVIEW_MODE"
- name: "🔑 Generate GitHub App token for PR review"
if: ${{ steps.resolve-target.outputs.review_mode == 'pr' }}
id: app-token
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.AI_CODE_REVIEW_APP_ID }}
private-key: ${{ secrets.AI_CODE_REVIEW_PRIVATE_KEY }}
owner: ${{ github.repository_owner }}
repositories: ${{ github.event.repository.name }}
- name: "🤖 Run Codex analysis"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
id: run-codex
timeout-minutes: 12
env:
REVIEW_MODE: ${{ steps.resolve-target.outputs.review_mode }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ steps.resolve-target.outputs.pr_number }}
GH_TOKEN: ${{ steps.app-token.outputs.token }}
uses: openai/codex-action@v1
with:
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
prompt-file: .github/workflows/ai-review.prompt.md
output-file: .github/codex/output/codex-final-message.md
model: gpt-5.3-codex
effort: high
codex-home: .github/codex/home
sandbox: danger-full-access
- name: "✅ Validate review output"
if: ${{ steps.resolve-target.outputs.review_mode == 'commit' }}
id: validate-review
run: |
set -euo pipefail
test -f .github/codex/output/review.md
test -s .github/codex/output/review.md
- name: "💰 Estimate token usage and cost"
if: ${{ steps.resolve-target.outputs.review_mode != 'skip' }}
id: estimate-cost
env:
CODEX_HOME: .github/codex/home
MODEL: gpt-5.3-codex
# Standard text-token rates per 1M tokens, sourced from:
# https://developers.openai.com/api/docs/models/gpt-5.3-codex
PRICE_INPUT_PER_M: "1.75"
PRICE_CACHED_INPUT_PER_M: "0.175"
PRICE_OUTPUT_PER_M: "14.0"
run: |
set -euo pipefail
echo "model=$MODEL" >> "$GITHUB_OUTPUT"
echo "price_source=https://developers.openai.com/api/docs/models/gpt-5.3-codex" >> "$GITHUB_OUTPUT"
SESSION_FILE=$(find "$CODEX_HOME" -type f -name '*.jsonl' -print 2>/dev/null | xargs ls -t 2>/dev/null | head -n1 || true)
if [ -z "$SESSION_FILE" ]; then
echo "Codex session log (*.jsonl) not found under $CODEX_HOME. Token usage unavailable."
echo "usage_found=false" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "Inspecting Codex session log: $SESSION_FILE"
USAGE_JSON=$(jq -cn '
reduce inputs as $line (null;
if ($line.type == "event_msg" and ($line.payload.type // "") == "token_count")
then $line.payload.info.total_token_usage
else .
end
)
' "$SESSION_FILE")
if [ -z "$USAGE_JSON" ] || [ "$USAGE_JSON" = "null" ]; then
echo "No token_count event found in Codex session log. Token usage unavailable."
echo "usage_found=false" >> "$GITHUB_OUTPUT"
exit 0
fi
INPUT_TOKENS=$(echo "$USAGE_JSON" | jq -r '.input_tokens // 0')
CACHED_INPUT_TOKENS=$(echo "$USAGE_JSON" | jq -r '.cached_input_tokens // 0')
OUTPUT_TOKENS=$(echo "$USAGE_JSON" | jq -r '.output_tokens // 0')
TOTAL_TOKENS=$(echo "$USAGE_JSON" | jq -r '.total_tokens // (.input_tokens + .output_tokens)')
COST_USD=$(awk -v i="$INPUT_TOKENS" -v ci="$CACHED_INPUT_TOKENS" -v o="$OUTPUT_TOKENS" \
-v pi="$PRICE_INPUT_PER_M" -v pci="$PRICE_CACHED_INPUT_PER_M" -v po="$PRICE_OUTPUT_PER_M" \
'BEGIN {
cost = (i/1000000.0)*pi + (ci/1000000.0)*pci + (o/1000000.0)*po;
printf "%.6f", cost
}')
echo "usage_found=true" >> "$GITHUB_OUTPUT"
echo "input_tokens=$INPUT_TOKENS" >> "$GITHUB_OUTPUT"
echo "cached_input_tokens=$CACHED_INPUT_TOKENS" >> "$GITHUB_OUTPUT"
echo "output_tokens=$OUTPUT_TOKENS" >> "$GITHUB_OUTPUT"
echo "total_tokens=$TOTAL_TOKENS" >> "$GITHUB_OUTPUT"
echo "estimated_cost_usd=$COST_USD" >> "$GITHUB_OUTPUT"
echo "Token usage: input=$INPUT_TOKENS cached_input=$CACHED_INPUT_TOKENS output=$OUTPUT_TOKENS total=$TOTAL_TOKENS"
echo "Estimated run cost (USD): $COST_USD"
- name: "💬 Post commit review comments"
if: ${{ steps.resolve-target.outputs.review_mode == 'commit' }}
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
COMMIT_SHA: ${{ steps.resolve-target.outputs.commit_sha }}
run: |
set -euo pipefail
jq -n --rawfile body .github/codex/output/review.md '{body: $body}' > /tmp/commit-comment.json
gh api "repos/$REPO/commits/$COMMIT_SHA/comments" \
--method POST \
--input /tmp/commit-comment.json
- name: "📝 Write workflow summary"
if: always()
run: |
{
echo "## AI Review"
echo
echo "- Subject: ${{ steps.resolve-target.outputs.subject_url }}"
echo "- Mode: ${{ steps.resolve-target.outputs.review_mode }}"
[ -n "${{ steps.resolve-target.outputs.skip_reason }}" ] && echo "- Skip reason: ${{ steps.resolve-target.outputs.skip_reason }}"
echo "- Review output: review.md"
echo "- Model: ${{ steps.estimate-cost.outputs.model || 'n/a' }}"
if [ "${{ steps.estimate-cost.outputs.usage_found || 'false' }}" = "true" ]; then
echo "- Token usage: input=${{ steps.estimate-cost.outputs.input_tokens }}, cached_input=${{ steps.estimate-cost.outputs.cached_input_tokens }}, output=${{ steps.estimate-cost.outputs.output_tokens }}, total=${{ steps.estimate-cost.outputs.total_tokens }}"
echo "- Estimated model cost (USD): ${{ steps.estimate-cost.outputs.estimated_cost_usd }}"
else
echo "- Token usage: unavailable"
echo "- Estimated model cost (USD): unavailable"
fi
[ -n "${{ steps.estimate-cost.outputs.price_source }}" ] && echo "- Pricing source: ${{ steps.estimate-cost.outputs.price_source }}"
echo
if [ -f .github/codex/output/review.md ]; then
cat .github/codex/output/review.md
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: "📦 Upload review log"
if: ${{ always() && steps.resolve-target.outputs.review_mode != 'skip' }}
uses: actions/upload-artifact@v4
with:
name: ai-review-log-${{ github.run_id }}
path: .github/codex/output
retention-days: 30