update-docs.yml

name: Update Claude Code Documentation

on:
  schedule:
    # Every 3 hours (00:00, 03:00, ... UTC)
    - cron: '0 */3 * * *'
  workflow_dispatch:
    inputs:
      confirm_removals:
        description: >-
          One-shot override: allow a removal of more than 10% of previously-live
          pages for THIS run only (confirmed upstream removal/rename). Logs every
          removed URL. The stale-share ceiling and fetched-OK floor still apply.
        type: boolean
        default: false

permissions:
  contents: write

concurrency:
  group: update-docs
  cancel-in-progress: false

jobs:
  update-docs:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
        ref: main

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install -r scripts/requirements.txt

    # Fetch discovers (llms.txt ∪ sitemap) and writes the v2 manifest at the repo
    # root; pages are fetched into the gitignored .doc_fetch/ scratch (never committed).
    # The fetcher's own safeguards (discovery >=200, <=10% removal of previously-
    # live pages, >=250 ok floor, <=25% stale/failed share) abort the run before
    # writing on a bad transition.
    - name: Fetch latest documentation (v2 manifest)
      id: fetch-docs
      env:
        GITHUB_REPOSITORY: ${{ github.repository }}
        GITHUB_REF_NAME: ${{ github.ref_name }}
        # '1' only on a manual dispatch that ticked the box; '0' on schedule.
        # Keep `inputs.` (a real boolean): `github.event.inputs.*` is a STRING,
        # so `== true` would always be false and the override could never fire.
        DOCS_CONFIRM_REMOVALS: ${{ inputs.confirm_removals == true && '1' || '0' }}
      run: python3 scripts/fetch_claude_docs.py

    # Build the prose-free search index from the scratch dir (floor guard refuses an
    # empty/tiny scratch, so a partial fetch can't produce a content-less index).
    - name: Build search index
      run: python3 scripts/build_search_index.py

    # Belt-and-suspenders shell safeguards, mirroring all three transition rules in
    # scripts/fetcher/safeguards.py so the run fails closed even if the Python guard
    # is bypassed: at least 250 documentation pages (changelog excluded) must have
    # fetched OK this run; no more than 25% of documentation pages may be stale/failed
    # carry-forwards;
    # and no more than 10% of the previously-live pages (HEAD manifest entries not
    # already stale/failed) may have been removed. tests/integration pins each
    # constant to scripts/fetcher/config.py.
    - name: Safeguard — page count, stale share, live removals
      env:
        DOCS_CONFIRM_REMOVALS: ${{ inputs.confirm_removals == true && '1' || '0' }}
      run: |
        # Scheduled runs must always show 0 here; only a ticked manual dispatch yields 1.
        echo "confirm_removals override: DOCS_CONFIRM_REMOVALS=${DOCS_CONFIRM_REMOVALS:-unset}"
        DOCS=$(jq '[.pages[] | select(.id != "changelog")] | length' paths_manifest.json)
        OK_DOCS=$(jq '[.pages[] | select(.id != "changelog" and .fetch_status == "ok")] | length' paths_manifest.json)
        NOT_OK=$((DOCS - OK_DOCS))
        echo "Manifest pages: $(jq '.pages | length' paths_manifest.json) (documentation pages fetched ok: $OK_DOCS of $DOCS)"
        if [ "$OK_DOCS" -lt 250 ]; then
          echo "::error::🚨 SAFEGUARD: only $OK_DOCS documentation pages fetched ok (<250). Refusing to commit."
          exit 1
        fi
        echo "Stale/failed documentation pages: $NOT_OK of $DOCS"
        if [ "$DOCS" -gt 0 ] && [ $((NOT_OK * 100)) -gt $((DOCS * 25)) ]; then
          echo "::error::🚨 SAFEGUARD: $NOT_OK of $DOCS pages are stale/failed (>25%). Refusing to commit."
          exit 1
        fi
        # Live-removal share: diff the committed (HEAD) manifest against the new one.
        # Mirrors load_manifest() + the Python guard: no HEAD file, or a JSON object
        # that is not a v2 manifest, skips the check (clean start); anything that is
        # not exactly one JSON object (unparsable, empty, array, scalar, concatenated
        # docs), or a v2 manifest with a non-object page entry, fails closed. A URL is
        # live if ANY of its rows is not stale/failed and dead only when every row is,
        # exactly like the Python set logic (conservative under duplicate rows).
        if git show HEAD:paths_manifest.json > /tmp/old_manifest.json 2>/dev/null; then
          OLD_KIND=$(jq -rs 'if length != 1 or (.[0] | type) != "object" then error("not a single JSON object")
                             elif .[0].schema_version == 2 and (.[0].pages | type) == "array" then
                               (if (.[0].pages | any(type != "object")) then error("page entry is not an object") else "v2" end)
                             else "other" end' /tmp/old_manifest.json 2>/tmp/old_manifest.err) || OLD_KIND=invalid
          [ -n "$OLD_KIND" ] || OLD_KIND=invalid
        else
          OLD_KIND=missing
        fi
        if [ "$OLD_KIND" = "invalid" ]; then
          echo "::error::🚨 SAFEGUARD: committed paths_manifest.json at HEAD is not a valid JSON object. Refusing to commit. jq: $(tr '\n' ' ' < /tmp/old_manifest.err 2>/dev/null)"
          exit 1
        elif [ "$OLD_KIND" = "v2" ]; then
          # A url counts only when it is a non-empty string (same rule as the Python guard).
          OLD_LIVE=$(jq '[.pages[] | select((.url | type) == "string" and .url != "" and .fetch_status != "stale" and .fetch_status != "failed") | .url]
            | unique | length' /tmp/old_manifest.json)
          REMOVED_LIVE=$(jq -n --slurpfile old /tmp/old_manifest.json --slurpfile new paths_manifest.json '
            ([$new[0].pages[] | .url | select(type == "string" and . != "")] | map({(.): true}) | add // {}) as $keep
            | [$old[0].pages[] | select((.url | type) == "string" and .url != "" and .fetch_status != "stale" and .fetch_status != "failed") | .url]
            | unique | map(select($keep[.] | not)) | length')
          echo "Previously-live pages removed: $REMOVED_LIVE of $OLD_LIVE"
          if [ "$OLD_LIVE" -gt 0 ] && [ $((REMOVED_LIVE * 100)) -gt $((OLD_LIVE * 10)) ]; then
            if [ "${DOCS_CONFIRM_REMOVALS:-0}" = "1" ]; then
              echo "::warning::$REMOVED_LIVE of $OLD_LIVE previously-live pages removed (>10%) — allowed by the confirm_removals override for this run."
              # Same live-URL set as the count above, so the listing equals REMOVED_LIVE exactly.
              jq -r --slurpfile new paths_manifest.json '
                ([$new[0].pages[] | .url | select(type == "string" and . != "")] | map({(.): true}) | add // {}) as $keep
                | [.pages[] | select((.url | type) == "string" and .url != "" and .fetch_status != "stale" and .fetch_status != "failed") | .url]
                | unique | map(select($keep[.] | not)) | sort[] | "  removed: \(.)"' /tmp/old_manifest.json
            else
              echo "::error::🚨 SAFEGUARD: $REMOVED_LIVE of $OLD_LIVE previously-live pages removed (>10%). Refusing to commit. Confirmed upstream removal? Re-run manually with confirm_removals=true."
              exit 1
            fi
          fi
        else
          echo "No committed v2 manifest at HEAD ($OLD_KIND) — skipping live-removal check (clean start)."
        fi

    # The fetcher rewrites generated_at on every run, so a raw `git diff` is never
    # clean. `git diff -I` ignores only the generated_at line (both files are
    # pretty-printed, one key per line), so a commit only lands when actual content
    # (pages, index entries) changed. Unlike a process-substitution diff, a failure
    # here (bad object, unreadable file) propagates as a non-zero exit instead of
    # silently reading as "no change". First run (file absent in HEAD) shows as an
    # addition and correctly reports changed=true.
    - name: Check for changes
      id: verify-changed-files
      run: |
        set -euo pipefail
        git add paths_manifest.json search_index.json
        if git diff --cached --quiet -I '^  "generated_at":' -- paths_manifest.json search_index.json; then
          CHANGED=false
          echo "no content changes (generated_at-only churn ignored)"
        else
          CHANGED=true
          git diff --cached --stat -- paths_manifest.json search_index.json
        fi
        echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"

    - name: Generate commit message from manifest diff
      if: steps.verify-changed-files.outputs.changed == 'true'
      id: commit-msg
      run: |
        git show HEAD:paths_manifest.json > /tmp/old_manifest.json 2>/dev/null || echo '{"pages":[]}' > /tmp/old_manifest.json
        SUMMARY=$(jq -rn --slurpfile old /tmp/old_manifest.json --slurpfile new paths_manifest.json '
          ($old[0].pages // [] | map({(.id): .}) | add // {}) as $o
          | ($new[0].pages // [] | map({(.id): .}) | add // {}) as $n
          | ([$n | to_entries[] | select($o[.key] == null)] | length) as $added
          | ([$n | to_entries[] | select(($o[.key] != null) and ($o[.key].sha256 != .value.sha256))] | length) as $changed
          | ([$o | to_entries[] | select($n[.key] == null)] | length) as $removed
          | "\($added) added, \($changed) changed, \($removed) removed"')
        MSG="Update Claude docs manifest - $(date -u +'%Y-%m-%d') | ${SUMMARY}"
        {
          echo "message<<EOF"
          echo "${MSG}"
          echo "EOF"
        } >> "$GITHUB_OUTPUT"

    - name: Commit and push if changed
      if: steps.verify-changed-files.outputs.changed == 'true'
      env:
        COMMIT_MESSAGE: ${{ steps.commit-msg.outputs.message }}
      run: |
        git config --local user.email "github-actions[bot]@users.noreply.github.com"
        git config --local user.name "github-actions[bot]"
        # Only paths_manifest.json + search_index.json are staged (no prose committed).
        git commit -m "${COMMIT_MESSAGE}"
        git pull --rebase origin main
        git push