Sync typeshed #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync typeshed | |
| # How this works: | |
| # | |
| # 1. A Linux worker: | |
| # a. Checks out Ruff and typeshed | |
| # b. Deletes the vendored typeshed stdlib stubs from Ruff | |
| # c. Copies the latest versions of the stubs from typeshed | |
| # d. Uses docstring-adder to sync all docstrings available on Linux | |
| # e. Creates a new branch on the upstream astral-sh/ruff repository | |
| # f. Commits the changes it's made and pushes them to the new upstream branch | |
| # 2. Once the Linux worker is done, a Windows worker: | |
| # a. Checks out the branch created by the Linux worker | |
| # b. Syncs all docstrings available on Windows that are not available on Linux | |
| # c. Commits the changes and pushes them to the same upstream branch | |
| # 3. Once the Windows worker is done, a MacOS worker: | |
| # a. Checks out the branch created by the Linux worker | |
| # b. Syncs all docstrings available on MacOS that are not available on Linux or Windows | |
| # c. Attempts to update any snapshots that might have changed | |
| # (this sub-step is allowed to fail) | |
| # d. Commits the changes and pushes them to the same upstream branch | |
| # e. Creates a PR against the `main` branch using the branch all three workers have pushed to | |
| # 4. If any of steps 1-3 failed, an issue is created in the `astral-sh/ruff` repository | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run on the 1st and the 15th of every month: | |
| - cron: "0 0 1,15 * *" | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| # Don't set this flag globally for the workflow: it does strange things | |
| # to the snapshots in the `cargo insta test --accept` step in the MacOS job. | |
| # | |
| # FORCE_COLOR: 1 | |
| CARGO_TERM_COLOR: always | |
| NEXTEST_PROFILE: "ci" | |
| GH_TOKEN: ${{ github.token }} | |
| # The name of the upstream branch that the first worker creates, | |
| # and which all three workers push to. | |
| UPSTREAM_BRANCH: typeshedbot/sync-typeshed | |
| # The path to the directory that contains the vendored typeshed stubs, | |
| # relative to the root of the Ruff repository. | |
| VENDORED_TYPESHED: crates/ty_vendored/vendor/typeshed | |
| jobs: | |
| # Sync typeshed stubs, and sync all docstrings available on Linux. | |
| # Push the changes to a new branch on the upstream repository. | |
| sync: | |
| name: Sync typeshed | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # Don't run the cron job on forks: | |
| if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| name: Checkout Ruff | |
| with: | |
| path: ruff | |
| persist-credentials: true | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| name: Checkout typeshed | |
| with: | |
| repository: python/typeshed | |
| path: typeshed | |
| persist-credentials: false | |
| - name: Setup git | |
| run: | | |
| git config --global user.name typeshedbot | |
| git config --global user.email '<>' | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Sync typeshed stubs | |
| run: | | |
| rm -rf "ruff/${VENDORED_TYPESHED}" | |
| mkdir "ruff/${VENDORED_TYPESHED}" | |
| cp typeshed/README.md "ruff/${VENDORED_TYPESHED}" | |
| cp typeshed/LICENSE "ruff/${VENDORED_TYPESHED}" | |
| # The pyproject.toml file is needed by a later job for the black configuration. | |
| # It's deleted before creating the PR. | |
| cp typeshed/pyproject.toml "ruff/${VENDORED_TYPESHED}" | |
| cp -r typeshed/stdlib "ruff/${VENDORED_TYPESHED}/stdlib" | |
| rm -rf "ruff/${VENDORED_TYPESHED}/stdlib/@tests" | |
| git -C typeshed rev-parse HEAD > "ruff/${VENDORED_TYPESHED}/source_commit.txt" | |
| cd ruff | |
| git checkout -b "${UPSTREAM_BRANCH}" | |
| git add . | |
| git commit -m "Sync typeshed. Source commit: https://github.com/python/typeshed/commit/$(git -C ../typeshed rev-parse HEAD)" --allow-empty | |
| - name: Sync Linux docstrings | |
| if: ${{ success() }} | |
| env: | |
| FORCE_COLOR: 1 | |
| run: | | |
| cd ruff | |
| ./scripts/codemod_docstrings.sh | |
| git commit -am "Sync Linux docstrings" --allow-empty | |
| - name: Push the changes | |
| id: commit | |
| if: ${{ success() }} | |
| run: git -C ruff push --force --set-upstream origin "${UPSTREAM_BRANCH}" | |
| # Checkout the branch created by the sync job, | |
| # and sync all docstrings available on Windows that are not available on Linux. | |
| # Commit the changes and push them to the same branch. | |
| docstrings-windows: | |
| runs-on: windows-latest | |
| timeout-minutes: 20 | |
| needs: [sync] | |
| # Don't run the cron job on forks. | |
| # The job will also be skipped if the sync job failed, because it's specified in `needs` above, | |
| # and we haven't used `always()` in the `if` condition here | |
| # (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs) | |
| if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| name: Checkout Ruff | |
| with: | |
| persist-credentials: true | |
| ref: ${{ env.UPSTREAM_BRANCH}} | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Setup git | |
| run: | | |
| git config --global user.name typeshedbot | |
| git config --global user.email '<>' | |
| - name: Sync Windows docstrings | |
| id: docstrings | |
| env: | |
| FORCE_COLOR: 1 | |
| run: ./scripts/codemod_docstrings.sh | |
| - name: Commit the changes | |
| if: ${{ steps.docstrings.outcome == 'success' }} | |
| run: | | |
| git commit -am "Sync Windows docstrings" --allow-empty | |
| git push | |
| # Checkout the branch created by the sync job, | |
| # and sync all docstrings available on macOS that are not available on Linux or Windows. | |
| # Push the changes to the same branch and create a PR against the `main` branch using that branch. | |
| docstrings-macos-and-pr: | |
| runs-on: macos-latest | |
| timeout-minutes: 20 | |
| needs: [sync, docstrings-windows] | |
| # Don't run the cron job on forks. | |
| # The job will also be skipped if the sync or docstrings-windows jobs failed, | |
| # because they're specified in `needs` above and we haven't used an `always()` condition in the `if` here | |
| # (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs) | |
| if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| name: Checkout Ruff | |
| with: | |
| persist-credentials: true | |
| ref: ${{ env.UPSTREAM_BRANCH}} | |
| - uses: astral-sh/setup-uv@5a7eac68fb9809dea845d802897dc5c723910fa3 # v7.1.3 | |
| - name: Setup git | |
| run: | | |
| git config --global user.name typeshedbot | |
| git config --global user.email '<>' | |
| - name: Sync macOS docstrings | |
| if: ${{ success() }} | |
| env: | |
| FORCE_COLOR: 1 | |
| run: | | |
| ./scripts/codemod_docstrings.sh | |
| git commit -am "Sync macOS docstrings" --allow-empty | |
| - name: Format the changes | |
| if: ${{ success() }} | |
| env: | |
| FORCE_COLOR: 1 | |
| run: | | |
| # Here we just reformat the codemodded stubs so that they are | |
| # consistent with the other typeshed stubs around them. | |
| # Typeshed formats code using black in their CI, so we just invoke | |
| # black on the stubs the same way that typeshed does. | |
| uvx black "${VENDORED_TYPESHED}/stdlib" --config "${VENDORED_TYPESHED}/pyproject.toml" || true | |
| git commit -am "Format codemodded docstrings" --allow-empty | |
| - name: Remove typeshed pyproject.toml file | |
| if: ${{ success() }} | |
| run: | | |
| rm "${VENDORED_TYPESHED}/pyproject.toml" | |
| git commit -am "Remove pyproject.toml file" | |
| - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 | |
| - name: "Install Rust toolchain" | |
| if: ${{ success() }} | |
| run: rustup show | |
| - name: "Install mold" | |
| if: ${{ success() }} | |
| uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 | |
| - name: "Install cargo nextest" | |
| if: ${{ success() }} | |
| uses: taiki-e/install-action@537c30d2b45cc3aa3fb35e2bbcfb61ef93fd6f02 # v2.62.52 | |
| with: | |
| tool: cargo-nextest | |
| - name: "Install cargo insta" | |
| if: ${{ success() }} | |
| uses: taiki-e/install-action@537c30d2b45cc3aa3fb35e2bbcfb61ef93fd6f02 # v2.62.52 | |
| with: | |
| tool: cargo-insta | |
| - name: Update snapshots | |
| if: ${{ success() }} | |
| run: | | |
| cargo r \ | |
| --profile=profiling \ | |
| -p ty_completion_eval \ | |
| -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv | |
| # The `cargo insta` docs indicate that `--unreferenced=delete` might be a good option, | |
| # but from local testing it appears to just revert all changes made by `cargo insta test --accept`. | |
| # | |
| # If there were only snapshot-related failures, `cargo insta test --accept` will have exit code 0, | |
| # but if there were also other mdtest failures (for example), it will return a nonzero exit code. | |
| # We don't care about other tests failing here, we just want snapshots updated where possible, | |
| # so we use `|| true` here to ignore the exit code. | |
| cargo insta test --accept --color=always --all-features --test-runner=nextest || true | |
| - name: Commit snapshot changes | |
| if: ${{ success() }} | |
| run: git commit -am "Update snapshots" || echo "No snapshot changes to commit" | |
| - name: Push changes upstream and create a PR | |
| if: ${{ success() }} | |
| run: | | |
| git push | |
| gh pr list --repo "${GITHUB_REPOSITORY}" --head "${UPSTREAM_BRANCH}" --json id --jq length | grep 1 && exit 0 # exit if there is existing pr | |
| gh pr create --title "[ty] Sync vendored typeshed stubs" --body "Close and reopen this PR to trigger CI" --label "ty" | |
| create-issue-on-failure: | |
| name: Create an issue if the typeshed sync failed | |
| runs-on: ubuntu-latest | |
| needs: [sync, docstrings-windows, docstrings-macos-and-pr] | |
| if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && (needs.sync.result == 'failure' || needs.docstrings-windows.result == 'failure' || needs.docstrings-macos-and-pr.result == 'failure') }} | |
| permissions: | |
| issues: write | |
| steps: | |
| - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| await github.rest.issues.create({ | |
| owner: "astral-sh", | |
| repo: "ruff", | |
| title: `Automated typeshed sync failed on ${new Date().toDateString()}`, | |
| body: "Run listed here: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| labels: ["bug", "ty"], | |
| }) |