Merge pull request winpython#2103 from stonebig/changelog-pull-request

File a cycle's changelogs as a pull request

Author: stonebig

.github/scripts/changelog_files.py

@@ -0,0 +1,100 @@
+"""File a cycle's build output into changelogs/, and write the histories.
+
+    python .github/scripts/changelog_files.py <downloaded metadata> changelogs
+
+A build leg produces four metadata files. Three of them belong in changelogs/
+and are copied there as they are: the package index
+(WinPython<flavor>-64bit-<version>.md), the lock file and the requirements.
+The fourth, hashes_<winpyver>.md, describes the binaries of one build rather
+than the release, and stays out.
+
+The _History.md companions are then written here rather than shipped from the
+build, because a history is a comparison against the *previous* release, and
+only a checkout of the repository has that to compare against. Ordering is
+`wppm.diff`'s job; this decides what to hand it.
+
+"""
+import re
+import shutil
+import sys
+from pathlib import Path
+
+# Running a script puts the script's own directory on sys.path, not the
+# checkout root, so wppm has to be found deliberately: this file is
+# .github/scripts/changelog_files.py, hence two levels up. Without it the
+# import finds whatever wppm happens to be installed, or -- as on a CI runner,
+# which installs none -- nothing at all.
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from wppm import diff  # noqa: E402  the path above has to be set first
+from wppm.diff import version  # noqa: E402  packaging, or pip's vendored copy
+
+# WinPythonslim-64bit-3.15.0.5b1.md -- the flavor may be empty, and the version
+# may carry a release level, which is why the parser decides and not the regex
+CHANGELOG = re.compile(r"^WinPython(?P<flavor>[A-Za-z0-9]*)-(?P<arch>\d+)bit-(?P<version>.+)\.md$")
+
+
+def parse_changelog_name(name: str):
+    """(flavor, architecture, version) for a package index, else None."""
+    match = CHANGELOG.match(name)
+    if not match:
+        return None
+    try:
+        version.parse(match.group("version"))
+    except version.InvalidVersion:
+        return None  # the _History companions land here, as they should
+    return match.group("flavor"), int(match.group("arch")), match.group("version")
+
+
+def files_to_file(source: Path):
+    """The build output that belongs in changelogs/.
+
+    The package index, the lock file and the requirements -- and so not
+    hashes_<winpyver>.md, which describes one build's binaries rather than the
+    release. It is excluded by being neither: it is not named for a version,
+    and it is not a pylock or a requir.
+    """
+    for path in sorted(source.iterdir()):
+        if not path.is_file():
+            continue
+        if parse_changelog_name(path.name) or path.name.startswith(("pylock.", "requir.")):
+            yield path
+
+
+def main(argv: list[str]) -> None:
+    if len(argv) != 3:
+        raise SystemExit(f"usage: {Path(argv[0]).name} <metadata dir> <changelogs dir>")
+    source, changelogs = Path(argv[1]), Path(argv[2])
+    if not source.is_dir():
+        raise SystemExit(f"no such directory: {source}")
+    if not changelogs.is_dir():
+        raise SystemExit(f"no such directory: {changelogs}")
+
+    filed = []
+    for path in files_to_file(source):
+        shutil.copyfile(path, changelogs / path.name)
+        filed.append(path.name)
+    if not filed:
+        raise SystemExit(f"{source} held no changelog, lock file or requirements")
+    for name in filed:
+        print(f"filed {name}")
+
+    # every package index has to be in place before any history is written: a
+    # history reads the index of the release it compares against, which for the
+    # second flavor of a cycle may well be the one just copied
+    histories = 0
+    for name in filed:
+        parsed = parse_changelog_name(name)
+        if not parsed:
+            continue
+        flavor, architecture, ver = parsed
+        previous = diff.find_previous_version(ver, changelogs, flavor, architecture)
+        diff.write_changelog(ver, None, changelogs, flavor, architecture)
+        histories += 1
+        against = "nothing earlier" if previous == ver else previous
+        print(f"history WinPython{flavor}-{architecture}bit-{ver} vs {against}")
+    print(f"\n{len(filed)} file(s) filed, {histories} history file(s) written")
+
+
+if __name__ == "__main__":
+    main(sys.argv)

.github/workflows/build_winpython_cycle.yml

@@ -208,3 +208,68 @@ jobs:
           name: ${{ matrix.leg.artifact_name }}
           path: publish_output
           retention-days: 66  # keeps artifact for 66 days
+
+  changelogs:
+    # A cycle's changelogs, filed and offered as one reviewable pull request.
+    #
+    # Whole-cycle builds only. A re-run of a single leg would otherwise reduce
+    # the branch to that leg's files, and it has nothing to add anyway: the
+    # same lockfile produces the same package list, so a rebuilt leg cannot
+    # change a changelog.
+    needs: [config, build-winpython]
+    if: ${{ inputs.publish && inputs.python_versionf == 'all' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        # credentials are kept here, unlike in the build legs: this job pushes
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+
+      - name: Install pinned dependencies
+        # the same hash-pinned set the test suite uses; diff.py wants packaging
+        run: python -m pip install --no-deps --require-hashes -r tests/requir.wppmtest.txt
+
+      - name: Collect the metadata every leg produced
+        uses: actions/download-artifact@v6
+        with:
+          pattern: publish_*
+          merge-multiple: true
+          path: release_metadata
+
+      - name: File the changelogs and write the histories
+        run: python .github/scripts/changelog_files.py release_metadata changelogs
+
+      - name: Open the changelog pull request
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          TAG: ${{ needs.config.outputs.release_tag }}
+          TITLE: ${{ needs.config.outputs.release_title }}
+          BASE: ${{ github.event.repository.default_branch }}
+        run: |
+          branch="changelogs/$TAG"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git switch -c "$branch"
+          git add changelogs
+          if git diff --cached --quiet; then
+            echo "changelogs/ already holds this cycle; nothing to open"
+            exit 0
+          fi
+          git commit -m "Changelogs for $TITLE"
+          # the branch is generated wholly by this job, so a re-run replaces it
+          git push --force origin "$branch"
+          if gh pr view "$branch" --json number >/dev/null 2>&1; then
+            echo "pull request for $branch is open; the push updated it"
+          else
+            gh pr create --base "$BASE" --head "$branch" \
+              --title "Changelogs for $TITLE" \
+              --body "Package indexes, lock files and requirements for every leg of \`$TAG\`, with the \`_History.md\` companions written against the previous release. Filed by the build that produced them, rather than copied by hand."
+          fi

tests/test_changelog_files.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+"""What a build's output contributes to changelogs/, and what it must not.
+
+A leg produces four metadata files. Three belong in `changelogs/`; the fourth,
+`hashes_<winpyver>.md`, describes one build's binaries rather than the release
+and has never been kept there. Getting that wrong is quiet: the wrong file is
+committed and nothing complains, so the selection is pinned here.
+
+The `_History.md` companions are written from the checkout rather than shipped
+by the build, because a history compares against the *previous* release, which
+only the repository has.
+"""
+import importlib.util
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[1]
+SCRIPT = REPO / ".github/scripts/changelog_files.py"
+
+needs_script = pytest.mark.skipif(not SCRIPT.is_file(), reason="changelog_files.py is gone")
+
+
+@pytest.fixture(scope="module")
+def changelog_files():
+    if not SCRIPT.is_file():
+        pytest.skip("changelog_files.py is gone")
+    spec = importlib.util.spec_from_file_location("changelog_files", SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# what one leg of 2026-04 b1 actually uploaded, names verbatim
+LEG_OUTPUT = [
+    "WinPythonslim-64bit-3.15.0.5b1.md",
+    "pylock.64-3_15_0_5slimb1.toml",
+    "requir.64-3_15_0_5slimb1.txt",
+    "hashes_3.15.0.5slimb1.md",
+]
+
+
+@needs_script
+class TestParseChangelogName:
+    @pytest.mark.parametrize("name,expected", [
+        ("WinPythonslim-64bit-3.15.0.5b1.md", ("slim", 64, "3.15.0.5b1")),
+        ("WinPythondotf-64bit-3.14.7.1b1.md", ("dotf", 64, "3.14.7.1b1")),
+        ("WinPythondot-64bit-3.13.15.0.md", ("dot", 64, "3.13.15.0")),
+        ("WinPython-64bit-3.9.8.0.md", ("", 64, "3.9.8.0")),  # the unflavored ones
+        ("WinPythondot-32bit-3.9.0.0b1.md", ("dot", 32, "3.9.0.0b1")),
+    ])
+    def test_reads_flavor_architecture_and_version(self, changelog_files, name, expected):
+        assert changelog_files.parse_changelog_name(name) == expected
+
+    @pytest.mark.parametrize("name", [
+        "WinPythonslim-64bit-3.15.0.5b1_History.md",  # the companion, not an index
+        "hashes_3.15.0.5slimb1.md",
+        "pylock.64-3_15_0_5slimb1.toml",
+        "README.md",
+        "WinPythonslim-64bit-.md",
+    ])
+    def test_rejects_everything_that_is_not_a_package_index(self, changelog_files, name):
+        assert changelog_files.parse_changelog_name(name) is None
+
+
+@needs_script
+class TestSelection:
+    def test_hashes_are_left_behind(self, changelog_files, tmp_path):
+        """They describe one build's binaries; changelogs/ has never held them."""
+        for name in LEG_OUTPUT:
+            (tmp_path / name).write_text("", encoding="utf-8")
+        chosen = sorted(p.name for p in changelog_files.files_to_file(tmp_path))
+        assert chosen == [
+            "WinPythonslim-64bit-3.15.0.5b1.md",
+            "pylock.64-3_15_0_5slimb1.toml",
+            "requir.64-3_15_0_5slimb1.txt",
+        ]
+
+    def test_directories_are_skipped(self, changelog_files, tmp_path):
+        (tmp_path / "WinPythonslim-64bit-3.15.0.5b1.md").write_text("", encoding="utf-8")
+        (tmp_path / "pylock.64-nested").mkdir()
+        assert [p.name for p in changelog_files.files_to_file(tmp_path)] == [
+            "WinPythonslim-64bit-3.15.0.5b1.md"
+        ]
+
+
+def run_script(*args, cwd, env=None):
+    """The script by absolute path, from an unrelated directory.
+
+    It must not need to be run from the checkout: the workflow's own step and
+    these tests both invoke it as a path, which puts .github/scripts on
+    sys.path rather than the repository root.
+    """
+    return subprocess.run(
+        [sys.executable, str(SCRIPT), *map(str, args)],
+        cwd=str(cwd), capture_output=True, text=True, env=env,
+    )
+
+
+@needs_script
+class TestEndToEnd:
+    """Run the script the way the workflow runs it."""
+
+    @pytest.fixture
+    def cycle(self, tmp_path):
+        """A metadata directory and a changelogs/ holding one earlier release."""
+        source = tmp_path / "release_metadata"
+        source.mkdir()
+        changelogs = tmp_path / "changelogs"
+        changelogs.mkdir()
+
+        # a real package index makes a real diff; reuse two the repo already has
+        previous = REPO / "changelogs" / "WinPythonslim-64bit-3.15.0.4.md"
+        current = REPO / "changelogs" / "WinPythonslim-64bit-3.14.7.0.md"
+        if not (previous.is_file() and current.is_file()):
+            pytest.skip("the changelogs this test reads from are gone")
+        shutil.copyfile(previous, changelogs / previous.name)
+        shutil.copyfile(current, source / "WinPythonslim-64bit-3.15.0.5b1.md")
+        (source / "pylock.64-3_15_0_5slimb1.toml").write_text("x", encoding="utf-8")
+        (source / "requir.64-3_15_0_5slimb1.txt").write_text("x", encoding="utf-8")
+        (source / "hashes_3.15.0.5slimb1.md").write_text("x", encoding="utf-8")
+        return source, changelogs
+
+    def test_wppm_comes_from_the_checkout(self, cycle, tmp_path):
+        """A decoy wppm on PYTHONPATH must lose to the one being released.
+
+        Running a script puts the script's directory on sys.path, not the
+        checkout root, so without a deliberate insert the import falls through
+        to whatever else is reachable. On a machine with wppm installed that
+        looks fine -- which is how it once slipped past a green local run --
+        and on a CI runner, which installs none, it is ModuleNotFoundError.
+        The decoy makes the difference visible either way.
+        """
+        source, changelogs = cycle
+        decoy = tmp_path / "decoy"
+        (decoy / "wppm").mkdir(parents=True)
+        (decoy / "wppm" / "__init__.py").write_text(
+            "raise RuntimeError('decoy wppm imported')", encoding="utf-8"
+        )
+        env = {**os.environ, "PYTHONPATH": str(decoy)}
+        proc = run_script(source, changelogs, cwd=tmp_path, env=env)
+        assert proc.returncode == 0, proc.stderr
+        assert "decoy" not in proc.stderr
+
+    def test_files_the_three_and_writes_the_history(self, cycle, tmp_path):
+        source, changelogs = cycle
+        proc = run_script(source, changelogs, cwd=tmp_path)
+        assert proc.returncode == 0, proc.stderr
+        landed = sorted(p.name for p in changelogs.iterdir())
+        assert landed == [
+            "WinPythonslim-64bit-3.15.0.4.md",           # was already there
+            "WinPythonslim-64bit-3.15.0.5b1.md",         # filed
+            "WinPythonslim-64bit-3.15.0.5b1_History.md",  # written
+            "pylock.64-3_15_0_5slimb1.toml",
+            "requir.64-3_15_0_5slimb1.txt",
+        ]
+
+    def test_the_history_names_the_release_it_compares_against(self, cycle, tmp_path):
+        source, changelogs = cycle
+        proc = run_script(source, changelogs, cwd=tmp_path)
+        assert proc.returncode == 0, proc.stderr
+        history = (changelogs / "WinPythonslim-64bit-3.15.0.5b1_History.md").read_text(
+            encoding="utf-8"
+        )
+        assert "since version 3.15.0.4slim" in history
+        assert "3.15.0.5b1slim" in history
+
+    def test_an_empty_metadata_directory_is_an_error(self, tmp_path):
+        """Silence here would commit nothing and call it a success."""
+        source, changelogs = tmp_path / "src", tmp_path / "changelogs"
+        source.mkdir()
+        changelogs.mkdir()
+        proc = run_script(source, changelogs, cwd=tmp_path)
+        assert proc.returncode != 0
+        assert "held no changelog" in proc.stdout + proc.stderr
+
+    def test_a_missing_directory_is_an_error(self, tmp_path):
+        proc = run_script(tmp_path / "nope", tmp_path, cwd=tmp_path)
+        assert proc.returncode != 0
+        assert "no such directory" in proc.stdout + proc.stderr

tests/test_cycle_config.py

@@ -308,6 +308,16 @@ def test_publishing_is_gated_both_ways(self, workflow_text):
         assert "if: ${{ inputs.publish }}" in workflow_text
         assert "if: ${{ !inputs.publish }}" in workflow_text
 
+    def test_the_changelog_pr_is_whole_cycle_only(self, workflow_text):
+        """A single-leg re-run would reduce the branch to that leg's files.
+
+        It has nothing to add either way: the same lockfile builds the same
+        package list, so a rebuilt leg cannot change a changelog.
+        """
+        assert "if: ${{ inputs.publish && inputs.python_versionf == 'all' }}" in workflow_text
+        assert ".github/scripts/changelog_files.py release_metadata changelogs" in workflow_text
+        assert (REPO / ".github/scripts/changelog_files.py").is_file()
+
     def test_the_release_title_is_built_by_the_script(self, workflow_text):
         """Ordinal dates are miserable in shell, and untestable there."""
         assert 'TITLE: ${{ needs.config.outputs.release_title }}' in workflow_text