diff --git a/.github/scripts/release_rust/README.md b/.github/scripts/release_rust/README.md index 4a751382..897d398f 100644 --- a/.github/scripts/release_rust/README.md +++ b/.github/scripts/release_rust/README.md @@ -5,7 +5,8 @@ These scripts define and publish the crates.io package set for Apache Reqsign. ## Publish plan `plan.py` reads `cargo metadata --no-deps`, selects workspace packages that can -publish to crates.io, and orders them by non-dev local dependencies. +publish to crates.io, and orders them by non-dev local dependencies using +Python's `TopologicalSorter`. ```bash python3 .github/scripts/release_rust/plan.py @@ -43,9 +44,8 @@ Trusted Publishing cannot create the first version of a crate. ownership or the exact Trusted Publisher because those APIs require authentication. -The protected `rust-bootstrap` workflow always runs the authenticated audit, -including when discovery finds no missing names. It never changes an -established crate. Existing crates must be migrated independently before the +The protected `rust-bootstrap` job audits the complete plan before any write, +including when no names are missing. It never changes an established crate. Existing crates must be migrated independently before the workflow can succeed. Published crate metadata may still reference the former repository URL or the @@ -56,6 +56,15 @@ New placeholders and releases use the current project name and repository URL. Version `0.0.0` is an irreversible namespace reservation. It is not an ASF software release and contains no implementation. +The local dispatch helper uses GitHub REST API version `2026-03-10` to obtain +the exact workflow run ID, then checks its commit and waits for completion. + +These are repository-specific scripts: run them with normal Python (without +`-O` or `PYTHONOPTIMIZE`, which disable assertions). Cargo and crates.io response +fields are used directly; unexpected package or publisher state fails an +assertion. Failed API requests stop the run; rerun after resolving the failure. +Only Cargo publish rate limits and public metadata propagation are retried. + ## Tests ```bash diff --git a/.github/scripts/release_rust/bootstrap.py b/.github/scripts/release_rust/bootstrap.py index 06e775c4..95bfb7d5 100644 --- a/.github/scripts/release_rust/bootstrap.py +++ b/.github/scripts/release_rust/bootstrap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -24,26 +23,18 @@ import tempfile import time import urllib.error -import urllib.parse import urllib.request -from dataclasses import dataclass from pathlib import Path -from plan import PROJECT_DIR -from plan import plan -from publish import parse_retry_after -from publish import should_retry +from plan import PROJECT_DIR, plan +from publish import parse_retry_after, should_retry +from trusted_publishing import REGISTRY_URL, USER_AGENT - -REGISTRY_URL = "https://crates.io" REPOSITORY = "https://github.com/apache/reqsign" -# Published versions retain their original manifest metadata after a repo rename. +# Published versions keep their original metadata after a repository rename. LEGACY_REPOSITORY = "https://github.com/apache/opendal-reqsign" PLACEHOLDER_VERSION = "0.0.0" -PLACEHOLDER_DESCRIPTION = ( - "Namespace reservation for a crate planned by Apache Reqsign." -) -# Published placeholders retain their original description after a project rename. +PLACEHOLDER_DESCRIPTION = "Namespace reservation for a crate planned by Apache Reqsign." LEGACY_PLACEHOLDER_DESCRIPTION = ( "Namespace reservation for a crate planned by Apache OpenDAL reqsign." ) @@ -53,339 +44,100 @@ "workflow_filename": "release.yml", "environment": "release", } -USER_AGENT = ( - "apache-reqsign-release-bootstrap/1.0 " - "(https://github.com/apache/reqsign)" -) - - -class ApiError(RuntimeError): - def __init__(self, operation: str, status: int, detail: str): - super().__init__(f"{operation} failed with HTTP {status}: {detail}") - self.status = status - - -@dataclass(frozen=True) -class PlannedCrate: - name: str - path: str - - -@dataclass(frozen=True) -class ReconcileResult: - name: str - actions: tuple[str, ...] - - -def planned_crates(project_dir: Path = PROJECT_DIR) -> list[PlannedCrate]: - packages = [ - PlannedCrate(name=package.name, path=package.path) - for package in plan(project_dir.resolve()) - ] - names = [package.name for package in packages] - if len(names) != len(set(names)): - raise RuntimeError("duplicate crates.io package name in publish plan") - return packages - - -def _api_error_detail(error: urllib.error.HTTPError) -> str: - try: - response = error.read().decode("utf-8", errors="replace") - finally: - error.close() - try: - body = json.loads(response) - errors = body.get("errors", []) - return "; ".join(item["detail"] for item in errors) - except (json.JSONDecodeError, AttributeError, KeyError, TypeError): - return response class CratesIoClient: - def __init__(self, registry_url: str = REGISTRY_URL, token: str | None = None): - self.registry_url = registry_url.rstrip("/") + def __init__(self, token: str | None = None): self.token = token - def _request( - self, - method: str, - path: str, - operation: str, - body: dict[str, object] | None = None, - authenticated: bool = False, - ) -> dict[str, object]: - headers = { - "Accept": "application/json", - "User-Agent": USER_AGENT, - } + def request(self, method, path, body=None, *, authenticated=False): + headers = {"User-Agent": USER_AGENT, "Content-Type": "application/json"} if authenticated: - if not self.token: - raise RuntimeError(f"{operation} requires a crates.io API token") + assert self.token, "a crates.io bootstrap token is required" headers["Authorization"] = self.token - - data = None - if body is not None: - data = json.dumps(body).encode() - headers["Content-Type"] = "application/json" - request = urllib.request.Request( - f"{self.registry_url}{path}", - data=data, + f"{REGISTRY_URL}/api/v1/{path}", + data=json.dumps(body).encode() if body is not None else None, headers=headers, method=method, ) - retryable = method in {"GET", "PATCH"} - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=30) as response: - result = json.load(response) - if not isinstance(result, dict): - raise RuntimeError(f"{operation} returned an invalid response") - return result - except urllib.error.HTTPError as error: - retry = retryable and error.code in {429, 502, 503, 504} - if retry and attempt < 4: - retry_after = ( - error.headers.get("Retry-After") if error.headers else None - ) - delay = ( - int(retry_after) - if retry_after and retry_after.isdigit() - else 2**attempt - ) - error.read() - error.close() - print( - f"{operation} returned HTTP {error.code}; retrying in {delay}s", - flush=True, - ) - time.sleep(delay) - continue - raise ApiError( - operation, error.code, _api_error_detail(error) - ) from None - except urllib.error.URLError as error: - if retryable and attempt < 4: - delay = 2**attempt - print( - f"{operation} failed: {error.reason}; retrying in {delay}s", - flush=True, - ) - time.sleep(delay) - continue - raise RuntimeError(f"{operation} failed: {error.reason}") from None - - raise AssertionError("unreachable") - - def get_crate(self, name: str) -> dict[str, object] | None: - encoded_name = urllib.parse.quote(name, safe="") - try: - response = self._request( - "GET", - f"/api/v1/crates/{encoded_name}", - f"reading crate {name}", - ) - except ApiError as error: - if error.status == 404: - return None - raise - krate = response.get("crate") - if not isinstance(krate, dict): - raise RuntimeError(f"crates.io returned invalid metadata for {name}") - return krate - - def get_version(self, name: str, version: str) -> dict[str, object]: - encoded_name = urllib.parse.quote(name, safe="") - encoded_version = urllib.parse.quote(version, safe="") - response = self._request( - "GET", - f"/api/v1/crates/{encoded_name}/{encoded_version}", - f"reading {name} {version}", - ) - version_data = response.get("version") - if not isinstance(version_data, dict): - raise RuntimeError( - f"crates.io returned invalid version metadata for {name} {version}" - ) - return version_data + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) - def list_github_configs(self, name: str) -> list[dict[str, object]]: - query = urllib.parse.urlencode({"crate": name, "per_page": 100}) - response = self._request( + def get_crate(self, name): + try: + return self.request("GET", f"crates/{name}")["crate"] + except urllib.error.HTTPError as error: + if error.code != 404: + raise + error.close() + return None + + def list_github_configs(self, name): + return self.request( "GET", - f"/api/v1/trusted_publishing/github_configs?{query}", - f"listing Trusted Publishers for {name}", + f"trusted_publishing/github_configs?crate={name}&per_page=100", authenticated=True, - ) - configs = response.get("github_configs") - if not isinstance(configs, list) or not all( - isinstance(config, dict) for config in configs - ): - raise RuntimeError( - f"crates.io returned invalid Trusted Publisher metadata for {name}" - ) - return configs + )["github_configs"] - def create_github_config(self, name: str) -> dict[str, object]: - config = {"crate": name, **PUBLISHER} - response = self._request( + def create_github_config(self, name): + return self.request( "POST", - "/api/v1/trusted_publishing/github_configs", - f"creating the Trusted Publisher for {name}", - body={"github_config": config}, + "trusted_publishing/github_configs", + {"github_config": {"crate": name, **PUBLISHER}}, authenticated=True, - ) - created = response.get("github_config") - if not isinstance(created, dict): - raise RuntimeError( - f"crates.io returned invalid Trusted Publisher metadata for {name}" - ) - return created + )["github_config"] - def set_trustpub_only(self, name: str) -> dict[str, object]: - encoded_name = urllib.parse.quote(name, safe="") - response = self._request( + def set_trustpub_only(self, name): + return self.request( "PATCH", - f"/api/v1/crates/{encoded_name}", - f"enabling Trusted Publishing only for {name}", - body={"crate": {"trustpub_only": True}}, + f"crates/{name}", + {"crate": {"trustpub_only": True}}, authenticated=True, - ) - krate = response.get("crate") - if not isinstance(krate, dict): - raise RuntimeError(f"crates.io returned invalid metadata for {name}") - return krate - - -def _normalized_repository(value: object) -> str: - if not isinstance(value, str): - return "" - normalized = value.rstrip("/") - if normalized.endswith(".git"): - normalized = normalized[:-4] - return normalized.lower() - - -def validate_crate_metadata( - planned: PlannedCrate, metadata: dict[str, object], client: CratesIoClient -) -> None: - if metadata.get("id") != planned.name: - raise RuntimeError( - f"crate name mismatch for {planned.name}: got {metadata.get('id')!r}" - ) - if _normalized_repository(metadata.get("repository")) not in { - _normalized_repository(REPOSITORY), - _normalized_repository(LEGACY_REPOSITORY), - }: - raise RuntimeError( - f"{planned.name} already exists with an unexpected repository: " - f"{metadata.get('repository')!r}" - ) - - if metadata.get("max_version") == PLACEHOLDER_VERSION: - if metadata.get("description") not in ( - PLACEHOLDER_DESCRIPTION, - LEGACY_PLACEHOLDER_DESCRIPTION, - ): - raise RuntimeError( - f"{planned.name} has an unexpected {PLACEHOLDER_VERSION} placeholder" - ) - version = client.get_version(planned.name, PLACEHOLDER_VERSION) - if version.get("num") != PLACEHOLDER_VERSION: - raise RuntimeError( - f"{planned.name} placeholder version could not be verified" - ) + )["crate"] -def _is_expected_config(name: str, config: dict[str, object]) -> bool: - repository_owner = config.get("repository_owner") - repository_name = config.get("repository_name") - return all( - ( - config.get("crate") == name, - isinstance(repository_owner, str) - and repository_owner.lower() == PUBLISHER["repository_owner"], - isinstance(repository_name, str) - and repository_name.lower() == PUBLISHER["repository_name"], - config.get("workflow_filename") == PUBLISHER["workflow_filename"], - config.get("environment") == PUBLISHER["environment"], - ) +def validate_crate(name, metadata): + assert metadata["id"] == name, f"crate name mismatch for {name}" + repository = (metadata["repository"] or "").rstrip("/").removesuffix(".git").lower() + assert repository in {REPOSITORY, LEGACY_REPOSITORY}, ( + f"{name} has an unexpected repository: {repository}" ) + if metadata["max_version"] == PLACEHOLDER_VERSION: + assert metadata["description"] in { + PLACEHOLDER_DESCRIPTION, + LEGACY_PLACEHOLDER_DESCRIPTION, + }, f"{name} has an unexpected placeholder" -def validate_github_configs(name: str, configs: list[dict[str, object]]) -> None: - if len(configs) != 1 or not _is_expected_config(name, configs[0]): - compact = [ - { - key: config.get(key) - for key in ( - "repository_owner", - "repository_name", - "workflow_filename", - "environment", - ) - } - for config in configs - ] - raise RuntimeError( - f"{name} has unexpected Trusted Publisher configurations: " - f"{json.dumps(compact, sort_keys=True)}" - ) +def validate_publisher(name, configs): + expected = {"crate": name, **PUBLISHER} + assert len(configs) == 1 and all( + configs[0][key] == value for key, value in expected.items() + ), f"{name} has unexpected Trusted Publishers: {configs}" -def preflight_authenticated( - packages: list[PlannedCrate], - candidate_names: set[str], - client: CratesIoClient, -) -> list[str]: - verified: list[str] = [] - for planned in packages: - metadata = client.get_crate(planned.name) - is_candidate = planned.name in candidate_names +def audit(names, client, *, ready=False): + """Check every crate before any write; return names eligible for bootstrap.""" + candidates = [] + for name in names: + metadata = client.get_crate(name) if metadata is None: - if not is_candidate: - raise RuntimeError( - f"{planned.name} is missing but was not selected for bootstrap" - ) - verified.append(planned.name) + assert not ready, f"{name} does not exist on crates.io" + candidates.append(name) continue - - validate_crate_metadata(planned, metadata, client) - is_placeholder = metadata.get("max_version") == PLACEHOLDER_VERSION - if is_placeholder != is_candidate: - state = "a placeholder" if is_placeholder else "an established crate" - raise RuntimeError( - f"{planned.name} is now {state}, which does not match discovery" - ) - - configs = client.list_github_configs(planned.name) - if is_placeholder: + validate_crate(name, metadata) + configs = client.list_github_configs(name) + if not ready and metadata["max_version"] == PLACEHOLDER_VERSION: if configs: - validate_github_configs(planned.name, configs) + validate_publisher(name, configs) + candidates.append(name) else: - validate_github_configs(planned.name, configs) - if metadata.get("trustpub_only") is not True: - raise RuntimeError( - f"{planned.name} does not require Trusted Publishing" - ) - verified.append(planned.name) - return verified - - -def verify_authenticated( - packages: list[PlannedCrate], client: CratesIoClient -) -> list[str]: - verified: list[str] = [] - for planned in packages: - metadata = client.get_crate(planned.name) - if metadata is None: - raise RuntimeError(f"{planned.name} does not exist on crates.io") - validate_crate_metadata(planned, metadata, client) - validate_github_configs(planned.name, client.list_github_configs(planned.name)) - if metadata.get("trustpub_only") is not True: - raise RuntimeError(f"{planned.name} does not require Trusted Publishing") - verified.append(planned.name) - return verified + validate_publisher(name, configs) + assert metadata["trustpub_only"] is True, ( + f"{name} does not require Trusted Publishing" + ) + return candidates def _placeholder_manifest(name: str) -> str: @@ -452,313 +204,104 @@ def _placeholder_readme(name: str) -> str: """ -def write_placeholder_package( - project_dir: Path, planned: PlannedCrate, package_dir: Path -) -> None: - source_dir = package_dir / "src" - source_dir.mkdir() +def write_placeholder_package(name: str, package_dir: Path) -> None: + (package_dir / "src").mkdir() (package_dir / "Cargo.toml").write_text( - _placeholder_manifest(planned.name), encoding="utf-8" - ) - (package_dir / "README.md").write_text( - _placeholder_readme(planned.name), encoding="utf-8" + _placeholder_manifest(name), encoding="utf-8" ) - (source_dir / "lib.rs").write_text(PLACEHOLDER_LIB, encoding="utf-8") - shutil.copyfile(project_dir / "LICENSE", package_dir / "LICENSE") - shutil.copyfile(project_dir / "NOTICE", package_dir / "NOTICE") + (package_dir / "README.md").write_text(_placeholder_readme(name), encoding="utf-8") + (package_dir / "src/lib.rs").write_text(PLACEHOLDER_LIB, encoding="utf-8") + for filename in ("LICENSE", "NOTICE"): + shutil.copyfile(PROJECT_DIR / filename, package_dir / filename) -def publish_placeholder(project_dir: Path, planned: PlannedCrate, token: str) -> None: - with tempfile.TemporaryDirectory(prefix=f"{planned.name}-bootstrap-") as tmpdir: +def publish_placeholder(name, token): + with tempfile.TemporaryDirectory(prefix=f"{name}-bootstrap-") as tmpdir: package_dir = Path(tmpdir) - write_placeholder_package(project_dir, planned, package_dir) - - env = os.environ.copy() - env["CARGO_REGISTRY_TOKEN"] = token + write_placeholder_package(name, package_dir) command = ["cargo", "publish", "--manifest-path", "Cargo.toml"] while True: process = subprocess.run( command, cwd=package_dir, - env=env, check=False, + env={**os.environ, "CARGO_REGISTRY_TOKEN": token}, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - output = process.stdout or "" - print(output, end="", flush=True) + print(process.stdout, end="", flush=True) if process.returncode == 0: return - if should_retry(output): - delay = parse_retry_after(output) - print( - f"crates.io rate limited {planned.name}; sleeping {delay}s", - flush=True, - ) - time.sleep(delay) - continue - raise subprocess.CalledProcessError( - process.returncode, command, output=output - ) - - -def wait_for_crate( - client: CratesIoClient, name: str, timeout: int = 120 -) -> dict[str, object]: - deadline = time.monotonic() + timeout - while True: - metadata = client.get_crate(name) - if metadata is not None: - return metadata - if time.monotonic() >= deadline: - raise RuntimeError( - f"timed out waiting for {name} to become visible on crates.io" - ) - time.sleep(2) + if not should_retry(process.stdout): + process.check_returncode() + delay = parse_retry_after(process.stdout) + print(f"crates.io rate limited {name}; sleeping {delay}s", flush=True) + time.sleep(delay) -def wait_for_trustpub_only( - client: CratesIoClient, name: str, timeout: int = 120 -) -> dict[str, object]: - deadline = time.monotonic() + timeout +def wait_for_crate(client, name, *, ready=False): + # Public metadata may lag behind a successful publish or settings update. + deadline = time.monotonic() + 120 while True: metadata = client.get_crate(name) - if metadata is not None and metadata.get("trustpub_only") is True: + if metadata is not None and (not ready or metadata["trustpub_only"] is True): + validate_crate(name, metadata) return metadata - if time.monotonic() >= deadline: - raise RuntimeError( - f"timed out waiting for {name} to require Trusted Publishing" - ) + assert time.monotonic() < deadline, f"timed out waiting for {name} on crates.io" time.sleep(2) -def wait_for_expected_config( - client: CratesIoClient, name: str, timeout: int = 120 -) -> None: - deadline = time.monotonic() + timeout - while True: - configs = client.list_github_configs(name) - if configs: - validate_github_configs(name, configs) - return - if time.monotonic() >= deadline: - raise RuntimeError( - f"timed out waiting for the Trusted Publisher for {name}" - ) - time.sleep(2) - - -def reconcile_crate( - project_dir: Path, - planned: PlannedCrate, - client: CratesIoClient, - token: str, -) -> ReconcileResult: - actions: list[str] = [] - metadata = client.get_crate(planned.name) +def reconcile(name, client): + metadata = client.get_crate(name) if metadata is None: - publish_placeholder(project_dir, planned, token) - metadata = wait_for_crate(client, planned.name) - actions.append("created placeholder") - elif metadata.get("max_version") != PLACEHOLDER_VERSION: - raise RuntimeError( - f"{planned.name} became an established crate after discovery; " - "refusing to modify it in the bootstrap workflow" - ) - - validate_crate_metadata(planned, metadata, client) - - configs = client.list_github_configs(planned.name) + publish_placeholder(name, client.token) + metadata = wait_for_crate(client, name) + validate_crate(name, metadata) + assert metadata["max_version"] == PLACEHOLDER_VERSION, ( + f"{name} is an established crate; bootstrap must not modify it" + ) + configs = client.list_github_configs(name) if not configs: - created = client.create_github_config(planned.name) - if not _is_expected_config(planned.name, created): - raise RuntimeError( - f"crates.io created an unexpected Trusted Publisher for {planned.name}" - ) - actions.append("configured Trusted Publishing") - else: - validate_github_configs(planned.name, configs) - - if metadata.get("trustpub_only") is not True: - updated = client.set_trustpub_only(planned.name) - if updated.get("trustpub_only") is not True: - raise RuntimeError( - f"crates.io did not enable Trusted Publishing only for {planned.name}" - ) - actions.append("enabled Trusted Publishing only") - - verified_metadata = wait_for_trustpub_only(client, planned.name) - validate_crate_metadata(planned, verified_metadata, client) - wait_for_expected_config(client, planned.name) - - if not actions: - actions.append("verified") - return ReconcileResult(planned.name, tuple(actions)) - - -def discover( - project_dir: Path, client: CratesIoClient -) -> tuple[list[PlannedCrate], list[str], list[str]]: - packages = planned_crates(project_dir) - missing: list[str] = [] - placeholders: list[str] = [] - for planned in packages: - metadata = client.get_crate(planned.name) - if metadata is None: - missing.append(planned.name) - continue - validate_crate_metadata(planned, metadata, client) - if metadata.get("max_version") == PLACEHOLDER_VERSION: - placeholders.append(planned.name) - return packages, missing, placeholders - - -def verify_public(project_dir: Path, client: CratesIoClient) -> list[str]: - verified: list[str] = [] - for planned in planned_crates(project_dir): - metadata = client.get_crate(planned.name) - if metadata is None: - raise RuntimeError(f"{planned.name} does not exist on crates.io") - validate_crate_metadata(planned, metadata, client) - if metadata.get("trustpub_only") is not True: - raise RuntimeError(f"{planned.name} does not require Trusted Publishing") - verified.append(planned.name) - return verified - - -def _write_summary(title: str, lines: list[str]) -> None: - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if not summary_path: + configs = [client.create_github_config(name)] + validate_publisher(name, configs) + if metadata["trustpub_only"] is not True: + updated = client.set_trustpub_only(name) + assert updated["trustpub_only"] is True, f"failed to restrict {name}" + wait_for_crate(client, name, ready=True) + print(f"{name}: bootstrapped", flush=True) + + +def apply(names, client): + candidates = audit(names, client) + print(f"Authenticated audit passed; bootstrap candidates: {candidates}", flush=True) + for name in candidates: + reconcile(name, client) + audit(names, client, ready=True) + print(f"Verified all {len(names)} crates", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description="Bootstrap reqsign crates.io names.") + parser.add_argument("command", choices=("discover", "apply", "verify")) + command = parser.parse_args().command + names = [package.name for package in plan()] + if command == "apply": + apply(names, CratesIoClient(os.environ["CARGO_REGISTRY_BOOTSTRAP_TOKEN"])) return - source_commit = os.environ.get("GITHUB_SHA", "unknown") - with Path(summary_path).open("a", encoding="utf-8") as summary: - summary.write(f"## {title}\n\n") - summary.write(f"Source commit: `{source_commit}`\n\n") - for line in lines: - summary.write(f"- {line}\n") - summary.write("\n") - - -def run_discover(args: argparse.Namespace) -> int: - client = CratesIoClient(args.registry_url) - packages, missing, placeholders = discover(args.project_dir, client) - candidates = [*missing, *placeholders] - result = { - "packages": len(packages), - "missing": missing, - "placeholders": placeholders, - "bootstrap_candidates": candidates, - } - print(json.dumps(result, indent=2)) - _write_summary( - "Rust crate bootstrap discovery", - [ - f"Publishable crates: {len(packages)}", - f"Missing crates: {', '.join(missing) if missing else 'none'}", - f"Placeholder crates: {', '.join(placeholders) if placeholders else 'none'}", - ], - ) - return 0 - - -def run_apply(args: argparse.Namespace) -> int: - token = os.environ.get("CARGO_REGISTRY_BOOTSTRAP_TOKEN") - if not token: - raise RuntimeError("CARGO_REGISTRY_BOOTSTRAP_TOKEN is not set") - - client = CratesIoClient(args.registry_url, token=token) - packages, missing, placeholders = discover(args.project_dir, client) - candidate_set = {*missing, *placeholders} - candidates = [package for package in packages if package.name in candidate_set] - - results: list[ReconcileResult] = [] - authenticated: list[str] = [] - try: - preflight_authenticated(packages, candidate_set, client) - print( - f"authenticated preflight passed for {len(packages)} planned crates", - flush=True, - ) - print(f"bootstrap candidates: {len(candidates)}", flush=True) - for planned in candidates: - result = reconcile_crate(args.project_dir, planned, client, token) - results.append(result) - print(f"{result.name}: {', '.join(result.actions)}", flush=True) - authenticated = verify_authenticated(packages, client) - print( - f"authenticated final audit passed for {len(authenticated)} planned crates", - flush=True, - ) - except Exception as error: - _write_summary( - "Rust crate bootstrap", - [ - *( - f"`{result.name}`: {', '.join(result.actions)}" - for result in results - ), - f"Failed: {error}", - ], - ) - raise - - _write_summary( - "Rust crate bootstrap", - [ - f"Authenticated preflight: {len(packages)} planned crates", - *(f"`{result.name}`: {', '.join(result.actions)}" for result in results), - f"Authenticated final audit: {len(authenticated)} planned crates", - ], - ) - return 0 - - -def run_verify(args: argparse.Namespace) -> int: - verified = verify_public(args.project_dir, CratesIoClient(args.registry_url)) - print(json.dumps({"verified": verified}, indent=2)) - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser( - description=( - "Create and secure crates.io names in the reqsign Rust publish plan." - ) - ) - parser.add_argument( - "--project-dir", - type=Path, - default=PROJECT_DIR, - help="Path to the repository root.", - ) - parser.add_argument( - "--registry-url", - default=REGISTRY_URL, - help="crates.io-compatible registry API URL.", - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - discover_parser = subparsers.add_parser( - "discover", help="Report missing and placeholder crate names." - ) - discover_parser.set_defaults(run=run_discover) - - apply_parser = subparsers.add_parser( - "apply", - help="Audit all planned crates and reconcile missing names and placeholders.", - ) - apply_parser.set_defaults(run=run_apply) - - verify_parser = subparsers.add_parser( - "verify", help="Verify public crate existence and Trusted Publishing only." - ) - verify_parser.set_defaults(run=run_verify) - - args = parser.parse_args() - args.project_dir = args.project_dir.resolve() - return args.run(args) + client = CratesIoClient() + for name in names: + metadata = client.get_crate(name) + if metadata is not None: + validate_crate(name, metadata) + if command == "verify": + assert metadata is not None, f"{name} does not exist on crates.io" + assert metadata["trustpub_only"] is True, ( + f"{name} does not require Trusted Publishing" + ) + version = metadata["max_version"] if metadata else "missing" + print(f"{name}: {version}") if __name__ == "__main__": - raise SystemExit(main()) + main() diff --git a/.github/scripts/release_rust/plan.py b/.github/scripts/release_rust/plan.py index 5654a6e6..57378405 100644 --- a/.github/scripts/release_rust/plan.py +++ b/.github/scripts/release_rust/plan.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,181 +15,58 @@ # specific language governing permissions and limitations # under the License. -import argparse -import heapq import json import subprocess -from dataclasses import asdict -from dataclasses import dataclass +from dataclasses import asdict, dataclass +from graphlib import TopologicalSorter from pathlib import Path - -SCRIPT_PATH = Path(__file__).resolve() -PROJECT_DIR = SCRIPT_PATH.parents[3] +PROJECT_DIR = Path(__file__).resolve().parents[3] @dataclass(frozen=True) class Package: name: str version: str - path: str - -def load_metadata(project_dir: Path = PROJECT_DIR) -> dict[str, object]: - process = subprocess.run( - ["cargo", "metadata", "--no-deps", "--format-version", "1"], - cwd=project_dir, - check=True, - text=True, - stdout=subprocess.PIPE, - ) - metadata = json.loads(process.stdout) - if not isinstance(metadata, dict): - raise RuntimeError("cargo metadata returned an invalid document") - return metadata - -def is_publishable(package: dict[str, object]) -> bool: - publish = package.get("publish") - if publish is None: - return True - if not isinstance(publish, list): - raise RuntimeError( - f"cargo metadata returned invalid publish settings for {package.get('name')}" +def plan() -> list[Package]: + metadata = json.loads( + subprocess.check_output( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=PROJECT_DIR, + text=True, ) - return "crates-io" in publish - - -def plan_from_metadata(metadata: dict[str, object], project_dir: Path) -> list[Package]: - raw_packages = metadata.get("packages") - workspace_members = metadata.get("workspace_members") - if not isinstance(raw_packages, list) or not isinstance(workspace_members, list): - raise RuntimeError("cargo metadata omitted packages or workspace members") - - members = set(workspace_members) - packages_by_id: dict[str, dict[str, object]] = {} - publishable_by_dir: dict[Path, dict[str, object]] = {} - local_by_dir: dict[Path, dict[str, object]] = {} - - for package in raw_packages: - if not isinstance(package, dict): - raise RuntimeError("cargo metadata returned an invalid package") - package_id = package.get("id") - manifest_path = package.get("manifest_path") - if not isinstance(package_id, str) or not isinstance(manifest_path, str): - raise RuntimeError("cargo metadata returned an invalid package identity") - if package_id not in members: - continue - - manifest_dir = Path(manifest_path).resolve().parent - packages_by_id[package_id] = package - local_by_dir[manifest_dir] = package - if is_publishable(package): - publishable_by_dir[manifest_dir] = package - - if set(packages_by_id) != members: - missing = sorted(members - set(packages_by_id)) - raise RuntimeError(f"cargo metadata omitted workspace packages: {missing}") - - graph: dict[Path, set[Path]] = { - manifest_dir: set() for manifest_dir in publishable_by_dir + ) + # --no-deps returns workspace members; Cargo validates their identities. + local = { + Path(package["manifest_path"]).parent: package + for package in metadata["packages"] } - indegree = {manifest_dir: 0 for manifest_dir in publishable_by_dir} - - for manifest_dir, package in publishable_by_dir.items(): - dependencies = package.get("dependencies") - if not isinstance(dependencies, list): - raise RuntimeError( - f"cargo metadata omitted dependencies for {package.get('name')}" - ) - - for dependency in dependencies: - if not isinstance(dependency, dict): - raise RuntimeError("cargo metadata returned an invalid dependency") - if dependency.get("kind") == "dev": - continue - dependency_path = dependency.get("path") - if not isinstance(dependency_path, str): - continue - - dependency_dir = Path(dependency_path).resolve() - if dependency_dir not in local_by_dir: + packages = { + path: package + for path, package in local.items() + if package["publish"] is None or "crates-io" in package["publish"] + } + graph = {} + for path, package in packages.items(): + dependencies = set() + for dependency in package["dependencies"]: + if dependency["kind"] == "dev" or "path" not in dependency: continue - if dependency_dir not in publishable_by_dir: - raise RuntimeError( - f"{package.get('name')} depends on unpublished workspace package " - f"{local_by_dir[dependency_dir].get('name')}" + dependency_path = Path(dependency["path"]) + if dependency_path in local: + assert dependency_path in packages, ( + f"{package['name']} depends on unpublished workspace package " + f"{local[dependency_path]['name']}" ) - if manifest_dir in graph[dependency_dir]: - continue - - graph[dependency_dir].add(manifest_dir) - indegree[manifest_dir] += 1 - - queue = [ - (manifest_dir.relative_to(project_dir).as_posix(), manifest_dir) - for manifest_dir, degree in indegree.items() - if degree == 0 + dependencies.add(dependency_path) + graph[path] = dependencies + return [ + Package(packages[path]["name"], packages[path]["version"]) + for path in TopologicalSorter(graph).static_order() ] - heapq.heapify(queue) - - ordered: list[Package] = [] - while queue: - _, manifest_dir = heapq.heappop(queue) - package = publishable_by_dir[manifest_dir] - name = package.get("name") - version = package.get("version") - if not isinstance(name, str) or not isinstance(version, str): - raise RuntimeError("cargo metadata returned invalid package metadata") - - ordered.append( - Package( - name=name, - version=version, - path=manifest_dir.relative_to(project_dir).as_posix(), - ) - ) - for dependent in graph[manifest_dir]: - indegree[dependent] -= 1 - if indegree[dependent] == 0: - heapq.heappush( - queue, - (dependent.relative_to(project_dir).as_posix(), dependent), - ) - - if len(ordered) != len(publishable_by_dir): - cyclic = sorted( - publishable_by_dir[path].get("name") - for path, degree in indegree.items() - if degree > 0 - ) - raise RuntimeError(f"publishable workspace dependency cycle: {cyclic}") - - names = [package.name for package in ordered] - if len(names) != len(set(names)): - raise RuntimeError("duplicate crates.io package name in publish plan") - return ordered - - -def plan(project_dir: Path = PROJECT_DIR) -> list[Package]: - project_dir = project_dir.resolve() - return plan_from_metadata(load_metadata(project_dir), project_dir) - - -def main() -> int: - parser = argparse.ArgumentParser( - description="Print the reqsign crates.io publish plan in dependency order." - ) - parser.add_argument( - "--project-dir", - type=Path, - default=PROJECT_DIR, - help="Path to the repository root.", - ) - args = parser.parse_args() - print(json.dumps([asdict(package) for package in plan(args.project_dir)], indent=2)) - return 0 if __name__ == "__main__": - raise SystemExit(main()) + print(json.dumps([asdict(package) for package in plan()], indent=2)) diff --git a/.github/scripts/release_rust/publish.py b/.github/scripts/release_rust/publish.py index 00671caa..e25f2f4d 100644 --- a/.github/scripts/release_rust/publish.py +++ b/.github/scripts/release_rust/publish.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,19 +15,14 @@ # specific language governing permissions and limitations # under the License. -import argparse import os import re import subprocess import time -from datetime import datetime -from datetime import timezone +from datetime import datetime, timezone from email.utils import parsedate_to_datetime -from pathlib import Path -from plan import PROJECT_DIR -from plan import Package -from plan import plan +from plan import PROJECT_DIR, Package, plan from trusted_publishing import temporary_trusted_publishing_token @@ -37,7 +31,7 @@ def parse_retry_after(output: str) -> int: if match: value = match.group(1).strip().rstrip(".") for parser in ( - lambda text: parsedate_to_datetime(text), + parsedate_to_datetime, lambda text: datetime.fromisoformat(text.replace("Z", "+00:00")), ): try: @@ -70,7 +64,7 @@ def already_published(output: str, package: Package) -> bool: ) -def publish_package(project_dir: Path, package: Package) -> str: +def publish_package(package: Package) -> str: command = [ "cargo", "publish", @@ -86,7 +80,7 @@ def publish_package(project_dir: Path, package: Package) -> str: env["CARGO_REGISTRY_TOKEN"] = token process = subprocess.run( command, - cwd=project_dir, + cwd=PROJECT_DIR, check=False, env=env, text=True, @@ -116,21 +110,7 @@ def publish_package(project_dir: Path, package: Package) -> str: def main() -> int: - parser = argparse.ArgumentParser( - description="Publish reqsign crates with fresh OIDC credentials." - ) - parser.add_argument( - "--project-dir", - type=Path, - default=PROJECT_DIR, - help="Path to the repository root.", - ) - args = parser.parse_args() - project_dir = args.project_dir.resolve() - - results: list[tuple[Package, str]] = [] - for package in plan(project_dir): - results.append((package, publish_package(project_dir, package))) + results = [(package, publish_package(package)) for package in plan()] print("Publish plan completed:", flush=True) for package, result in results: diff --git a/.github/scripts/release_rust/test_bootstrap.py b/.github/scripts/release_rust/test_bootstrap.py index 1f7db8c8..e0ff06f2 100644 --- a/.github/scripts/release_rust/test_bootstrap.py +++ b/.github/scripts/release_rust/test_bootstrap.py @@ -15,488 +15,162 @@ # specific language governing permissions and limitations # under the License. -import argparse -import io import subprocess import tempfile -import tomllib import unittest -import urllib.error from pathlib import Path from unittest import mock -from bootstrap import CratesIoClient -from bootstrap import LEGACY_PLACEHOLDER_DESCRIPTION -from bootstrap import LEGACY_REPOSITORY -from bootstrap import PLACEHOLDER_DESCRIPTION -from bootstrap import PLACEHOLDER_VERSION -from bootstrap import PUBLISHER -from bootstrap import PROJECT_DIR -from bootstrap import REPOSITORY -from bootstrap import PlannedCrate -from bootstrap import ReconcileResult -from bootstrap import _placeholder_manifest -from bootstrap import discover -from bootstrap import preflight_authenticated -from bootstrap import reconcile_crate -from bootstrap import run_apply -from bootstrap import verify_authenticated -from bootstrap import write_placeholder_package +from bootstrap import ( + LEGACY_PLACEHOLDER_DESCRIPTION, + LEGACY_REPOSITORY, + PLACEHOLDER_DESCRIPTION, + PUBLISHER, + REPOSITORY, + apply, + write_placeholder_package, +) -def metadata(name: str, *, version: str = "1.0.0", trustpub_only: bool = False): +def metadata(name, version="1.0.0", *, ready=True): return { "id": name, "max_version": version, "repository": REPOSITORY, - "description": ( - PLACEHOLDER_DESCRIPTION - if version == PLACEHOLDER_VERSION - else "An Apache Reqsign crate" - ), - "trustpub_only": trustpub_only, + "description": PLACEHOLDER_DESCRIPTION, + "trustpub_only": ready, } -def expected_config(name: str): +def publisher(name): return {"crate": name, **PUBLISHER} -class FakeClient: - def __init__(self, name: str, krate=None, configs=None): - self.name = name - self.krate = krate - self.configs = list(configs or []) - self.created_configs = 0 - self.restricted = 0 +class FakeRegistry: + token = "bootstrap-token" - def get_crate(self, name: str): - self._assert_name(name) - return None if self.krate is None else dict(self.krate) + def __init__(self, crates, configs): + self.crates = crates + self.configs = configs + self.writes = [] - def get_version(self, name: str, version: str): - self._assert_name(name) - return {"num": version} + def get_crate(self, name): + crate = self.crates.get(name) + return dict(crate) if crate else None - def list_github_configs(self, name: str): - self._assert_name(name) - return [dict(config) for config in self.configs] + def list_github_configs(self, name): + return self.configs.get(name, []) - def create_github_config(self, name: str): - self._assert_name(name) - config = expected_config(name) - self.configs.append(config) - self.created_configs += 1 - return dict(config) + def create_github_config(self, name): + self.writes.append(("publisher", name)) + config = publisher(name) + self.configs[name] = [config] + return config - def set_trustpub_only(self, name: str): - self._assert_name(name) - self.krate["trustpub_only"] = True - self.restricted += 1 - return dict(self.krate) - - def _assert_name(self, name: str): - if name != self.name: - raise AssertionError(f"expected {self.name}, got {name}") - - -class JsonResponse(io.BytesIO): - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() + def set_trustpub_only(self, name): + self.writes.append(("restrict", name)) + self.crates[name]["trustpub_only"] = True + return self.get_crate(name) class BootstrapTest(unittest.TestCase): - def test_bootstrap_workflow_is_input_free_and_always_protected(self): - workflow = ( - PROJECT_DIR / ".github/workflows/bootstrap_rust_crates.yml" - ).read_text(encoding="utf-8") - dispatch = workflow.split("\non:\n", 1)[1].split("\npermissions:\n", 1)[0] - - self.assertEqual(dispatch, " workflow_dispatch:\n") - self.assertIn(" environment: rust-bootstrap\n", workflow) - self.assertNotIn("candidate_count", workflow) - - def test_publisher_matches_the_release_workflow(self): - workflow = (PROJECT_DIR / ".github/workflows/release.yml").read_text( - encoding="utf-8" + def test_entire_plan_is_checked_before_any_write(self): + for invalid in ( + "repository", + "publisher", + "multiple publishers", + "trustpub_only", + ): + with self.subTest(invalid=invalid): + crate = metadata("established") + configs = [publisher("established")] + if invalid == "repository": + crate["repository"] = "https://github.com/other/reqsign" + elif invalid == "publisher": + configs[0]["repository_name"] = "opendal-reqsign" + elif invalid == "multiple publishers": + configs.append(publisher("established")) + else: + crate["trustpub_only"] = False + registry = FakeRegistry( + {"established": crate}, {"established": configs} + ) + with ( + mock.patch("bootstrap.publish_placeholder") as publish, + self.assertRaises(AssertionError), + ): + apply(["missing", "established"], registry) + publish.assert_not_called() + self.assertEqual(registry.writes, []) + + def test_bootstrap_resumes_partial_placeholders_and_is_safe_to_repeat(self): + crates = { + "partial": metadata("partial", "0.0.0", ready=False), + "configured": metadata("configured", "0.0.0", ready=False), + "ready": metadata("ready", "0.0.0"), + "established": metadata("established"), + } + crates["partial"]["description"] = LEGACY_PLACEHOLDER_DESCRIPTION + crates["established"]["repository"] = LEGACY_REPOSITORY + registry = FakeRegistry( + crates, + { + name: [publisher(name)] + for name in ("configured", "ready", "established") + }, ) - publish_job = workflow.split("\n publish:\n", 1)[1] + published = [] - self.assertEqual(PUBLISHER["workflow_filename"], "release.yml") - self.assertEqual(PUBLISHER["repository_owner"], "apache") - self.assertEqual(PUBLISHER["repository_name"], "reqsign") - self.assertEqual(PUBLISHER["environment"], "release") - self.assertIn(" environment: release\n", publish_job) - self.assertIn("github.event_name == 'push'", publish_job) - self.assertIn("!contains(github.ref, '-')", publish_job) - self.assertIn(" id-token: write\n", publish_job) - self.assertNotIn("secrets.CARGO_REGISTRY_TOKEN", publish_job) - self.assertNotIn("CARGO_REGISTRY_BOOTSTRAP_TOKEN", workflow) + def publish(name, token): + published.append(name) + registry.crates[name] = metadata(name, "0.0.0", ready=False) - def test_placeholder_manifest_is_dependency_free(self): - manifest = tomllib.loads(_placeholder_manifest("reqsign-new")) + names = ["missing", *crates] + with mock.patch("bootstrap.publish_placeholder", publish): + apply(names, registry) + apply(names, registry) + self.assertEqual(published, ["missing"]) + self.assertEqual( + registry.writes, + [ + ("publisher", "missing"), + ("restrict", "missing"), + ("publisher", "partial"), + ("restrict", "partial"), + ("restrict", "configured"), + ], + ) - self.assertEqual(manifest["package"]["name"], "reqsign-new") - self.assertEqual(manifest["package"]["version"], PLACEHOLDER_VERSION) - self.assertNotIn("dependencies", manifest) - self.assertNotIn("dev-dependencies", manifest) - self.assertNotIn("build-dependencies", manifest) + def test_crate_that_becomes_established_after_audit_is_not_modified(self): + registry = FakeRegistry({}, {}) + with ( + mock.patch.object( + registry, "get_crate", side_effect=[None, metadata("new")] + ), + mock.patch("bootstrap.publish_placeholder") as publish, + self.assertRaisesRegex(AssertionError, "established crate"), + ): + apply(["new"], registry) + publish.assert_not_called() + self.assertEqual(registry.writes, []) - def test_placeholder_can_be_packaged_offline(self): - planned = PlannedCrate("reqsign-new", "services/new") + def test_placeholder_can_be_packaged_and_built_offline(self): with tempfile.TemporaryDirectory() as tmpdir: package_dir = Path(tmpdir) - write_placeholder_package(PROJECT_DIR, planned, package_dir) - + write_placeholder_package("reqsign-new", package_dir) process = subprocess.run( - [ - "cargo", - "package", - "--manifest-path", - str(package_dir / "Cargo.toml"), - "--no-verify", - "--offline", - ], + ["cargo", "package", "--offline"], + cwd=package_dir, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - self.assertEqual(process.returncode, 0, process.stdout) self.assertTrue( - ( - package_dir - / "target" - / "package" - / f"{planned.name}-{PLACEHOLDER_VERSION}.crate" - ).is_file() + (package_dir / "target/package/reqsign-new-0.0.0.crate").is_file() ) - def test_public_read_retries_a_transient_registry_failure(self): - error = urllib.error.HTTPError( - "https://crates.io/api/v1/crates/reqsign", - 503, - "unavailable", - {}, - io.BytesIO(b""), - ) - response = JsonResponse( - b'{"crate":{"id":"reqsign","repository":' - b'"https://github.com/apache/reqsign"}}' - ) - - with ( - mock.patch( - "bootstrap.urllib.request.urlopen", - side_effect=(error, response), - ), - mock.patch("bootstrap.time.sleep") as sleep, - ): - krate = CratesIoClient().get_crate("reqsign") - - self.assertEqual(krate["id"], "reqsign") - sleep.assert_called_once_with(1) - - def test_discovery_does_not_migrate_established_crates(self): - planned = PlannedCrate("reqsign-core", "core") - client = FakeClient(planned.name, metadata(planned.name)) - - with mock.patch("bootstrap.planned_crates", return_value=[planned]): - packages, missing, placeholders = discover(Path(), client) - - self.assertEqual(packages, [planned]) - self.assertEqual(missing, []) - self.assertEqual(placeholders, []) - - def test_discovery_selects_a_placeholder(self): - planned = PlannedCrate("reqsign-new", "services/new") - client = FakeClient( - planned.name, - metadata(planned.name, version=PLACEHOLDER_VERSION), - ) - - with mock.patch("bootstrap.planned_crates", return_value=[planned]): - _, missing, placeholders = discover(Path(), client) - - self.assertEqual(missing, []) - self.assertEqual(placeholders, [planned.name]) - - def test_authenticated_preflight_audits_an_established_crate(self): - planned = PlannedCrate("reqsign-core", "core") - client = FakeClient( - planned.name, - metadata(planned.name, trustpub_only=True), - [expected_config(planned.name)], - ) - - verified = preflight_authenticated([planned], set(), client) - - self.assertEqual(verified, [planned.name]) - self.assertEqual(client.created_configs, 0) - self.assertEqual(client.restricted, 0) - - def test_authenticated_preflight_rejects_an_unexpected_publisher(self): - planned = PlannedCrate("reqsign-core", "core") - unexpected = { - **expected_config(planned.name), - "workflow_filename": "other.yml", - } - client = FakeClient( - planned.name, - metadata(planned.name, trustpub_only=True), - [unexpected], - ) - - with self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"): - preflight_authenticated([planned], set(), client) - - self.assertEqual(client.created_configs, 0) - self.assertEqual(client.restricted, 0) - - def test_authenticated_preflight_accepts_historical_repository_metadata(self): - planned = PlannedCrate("reqsign-core", "core") - krate = metadata(planned.name, trustpub_only=True) - krate["repository"] = LEGACY_REPOSITORY - client = FakeClient(planned.name, krate, [expected_config(planned.name)]) - - self.assertEqual( - preflight_authenticated([planned], set(), client), [planned.name] - ) - self.assertEqual(verify_authenticated([planned], client), [planned.name]) - self.assertEqual(client.created_configs, 0) - - def test_authenticated_preflight_rejects_historical_publisher(self): - planned = PlannedCrate("reqsign-core", "core") - krate = metadata(planned.name, trustpub_only=True) - krate["repository"] = LEGACY_REPOSITORY - config = {**expected_config(planned.name), "repository_name": "opendal-reqsign"} - client = FakeClient(planned.name, krate, [config]) - - with self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"): - preflight_authenticated([planned], set(), client) - - self.assertEqual(client.created_configs, 0) - - def test_discovery_rejects_an_unrelated_repository(self): - planned = PlannedCrate("reqsign-core", "core") - krate = metadata(planned.name) - krate["repository"] = "https://github.com/other/reqsign" - client = FakeClient(planned.name, krate) - - with ( - mock.patch("bootstrap.planned_crates", return_value=[planned]), - self.assertRaisesRegex(RuntimeError, "unexpected repository"), - ): - discover(Path(), client) - - def test_authenticated_preflight_rejects_unmigrated_established_crate(self): - planned = PlannedCrate("reqsign-core", "core") - client = FakeClient( - planned.name, - metadata(planned.name), - [expected_config(planned.name)], - ) - - with self.assertRaisesRegex(RuntimeError, "does not require"): - preflight_authenticated([planned], set(), client) - - def test_authenticated_verification_requires_the_exact_publisher(self): - planned = PlannedCrate("reqsign-core", "core") - unexpected = { - **expected_config(planned.name), - "environment": "other", - } - client = FakeClient( - planned.name, - metadata(planned.name, trustpub_only=True), - [unexpected], - ) - - with self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"): - verify_authenticated([planned], client) - - def test_apply_preflights_the_complete_plan_before_reconciling(self): - candidate = PlannedCrate("reqsign-new", "services/new") - established = PlannedCrate("reqsign-core", "core") - packages = [candidate, established] - client = object() - calls = [] - - def preflight(actual_packages, candidate_names, actual_client): - calls.append(("preflight", actual_packages, candidate_names, actual_client)) - - def reconcile(project_dir, planned, actual_client, token): - calls.append(("reconcile", planned, actual_client, token)) - return ReconcileResult(planned.name, ("verified",)) - - def verify(actual_packages, actual_client): - calls.append(("verify", actual_packages, actual_client)) - return [package.name for package in actual_packages] - - args = argparse.Namespace(project_dir=Path(), registry_url="registry") - with ( - mock.patch.dict( - "bootstrap.os.environ", - {"CARGO_REGISTRY_BOOTSTRAP_TOKEN": "bootstrap-token"}, - clear=True, - ), - mock.patch("bootstrap.CratesIoClient", return_value=client), - mock.patch( - "bootstrap.discover", - return_value=(packages, [candidate.name], []), - ), - mock.patch("bootstrap.preflight_authenticated", preflight), - mock.patch("bootstrap.reconcile_crate", reconcile), - mock.patch("bootstrap.verify_authenticated", verify), - ): - self.assertEqual(run_apply(args), 0) - - self.assertEqual( - [call[0] for call in calls], - ["preflight", "reconcile", "verify"], - ) - self.assertEqual(calls[0][1], packages) - self.assertEqual(calls[0][2], {candidate.name}) - self.assertEqual(calls[2][1], packages) - - def test_apply_does_not_mutate_when_authenticated_preflight_fails(self): - candidate = PlannedCrate("reqsign-new", "services/new") - args = argparse.Namespace(project_dir=Path(), registry_url="registry") - - with ( - mock.patch.dict( - "bootstrap.os.environ", - {"CARGO_REGISTRY_BOOTSTRAP_TOKEN": "bootstrap-token"}, - clear=True, - ), - mock.patch("bootstrap.CratesIoClient"), - mock.patch( - "bootstrap.discover", - return_value=([candidate], [candidate.name], []), - ), - mock.patch( - "bootstrap.preflight_authenticated", - side_effect=RuntimeError("audit failed"), - ), - mock.patch("bootstrap.reconcile_crate") as reconcile, - mock.patch("bootstrap.verify_authenticated") as verify, - self.assertRaisesRegex(RuntimeError, "audit failed"), - ): - run_apply(args) - - reconcile.assert_not_called() - verify.assert_not_called() - - def test_missing_crate_is_created_configured_and_restricted(self): - planned = PlannedCrate("reqsign-new", "services/new") - client = FakeClient(planned.name) - - def publish(project_dir, package, token): - self.assertEqual(package, planned) - self.assertEqual(token, "bootstrap-token") - client.krate = metadata(planned.name, version=PLACEHOLDER_VERSION) - - with ( - tempfile.TemporaryDirectory() as tmpdir, - mock.patch("bootstrap.publish_placeholder", publish), - ): - result = reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token") - - self.assertEqual( - result.actions, - ( - "created placeholder", - "configured Trusted Publishing", - "enabled Trusted Publishing only", - ), - ) - self.assertEqual(client.created_configs, 1) - self.assertEqual(client.restricted, 1) - - def test_partial_placeholder_resumes_without_republishing(self): - planned = PlannedCrate("reqsign-new", "services/new") - for description in ( - PLACEHOLDER_DESCRIPTION, - LEGACY_PLACEHOLDER_DESCRIPTION, - ): - with self.subTest(description=description): - krate = metadata(planned.name, version=PLACEHOLDER_VERSION) - krate["description"] = description - client = FakeClient(planned.name, krate) - - with ( - tempfile.TemporaryDirectory() as tmpdir, - mock.patch("bootstrap.publish_placeholder") as publish, - ): - result = reconcile_crate( - Path(tmpdir), planned, client, "bootstrap-token" - ) - - publish.assert_not_called() - self.assertEqual( - result.actions, - ( - "configured Trusted Publishing", - "enabled Trusted Publishing only", - ), - ) - - def test_ready_placeholder_is_a_noop(self): - planned = PlannedCrate("reqsign-new", "services/new") - client = FakeClient( - planned.name, - metadata( - planned.name, - version=PLACEHOLDER_VERSION, - trustpub_only=True, - ), - [expected_config(planned.name)], - ) - - with tempfile.TemporaryDirectory() as tmpdir: - result = reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token") - - self.assertEqual(result.actions, ("verified",)) - self.assertEqual(client.created_configs, 0) - self.assertEqual(client.restricted, 0) - - def test_established_crate_is_never_migrated(self): - planned = PlannedCrate("reqsign-core", "core") - client = FakeClient(planned.name, metadata(planned.name)) - - with ( - tempfile.TemporaryDirectory() as tmpdir, - self.assertRaisesRegex(RuntimeError, "established crate"), - ): - reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token") - - self.assertEqual(client.created_configs, 0) - self.assertEqual(client.restricted, 0) - - def test_unexpected_publisher_fails_closed(self): - planned = PlannedCrate("reqsign-new", "services/new") - unexpected = { - **expected_config(planned.name), - "workflow_filename": "other.yml", - } - client = FakeClient( - planned.name, - metadata(planned.name, version=PLACEHOLDER_VERSION), - [unexpected], - ) - - with ( - tempfile.TemporaryDirectory() as tmpdir, - self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"), - ): - reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token") - - self.assertEqual(client.restricted, 0) - if __name__ == "__main__": unittest.main() diff --git a/.github/scripts/release_rust/test_plan.py b/.github/scripts/release_rust/test_plan.py index c064094f..0147e46f 100644 --- a/.github/scripts/release_rust/test_plan.py +++ b/.github/scripts/release_rust/test_plan.py @@ -15,129 +15,66 @@ # specific language governing permissions and limitations # under the License. -import tempfile +import json import unittest -from pathlib import Path +from unittest import mock -from plan import PROJECT_DIR -from plan import Package from plan import plan -from plan import plan_from_metadata -def package( - root: Path, - name: str, - *, - dependencies: list[dict[str, object]] | None = None, - publish: list[str] | None = None, -) -> dict[str, object]: +def package(name, *, dependencies=(), publish=None): return { - "id": f"path+file://{root / name}#{name}@1.0.0", "name": name, "version": "1.0.0", - "manifest_path": str(root / name / "Cargo.toml"), "publish": publish, - "dependencies": dependencies or [], + "manifest_path": f"/workspace/{name}/Cargo.toml", + "dependencies": list(dependencies), } -def dependency(root: Path, name: str, *, kind: str | None = None): - return { - "name": name, - "kind": kind, - "path": str(root / name), - } - - -class ReleaseRustPlanTest(unittest.TestCase): - def test_current_workspace_plan_contains_new_aws_crates(self): - packages = plan(PROJECT_DIR) - positions = {package.name: index for index, package in enumerate(packages)} - - self.assertIn("reqsign-aws-core", positions) - self.assertIn("reqsign-aws-v4a", positions) - self.assertLess(positions["reqsign-core"], positions["reqsign-aws-core"]) - self.assertLess(positions["reqsign-aws-core"], positions["reqsign-aws-v4"]) - self.assertLess(positions["reqsign-aws-core"], positions["reqsign-aws-v4a"]) - self.assertLess(positions["reqsign-aws-v4"], positions["reqsign-google"]) - self.assertEqual(packages[-1].name, "reqsign") - - def test_plan_is_topological_and_ignores_dev_dependencies(self): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir).resolve() - core = package(root, "core") - service = package( - root, - "service", - dependencies=[dependency(root, "core")], - ) - facade = package( - root, - "facade", - dependencies=[ - dependency(root, "service"), - dependency(root, "dev-only", kind="dev"), - ], - ) - dev_only = package(root, "dev-only") - packages = [facade, service, dev_only, core] - metadata = { - "packages": packages, - "workspace_members": [package["id"] for package in packages], - } - - result = plan_from_metadata(metadata, root) - - self.assertEqual( - result, - [ - Package("core", "1.0.0", "core"), - Package("dev-only", "1.0.0", "dev-only"), - Package("service", "1.0.0", "service"), - Package("facade", "1.0.0", "facade"), - ], - ) - - def test_unpublishable_workspace_dependency_fails_closed(self): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir).resolve() - internal = package(root, "internal", publish=[]) - public = package( - root, - "public", - dependencies=[dependency(root, "internal")], - ) - packages = [internal, public] - metadata = { - "packages": packages, - "workspace_members": [package["id"] for package in packages], - } - - with self.assertRaisesRegex(RuntimeError, "unpublished workspace package"): - plan_from_metadata(metadata, root) - - def test_cycle_fails_closed(self): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir).resolve() - first = package( - root, - "first", - dependencies=[dependency(root, "second")], - ) - second = package( - root, - "second", - dependencies=[dependency(root, "first")], - ) - packages = [first, second] - metadata = { - "packages": packages, - "workspace_members": [package["id"] for package in packages], - } - - with self.assertRaisesRegex(RuntimeError, "dependency cycle"): - plan_from_metadata(metadata, root) +def dependency(name, kind=None): + return {"path": f"/workspace/{name}", "kind": kind} + + +class PublishPlanTest(unittest.TestCase): + def test_local_dependencies_precede_dependents_and_dev_cycles_are_ignored(self): + metadata = { + "packages": [ + package("facade", dependencies=[dependency("service")]), + package( + "service", + dependencies=[ + dependency("core"), + dependency("core", "build"), + dependency("internal", "dev"), + ], + ), + package("core", dependencies=[dependency("facade", "dev")]), + package("internal", publish=[]), + package("private-registry", publish=["private"]), + ] + } + with mock.patch( + "plan.subprocess.check_output", return_value=json.dumps(metadata) + ): + self.assertEqual([p.name for p in plan()], ["core", "service", "facade"]) + + def test_unpublishable_local_dependency_is_rejected(self): + metadata = { + "packages": [ + package("public", dependencies=[dependency("internal")]), + package("internal", publish=[]), + ] + } + with ( + mock.patch( + "plan.subprocess.check_output", return_value=json.dumps(metadata) + ), + self.assertRaisesRegex( + AssertionError, "unpublished workspace package internal" + ), + ): + plan() if __name__ == "__main__": diff --git a/.github/scripts/release_rust/test_publish.py b/.github/scripts/release_rust/test_publish.py index 0fb7a4f3..f3d13f28 100644 --- a/.github/scripts/release_rust/test_publish.py +++ b/.github/scripts/release_rust/test_publish.py @@ -18,12 +18,10 @@ import subprocess import unittest from contextlib import contextmanager -from pathlib import Path from unittest import mock from plan import Package -from publish import already_published -from publish import publish_package +from publish import already_published, publish_package def subprocess_result(returncode: int, output: str): @@ -32,11 +30,10 @@ def subprocess_result(returncode: int, output: str): class ReleaseRustPublishTest(unittest.TestCase): def test_live_publish_fetches_a_new_token_for_every_attempt(self): - package = Package("reqsign-test", "1.0.0", "test") + package = Package("reqsign-test", "1.0.0") tokens = iter(("first-token", "second-token")) revoked: list[str] = [] cargo_tokens: list[str | None] = [] - cargo_commands: list[list[str]] = [] @contextmanager def token_provider(): @@ -56,7 +53,6 @@ def token_provider(): ) def run(*args, **kwargs): - cargo_commands.append(args[0]) cargo_tokens.append(kwargs["env"].get("CARGO_REGISTRY_TOKEN")) return next(results) @@ -70,34 +66,15 @@ def run(*args, **kwargs): mock.patch("publish.subprocess.run", run), mock.patch("publish.time.sleep") as sleep, ): - result = publish_package(Path(), package) + result = publish_package(package) self.assertEqual(result, "published") - self.assertEqual( - cargo_commands, - [ - [ - "cargo", - "publish", - "--package", - "reqsign-test", - "--no-verify", - ], - [ - "cargo", - "publish", - "--package", - "reqsign-test", - "--no-verify", - ], - ], - ) self.assertEqual(cargo_tokens, ["first-token", "second-token"]) self.assertEqual(revoked, ["first-token", "second-token"]) sleep.assert_called_once_with(610) def test_already_published_package_is_recoverable(self): - package = Package("reqsign-test", "1.0.0", "test") + package = Package("reqsign-test", "1.0.0") @contextmanager def token_provider(): @@ -113,12 +90,12 @@ def token_provider(): ), ), ): - result = publish_package(Path(), package) + result = publish_package(package) self.assertEqual(result, "already published") def test_unrelated_already_exists_error_is_not_ignored(self): - package = Package("reqsign-test", "1.0.0", "test") + package = Package("reqsign-test", "1.0.0") self.assertFalse( already_published( "crate another@1.0.0 already exists on crates.io index", package @@ -126,7 +103,7 @@ def test_unrelated_already_exists_error_is_not_ignored(self): ) def test_non_retryable_failure_is_reported(self): - package = Package("reqsign-test", "1.0.0", "test") + package = Package("reqsign-test", "1.0.0") @contextmanager def token_provider(): @@ -140,7 +117,7 @@ def token_provider(): ), self.assertRaises(subprocess.CalledProcessError), ): - publish_package(Path(), package) + publish_package(package) if __name__ == "__main__": diff --git a/.github/scripts/release_rust/test_trusted_publishing.py b/.github/scripts/release_rust/test_trusted_publishing.py index 4b982d3b..a3acb20f 100644 --- a/.github/scripts/release_rust/test_trusted_publishing.py +++ b/.github/scripts/release_rust/test_trusted_publishing.py @@ -19,45 +19,24 @@ import json import unittest from unittest import mock -from urllib.parse import parse_qs -from urllib.parse import urlsplit +from urllib.parse import parse_qs, urlsplit -from trusted_publishing import _oidc_request_url -from trusted_publishing import request_trusted_publishing_token -from trusted_publishing import revoke_trusted_publishing_token from trusted_publishing import temporary_trusted_publishing_token -class JsonResponse(io.BytesIO): - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - class TrustedPublishingTest(unittest.TestCase): - def test_oidc_request_url_preserves_existing_query(self): - result = _oidc_request_url( - "https://example.test/token?api-version=1", "crates.io" - ) - - query = parse_qs(urlsplit(result).query) - self.assertEqual(query["api-version"], ["1"]) - self.assertEqual(query["audience"], ["crates.io"]) - - def test_token_exchange_uses_github_oidc_and_crates_io(self): + def test_exchange_masks_and_revokes_credentials_even_when_publishing_fails(self): requests = [] responses = iter( ( - JsonResponse(b'{"value":"github-jwt"}'), - JsonResponse(b'{"token":"crates-token"}'), + io.BytesIO(b'{"value":"github-jwt"}'), + io.BytesIO(b'{"token":"crates-token"}'), + io.BytesIO(b""), ) ) def urlopen(request, timeout): requests.append(request) - self.assertEqual(timeout, 30) return next(responses) environment = { @@ -65,51 +44,30 @@ def urlopen(request, timeout): "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", } with ( - mock.patch.dict("trusted_publishing.os.environ", environment, clear=True), + mock.patch.dict("os.environ", environment, clear=True), mock.patch("trusted_publishing.urllib.request.urlopen", urlopen), + mock.patch("sys.stdout", new_callable=io.StringIO) as output, + self.assertRaisesRegex(RuntimeError, "publish failed"), + temporary_trusted_publishing_token() as token, ): - token = request_trusted_publishing_token() + self.assertEqual(token, "crates-token") + self.assertIn("::add-mask::github-jwt", output.getvalue()) + self.assertIn("::add-mask::crates-token", output.getvalue()) + raise RuntimeError("publish failed") - self.assertEqual(token, "crates-token") - self.assertEqual(requests[0].get_method(), "GET") self.assertEqual( - parse_qs(urlsplit(requests[0].full_url).query)["audience"], - ["crates.io"], + parse_qs(urlsplit(requests[0].full_url).query), + { + "api-version": ["1"], + "audience": ["crates.io"], + }, ) self.assertEqual( requests[0].get_header("Authorization"), "Bearer request-token" ) - self.assertEqual(requests[1].get_method(), "POST") self.assertEqual(json.loads(requests[1].data), {"jwt": "github-jwt"}) - - def test_revoke_uses_the_temporary_token(self): - requests = [] - - def urlopen(request, timeout): - requests.append(request) - self.assertEqual(timeout, 30) - return JsonResponse(b"") - - with mock.patch("trusted_publishing.urllib.request.urlopen", urlopen): - revoke_trusted_publishing_token("crates-token") - - self.assertEqual(requests[0].get_method(), "DELETE") - self.assertEqual(requests[0].get_header("Authorization"), "Bearer crates-token") - - def test_temporary_token_is_always_revoked(self): - with ( - mock.patch( - "trusted_publishing.request_trusted_publishing_token", - return_value="temporary-token", - ), - mock.patch("trusted_publishing.revoke_trusted_publishing_token") as revoke, - ): - with self.assertRaisesRegex(RuntimeError, "publish failed"): - with temporary_trusted_publishing_token() as token: - self.assertEqual(token, "temporary-token") - raise RuntimeError("publish failed") - - revoke.assert_called_once_with("temporary-token", "https://crates.io") + self.assertEqual(requests[2].get_method(), "DELETE") + self.assertEqual(requests[2].get_header("Authorization"), "Bearer crates-token") if __name__ == "__main__": diff --git a/.github/scripts/release_rust/trusted_publishing.py b/.github/scripts/release_rust/trusted_publishing.py index 93372cd2..0831acf7 100644 --- a/.github/scripts/release_rust/trusted_publishing.py +++ b/.github/scripts/release_rust/trusted_publishing.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -18,143 +17,51 @@ import json import os -import urllib.error -import urllib.parse import urllib.request -from collections.abc import Iterator from contextlib import contextmanager - -DEFAULT_REGISTRY_URL = "https://crates.io" +REGISTRY_URL = "https://crates.io" USER_AGENT = "apache-reqsign-release/1.0 (https://github.com/apache/reqsign)" -def _response_error(operation: str, error: urllib.error.HTTPError) -> RuntimeError: - try: - response = error.read().decode("utf-8", errors="replace") - finally: - error.close() - try: - details = json.loads(response) - errors = details.get("errors", []) - response = "; ".join(item["detail"] for item in errors) - except (json.JSONDecodeError, AttributeError, KeyError, TypeError): - pass - return RuntimeError(f"{operation} failed with HTTP {error.code}: {response}") - - -def _request_json(request: urllib.request.Request, operation: str) -> dict[str, object]: - try: - with urllib.request.urlopen(request, timeout=30) as response: - result = json.load(response) - except urllib.error.HTTPError as error: - raise _response_error(operation, error) from None - except urllib.error.URLError as error: - raise RuntimeError(f"{operation} failed: {error.reason}") from None - if not isinstance(result, dict): - raise RuntimeError(f"{operation} returned an invalid response") - return result - - -def _oidc_request_url(request_url: str, audience: str) -> str: - parsed = urllib.parse.urlsplit(request_url) - query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) - query.append(("audience", audience)) - return urllib.parse.urlunsplit( - ( - parsed.scheme, - parsed.netloc, - parsed.path, - urllib.parse.urlencode(query), - parsed.fragment, - ) - ) - - -def _mask_secret(value: str) -> None: - if os.environ.get("GITHUB_ACTIONS") == "true": - print(f"::add-mask::{value}", flush=True) - - -def request_trusted_publishing_token( - registry_url: str = DEFAULT_REGISTRY_URL, -) -> str: - request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL") - request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") - if not request_url or not request_token: - raise RuntimeError( - "GitHub OIDC is unavailable; grant this job `id-token: write`" - ) - - audience = registry_url.rstrip("/") - for prefix in ("https://", "http://"): - if audience.startswith(prefix): - audience = audience.removeprefix(prefix) - break - oidc_request = urllib.request.Request( - _oidc_request_url(request_url, audience), +def request_json(url: str, *, headers=None, body=None): + request = urllib.request.Request( + url, + data=json.dumps(body).encode() if body is not None else None, headers={ - "Authorization": f"Bearer {request_token}", "User-Agent": USER_AGENT, - }, - ) - oidc_response = _request_json(oidc_request, "requesting a GitHub OIDC token") - jwt = oidc_response.get("value") - if not isinstance(jwt, str) or not jwt: - raise RuntimeError("GitHub OIDC response did not contain a token") - _mask_secret(jwt) - - token_request = urllib.request.Request( - f"{registry_url.rstrip('/')}/api/v1/trusted_publishing/tokens", - data=json.dumps({"jwt": jwt}).encode(), - headers={ "Content-Type": "application/json", - "User-Agent": USER_AGENT, + **(headers or {}), }, - method="POST", ) - token_response = _request_json( - token_request, "exchanging a crates.io Trusted Publishing token" - ) - token = token_response.get("token") - if not isinstance(token, str) or not token: - raise RuntimeError( - "crates.io Trusted Publishing response did not contain a token" - ) - _mask_secret(token) - return token + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) -def revoke_trusted_publishing_token( - token: str, registry_url: str = DEFAULT_REGISTRY_URL -) -> None: - request = urllib.request.Request( - f"{registry_url.rstrip('/')}/api/v1/trusted_publishing/tokens", +@contextmanager +def temporary_trusted_publishing_token(): + request_url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + separator = "&" if "?" in request_url else "?" + jwt = request_json( + f"{request_url}{separator}audience=crates.io", headers={ - "Authorization": f"Bearer {token}", - "User-Agent": USER_AGENT, + "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}" }, - method="DELETE", - ) - try: - with urllib.request.urlopen(request, timeout=30): - return - except urllib.error.HTTPError as error: - raise _response_error( - "revoking a crates.io Trusted Publishing token", error - ) from None - except urllib.error.URLError as error: - raise RuntimeError( - f"revoking a crates.io Trusted Publishing token failed: {error.reason}" - ) from None - - -@contextmanager -def temporary_trusted_publishing_token( - registry_url: str = DEFAULT_REGISTRY_URL, -) -> Iterator[str]: - token = request_trusted_publishing_token(registry_url) + )["value"] + assert jwt, "GitHub returned an empty OIDC token" + print(f"::add-mask::{jwt}", flush=True) + token = request_json( + f"{REGISTRY_URL}/api/v1/trusted_publishing/tokens", body={"jwt": jwt} + )["token"] + assert token, "crates.io returned an empty Trusted Publishing token" + print(f"::add-mask::{token}", flush=True) try: yield token finally: - revoke_trusted_publishing_token(token, registry_url) + request = urllib.request.Request( + f"{REGISTRY_URL}/api/v1/trusted_publishing/tokens", + headers={"Authorization": f"Bearer {token}", "User-Agent": USER_AGENT}, + method="DELETE", + ) + with urllib.request.urlopen(request, timeout=30): + pass diff --git a/.github/workflows/bootstrap_rust_crates.yml b/.github/workflows/bootstrap_rust_crates.yml index 739f5c7e..5df96659 100644 --- a/.github/workflows/bootstrap_rust_crates.yml +++ b/.github/workflows/bootstrap_rust_crates.yml @@ -29,29 +29,7 @@ concurrency: cancel-in-progress: false jobs: - discover: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ github.sha }} - - name: Validate main dispatch - run: | - set -euo pipefail - if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then - echo "Bootstrap Rust Crates must be dispatched from main" >&2 - exit 1 - fi - test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.11" - - name: Discover Rust crates - run: python3 .github/scripts/release_rust/bootstrap.py discover - bootstrap: - needs: discover runs-on: ubuntu-24.04 environment: rust-bootstrap steps: @@ -61,11 +39,7 @@ jobs: ref: ${{ github.sha }} - name: Validate main dispatch run: | - set -euo pipefail - if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then - echo "Bootstrap Rust Crates must be dispatched from main" >&2 - exit 1 - fi + test "${GITHUB_REF}" = "refs/heads/main" test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71ace79f..3df67998 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,10 +66,8 @@ jobs: publish: name: Publish Workspace - if: >- - github.event_name == 'push' && - startsWith(github.ref, 'refs/tags/') && - !contains(github.ref, '-') + # The push trigger accepts only formal release tags. + if: github.event_name == 'push' needs: validate runs-on: ubuntu-24.04 environment: release diff --git a/release/scripts/bootstrap-rust-crates.sh b/release/scripts/bootstrap-rust-crates.sh index eb63b8e2..a466613d 100755 --- a/release/scripts/bootstrap-rust-crates.sh +++ b/release/scripts/bootstrap-rust-crates.sh @@ -19,134 +19,34 @@ set -euo pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -repo_dir="$(git -C "${script_dir}" rev-parse --show-toplevel)" -cd "${repo_dir}" - -for command in gh git jq python3; do - command -v "${command}" >/dev/null || { - echo "required command is unavailable: ${command}" >&2 - exit 1 - } -done - -if [[ "$#" -ne 0 ]]; then - echo "usage: $0" >&2 - exit 1 -fi -if [[ -n "$(git status --porcelain)" ]]; then - echo "the release checkout must be clean" >&2 - exit 1 -fi - -apache_remote="$( - git remote -v | - awk '$2 ~ /github.com[:\/]apache\/reqsign(\.git)?$/ && $3 == "(fetch)" { print $1; exit }' -)" -if [[ -z "${apache_remote}" ]]; then - echo "cannot find a git remote for apache/reqsign" >&2 - exit 1 -fi - -git fetch "${apache_remote}" main -source_commit="$(git rev-parse FETCH_HEAD)" -if [[ "$(git rev-parse HEAD)" != "${source_commit}" ]]; then - echo "the release checkout must be at the current apache/reqsign main: ${source_commit}" >&2 - exit 1 -fi - -git cat-file -e \ - "${source_commit}:.github/workflows/bootstrap_rust_crates.yml" -git cat-file -e \ - "${source_commit}:.github/scripts/release_rust/bootstrap.py" +cd "$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +test "$#" -eq 0 +test -z "$(git status --porcelain)" repo="apache/reqsign" workflow="bootstrap_rust_crates.yml" -environment="rust-bootstrap" -if ! environment_json="$(gh api "repos/${repo}/environments/${environment}")"; then - echo "GitHub environment ${environment} is not configured" >&2 - exit 1 -fi -if ! jq -e \ - '[.protection_rules[]? | select(.type == "required_reviewers")] | length > 0' \ - >/dev/null <<<"${environment_json}"; then - echo "GitHub environment ${environment} must require reviewers" >&2 - exit 1 -fi - -run_title="Bootstrap Rust crates at ${source_commit}" -started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +git fetch "https://github.com/${repo}.git" main +source_commit="$(git rev-parse FETCH_HEAD)" +test "$(git rev-parse HEAD)" = "${source_commit}" -if ! dispatch_output="$( - gh workflow run "${workflow}" \ - --repo "${repo}" \ - --ref main 2>&1 -)"; then - echo "${dispatch_output}" >&2 - exit 1 -fi -echo "${dispatch_output}" +gh api "repos/${repo}/environments/rust-bootstrap" | + jq -e '[.protection_rules[] | select(.type == "required_reviewers")] | length > 0' >/dev/null +# The versioned dispatch API returns the exact run; no polling or title matching. run_id="$( - sed -nE \ - 's#.*github\.com/apache/reqsign/actions/runs/([0-9]+).*#\1#p' \ - <<<"${dispatch_output}" | - tail -n 1 -)" -deadline=$((SECONDS + 120)) -while [[ -z "${run_id}" && ${SECONDS} -lt ${deadline} ]]; do - runs="$( - gh run list \ - --repo "${repo}" \ - --workflow "${workflow}" \ - --event workflow_dispatch \ - --limit 50 \ - --json databaseId,displayTitle,headSha,createdAt - )" - run_id="$( - jq -r \ - --arg title "${run_title}" \ - --arg head_sha "${source_commit}" \ - --arg started_at "${started_at}" \ - '[.[] - | select( - .displayTitle == $title - and .headSha == $head_sha - and .createdAt >= $started_at - ) - ] - | sort_by(.createdAt) - | last - | .databaseId // empty' \ - <<<"${runs}" - )" - if [[ -z "${run_id}" ]]; then - sleep 2 - fi -done - -if [[ -z "${run_id}" ]]; then - echo "could not resolve the dispatched ${workflow} run" >&2 - exit 1 -fi - -run_head_sha="$( - gh run view "${run_id}" --repo "${repo}" --json headSha --jq '.headSha' + gh api --method POST \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "repos/${repo}/actions/workflows/${workflow}/dispatches" \ + -f ref=main \ + -F return_run_details=true | + jq -er '.workflow_run_id' )" -if [[ "${run_head_sha}" != "${source_commit}" ]]; then - echo "workflow run ${run_id} uses ${run_head_sha}, expected ${source_commit}" >&2 - exit 1 -fi +run_head_sha="$(gh run view "${run_id}" --repo "${repo}" --json headSha --jq '.headSha')" +test "${run_head_sha}" = "${source_commit}" echo "Waiting for crates.io bootstrap run ${run_id}" if ! gh run watch "${run_id}" --repo "${repo}" --exit-status; then gh run view "${run_id}" --repo "${repo}" --log-failed exit 1 fi - -gh run view "${run_id}" \ - --repo "${repo}" \ - --json displayTitle,headSha,status,conclusion,url \ - --jq '{title: .displayTitle, head_sha: .headSha, status, conclusion, url}' - python3 .github/scripts/release_rust/bootstrap.py verify