From 52d30d5c75ce8b5a8781a9d4e7f90b2986120fb5 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 23:44:19 -0400 Subject: [PATCH] CI: retry transient staged release downloads Package promotion currently fails an entire prepare-merge run when a single GitHub release read transiently reports that an existing staging release is missing. Retry exact asset downloads in private attempt directories and publish only a complete archive into the promotion directory. Add focused fixture coverage for transient and permanent failures, partial files, exact arguments, and unsafe input. --- .../download-verified-release-asset.sh | 63 +++++++++++++-- .../test-download-verified-release-asset.sh | 77 ++++++++++++++++++- .../scripts/test-merge-candidate-workflows.sh | 8 +- .github/workflows/prepare-merge.yml | 16 ++-- 4 files changed, 147 insertions(+), 17 deletions(-) diff --git a/.github/scripts/download-verified-release-asset.sh b/.github/scripts/download-verified-release-asset.sh index e0866029bd..f5c03d56e2 100755 --- a/.github/scripts/download-verified-release-asset.sh +++ b/.github/scripts/download-verified-release-asset.sh @@ -31,6 +31,17 @@ REPOSITORY="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY required}" TMP_ROOT="$(mktemp -d)" trap 'rm -rf "$TMP_ROOT"' EXIT +ATTEMPTS="${RELEASE_DOWNLOAD_ATTEMPTS:-4}" +RETRY_SECONDS="${RELEASE_DOWNLOAD_RETRY_SECONDS:-2}" +if ! [[ "$ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then + echo "download-verified-release-asset: attempts must be positive: $ATTEMPTS" >&2 + exit 2 +fi +if ! [[ "$RETRY_SECONDS" =~ ^[0-9]+$ ]]; then + echo "download-verified-release-asset: retry seconds must be non-negative: $RETRY_SECONDS" >&2 + exit 2 +fi + sha256_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -39,15 +50,51 @@ sha256_file() { fi } -gh release download "$TAG" \ - --repo "$REPOSITORY" \ - --pattern "$ASSET" \ - --dir "$TMP_ROOT" \ - --clobber +attempt=1 +delay="$RETRY_SECONDS" +downloaded="" +while [ "$attempt" -le "$ATTEMPTS" ]; do + attempt_dir="$TMP_ROOT/attempt-$attempt" + stdout_file="$TMP_ROOT/stdout-$attempt" + stderr_file="$TMP_ROOT/stderr-$attempt" + mkdir "$attempt_dir" + + if gh release download "$TAG" \ + --repo "$REPOSITORY" \ + --pattern "$ASSET" \ + --dir "$attempt_dir" \ + --clobber >"$stdout_file" 2>"$stderr_file"; then + candidate="$attempt_dir/$ASSET" + if [ -f "$candidate" ] && [ ! -L "$candidate" ]; then + cat "$stdout_file" + downloaded="$candidate" + break + fi + echo "download-verified-release-asset: GitHub reported success without a regular $ASSET" >&2 + rc=1 + else + rc=$? + fi + + cat "$stderr_file" >&2 + if [ -s "$stdout_file" ]; then + cat "$stdout_file" >&2 + fi + rm -rf "$attempt_dir" + + if [ "$attempt" -ge "$ATTEMPTS" ]; then + echo "download-verified-release-asset: failed after $ATTEMPTS attempts: $TAG/$ASSET" >&2 + exit "$rc" + fi + + echo "::warning::release asset download failed (attempt $attempt/$ATTEMPTS); retrying in ${delay}s: $TAG/$ASSET" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) +done -downloaded="$TMP_ROOT/$ASSET" -if [ ! -f "$downloaded" ]; then - echo "download-verified-release-asset: $TAG did not yield $ASSET" >&2 +if [ -z "$downloaded" ]; then + echo "download-verified-release-asset: no verified download candidate for $TAG/$ASSET" >&2 exit 1 fi actual_size=$(wc -c < "$downloaded" | tr -d '[:space:]') diff --git a/.github/scripts/test-download-verified-release-asset.sh b/.github/scripts/test-download-verified-release-asset.sh index 4d5f2d53bb..65dc944ffa 100755 --- a/.github/scripts/test-download-verified-release-asset.sh +++ b/.github/scripts/test-download-verified-release-asset.sh @@ -15,10 +15,10 @@ set -euo pipefail shift 2 tag="$1" shift -asset=""; dir="" +repository=""; asset=""; dir="" while [ "$#" -gt 0 ]; do case "$1" in - --repo) shift 2 ;; + --repo) repository="$2"; shift 2 ;; --pattern) asset="$2"; shift 2 ;; --dir) dir="$2"; shift 2 ;; --clobber) shift ;; @@ -26,6 +26,21 @@ while [ "$#" -gt 0 ]; do esac done [ "$tag" = binaries-abi-v39 ] +printf '%s\t%s\t%s\n' "$repository" "$tag" "$asset" >> "$GH_FAKE_ARGS_FILE" +count=0 +if [ -f "$GH_FAKE_COUNT_FILE" ]; then + count="$(cat "$GH_FAKE_COUNT_FILE")" +fi +count=$((count + 1)) +printf '%s\n' "$count" > "$GH_FAKE_COUNT_FILE" +if [ "$count" -lt "$GH_FAKE_SUCCEED_ON" ]; then + printf 'partial archive bytes\n' > "$dir/$asset" + echo "release not found" >&2 + exit 1 +fi +if [ "${GH_FAKE_OMIT_ON_SUCCESS_COUNT:-0}" = "$count" ]; then + exit 0 +fi cp "$GH_ASSET_FIXTURE" "$dir/$asset" EOF chmod +x "$BIN/gh" @@ -35,6 +50,11 @@ printf 'canonical archive bytes\n' > "$fixture" sha=$(shasum -a 256 "$fixture" | awk '{print $1}') size=$(wc -c < "$fixture" | tr -d '[:space:]') asset="libcurl-8.11.1-rev3-abi39-wasm32-d0c9d681.tar.zst" +export GH_FAKE_COUNT_FILE="$TMP_ROOT/count" +export GH_FAKE_ARGS_FILE="$TMP_ROOT/args" +export GH_FAKE_OMIT_ON_SUCCESS_COUNT=0 +export GH_FAKE_SUCCEED_ON=1 +export RELEASE_DOWNLOAD_RETRY_SECONDS=0 run_download() { GH_ASSET_FIXTURE="$fixture" GITHUB_REPOSITORY=example/repo PATH="$BIN:$PATH" \ @@ -46,10 +66,49 @@ run_download() { --output "$3" } +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" run_download "$sha" "$size" "$TMP_ROOT/good/$asset" >/dev/null cmp "$fixture" "$TMP_ROOT/good/$asset" +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ] + +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" +export GH_FAKE_SUCCEED_ON=3 +export RELEASE_DOWNLOAD_ATTEMPTS=4 +run_download "$sha" "$size" "$TMP_ROOT/retried/$asset" \ + >"$TMP_ROOT/retried.out" 2>"$TMP_ROOT/retried.err" +cmp "$fixture" "$TMP_ROOT/retried/$asset" +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 3 ] +[ "$(wc -l < "$GH_FAKE_ARGS_FILE" | tr -d '[:space:]')" = 3 ] +[ "$(sort -u "$GH_FAKE_ARGS_FILE")" = $'example/repo\tbinaries-abi-v39\tlibcurl-8.11.1-rev3-abi39-wasm32-d0c9d681.tar.zst' ] +grep -q 'attempt 2/4' "$TMP_ROOT/retried.err" + +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" +export GH_FAKE_OMIT_ON_SUCCESS_COUNT=1 +export GH_FAKE_SUCCEED_ON=1 +export RELEASE_DOWNLOAD_ATTEMPTS=3 +run_download "$sha" "$size" "$TMP_ROOT/missing-after-success/$asset" \ + >"$TMP_ROOT/missing-after-success.out" 2>"$TMP_ROOT/missing-after-success.err" +cmp "$fixture" "$TMP_ROOT/missing-after-success/$asset" +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 2 ] +grep -q 'reported success without a regular' "$TMP_ROOT/missing-after-success.err" +export GH_FAKE_OMIT_ON_SUCCESS_COUNT=0 + +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" +export GH_FAKE_SUCCEED_ON=99 +export RELEASE_DOWNLOAD_ATTEMPTS=3 +if run_download "$sha" "$size" "$TMP_ROOT/missing/$asset" \ + >"$TMP_ROOT/missing.out" 2>"$TMP_ROOT/missing.err" +then + echo "download verifier accepted a permanently unavailable asset" >&2 + exit 1 +fi +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 3 ] +[ ! -e "$TMP_ROOT/missing/$asset" ] +grep -q 'failed after 3 attempts' "$TMP_ROOT/missing.err" bad_sha=$(printf '0%.0s' {1..64}) +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" +export GH_FAKE_SUCCEED_ON=1 if run_download "$bad_sha" "$size" "$TMP_ROOT/bad-sha/$asset" \ >"$TMP_ROOT/bad-sha.out" 2>"$TMP_ROOT/bad-sha.err" then @@ -57,8 +116,10 @@ then exit 1 fi [ ! -e "$TMP_ROOT/bad-sha/$asset" ] +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ] grep -q 'does not match snapshot' "$TMP_ROOT/bad-sha.err" +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" if run_download "$sha" "$((size + 1))" "$TMP_ROOT/bad-size/$asset" \ >"$TMP_ROOT/bad-size.out" 2>"$TMP_ROOT/bad-size.err" then @@ -66,6 +127,18 @@ then exit 1 fi [ ! -e "$TMP_ROOT/bad-size/$asset" ] +[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ] grep -q 'does not match snapshot' "$TMP_ROOT/bad-size.err" +rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE" +export RELEASE_DOWNLOAD_ATTEMPTS=0 +if run_download "$sha" "$size" "$TMP_ROOT/bad-attempts/$asset" \ + >"$TMP_ROOT/bad-attempts.out" 2>"$TMP_ROOT/bad-attempts.err" +then + echo "download verifier accepted an invalid retry count" >&2 + exit 1 +fi +[ ! -e "$GH_FAKE_COUNT_FILE" ] +grep -q 'attempts must be positive' "$TMP_ROOT/bad-attempts.err" + echo "verified release asset download tests passed" diff --git a/.github/scripts/test-merge-candidate-workflows.sh b/.github/scripts/test-merge-candidate-workflows.sh index c082cd5e26..fb02009132 100755 --- a/.github/scripts/test-merge-candidate-workflows.sh +++ b/.github/scripts/test-merge-candidate-workflows.sh @@ -170,8 +170,14 @@ grep -Fq 'select-package-archive-source.sh' "$PREPARE" || \ fail "Prepare merge must prefer an existing canonical cache-key asset" grep -Fq 'download-verified-release-asset.sh' "$PREPARE" || \ fail "Prepare merge must verify snapshotted source asset bytes before promotion" -grep -Fq -- '--tag "${{ matrix.source_tag }}"' "$PREPARE" || \ +grep -Fq 'ARCHIVE_NAME: ${{ matrix.archive_name }}' "$PREPARE" || \ + fail "Prepare merge must pass the selected archive name through the environment" +grep -Fq 'SOURCE_TAG: ${{ matrix.source_tag }}' "$PREPARE" || \ + fail "Prepare merge must pass the selected source release through the environment" +grep -Fq -- '--tag "$SOURCE_TAG"' "$PREPARE" || \ fail "Prepare merge promotion must download from the selected source release" +grep -Fq -- '--asset "$ARCHIVE_NAME"' "$PREPARE" || \ + fail "Prepare merge promotion must download the selected archive without shell interpolation" grep -Fq 'if select_match "$CANONICAL_ASSETS" canonical' "$ARCHIVE_SOURCE_SCRIPT" || \ fail "canonical cache-key bytes must take precedence over PR staging bytes" grep -Fq 'actual_sha256' "$ARCHIVE_DOWNLOAD_SCRIPT" || \ diff --git a/.github/workflows/prepare-merge.yml b/.github/workflows/prepare-merge.yml index 328ca962c1..374a3b3085 100644 --- a/.github/workflows/prepare-merge.yml +++ b/.github/workflows/prepare-merge.yml @@ -978,20 +978,24 @@ jobs: - name: Download verified source archive id: archive env: + ARCHIVE_NAME: ${{ matrix.archive_name }} + ARCHIVE_SHA256: ${{ matrix.archive_sha256 }} + ARCHIVE_SIZE: ${{ matrix.archive_size }} GH_TOKEN: ${{ github.token }} GITHUB_REPOSITORY: ${{ github.repository }} + SOURCE_TAG: ${{ matrix.source_tag }} run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/promote" - archive_path="$RUNNER_TEMP/promote/${{ matrix.archive_name }}" + archive_path="$RUNNER_TEMP/promote/$ARCHIVE_NAME" bash .github/scripts/download-verified-release-asset.sh \ - --tag "${{ matrix.source_tag }}" \ - --asset "${{ matrix.archive_name }}" \ - --sha256 "${{ matrix.archive_sha256 }}" \ - --size "${{ matrix.archive_size }}" \ + --tag "$SOURCE_TAG" \ + --asset "$ARCHIVE_NAME" \ + --sha256 "$ARCHIVE_SHA256" \ + --size "$ARCHIVE_SIZE" \ --output "$archive_path" echo "archive-path=$archive_path" >> "$GITHUB_OUTPUT" - echo "archive-name=${{ matrix.archive_name }}" >> "$GITHUB_OUTPUT" + echo "archive-name=$ARCHIVE_NAME" >> "$GITHUB_OUTPUT" - name: Promote archive + index entry env: