CI: retry verified staged release downloads

.github/scripts/download-verified-release-asset.sh

@@ -31,6 +31,17 @@ REPOSITORY="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY required}"
 TMP_ROOT="$(mktemp -d)"
 trap 'rm -rf "$TMP_ROOT"' EXIT
 
+ATTEMPTS="${RELEASE_DOWNLOAD_ATTEMPTS:-4}"
+RETRY_SECONDS="${RELEASE_DOWNLOAD_RETRY_SECONDS:-2}"
+if ! [[ "$ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then
+  echo "download-verified-release-asset: attempts must be positive: $ATTEMPTS" >&2
+  exit 2
+fi
+if ! [[ "$RETRY_SECONDS" =~ ^[0-9]+$ ]]; then
+  echo "download-verified-release-asset: retry seconds must be non-negative: $RETRY_SECONDS" >&2
+  exit 2
+fi
+
 sha256_file() {
   if command -v sha256sum >/dev/null 2>&1; then
     sha256sum "$1" | awk '{print $1}'
@@ -39,15 +50,51 @@ sha256_file() {
   fi
 }
 
-gh release download "$TAG" \
-  --repo "$REPOSITORY" \
-  --pattern "$ASSET" \
-  --dir "$TMP_ROOT" \
-  --clobber
+attempt=1
+delay="$RETRY_SECONDS"
+downloaded=""
+while [ "$attempt" -le "$ATTEMPTS" ]; do
+  attempt_dir="$TMP_ROOT/attempt-$attempt"
+  stdout_file="$TMP_ROOT/stdout-$attempt"
+  stderr_file="$TMP_ROOT/stderr-$attempt"
+  mkdir "$attempt_dir"
+
+  if gh release download "$TAG" \
+      --repo "$REPOSITORY" \
+      --pattern "$ASSET" \
+      --dir "$attempt_dir" \
+      --clobber >"$stdout_file" 2>"$stderr_file"; then
+    candidate="$attempt_dir/$ASSET"
+    if [ -f "$candidate" ] && [ ! -L "$candidate" ]; then
+      cat "$stdout_file"
+      downloaded="$candidate"
+      break
+    fi
+    echo "download-verified-release-asset: GitHub reported success without a regular $ASSET" >&2
+    rc=1
+  else
+    rc=$?
+  fi
+
+  cat "$stderr_file" >&2
+  if [ -s "$stdout_file" ]; then
+    cat "$stdout_file" >&2
+  fi
+  rm -rf "$attempt_dir"
+
+  if [ "$attempt" -ge "$ATTEMPTS" ]; then
+    echo "download-verified-release-asset: failed after $ATTEMPTS attempts: $TAG/$ASSET" >&2
+    exit "$rc"
+  fi
+
+  echo "::warning::release asset download failed (attempt $attempt/$ATTEMPTS); retrying in ${delay}s: $TAG/$ASSET" >&2
+  sleep "$delay"
+  attempt=$((attempt + 1))
+  delay=$((delay * 2))
+done
 
-downloaded="$TMP_ROOT/$ASSET"
-if [ ! -f "$downloaded" ]; then
-  echo "download-verified-release-asset: $TAG did not yield $ASSET" >&2
+if [ -z "$downloaded" ]; then
+  echo "download-verified-release-asset: no verified download candidate for $TAG/$ASSET" >&2
   exit 1
 fi
 actual_size=$(wc -c < "$downloaded" | tr -d '[:space:]')

.github/scripts/test-download-verified-release-asset.sh

@@ -15,17 +15,32 @@ set -euo pipefail
 shift 2
 tag="$1"
 shift
-asset=""; dir=""
+repository=""; asset=""; dir=""
 while [ "$#" -gt 0 ]; do
   case "$1" in
-    --repo) shift 2 ;;
+    --repo) repository="$2"; shift 2 ;;
     --pattern) asset="$2"; shift 2 ;;
     --dir) dir="$2"; shift 2 ;;
     --clobber) shift ;;
     *) exit 99 ;;
   esac
 done
 [ "$tag" = binaries-abi-v39 ]
+printf '%s\t%s\t%s\n' "$repository" "$tag" "$asset" >> "$GH_FAKE_ARGS_FILE"
+count=0
+if [ -f "$GH_FAKE_COUNT_FILE" ]; then
+  count="$(cat "$GH_FAKE_COUNT_FILE")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" > "$GH_FAKE_COUNT_FILE"
+if [ "$count" -lt "$GH_FAKE_SUCCEED_ON" ]; then
+  printf 'partial archive bytes\n' > "$dir/$asset"
+  echo "release not found" >&2
+  exit 1
+fi
+if [ "${GH_FAKE_OMIT_ON_SUCCESS_COUNT:-0}" = "$count" ]; then
+  exit 0
+fi
 cp "$GH_ASSET_FIXTURE" "$dir/$asset"
 EOF
 chmod +x "$BIN/gh"
@@ -35,6 +50,11 @@ printf 'canonical archive bytes\n' > "$fixture"
 sha=$(shasum -a 256 "$fixture" | awk '{print $1}')
 size=$(wc -c < "$fixture" | tr -d '[:space:]')
 asset="libcurl-8.11.1-rev3-abi39-wasm32-d0c9d681.tar.zst"
+export GH_FAKE_COUNT_FILE="$TMP_ROOT/count"
+export GH_FAKE_ARGS_FILE="$TMP_ROOT/args"
+export GH_FAKE_OMIT_ON_SUCCESS_COUNT=0
+export GH_FAKE_SUCCEED_ON=1
+export RELEASE_DOWNLOAD_RETRY_SECONDS=0
 
 run_download() {
   GH_ASSET_FIXTURE="$fixture" GITHUB_REPOSITORY=example/repo PATH="$BIN:$PATH" \
@@ -46,26 +66,79 @@ run_download() {
       --output "$3"
 }
 
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
 run_download "$sha" "$size" "$TMP_ROOT/good/$asset" >/dev/null
 cmp "$fixture" "$TMP_ROOT/good/$asset"
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ]
+
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
+export GH_FAKE_SUCCEED_ON=3
+export RELEASE_DOWNLOAD_ATTEMPTS=4
+run_download "$sha" "$size" "$TMP_ROOT/retried/$asset" \
+  >"$TMP_ROOT/retried.out" 2>"$TMP_ROOT/retried.err"
+cmp "$fixture" "$TMP_ROOT/retried/$asset"
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 3 ]
+[ "$(wc -l < "$GH_FAKE_ARGS_FILE" | tr -d '[:space:]')" = 3 ]
+[ "$(sort -u "$GH_FAKE_ARGS_FILE")" = $'example/repo\tbinaries-abi-v39\tlibcurl-8.11.1-rev3-abi39-wasm32-d0c9d681.tar.zst' ]
+grep -q 'attempt 2/4' "$TMP_ROOT/retried.err"
+
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
+export GH_FAKE_OMIT_ON_SUCCESS_COUNT=1
+export GH_FAKE_SUCCEED_ON=1
+export RELEASE_DOWNLOAD_ATTEMPTS=3
+run_download "$sha" "$size" "$TMP_ROOT/missing-after-success/$asset" \
+  >"$TMP_ROOT/missing-after-success.out" 2>"$TMP_ROOT/missing-after-success.err"
+cmp "$fixture" "$TMP_ROOT/missing-after-success/$asset"
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 2 ]
+grep -q 'reported success without a regular' "$TMP_ROOT/missing-after-success.err"
+export GH_FAKE_OMIT_ON_SUCCESS_COUNT=0
+
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
+export GH_FAKE_SUCCEED_ON=99
+export RELEASE_DOWNLOAD_ATTEMPTS=3
+if run_download "$sha" "$size" "$TMP_ROOT/missing/$asset" \
+    >"$TMP_ROOT/missing.out" 2>"$TMP_ROOT/missing.err"
+then
+  echo "download verifier accepted a permanently unavailable asset" >&2
+  exit 1
+fi
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 3 ]
+[ ! -e "$TMP_ROOT/missing/$asset" ]
+grep -q 'failed after 3 attempts' "$TMP_ROOT/missing.err"
 
 bad_sha=$(printf '0%.0s' {1..64})
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
+export GH_FAKE_SUCCEED_ON=1
 if run_download "$bad_sha" "$size" "$TMP_ROOT/bad-sha/$asset" \
     >"$TMP_ROOT/bad-sha.out" 2>"$TMP_ROOT/bad-sha.err"
 then
   echo "download verifier accepted bytes that disagreed with the snapshot digest" >&2
   exit 1
 fi
 [ ! -e "$TMP_ROOT/bad-sha/$asset" ]
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ]
 grep -q 'does not match snapshot' "$TMP_ROOT/bad-sha.err"
 
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
 if run_download "$sha" "$((size + 1))" "$TMP_ROOT/bad-size/$asset" \
     >"$TMP_ROOT/bad-size.out" 2>"$TMP_ROOT/bad-size.err"
 then
   echo "download verifier accepted bytes that disagreed with the snapshot size" >&2
   exit 1
 fi
 [ ! -e "$TMP_ROOT/bad-size/$asset" ]
+[ "$(cat "$GH_FAKE_COUNT_FILE")" = 1 ]
 grep -q 'does not match snapshot' "$TMP_ROOT/bad-size.err"
 
+rm -f "$GH_FAKE_COUNT_FILE" "$GH_FAKE_ARGS_FILE"
+export RELEASE_DOWNLOAD_ATTEMPTS=0
+if run_download "$sha" "$size" "$TMP_ROOT/bad-attempts/$asset" \
+    >"$TMP_ROOT/bad-attempts.out" 2>"$TMP_ROOT/bad-attempts.err"
+then
+  echo "download verifier accepted an invalid retry count" >&2
+  exit 1
+fi
+[ ! -e "$GH_FAKE_COUNT_FILE" ]
+grep -q 'attempts must be positive' "$TMP_ROOT/bad-attempts.err"
+
 echo "verified release asset download tests passed"

.github/scripts/test-merge-candidate-workflows.sh

@@ -170,8 +170,14 @@ grep -Fq 'select-package-archive-source.sh' "$PREPARE" || \
   fail "Prepare merge must prefer an existing canonical cache-key asset"
 grep -Fq 'download-verified-release-asset.sh' "$PREPARE" || \
   fail "Prepare merge must verify snapshotted source asset bytes before promotion"
-grep -Fq -- '--tag "${{ matrix.source_tag }}"' "$PREPARE" || \
+grep -Fq 'ARCHIVE_NAME: ${{ matrix.archive_name }}' "$PREPARE" || \
+  fail "Prepare merge must pass the selected archive name through the environment"
+grep -Fq 'SOURCE_TAG: ${{ matrix.source_tag }}' "$PREPARE" || \
+  fail "Prepare merge must pass the selected source release through the environment"
+grep -Fq -- '--tag "$SOURCE_TAG"' "$PREPARE" || \
   fail "Prepare merge promotion must download from the selected source release"
+grep -Fq -- '--asset "$ARCHIVE_NAME"' "$PREPARE" || \
+  fail "Prepare merge promotion must download the selected archive without shell interpolation"
 grep -Fq 'if select_match "$CANONICAL_ASSETS" canonical' "$ARCHIVE_SOURCE_SCRIPT" || \
   fail "canonical cache-key bytes must take precedence over PR staging bytes"
 grep -Fq 'actual_sha256' "$ARCHIVE_DOWNLOAD_SCRIPT" || \

.github/workflows/prepare-merge.yml

@@ -978,20 +978,24 @@ jobs:
       - name: Download verified source archive
         id: archive
         env:
+          ARCHIVE_NAME: ${{ matrix.archive_name }}
+          ARCHIVE_SHA256: ${{ matrix.archive_sha256 }}
+          ARCHIVE_SIZE: ${{ matrix.archive_size }}
           GH_TOKEN: ${{ github.token }}
           GITHUB_REPOSITORY: ${{ github.repository }}
+          SOURCE_TAG: ${{ matrix.source_tag }}
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/promote"
-          archive_path="$RUNNER_TEMP/promote/${{ matrix.archive_name }}"
+          archive_path="$RUNNER_TEMP/promote/$ARCHIVE_NAME"
           bash .github/scripts/download-verified-release-asset.sh \
-            --tag "${{ matrix.source_tag }}" \
-            --asset "${{ matrix.archive_name }}" \
-            --sha256 "${{ matrix.archive_sha256 }}" \
-            --size "${{ matrix.archive_size }}" \
+            --tag "$SOURCE_TAG" \
+            --asset "$ARCHIVE_NAME" \
+            --sha256 "$ARCHIVE_SHA256" \
+            --size "$ARCHIVE_SIZE" \
             --output "$archive_path"
           echo "archive-path=$archive_path" >> "$GITHUB_OUTPUT"
-          echo "archive-name=${{ matrix.archive_name }}" >> "$GITHUB_OUTPUT"
+          echo "archive-name=$ARCHIVE_NAME" >> "$GITHUB_OUTPUT"
 
       - name: Promote archive + index entry
         env: