diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index d622c1291c00..12afad4bb933 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -125,12 +125,19 @@ env: IMAGE: ubuntu-26.04 SSH_KEY_ID: '107239874' EPHEMERAL_RUNNER_NAME: ps-arm64-${{ github.run_id }}-${{ github.run_attempt }} + # x86_64 chain uses a hostname-safe name (no underscore) for the Hetzner + # server and GitHub runner; the arch tag is "amd64", not "x86_64". + EPHEMERAL_RUNNER_NAME_AMD64: ps-amd64-${{ github.run_id }}-${{ github.run_attempt }} # PS-11179: EC2 fallback knobs. eu-central-1 chosen to match the existing # Jenkins eu-central-1 footprint (low latency to EU contributors, no new # cross-region S3/ECR traffic for boost / ccache restore). c7g.4xlarge gives # 16 vCPU + 32 GB Graviton3, which is parity with the cax41 default. AWS_REGION: eu-central-1 AWS_INSTANCE_TYPE: c7g.4xlarge + # PS-11298: x86_64 nightly gcc-16 sibling chain. c7i.4xlarge gives 16 vCPU + # + 32 GB (cax41/c7g.4xlarge parity on the Intel side). Added as a new key so + # the arm64 AWS_INSTANCE_TYPE default stays unchanged. + AWS_INSTANCE_TYPE_X86_64: c7i.4xlarge jobs: # TRUSTED authorization gate (PS-11254). Root of the job graph: dispatch @@ -1168,6 +1175,7 @@ jobs: "Name=tag:github_run_attempt,Values=${{ github.run_attempt }}" \ "Name=tag:github_repository,Values=${{ github.repository }}" \ "Name=tag:iit-billing-tag,Values=percona-server-gha-fallback" \ + "Name=tag:Name,Values=ps-arm64-${{ github.run_id }}-${{ github.run_attempt }}" \ "Name=instance-state-name,Values=pending,running,stopping,stopped" \ --query 'Reservations[].Instances[].InstanceId' --output text 2>/dev/null || true) # Union with whatever the create job published; dedupe via sort -u. @@ -1199,3 +1207,894 @@ jobs: else echo "Runner $EPHEMERAL_RUNNER_NAME already de-registered (ephemeral self-removal)" fi + + # =========================================================================== + # PS-11298: x86_64 gcc-16 nightly chain. Sibling jobs that mirror the arm64 + # chain above (pick-target / create-runner-* / build / delete-runner-*) with + # x86-specific deltas. ALL gated to the nightly/RelWithDebInfo dispatch path + # (workflow_dispatch + build_type RelWithDebInfo) so they NEVER run on the + # per-PR Debug builds; the arm64 chain above is unchanged. + # + # Deltas vs arm64: + # - Hetzner probe: cpx62 (single x86 tier) instead of cax41/cax31/cax21. + # - EC2: c7i.4xlarge + amd64 AMI + linux-x64 runner tarball. + # - Compiler: COMPILER_VER='16' -> gcc-16/g++-16 (Ubuntu 26.04 archive; no PPA). + # - Shared-state isolation so the two chains can co-run in one nightly: + # * distinct ephemeral runner name (ps-amd64-... suffix), and + # * ccache key includes COMPILER_VER + x86_64 so the gcc-16 build does + # not share a cache slot with the arm64 gcc-15 build. + # + # Trust split is identical to the arm64 chain: pick-target-x86_64 / + # create-runner-*-x86_64 / delete-runner-*-x86_64 do NO checkout and hold the + # infra secrets; build-x86_64 is the ONLY job that checks out PR code and it + # holds no secrets (read-only GITHUB_TOKEN). + # =========================================================================== + + # TRUSTED. x86_64 capacity probe (mirror of pick-target). Single cpx62 tier + # swept across fsn1/hel1/nbg1. Emits provider/location/server_type identically. + pick-target-x86_64: + needs: dispatch + if: github.event_name == 'workflow_dispatch' && inputs.build_type == 'RelWithDebInfo' + runs-on: ubuntu-latest + permissions: {} + timeout-minutes: 30 + outputs: + provider: ${{ steps.probe.outputs.provider }} + location: ${{ steps.probe.outputs.location }} + server_type: ${{ steps.probe.outputs.server_type }} + steps: + - id: probe + env: + HCLOUD_TOKEN: ${{ secrets.GHA_RUNNER_HCLOUD_TOKEN }} + FORCE_PROVIDER: ${{ github.event_name == 'workflow_dispatch' && inputs.force_provider || 'auto' }} + run: | + set -euo pipefail + + # PS-11179: force_provider override (workflow_dispatch only; defaults + # to "auto" for pull_request / pull_request_target, and for the nightly + # dispatched as workflow_dispatch). Lets operators validate the + # EC2 path on demand without waiting for a real Hetzner outage, or + # pin to Hetzner-only when AWS spend must be avoided. + case "$FORCE_PROVIDER" in + aws) + echo "::notice::force_provider=aws -> skipping Hetzner probing, going straight to EC2 fallback" + echo "### Provider override: AWS (forced via workflow_dispatch input)" >> "$GITHUB_STEP_SUMMARY" + { + echo "provider=aws" + echo "location=" + echo "server_type=" + } >> "$GITHUB_OUTPUT" + exit 0 + ;; + hetzner) + echo "::notice::force_provider=hetzner -> EC2 fallback disabled; will fail if Hetzner exhausted" + ;; + auto) + ;; + *) + echo "::error::Unrecognised force_provider value: $FORCE_PROVIDER (expected auto|hetzner|aws)" + exit 1 + ;; + esac + + # PS-11179: backoff curve in minutes between sweep attempts. + # Index 0 is the gap AFTER sweep 1 fails (before sweep 2), etc. + # PS-11254: 3 entries == 3 retries after the initial sweep == 4 sweeps + # total (~17m: 2+5+10), so the AWS EC2 fallback takes over fast instead + # of burning ~3h on a Hetzner capacity outage. + BACKOFF_MIN=(2 5 10) + MAX_SWEEPS=4 + + for sweep in $(seq 1 "$MAX_SWEEPS"); do + echo "::group::Hetzner capacity sweep $sweep/$MAX_SWEEPS" + for type in cpx62; do + for dc in fsn1 hel1 nbg1; do + PROBE_NAME="cap-probe-x86-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-s${sweep}-${type}-${dc}" + HTTP=$(curl -sS -o /tmp/resp.json -w "%{http_code}" -X POST \ + -H "Authorization: Bearer $HCLOUD_TOKEN" -H "Content-Type: application/json" \ + https://api.hetzner.cloud/v1/servers \ + -d "{\"name\":\"$PROBE_NAME\",\"server_type\":\"$type\",\"image\":\"$IMAGE\",\"location\":\"$dc\",\"start_after_create\":false}") + if [ "$HTTP" = "201" ]; then + ID=$(jq -r '.server.id // empty' /tmp/resp.json) + [ -n "$ID" ] || { echo "::error::201 without server id"; exit 1; } + curl -fsS -X DELETE -H "Authorization: Bearer $HCLOUD_TOKEN" \ + "https://api.hetzner.cloud/v1/servers/$ID" >/dev/null + if [ "$sweep" -gt 1 ]; then + echo "::notice::Hetzner capacity recovered on sweep $sweep" + fi + echo "Hetzner target: $type in $dc (sweep $sweep/$MAX_SWEEPS)" >> "$GITHUB_STEP_SUMMARY" + { + echo "provider=hetzner" + echo "location=$dc" + echo "server_type=$type" + } >> "$GITHUB_OUTPUT" + echo "::endgroup::" + exit 0 + fi + ERR=$(jq -r '.error.code // "?"' /tmp/resp.json 2>/dev/null || echo "?") + echo "::warning::sweep $sweep: $type/$dc: HTTP $HTTP ($ERR)" + done + done + echo "::endgroup::" + + # Sleep before the next sweep unless this was the last one. + if [ "$sweep" -lt "$MAX_SWEEPS" ]; then + idx=$((sweep - 1)) + wait_min="${BACKOFF_MIN[$idx]}" + echo "::notice::All cpx62 combos exhausted on sweep $sweep; sleeping ${wait_min}m before sweep $((sweep + 1))" + sleep $((wait_min * 60)) + fi + done + + # PS-11179: All sweeps exhausted; switch to EC2 path (unless + # force_provider=hetzner explicitly disabled fallback). + if [ "$FORCE_PROVIDER" = "hetzner" ]; then + echo "::error::Hetzner cpx62 exhausted after $MAX_SWEEPS sweeps AND force_provider=hetzner; failing per request" + exit 1 + fi + echo "::warning::Hetzner cpx62 capacity exhausted after $MAX_SWEEPS sweeps (~17m wall time); falling back to AWS EC2 c7i.4xlarge in $AWS_REGION" + { + echo "### EC2 fallback fired (PS-11298 x86_64)" + echo "" + echo "Hetzner cpx62 across fsn1/hel1/nbg1 had no capacity" + echo "across $MAX_SWEEPS sweeps spanning ~17m. Falling back to AWS" + echo "EC2 c7i.4xlarge (x86_64) in eu-central-1." + } >> "$GITHUB_STEP_SUMMARY" + { + echo "provider=aws" + echo "location=" + echo "server_type=" + } >> "$GITHUB_OUTPUT" + + # TRUSTED. Provision the ephemeral Hetzner x86_64 runner (mirror of + # create-runner-hetzner). image stays ubuntu-26.04: Hetzner resolves the x86 + # image variant from the x86 server_type (cpx62). + create-runner-hetzner-x86_64: + needs: [dispatch, pick-target-x86_64] + if: needs.pick-target-x86_64.outputs.provider == 'hetzner' + runs-on: ubuntu-latest + permissions: {} + outputs: + runner_label: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + server_id: ${{ steps.create.outputs.server_id }} + server_type: ${{ needs.pick-target-x86_64.outputs.server_type }} + steps: + - id: create + uses: olexandr-havryliak/hcloud-github-runner@bb1089d8b718a06493cb37c51dfe596e44baefc2 + with: + mode: create + name: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + hcloud_token: ${{ secrets.GHA_RUNNER_HCLOUD_TOKEN }} + github_token: ${{ secrets.GHA_RUNNER_PAT }} + image: ${{ env.IMAGE }} + location: ${{ needs.pick-target-x86_64.outputs.location }} + server_type: ${{ needs.pick-target-x86_64.outputs.server_type }} + ssh_key: ${{ env.SSH_KEY_ID }} + # Default budget is 60x10s = 10 min; bumped to 20 min absorbs transient + # apt-mirror or GitHub-release slowness during bootstrap. + runner_wait: '120' + # Retry apt with backoff for transient apt-mirror hiccups on first + # cloud-init (Codex diagnosis 2026-05-22). + pre_runner_script: | + set -euxo pipefail + export DEBIAN_FRONTEND=noninteractive + ok=0 + for i in 1 2 3; do + if apt-get update -y && apt-get install -y --no-install-recommends curl ca-certificates jq; then + ok=1 + break + fi + sleep $((i * 15)) + done + [ "$ok" = 1 ] + + # On create-runner failure with workflow_dispatch `debug_keep_vm` set, + # surface VM IP + SSH instructions in the run summary BEFORE delete-runner + # reaps the VM. Anderson SSHs in manually with his personal key + # (107239874) to grab cloud-init logs and the runner _diag dir. + - name: Preserve VM for manual diagnosis (on failure, debug-only) + if: failure() && inputs.debug_keep_vm == true + env: + HCLOUD_TOKEN: ${{ secrets.GHA_RUNNER_HCLOUD_TOKEN }} + SERVER_ID: ${{ steps.create.outputs.server_id }} + run: | + set -euo pipefail + [ -n "${SERVER_ID:-}" ] || { echo "::warning::No server_id; nothing to preserve"; exit 0; } + IP=$(curl -fsS -H "Authorization: Bearer $HCLOUD_TOKEN" \ + "https://api.hetzner.cloud/v1/servers/$SERVER_ID" \ + | jq -r '.server.public_net.ipv4.ip') + { + echo "### Debug VM kept alive for manual diagnosis" + echo "" + echo "Server ID: \`$SERVER_ID\`" + echo "Public IPv4: \`$IP\`" + echo "" + echo "**SSH** (key 107239874, \`anderson@percona\`):" + echo "\`\`\`bash" + echo "ssh root@$IP" + echo "\`\`\`" + echo "" + echo "**Files to grab:**" + echo "- \`/var/log/cloud-init.log\`" + echo "- \`/var/log/cloud-init-output.log\`" + echo "- \`/actions-runner/_diag/\` (if it exists)" + echo "- \`/actions-runner/.runner\` + \`.credentials\` (if registration partially completed)" + echo "" + echo "**Cleanup when done:**" + echo "\`\`\`bash" + echo "curl -X DELETE -H \"Authorization: Bearer \$HCLOUD_TOKEN\" \\" + echo " https://api.hetzner.cloud/v1/servers/$SERVER_ID" + echo "\`\`\`" + echo "" + echo "orphan-sweep.yml will reap this VM after 6h regardless." + } >> "$GITHUB_STEP_SUMMARY" + + # TRUSTED. Provision the ephemeral EC2 x86_64 runner (mirror of + # create-runner-aws). c7i.4xlarge + amd64 AMI + linux-x64 runner tarball. + # No checkout; no instance profile; OIDC -> STS only. + create-runner-aws-x86_64: + needs: [dispatch, pick-target-x86_64] + if: needs.pick-target-x86_64.outputs.provider == 'aws' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + outputs: + runner_label: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + instance_id: ${{ steps.run.outputs.instance_id }} + az: ${{ steps.run.outputs.az }} + market: ${{ steps.run.outputs.market }} + steps: + - name: Configure AWS credentials via OIDC + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: gha-${{ github.run_id }}-${{ github.run_attempt }}-x86_64 + aws-region: ${{ env.AWS_REGION }} + + - name: Mint GitHub runner registration token + id: regtoken + env: + GH_TOKEN: ${{ secrets.GHA_RUNNER_PAT }} + run: | + set -euo pipefail + TOKEN=$(gh api -X POST /repos/${{ github.repository }}/actions/runners/registration-token --jq .token) + if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then + echo "::error::Failed to mint runner registration token" + exit 1 + fi + # Mark as a workflow secret so accidental log echoes are scrubbed. + echo "::add-mask::$TOKEN" + echo "token=$TOKEN" >> "$GITHUB_OUTPUT" + + # Resolve the current Canonical Ubuntu 26.04 amd64 AMI from SSM at + # runtime. Canonical refreshes this parameter weekly with security + # patches; never hardcode the AMI ID in this workflow. + - name: Resolve Ubuntu 26.04 amd64 AMI + id: ami + run: | + set -euo pipefail + AMI=$(aws ssm get-parameter \ + --name /aws/service/canonical/ubuntu/server/26.04/stable/current/amd64/hvm/ebs-gp3/ami-id \ + --region "$AWS_REGION" \ + --query Parameter.Value --output text) + if [ -z "$AMI" ] || [ "$AMI" = "None" ]; then + echo "::error::Failed to resolve Ubuntu 26.04 amd64 AMI from SSM" + exit 1 + fi + echo "Resolved AMI: $AMI" + echo "ami=$AMI" >> "$GITHUB_OUTPUT" + + # Build the userData payload inline (P0-1 fix). See create-runner-aws + # for the full trust-split rationale; identical here. + # + # Variables exported into this step: + # RUNNER_TOKEN one-shot ~1h-TTL GitHub registration token + # RUNNER_LABEL unique runner name (=== EPHEMERAL_RUNNER_NAME-x86_64) + # RUNNER_URL https://github.com// + # RUNNER_VERSION pinned actions/runner release (without leading v) + # RUNNER_SHA256 SHA256 of actions-runner-linux-x64-.tar.gz + - name: Render userData (inline; no checkout) + env: + RUNNER_TOKEN: ${{ steps.regtoken.outputs.token }} + RUNNER_LABEL: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + RUNNER_URL: https://github.com/${{ github.repository }} + # Pinned actions/runner release. Bump together with RUNNER_SHA256. + # GitHub deprecates runner versions (typically 4-6 months after + # release); a deprecated runner registers fine but is immediately + # rejected with "cannot receive messages" and exits. Verify the pin + # against https://github.com/actions/runner/releases periodically. + # Last bump 2026-05-28 (v2.328.0 -> v2.334.0; 2.328.0 was deprecated). + RUNNER_VERSION: '2.334.0' + # SHA256 of actions-runner-linux-x64-2.334.0.tar.gz from the + # actions/runner GitHub release page (BEGIN/END SHA linux-x64 markers). + RUNNER_SHA256: '048024cd2c848eb6f14d5646d56c13a4def2ae7ee3ad12122bee960c56f3d271' + run: | + set -euo pipefail + mkdir -p /tmp/ec2 + # Fail fast if any required variable is empty; we'd rather not + # ship a half-rendered userData to AWS. + : "${RUNNER_TOKEN:?registration token missing}" + : "${RUNNER_LABEL:?runner name missing}" + : "${RUNNER_URL:?runner url missing}" + : "${RUNNER_VERSION:?runner version pin missing}" + : "${RUNNER_SHA256:?runner sha256 pin missing}" + + # Heredoc WITHOUT a quoted delimiter -> the workflow shell expands + # ${VAR} at write time. PR code never touches this content. + # YAML forces a multi-line block to share an indentation prefix; + # we strip exactly 10 leading spaces on every heredoc body line + # via sed before writing so cloud-init sees a script starting at + # column 1 (the `#!` shebang must be column 1 or cloud-init + # refuses to exec it). + sed -E 's/^ //' < /tmp/ec2/userdata.sh + #!/bin/bash + # Inlined from .github/workflows/builds.yml (create-runner-aws-x86_64). + # This heredoc is the single source of truth for the runner + # bootstrap script. Do not duplicate it to a separate file + # (would be a PR-poisonable trusted-bootstrap path; PS-11219). + set -euxo pipefail + exec > >(tee -a /var/log/gha-runner.log) 2>&1 + + # Self-shutdown on any exit path. RunInstances was called with + # --instance-initiated-shutdown-behavior terminate, so this + # reaps the instance rather than just powering it off. + trap '/sbin/shutdown -h now' EXIT + + echo "Starting userData at \$(date -u +%FT%TZ)" + echo "Resolved RUNNER_VERSION=${RUNNER_VERSION}" + echo "Resolved RUNNER_LABEL=${RUNNER_LABEL}" + + export DEBIAN_FRONTEND=noninteractive + + # Install runtime deps with retry-on-flake. + ok=0 + for i in 1 2 3; do + if apt-get update -y \\ + && apt-get install -y --no-install-recommends \\ + curl ca-certificates jq tar libicu78 git sudo; then + ok=1 + break + fi + sleep \$((i * 15)) + done + [ "\$ok" = 1 ] || { echo "apt install failed after 3 attempts"; exit 1; } + + # Unprivileged runner user; /home/runner matches the Hetzner + # action's convention and the CCACHE_DIR baked into builds.yml. + if ! id -u runner >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash --home-dir /home/runner runner + fi + + # Passwordless sudo for the runner. percona-server build steps + # invoke \`sudo apt-get\` extensively (Install Build Dependencies); + # without NOPASSWD the runner cannot install packages and the build + # fails immediately. The instance has no inbound network (no SSH); + # the runner only runs one ephemeral job; expanding sudo to NOPASSWD + # is bounded by the ephemeral-VM blast radius. + install -m 0440 -o root -g root /dev/stdin /etc/sudoers.d/90-runner <<'SUDOERS' + runner ALL=(ALL) NOPASSWD:ALL + SUDOERS + + RUNNER_DIR=/home/runner/actions-runner + mkdir -p "\${RUNNER_DIR}" + chown -R runner:runner /home/runner + + # Download the pinned actions/runner x64 tarball. + TARBALL="actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" + TARBALL_URL="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/\${TARBALL}" + cd "\${RUNNER_DIR}" + curl --fail --silent --show-error --location --retry 5 --retry-delay 5 \\ + --output "\${TARBALL}" "\${TARBALL_URL}" + + # FAIL HARD on checksum mismatch. + echo "${RUNNER_SHA256} \${TARBALL}" | sha256sum --check --status \\ + || { echo "SHA256 mismatch for \${TARBALL}; refusing to continue"; exit 1; } + + tar xzf "\${TARBALL}" + rm -f "\${TARBALL}" + chown -R runner:runner "\${RUNNER_DIR}" + + # Configure + run as the unprivileged user. + # --ephemeral : deregister after one job + # --disableupdate : never self-update mid-job (we pin the version) + # --unattended : non-interactive + sudo -u runner -H bash -lc " + set -euxo pipefail + cd '\${RUNNER_DIR}' + ./config.sh \\ + --url '${RUNNER_URL}' \\ + --token '${RUNNER_TOKEN}' \\ + --name '${RUNNER_LABEL}' \\ + --labels '${RUNNER_LABEL}' \\ + --runnergroup default \\ + --work _work \\ + --unattended \\ + --ephemeral \\ + --disableupdate + " + echo "Token registered, runner: ${RUNNER_LABEL}" + + runner_status=0 + sudo -u runner -H bash -lc "cd '\${RUNNER_DIR}' && ./run.sh" || runner_status=\$? + echo "Runner exited with status \${runner_status}" + + echo "Shutting down at \$(date -u +%FT%TZ)" + exit 0 + EOF + chmod 0644 /tmp/ec2/userdata.sh + # Sanity checks: + # (a) shebang at column 1 (cloud-init refuses otherwise), + # (b) all required values made it into the rendered script. + if [ "$(head -1 /tmp/ec2/userdata.sh)" != "#!/bin/bash" ]; then + echo "::error::Rendered userData does not start with #!/bin/bash at column 1" + head -3 /tmp/ec2/userdata.sh + exit 1 + fi + for needle in "${RUNNER_LABEL}" "${RUNNER_URL}" "${RUNNER_VERSION}" "${RUNNER_SHA256}"; do + if ! grep -qF "$needle" /tmp/ec2/userdata.sh; then + echo "::error::Rendered userData missing expected value (sanity check failed)" + exit 1 + fi + done + + # Spot-first across 3 AZs, then on-demand across the same 3 AZs. + - id: run + name: RunInstances (spot eu-central-1a/1b/1c, then on-demand) + env: + AMI_ID: ${{ steps.ami.outputs.ami }} + run: | + set -euo pipefail + + TAG_SPEC="ResourceType=instance,Tags=[\ + {Key=Name,Value=ps-amd64-${{ github.run_id }}-${{ github.run_attempt }}},\ + {Key=iit-billing-tag,Value=percona-server-gha-fallback},\ + {Key=PerconaKeep,Value=True},\ + {Key=github_run_id,Value=${{ github.run_id }}},\ + {Key=github_run_attempt,Value=${{ github.run_attempt }}},\ + {Key=github_workflow,Value=builds},\ + {Key=github_repository,Value=${{ github.repository }}}]" + + AZS=(eu-central-1a eu-central-1b eu-central-1c) + INSTANCE_ID="" + CHOSEN_AZ="" + CHOSEN_MARKET="" + + # Helper: try to launch in $1=az with market $2=spot|on-demand. + # Echoes the instance id on success, empty on InsufficientCapacity. + # Any other AWS error exits the job non-zero (fail fast). + try_launch() { + local az="$1" market="$2" + local market_opts=() + if [ "$market" = "spot" ]; then + market_opts=(--instance-market-options 'MarketType=spot') + fi + # Client-token: stable per (run_id, run_attempt, az, market) so a + # retried RunInstances call returns the same instance instead of + # launching a duplicate. AWS requires <= 64 chars; the run-id + + # attempt + az + market fits comfortably. + local client_token="ps-x86-${{ github.run_id }}-${{ github.run_attempt }}-${az}-${market}" + set +e + # Canonical's Ubuntu 26.04 amd64 AMI defaults the root volume to + # 8 GB which fills up partway through the percona-server build + # (boost + ccache + intermediate objects). Override to 80 GB gp3, + # delete-on-termination so the volume dies with the ephemeral + # instance and the cleanup Lambdas have nothing to do. + OUT=$(aws ec2 run-instances \ + --region "$AWS_REGION" \ + --image-id "$AMI_ID" \ + --instance-type "$AWS_INSTANCE_TYPE_X86_64" \ + --placement "AvailabilityZone=$az" \ + --instance-initiated-shutdown-behavior terminate \ + --user-data file:///tmp/ec2/userdata.sh \ + --tag-specifications "$TAG_SPEC" \ + --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":80,"VolumeType":"gp3","DeleteOnTermination":true}}]' \ + --client-token "$client_token" \ + --metadata-options 'HttpTokens=required,HttpPutResponseHopLimit=2,HttpEndpoint=enabled' \ + "${market_opts[@]}" \ + --query 'Instances[0].InstanceId' --output text 2>/tmp/ec2/run.err) + rc=$? + set -e + if [ $rc -eq 0 ] && [ -n "$OUT" ] && [ "$OUT" != "None" ]; then + echo "$OUT" + return 0 + fi + ERR=$(cat /tmp/ec2/run.err) + # Match the two capacity-shaped errors AWS returns; everything + # else (auth, quota, malformed request) is fatal because retrying + # will not help. + # IMPORTANT: warnings/errors must go to STDERR (>&2), otherwise the + # surrounding `$(try_launch ...)` capture swallows them silently. + if echo "$ERR" | grep -qE 'InsufficientInstanceCapacity|SpotMaxPriceTooLow|MaxSpotInstanceCountExceeded'; then + echo "::warning::$market launch failed in $az: $(echo "$ERR" | head -1)" >&2 + return 1 + fi + echo "::error::Fatal RunInstances error in $az ($market): $ERR" >&2 + return 2 + } + + # Pass 1: spot across 3 AZs. + for az in "${AZS[@]}"; do + if ID=$(try_launch "$az" spot); then + INSTANCE_ID="$ID" + CHOSEN_AZ="$az" + CHOSEN_MARKET="spot" + break + fi + rc=$? + [ $rc -eq 2 ] && exit 1 + done + + # Pass 2: on-demand across the same 3 AZs if spot is fully exhausted. + if [ -z "$INSTANCE_ID" ]; then + echo "::notice::All 3 AZs returned no spot capacity for $AWS_INSTANCE_TYPE_X86_64; retrying on-demand" + for az in "${AZS[@]}"; do + if ID=$(try_launch "$az" on-demand); then + INSTANCE_ID="$ID" + CHOSEN_AZ="$az" + CHOSEN_MARKET="on-demand" + break + fi + rc=$? + [ $rc -eq 2 ] && exit 1 + done + fi + + if [ -z "$INSTANCE_ID" ]; then + echo "::error::No $AWS_INSTANCE_TYPE_X86_64 capacity in eu-central-1a/1b/1c for either spot or on-demand. Giving up." + exit 1 + fi + + echo "Launched $INSTANCE_ID ($CHOSEN_MARKET) in $CHOSEN_AZ" + { + echo "### EC2 fallback runner launched (x86_64)" + echo "" + echo "- Instance: \`$INSTANCE_ID\`" + echo "- Type: \`$AWS_INSTANCE_TYPE_X86_64\`" + echo "- AZ: \`$CHOSEN_AZ\`" + echo "- Market: \`$CHOSEN_MARKET\`" + echo "- AMI: \`$AMI_ID\`" + } >> "$GITHUB_STEP_SUMMARY" + { + echo "instance_id=$INSTANCE_ID" + echo "az=$CHOSEN_AZ" + echo "market=$CHOSEN_MARKET" + } >> "$GITHUB_OUTPUT" + + - name: Wait for runner to come online + env: + GH_TOKEN: ${{ secrets.GHA_RUNNER_PAT }} + INSTANCE_ID: ${{ steps.run.outputs.instance_id }} + RUNNER_LABEL: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + run: | + set -euo pipefail + # 10 min budget. Cloud-init on a fresh c7i.4xlarge spot needs + # apt-update + runner download + extract + configure + register; + # 5 min was too tight (PS-11219 PoC iteration 3 timed out at 5m15s). + # First-boot worst case is ~6-7 min on cold AMI/runner-release cache. + deadline=$(( $(date +%s) + 600 )) + attempt=0 + while [ "$(date +%s)" -lt "$deadline" ]; do + attempt=$((attempt + 1)) + STATUS=$(gh api -X GET /repos/${{ github.repository }}/actions/runners --paginate \ + --jq ".runners[] | select(.name == \"$RUNNER_LABEL\") | .status" || true) + if [ "$STATUS" = "online" ]; then + echo "Runner $RUNNER_LABEL online after ${attempt} polls" + exit 0 + fi + echo "Attempt $attempt: runner status='${STATUS:-not-registered}' (waiting 10s)" + sleep 10 + done + # On timeout, capture EC2 console output BEFORE terminating so we + # have a post-mortem (no SSM available; the no-IAM-on-VM design + # means we can't shell in either). + echo "::error::Runner $RUNNER_LABEL did not come online within 10m; capturing console output and terminating $INSTANCE_ID" + aws ec2 get-console-output --region "$AWS_REGION" --instance-id "$INSTANCE_ID" --output text --query Output 2>&1 | tail -200 || true + aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids "$INSTANCE_ID" >/dev/null || true + exit 1 + + # UNTRUSTED. PR code is NOT built here (this chain is nightly-only); 8.4 HEAD + # (RelWithDebInfo nightly) runs here under gcc-16. Mirror of build-arm64 with + # COMPILER_VER=16 and x86_64-scoped ccache. Sole job that checks out code; no + # secrets (read-only GITHUB_TOKEN). + build-x86_64: + needs: [dispatch, pick-target-x86_64, create-runner-hetzner-x86_64, create-runner-aws-x86_64] + # GHA defaults to skipping a job when ANY needed job is skipped. Since + # exactly ONE of create-runner-hetzner-x86_64 / create-runner-aws-x86_64 is + # skipped (the other ran), we explicitly require: dispatch + pick-target + # green, AND at least one of the two create-runner jobs succeeded. + if: | + always() + && needs.dispatch.result == 'success' + && needs.pick-target-x86_64.result == 'success' + && (needs.create-runner-hetzner-x86_64.result == 'success' || needs.create-runner-aws-x86_64.result == 'success') + runs-on: ${{ needs.create-runner-hetzner-x86_64.outputs.runner_label || needs.create-runner-aws-x86_64.outputs.runner_label }} + permissions: + contents: read + timeout-minutes: 240 + env: + BUILD_TYPE: ${{ needs.dispatch.outputs.build_type }} + MTR_SUITE: ${{ needs.dispatch.outputs.mtr_suite }} + CCACHE_MAXSIZE: ${{ needs.dispatch.outputs.ccache_maxsize }} + CCACHE_DIR: /home/runner/ccache + BOOST_DIR: /tmp/boost + DEBIAN_FRONTEND: noninteractive + # PS-11298: gcc-16 (g++-16) from the Ubuntu 26.04 archive. The + # Install Build Dependencies step's SELECTED_CC/SELECTED_CXX logic + # appends `-16` automatically; no PPA needed. + COMPILER_VER: '16' + steps: + - name: System info + run: | + set -eux + # PS-11298: report whichever provider actually fired. + echo "Provider: ${{ needs.pick-target-x86_64.outputs.provider }}" + if [ "${{ needs.pick-target-x86_64.outputs.provider }}" = "hetzner" ]; then + echo "Server type: ${{ needs.create-runner-hetzner-x86_64.outputs.server_type }}" + else + echo "Instance: ${{ needs.create-runner-aws-x86_64.outputs.instance_id }} (${{ needs.create-runner-aws-x86_64.outputs.market }} in ${{ needs.create-runner-aws-x86_64.outputs.az }})" + fi + echo "Build type: $BUILD_TYPE" + echo "MTR suite: $MTR_SUITE" + uname -a + nproc + free -h + df -h / + + - name: Conditional swap (only when RAM < 16 GB) + run: | + set -eux + MEM_GB=$(awk '/MemTotal/ {print int($2/1024/1024)}' /proc/meminfo) + if [ "$MEM_GB" -lt 16 ]; then + sudo fallocate -l 16G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + fi + free -h + + # Azure: `*** Install Build Dependencies`. COMPILER_VER=16 selects + # gcc-16/g++-16 from the Ubuntu 26.04 archive. + - name: Install Build Dependencies + run: | + set -eux + SELECTED_CC="${COMPILER}${COMPILER_VER:+-${COMPILER_VER}}" + SELECTED_CXX="${COMPILER}++${COMPILER_VER:+-${COMPILER_VER}}" + [ "$COMPILER" = "gcc" ] && SELECTED_CXX="g++${COMPILER_VER:+-${COMPILER_VER}}" + echo "SELECTED_CC=$SELECTED_CC" >> "$GITHUB_ENV" + echo "SELECTED_CXX=$SELECTED_CXX" >> "$GITHUB_ENV" + PACKAGES="$SELECTED_CC" + [ "$COMPILER" = "gcc" ] && PACKAGES="$SELECTED_CXX" + sudo apt-get -yq update + # Versioned g++-16 provides cpp-16 but not the bare `cpp` that XCom's + # rpcgen invokes for xcom_vp.h codegen; install the unversioned cpp + # provider so codegen does not silently emit an empty header. + sudo apt-get -yq --no-install-suggests --no-install-recommends install \ + $PACKAGES cpp make pkg-config dpkg-dev unzip lz4 git cmake cmake-curses-gui ccache bison \ + libtirpc-dev libudev-dev libaio-dev libmecab-dev libnuma-dev \ + libssl-dev libreadline-dev libedit-dev libpam-dev \ + libcurl4-openssl-dev libldap2-dev libkrb5-dev libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libxml-simple-perl + # Azure: SYSTEM_LIBRARIES path (BUILD_PARAMS_TYPE != inverted). + sudo apt-get -yq --no-install-suggests --no-install-recommends install \ + libicu-dev libevent-dev liblz4-dev zlib1g-dev \ + protobuf-compiler libprotobuf-dev libprotoc-dev \ + libzstd-dev libfido2-dev + # libeatmydata1 needed for MTR LD_PRELOAD + # (Cirrus arm64 task uses it; Azure x86 jobs do not run MTR). + sudo apt-get -yq --no-install-suggests --no-install-recommends install libeatmydata1 + REAL_COMPILER_VER=$($SELECTED_CC --version | head -1 | awk '{print $4}') + echo "REAL_COMPILER_VER=$REAL_COMPILER_VER" >> "$GITHUB_ENV" + + # Azure ccache key shape: "ccache | PARENT_BRANCH | imageName-Compiler-VER-BuildType | BUILD_PARAMS_TYPE | " + # PS-11298: key includes COMPILER_VER + x86_64 so the gcc-16 x86_64 build + # does NOT share a cache slot with the arm64 gcc (system) build. + - name: ccache cache (Azure key shape; build_type-scoped) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.0 + with: + path: /home/runner/ccache + key: ccache-${{ env.PARENT_BRANCH }}-${{ env.IMAGE_NAME }}-x86_64-${{ env.COMPILER }}-${{ env.COMPILER_VER }}-${{ env.BUILD_TYPE }}-${{ env.BUILD_PARAMS_TYPE }}-${{ github.run_id }} + restore-keys: | + ccache-${{ env.PARENT_BRANCH }}-${{ env.IMAGE_NAME }}-x86_64-${{ env.COMPILER }}-${{ env.COMPILER_VER }}-${{ env.BUILD_TYPE }}-${{ env.BUILD_PARAMS_TYPE }}- + + - name: boost cache (Azure key shape) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.0 + with: + path: /tmp/boost + key: ${{ env.BOOST_VERSION }} + + - name: Checkout (fetchDepth 32 mirrors Azure) + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 + with: + fetch-depth: 32 + # This chain is workflow_dispatch-only (nightly RelWithDebInfo), so + # the default checkout ref (the dispatched branch HEAD) is correct; + # no pull_request_target head-sha override needed here. + + - name: Update git submodules + run: | + set -eux + git submodule sync + git submodule update --init --force --depth=256 + git submodule + + - name: Compiler and cmake info + run: | + set -eux + $SELECTED_CC -v + $SELECTED_CXX -v + ccache --version + mkdir -p "$CCACHE_DIR" "$BOOST_DIR" + ccache -p | grep -E "max_size|cache_dir" + ccache --zero-stats + + # Azure `*** cmake ...` step, 8.4 Linux non-inverted branch. + - name: cmake (Azure parity, 8.4 Linux non-inverted) + run: | + set -eux + mkdir bin && cd bin + COMPILE_OPT=( + -DCMAKE_C_FLAGS_DEBUG=-g1 + -DCMAKE_CXX_FLAGS_DEBUG=-g1 + ) + CMAKE_OPT=" + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + -DBUILD_CONFIG=mysql_release + -DWITH_PACKAGE_FLAGS=OFF + -DDOWNLOAD_BOOST=1 + -DWITH_BOOST=$BOOST_DIR + -DCMAKE_C_COMPILER=$SELECTED_CC + -DCMAKE_CXX_COMPILER=$SELECTED_CXX + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DWITH_ROCKSDB=ON + -DWITH_COREDUMPER=ON + -DWITH_COMPONENT_KEYRING_VAULT=ON + -DWITH_PAM=ON + -DMYSQL_MAINTAINER_MODE=ON + -DWITH_MECAB=system + -DWITH_NUMA=ON + -DWITH_EDITLINE=system + -DWITH_SYSTEM_LIBS=ON + " + cmake .. $CMAKE_OPT "${COMPILE_OPT[@]}" + rm -f "$BOOST_DIR/$BOOST_VERSION.tar.gz" + cmake -L . + + # Azure: `*** Compile` uses `make -j2` (MS-hosted has 2 cores). Scale to + # runner cores, capped at 16. + - name: Compile (make -j, capped at 16) + run: | + set -eux + cd bin + NPROC=$(nproc --all) + NTHREADS=$(( NPROC > 16 ? 16 : NPROC )) + echo "Using $NTHREADS/$NPROC threads" + make -j${NTHREADS} + ccache --show-stats + df -h / + + # Suite picked by dispatch (RelWithDebInfo nightly -> binlog_nogtid). + - name: MTR (suite per build_type) + run: | + set -eux + cd bin + NPROC=$(nproc --all) + NTHREADS=$(( NPROC > 16 ? 16 : NPROC )) + LIBEATMYDATA=$(whereis libeatmydata.so | awk '{print $2}') + if [ "$MTR_SUITE" = "main.1st" ]; then + mysql-test/mysql-test-run.pl main.1st \ + --parallel=$NTHREADS \ + --junit-output=/tmp/MTR_results.xml \ + --mysqld-env=LD_PRELOAD=${LIBEATMYDATA} \ + --force --max-test-fail=0 --retry-failure=0 \ + --debug-server + else + mysql-test/mysql-test-run.pl \ + --suite=$MTR_SUITE \ + --parallel=$NTHREADS \ + --junit-output=/tmp/MTR_results.xml \ + --mysqld-env=LD_PRELOAD=${LIBEATMYDATA} \ + --force --max-test-fail=0 --retry-failure=0 + fi + + - name: Upload MTR results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0 + with: + name: mtr-x86_64-${{ env.BUILD_TYPE }}-${{ env.MTR_SUITE }} + path: /tmp/MTR_results.xml + if-no-files-found: ignore + + # x86_64 Hetzner teardown (mirror of delete-runner-hetzner). + delete-runner-hetzner-x86_64: + needs: [dispatch, pick-target-x86_64, create-runner-hetzner-x86_64, build-x86_64] + # Skip delete when debug_keep_vm is set AND create-runner failed, so the + # broken VM stays alive for manual SSH diagnosis. orphan-sweep reaps it + # after 6h regardless. Also gated on provider==hetzner so this is a no-op + # on EC2 fallback runs. + if: | + always() + && needs.pick-target-x86_64.outputs.provider == 'hetzner' + && needs.create-runner-hetzner-x86_64.outputs.server_id != '' + && !(inputs.debug_keep_vm == true && needs.create-runner-hetzner-x86_64.result == 'failure') + runs-on: ubuntu-latest + permissions: {} + steps: + - uses: olexandr-havryliak/hcloud-github-runner@bb1089d8b718a06493cb37c51dfe596e44baefc2 + with: + mode: delete + name: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + server_id: ${{ needs.create-runner-hetzner-x86_64.outputs.server_id }} + hcloud_token: ${{ secrets.GHA_RUNNER_HCLOUD_TOKEN }} + github_token: ${{ secrets.GHA_RUNNER_PAT }} + + # x86_64 EC2 teardown safety net (mirror of delete-runner-aws). Discovers the + # instance by tag (github_run_id + the ps-amd64- Name tag) then unions + # with the published output, and de-registers any orphaned runner. + delete-runner-aws-x86_64: + needs: [dispatch, pick-target-x86_64, create-runner-aws-x86_64, build-x86_64] + if: | + always() + && needs.pick-target-x86_64.outputs.provider == 'aws' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Configure AWS credentials via OIDC + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: gha-${{ github.run_id }}-${{ github.run_attempt }}-x86_64-cleanup + aws-region: ${{ env.AWS_REGION }} + + # Tag-based discovery first (scoped by the ps-amd64- Name tag so it + # never touches the arm64 chain's instance in the same run), then union + # with the published output. + - name: Terminate EC2 instance (safety net) + env: + PUBLISHED_INSTANCE_ID: ${{ needs.create-runner-aws-x86_64.outputs.instance_id }} + run: | + set -euo pipefail + IDS=$(aws ec2 describe-instances \ + --region "$AWS_REGION" \ + --filters \ + "Name=tag:github_run_id,Values=${{ github.run_id }}" \ + "Name=tag:github_run_attempt,Values=${{ github.run_attempt }}" \ + "Name=tag:github_repository,Values=${{ github.repository }}" \ + "Name=tag:iit-billing-tag,Values=percona-server-gha-fallback" \ + "Name=tag:Name,Values=ps-amd64-${{ github.run_id }}-${{ github.run_attempt }}" \ + "Name=instance-state-name,Values=pending,running,stopping,stopped" \ + --query 'Reservations[].Instances[].InstanceId' --output text 2>/dev/null || true) + # Union with whatever the create job published; dedupe via sort -u. + ALL_IDS=$(printf '%s\n%s\n' "$IDS" "${PUBLISHED_INSTANCE_ID:-}" \ + | tr '\t' '\n' | grep -E '^i-' | sort -u || true) + if [ -z "$ALL_IDS" ]; then + echo "No live EC2 instance found for run ${{ github.run_id }} attempt ${{ github.run_attempt }} x86_64 (already terminated or never launched)" + exit 0 + fi + echo "Terminating: $ALL_IDS" + # || true: instance may race to terminated between describe and + # terminate; either way we want this job to succeed so the + # workflow status is not contaminated by cleanup-only errors. + # shellcheck disable=SC2086 + aws ec2 terminate-instances \ + --region "$AWS_REGION" \ + --instance-ids $ALL_IDS >/dev/null 2>&1 || true + + - name: De-register orphaned runner (if still present) + env: + GH_TOKEN: ${{ secrets.GHA_RUNNER_PAT }} + RUNNER_LABEL: ${{ env.EPHEMERAL_RUNNER_NAME_AMD64 }} + run: | + set -euo pipefail + ID=$(gh api -X GET /repos/${{ github.repository }}/actions/runners --paginate \ + --jq ".runners[] | select(.name == \"$RUNNER_LABEL\") | .id" || true) + if [ -n "$ID" ]; then + gh api -X DELETE "/repos/${{ github.repository }}/actions/runners/$ID" || true + echo "Removed orphan runner $RUNNER_LABEL (id=$ID)" + else + echo "Runner $RUNNER_LABEL already de-registered (ephemeral self-removal)" + fi