beta-e2e.yml

name: Beta E2E (browser)

# Manual gate for "is beta safe to promote to production?" and reusable job
# for the scheduled beta health check.
#
# The public lane is sharded one host per runner. Measured: a page load against
# a beta host costs 1-2s from a laptop and 20-40s from a GitHub runner, because
# the fleet sits behind one CDN that throttles bursty datacenter traffic. On a
# single runner the sweep took ~28 minutes with scheduling already 83%
# efficient, so the time was raw network wait rather than queueing. Sharding
# makes the fleet cost the slowest single host and spreads the requests over
# sixteen source addresses: measured 1704s -> 437s.
#
# Drives the deployed beta fleet with a real browser. Two gating lanes decide
# the answer; a third advisory lane reports real-but-not-blocking findings so a
# red run always means "do not promote".
#
# Shape of a run: `discover` and `gate` (typecheck and helper tests, once) start
# together; the public shards, fleet, and advisory lanes start as soon as
# `discover` is done, and the authenticated shards as soon as `gate` passes. The
# lanes are independent, so they overlap, with the matrix `max-parallel` caps
# below bounding how many runners hit the shared CDN and databases at once.
# Every job has a `timeout-minutes` sized from observed runtimes and, inside it,
# a Playwright `globalTimeout` that ends a hung run with a reported failure and
# an uploadable results file rather than a cancelled job with neither.
#
# The workflow itself has no push or schedule trigger. The authenticated lane
# spends model tokens against the credential selected by its caller, and most
# beta apps share a database with production, so the explicit manual entrypoint
# remains a deliberate promotion decision. The separate scheduled wrapper
# calls this workflow on a six-hour cadence with the hosted sites' shared
# credential and sends one rolled-up report to #qa-agent-native with run links.

on:
  workflow_call:
    inputs:
      apps:
        description: 'Beta apps to test ("all", or e.g. "slides,analytics")'
        required: false
        default: "all"
        type: string
      lane:
        description: "Which lanes to run (public, authed, public+authed, or signup)"
        required: false
        default: "public+authed"
        type: string
      signup_apps:
        description: "Apps for the signup canary"
        required: false
        default: "all"
        type: string
      signup_environments:
        description: "Environments for the signup canary"
        required: false
        default: "beta"
        type: string
      grep:
        description: "Only run tests whose title matches this (optional)"
        required: false
        default: ""
        type: string
      key_source:
        description: "Which OpenAI credential the agent turns bill"
        required: false
        default: dedicated
        type: string
    secrets:
      BETA_E2E_EMAIL:
        required: false
      BETA_E2E_SESSION_TOKENS:
        required: false
      BETA_E2E_SESSION_TOKEN_CRM:
        required: false
      BETA_E2E_SESSION_TOKEN_CHAT:
        required: false
      BETA_E2E_OPENAI_API_KEY:
        required: false
      OPENAI_API_KEY:
        required: false
      MAILOSAUR_API_KEY:
        required: false
      MAILOSAUR_SERVER_ID:
        required: false
      PAGERDUTY_ROUTING_KEY:
        required: false
      QA_SLACK_BOT_TOKEN:
        required: false
    outputs:
      signup_outcome:
        description: "Signup lane result: success, failure, or inconclusive."
        value: ${{ jobs.signup.outputs.outcome }}
  workflow_dispatch:
    inputs:
      apps:
        description: 'Beta apps to test ("all", or e.g. "slides,analytics,chat")'
        required: false
        default: "all"
      lane:
        description: "Which browser lanes to run"
        required: false
        default: "public+authed"
        type: choice
        options:
          - public
          - authed
          - public+authed
      key_source:
        description: "Which OpenAI credential the agent turns bill"
        required: false
        default: dedicated
        type: choice
        options:
          - dedicated
          - shared
      grep:
        description: "Only run tests whose title matches this (optional)"
        required: false
        default: ""

# One group per kind of run. Runs that sign in share the e2e account and its
# user-scoped OpenAI key, so they queue behind each other. The public lane (the
# production pre-flight) and the signup canary use no shared credential and
# must not wait behind a multi-minute scheduled run.
concurrency:
  group: beta-e2e-${{ inputs.lane == 'signup' && 'signup' || inputs.lane == 'public' && 'public' || 'authed' }}
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  discover:
    name: Resolve host shards
    if: ${{ inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      matrix: ${{ steps.matrix.outputs.matrix }}
      apps: ${{ steps.matrix.outputs.apps }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 1
          sparse-checkout: scripts/netlify-beta-sites.json
          sparse-checkout-cone-mode: false

      - name: Build the host matrix
        id: matrix
        env:
          REQUESTED_APPS: ${{ inputs.apps }}
        run: |
          set -euo pipefail
          node <<'NODE'
          const fs = require("node:fs");
          const sites = JSON.parse(
            fs.readFileSync("scripts/netlify-beta-sites.json", "utf8"),
          );
          const known = sites
            .filter((site) => site.e2e !== false)
            .map((site) => site.id);
          const raw = (process.env.REQUESTED_APPS || "all").trim();
          let apps;
          if (!raw || raw === "all") {
            apps = [...new Set(known)];
          } else {
            apps = [
              ...new Set(
                raw
                  .split(",")
                  .map((value) => value.trim().toLowerCase())
                  .filter(Boolean),
              ),
            ];
            const unknown = apps.filter((id) => !known.includes(id));
            if (unknown.length > 0) {
              throw new Error(
                `Unknown app(s): ${unknown.join(", ")}. Known: ${known.join(", ")}`,
              );
            }
          }
          if (apps.length === 0) {
            throw new Error("The app selection resolved to nothing to test.");
          }
          const matrix = { app: apps };
          fs.appendFileSync(
            process.env.GITHUB_OUTPUT,
            `matrix=${JSON.stringify(matrix)}\napps=${apps.join(",")}\n`,
          );
          console.log(`Sharding ${apps.length} host(s): ${apps.join(", ")}`);
          NODE

  # Typecheck and helper unit tests, once for the whole run. `e2e/` is not a
  # workspace package, so the repo-wide typecheck never sees it; failing here
  # stops the authenticated shards before any token is spent.
  gate:
    name: Typecheck and helper tests
    if: ${{ inputs.lane != 'public' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 8
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
        with:
          browser: "false"
      - name: Typecheck the suite
        run: pnpm typecheck:e2e
      - name: Test beta E2E helper contracts
        # Every pure-helper spec next to the helpers, so a new one is picked up
        # without editing this list.
        run: pnpm exec tsx --test e2e/beta/lib/*.spec.ts

  public:
    name: ${{ matrix.app }} public sweep
    needs: discover
    if: ${{ needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      fail-fast: false
      # Eight, not the sixteen the 437s measurement used: the fleet sits behind
      # one CDN that throttles bursty datacenter traffic.
      max-parallel: 8
      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
    env:
      BETA_E2E_APPS: ${{ matrix.app }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: public-${{ matrix.app }}
      BETA_E2E_GLOBAL_TIMEOUT_MINUTES: "7"
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Public sweep
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=public --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=public
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-public-${{ matrix.app }}-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  fleet:
    name: Fleet-wide checks
    needs: discover
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      BETA_E2E_APPS: ${{ needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: fleet
      BETA_E2E_GLOBAL_TIMEOUT_MINUTES: "7"
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Cross-host checks
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=fleet --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=fleet
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-fleet-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  # One job per slot, independent of each other and of the unauthenticated
  # lanes. `timeout` is the job limit; `global_timeout` is Playwright's own, a
  # few minutes under it, so a hang is reported rather than killed. Slow slots
  # come first so they start in the first wave of `max-parallel`.
  #
  # `chat` is split per app: each turn bills luna and each shard writes the
  # user-scoped OpenAI key only on its own host, so a hang or failure on one
  # app is isolated and named instead of taking the whole cluster with it. No
  # two slots may install the key on the same host at once.
  #
  # The journeys are split by area into `journeys-*` projects so no slot has to
  # fit ~75 serial tests in one limit. `shard` (`n/m`) splits one project across
  # several slots with Playwright's own `--shard`; a project's shards must be
  # 1/m..m/m exactly once. `e2e/beta/lib/suite-partition.spec.ts` (run by `gate`)
  # lists every slot's tests and fails unless the slots cover each project's
  # tests exactly once, so a spec cannot fall outside every slot.
  authed:
    name: Authenticated ${{ matrix.slot }}
    needs: [discover, gate]
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && needs.gate.result == 'success' && inputs.lane != 'public' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: ${{ matrix.timeout }}
    strategy:
      fail-fast: false
      max-parallel: 5
      matrix:
        include:
          # Slides carries the realtime test (>= 10 idle minutes, and it does not
          # retry), so it gets the most room.
          - slot: chat-slides
            cluster: chat
            project: chat
            app: slides
            timeout: 30
            global_timeout: 27
          - slot: chat-analytics
            cluster: chat
            project: chat
            app: analytics
            timeout: 30
            global_timeout: 25
          - slot: chat-chat
            cluster: chat
            project: chat
            app: chat
            timeout: 30
            global_timeout: 25
          # 13 tests of 5 to 10 page loads each, split over two runners.
          - slot: journeys-session-1
            cluster: journeys
            project: journeys-session
            app: ""
            shard: 1/2
            timeout: 30
            global_timeout: 25
          - slot: journeys-session-2
            cluster: journeys
            project: journeys-session
            app: ""
            shard: 2/2
            timeout: 30
            global_timeout: 25
          # [credentials] (12, one per app) and [settings-keys] (3).
          - slot: journeys-credentials
            cluster: journeys
            project: journeys-credentials
            app: ""
            timeout: 30
            global_timeout: 25
          # [slides-import], [forms], [design-systems], [dispatch-apps].
          - slot: journeys-flows
            cluster: journeys
            project: journeys-flows
            app: ""
            timeout: 30
            global_timeout: 25
          # The original journeys (38 tests, 4 to 11 minutes when healthy).
          - slot: journeys-core
            cluster: journeys
            project: journeys-core
            app: ""
            timeout: 20
            global_timeout: 15
          - slot: chat-content
            cluster: chat
            project: chat
            app: content
            timeout: 20
            global_timeout: 15
          - slot: chat-dispatch
            cluster: chat
            project: chat
            app: dispatch
            timeout: 20
            global_timeout: 15
          - slot: design
            cluster: design
            project: design
            app: design
            timeout: 15
            global_timeout: 11
          - slot: registry
            cluster: registry
            project: registry
            app: ""
            timeout: 10
            global_timeout: 7
    env:
      # A shard with `app` runs only that app, and only when the run selected
      # it; the unsharded slots run whenever something other than design alone
      # was selected (design has its own slot).
      SHARD_SELECTED: ${{ matrix.app == '' && needs.discover.outputs.apps != 'design' || matrix.app != '' && contains(format(',{0},', needs.discover.outputs.apps), format(',{0},', matrix.app)) }}
      BETA_E2E_APPS: ${{ matrix.app || needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      # True when the caller narrowed the run (an app list or a title filter), so
      # a slot with nothing to run is expected rather than a broken partition.
      BETA_E2E_NARROWED: ${{ (inputs.apps != '' && inputs.apps != 'all') || inputs.grep != '' }}
      BETA_E2E_PROJECT: ${{ matrix.project }}
      BETA_E2E_SHARD: ${{ matrix.shard || '' }}
      BETA_E2E_AUTHED: "1"
      BETA_E2E_CLUSTER: ${{ matrix.cluster }}
      BETA_E2E_REPORT_SLOT: authed-${{ matrix.slot }}
      BETA_E2E_GLOBAL_TIMEOUT_MINUTES: ${{ matrix.global_timeout }}
      BETA_E2E_EMAIL: ${{ secrets.BETA_E2E_EMAIL }}
      BETA_E2E_SESSION_TOKENS: ${{ secrets.BETA_E2E_SESSION_TOKENS }}
      BETA_E2E_SESSION_TOKEN_CRM: ${{ secrets.BETA_E2E_SESSION_TOKEN_CRM }}
      BETA_E2E_SESSION_TOKEN_CHAT: ${{ secrets.BETA_E2E_SESSION_TOKEN_CHAT }}
      BETA_E2E_OPENAI_API_KEY: ${{ secrets.BETA_E2E_OPENAI_API_KEY }}
      BETA_E2E_SHARED_OPENAI_API_KEY: ${{ inputs.key_source == 'shared' && secrets.OPENAI_API_KEY || '' }}
      BETA_E2E_ALLOW_SHARED_KEY: ${{ inputs.key_source == 'shared' }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        if: ${{ env.SHARD_SELECTED == 'true' }}
      - uses: ./.github/actions/beta-e2e-setup
        if: ${{ env.SHARD_SELECTED == 'true' }}
      - name: Authenticated ${{ matrix.slot }}
        if: ${{ env.SHARD_SELECTED == 'true' }}
        # Global setup fails loudly if a credential is missing rather than
        # degrading to an anonymous run that would pass without testing
        # anything. The clusters are separate so a provider failure cannot
        # hide registry or journey regressions.
        # A narrowed run (an app list or a title filter) can leave a slot with
        # nothing to run, though. Ask Playwright for the authenticated selection
        # first with auth disabled; --list does not run globalSetup, so an empty
        # selection skips the credentialed setup without turning a requested
        # auth run into a pass. Playwright reports an empty --shard as a
        # successful zero-test listing and any other empty selection as an
        # "Error: No tests found" failure; both are the same fact here. A
        # full-fleet run with nothing to run is a broken partition, so it fails.
        run: |
          set -euo pipefail
          select_args=("--project=$BETA_E2E_PROJECT")
          if [ -n "$BETA_E2E_SHARD" ]; then
            select_args+=("--shard=$BETA_E2E_SHARD")
          fi
          if [ -n "$BETA_E2E_GREP" ]; then
            select_args+=(--grep "$BETA_E2E_GREP")
          fi
          selection_file="$(mktemp)"
          trap 'rm -f "$selection_file"' EXIT
          set +e
          BETA_E2E_AUTHED=0 pnpm e2e:beta "${select_args[@]}" --list >"$selection_file" 2>&1
          selection_status="$?"
          set -e
          cat "$selection_file"
          if grep -q "Total: 0 tests in 0 files" "$selection_file" && { [ "$selection_status" -eq 0 ] || grep -q "Error: No tests found" "$selection_file"; }; then
            if [ "$BETA_E2E_NARROWED" = "true" ]; then
              echo "No authenticated tests in this slot match the app or title selection; skipping authenticated setup."
              # Playwright writes reporter output for the empty --list selection.
              # Keep that discovery error out of the uploaded test results.
              rm -rf "e2e/beta/playwright-report/$BETA_E2E_REPORT_SLOT" "e2e/beta/test-results/$BETA_E2E_REPORT_SLOT"
              exit 0
            fi
            echo "::error::Slot $BETA_E2E_REPORT_SLOT selected no tests for a full-fleet run. Its project or shard no longer matches any spec." >&2
            exit 1
          fi
          if [ "$selection_status" -ne 0 ]; then
            echo "Authenticated test selection failed before execution." >&2
            exit "$selection_status"
          fi
          pnpm e2e:beta "${select_args[@]}"
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() && env.SHARD_SELECTED == 'true' }}
        with:
          name: beta-e2e-authed-${{ matrix.slot }}-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  advisory:
    name: Advisory findings (non-gating)
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    needs: discover
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      BETA_E2E_APPS: ${{ needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: advisory
      BETA_E2E_GLOBAL_TIMEOUT_MINUTES: "7"
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Advisory findings
        continue-on-error: true
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=advisory --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=advisory
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-advisory-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  signup:
    name: Signup canary
    if: ${{ inputs.lane == 'signup' }}
    outputs:
      outcome: ${{ steps.classify.outputs.outcome }}
    runs-on: ubuntu-latest
    timeout-minutes: 180
    env:
      SIGNUP_E2E_APPS: ${{ inputs.signup_apps || 'all' }}
      SIGNUP_E2E_ENVIRONMENTS: ${{ inputs.signup_environments || 'beta' }}
      MAILOSAUR_API_KEY: ${{ secrets.MAILOSAUR_API_KEY }}
      MAILOSAUR_SERVER_ID: ${{ secrets.MAILOSAUR_SERVER_ID }}
      PAGERDUTY_ROUTING_KEY: ${{ secrets.PAGERDUTY_ROUTING_KEY }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Test signup email provider contracts
        run: pnpm exec tsx --test e2e/signup/lib/mailosaur.spec.ts
      - name: Typecheck signup suite
        run: pnpm typecheck:e2e:signup
      - name: Run full signup flow
        id: signup
        run: pnpm e2e:signup
      - name: Classify signup result
        id: classify
        if: ${{ always() }}
        run: |
          set -euo pipefail
          if [ "${{ steps.signup.outcome }}" != "success" ]; then
            echo "outcome=failure" >> "$GITHUB_OUTPUT"
          elif find e2e/signup/test-results -type f -name 'mailosaur-inconclusive.txt' -print -quit | grep -q .; then
            echo "outcome=inconclusive" >> "$GITHUB_OUTPUT"
            echo "INCONCLUSIVE: Mailosaur rate-limited the verification inbox; no product result was recorded." >> "$GITHUB_STEP_SUMMARY"
          else
            echo "outcome=success" >> "$GITHUB_OUTPUT"
          fi
      - name: Page signup health on-call
        if: ${{ failure() && steps.signup.outcome == 'failure' && steps.classify.outputs.outcome == 'failure' }}
        run: |
          set -euo pipefail
          if [ -z "${PAGERDUTY_ROUTING_KEY:-}" ]; then
            echo "::warning::PAGERDUTY_ROUTING_KEY is not configured; no page was sent."
            exit 0
          fi
          jq -n \
            --arg routing_key "$PAGERDUTY_ROUTING_KEY" \
            --arg dedup_key "agent-native-signup-e2e" \
            --arg source "${{ github.repository }}" \
            --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            '{routing_key: $routing_key, event_action: "trigger", dedup_key: $dedup_key, payload: {summary: "Agent-Native signup E2E is failing", source: $source, severity: "critical", custom_details: {run_url: $run_url, apps: env.SIGNUP_E2E_APPS, environments: env.SIGNUP_E2E_ENVIRONMENTS}}}' \
            | curl --fail-with-body --connect-timeout 10 --max-time 20 --silent --show-error \
                --header 'Content-Type: application/json' \
                --data-binary @- \
                https://events.pagerduty.com/v2/enqueue
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: signup-e2e-${{ github.run_id }}
          path: e2e/signup/test-results/
          if-no-files-found: warn
          retention-days: 14

  report:
    name: Report directly dispatched beta E2E status
    if: >-
      ${{ always()
      && github.event_name == 'workflow_dispatch'
      && github.workflow == 'Beta E2E (browser)' }}
    needs: [discover, gate, public, fleet, authed, advisory, signup]
    uses: ./.github/workflows/beta-e2e-report.yml
    with:
      run_result: ${{ contains(join(needs.*.result, ','), 'failure') && 'failure' || contains(join(needs.*.result, ','), 'cancelled') && 'cancelled' || 'success' }}
      update_baseline: ${{ inputs.apps == 'all' && inputs.lane == 'public+authed' && inputs.grep == '' }}
    secrets:
      QA_SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_BOT_TOKEN }}
    permissions:
      actions: read
      contents: read