mutation.yml

name: Mutation Testing (PIT)

# Weekly, and on demand. A full PIT run over se.deversity.asynctest.* is the longest job in
# the repository (about 2h locally with the e2e tier included), and its score moves
# slowly, so running it on every push would buy a number nobody reads. Weekly keeps the gate
# real: the job fails when the mutation score drops below the threshold in
# async-test-lib/pom.xml (76%, measured 77.5% on 2026-08-31; the margin absorbs TIMED_OUT
# jitter). Trigger it from Actions -> Mutation Testing (PIT) -> Run workflow when test
# strength is the actual question, for instance before a release or after a detector sweep.
#
# The pom's pitest configuration sets parseSurefireConfig=false and declares everything
# explicitly, so the e2e engine tier is always part of the measurement — locally and here.
# Before 2026-08-31 pitest inherited surefire's excludedGroups, which silently scored the
# fast tier only on developer machines (74%) while CI's auto-activated e2e profile scored
# the full suite (77.5%): the same command measured two different things.

on:
  schedule:
    - cron: '0 2 * * 0'   # Sundays 02:00 UTC
  workflow_dispatch:

concurrency:
  # One run per workflow per ref. cancel-in-progress only for pull requests: a superseded
  # push has nothing to say, while a run on main or a schedule is the record of that commit
  # and must finish. Without this a force-push left the previous run alive and both competed
  # for the same runners (#484).
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

jobs:
  mutation-testing:
    name: Mutation Testing (PIT)
    runs-on: ubuntu-latest
    timeout-minutes: 300

    steps:
      - name: Harden the runner (Block unexpected outbound calls)
        uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
        with:
          egress-policy: block
          allowed-endpoints: >
            agent.api.stepsecurity.io:443
            api.adoptium.net:443
            api.github.com:443
            github.com:443
            objects.githubusercontent.com:443
            repo.maven.apache.org:443
            repo1.maven.org:443
            results-receiver.actions.githubusercontent.com:443

      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up Java
        uses: actions/setup-java@77dd5b2ce225ea11469928ac317850245a74ae78 # v6.2.0
        with:
          java-version: '21'
          distribution: 'temurin'
          cache: maven

      # test-compile first so the goal runs against this tree, not a published artifact
      # (a bare goal in the reactor resolves siblings from the repository). Scoped to
      # async-test-lib: it is the only module with a pitest configuration, and the only one
      # whose tests are a mutation-killing suite.
      # The sampler and watchdog below are what diagnosed #479, and they stay. This job had
      # never once completed: three runs, on 2026-08-16, -23 and -30, each died about 19 minutes
      # in during the coverage phase, with the runner shut down rather than the job timing out,
      # and nothing in the log said why because nothing in it reported the machine's state.
      #
      # They print to this step's own stdout rather than to a file, deliberately: an uploaded
      # artifact needs a post-step, and there is no post-step after the VM is killed, so the last
      # SAMPLE line in the job log is the only evidence that survives that ending.
      #
      # The cause was virtual-thread carrier starvation, not the memory exhaustion first
      # suspected - see the pom's excludedTestClasses for the two classes that wedged six virtual
      # threads against a four-carrier pool. The load average is the field to read first: 0.00
      # through a quiet stretch means blocked, and a healthy mutation phase runs at 5-6.
      - name: Run mutation coverage (fails below the pom's mutationThreshold)
        run: |
          # The sampler, and a watchdog beside it.
          #
          # What the first instrumented run (33778474051) established, and what it ruled out:
          # over three hours after the coverage phase went quiet, used memory moved 1652 -> 2344
          # MB of 16 GB, swap stayed at 0, OS threads stayed near 320, and the load average sat
          # at 0.00. So the minion is BLOCKED, not thrashing, and the accumulation hypothesis in
          # #479 - deadlocked threads piling up until the kernel kills the VM - is refuted by its
          # own numbers. A local full run on a fast box does not reproduce it either: coverage
          # finishes in 77 seconds there.
          #
          # What is still unknown is what the minion is blocked ON, and the only artefact that
          # answers it is a thread dump of the blocked JVM. The watchdog below takes one, once,
          # after four minutes without a line from mvn, and prints it to this step's stdout. To
          # stdout rather than an uploaded artifact on purpose: there is no post-step after the
          # VM is killed, so the job log is the only place evidence survives that ending.
          LOG="$RUNNER_TEMP/mvn.log"
          DUMPED="$RUNNER_TEMP/jstack.done"
          : > "$LOG"

          (
            while true; do
              used=$(free -m | sed -n '2p' | awk '{print $3}')
              avail=$(free -m | sed -n '2p' | awk '{print $7}')
              swap=$(free -m | sed -n '3p' | awk '{print $3}')
              threads=$(ps -eLf | wc -l)
              javas=$(pgrep -c java || true)
              disk=$(df -m / | tail -1 | awk '{print $4}')
              load=$(cut -d' ' -f1-3 /proc/loadavg)
              quiet=$(( $(date +%s) - $(stat -c %Y "$LOG") ))
              echo "SAMPLE $(date -u +%H:%M:%S) used_mb=$used avail_mb=$avail swap_mb=$swap os_threads=$threads java_procs=$javas root_avail_mb=$disk quiet_for=${quiet}s load=$load"

              # Four minutes without a line from mvn. The three runs that died had thirteen
              # minutes of silence before the kill, and the coverage phase's own quiet stretches
              # are far shorter than this, so the threshold separates the hang from ordinary
              # work. Taken once - a second dump of the same blocked JVM says nothing new and
              # would bury the first.
              if [ "$quiet" -gt 240 ] && [ ! -f "$DUMPED" ]; then
                touch "$DUMPED"
                echo "=== WATCHDOG: no mvn output for ${quiet}s; dumping every JVM ==="
                for pid in $(jps -q 2>/dev/null || true); do
                  echo "########## jstack -l $pid ##########"
                  jstack -l "$pid" 2>&1 || echo "(jstack failed for $pid)"
                  # jstack shows carriers but not the virtual threads mounted on them, and the
                  # first dump ended exactly there: all four ForkJoinPool carriers reported
                  # "Carrying virtual thread #NNNN" inside Continuation.run, with no way to see
                  # what those four were blocked in. Thread.dump_to_file enumerates virtual
                  # threads with their stacks, which is what names the subjects that pin.
                  echo "########## jcmd $pid Thread.dump_to_file ##########"
                  if jcmd "$pid" Thread.dump_to_file -format=json -overwrite                           "$RUNNER_TEMP/threads-$pid.json" >/dev/null 2>&1; then
                    cat "$RUNNER_TEMP/threads-$pid.json" 2>&1 || true
                  else
                    echo "(Thread.dump_to_file failed for $pid)"
                  fi
                done
                echo "=== WATCHDOG: dump complete ==="
              fi
              sleep 15
            done
          ) &
          sampler=$!
          trap 'kill $sampler 2>/dev/null || true' EXIT

          # tee so the watchdog can measure silence by the log's mtime. pipefail because a pipe
          # otherwise reports tee's status and would turn a failed mutation run green.
          set -o pipefail
          mvn -B -pl async-test-lib -am test-compile org.pitest:pitest-maven:mutationCoverage -Djacoco.skip=true 2>&1 | tee "$LOG"

      # The full run cannot say whether a @ConcurrencyTestFor test kills its class's mutants: without
      # the full mutation matrix, mutations.xml names only the first killer, and the slow @AsyncTest
      # tests are rarely first. One scoped PIT run per pair asks it directly and checks the floors in
      # .github/concurrency-kill-floors.txt (#909). Runs even when the full run missed its threshold.
      - name: Check each concurrency test kills its class's mutants
        if: success() || failure()
        run: |
          python3 .github/scripts/concurrency_test_kills.py . --self-test
          python3 .github/scripts/concurrency_test_kills.py .

      - name: Upload PIT mutation report
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: pitest-report-${{ github.run_id }}
          path: |
            async-test-lib/target/pit-reports/**
            async-test-lib/target/pit-concurrency/**
          if-no-files-found: warn
          retention-days: 90