Add GNU awk testsuite runner and result comparison

.github/workflows/GnuComment.yml

@@ -0,0 +1,83 @@
+name: GnuComment
+
+# Post the GNU awk testsuite comparison (produced by the GnuTests workflow)
+# as a comment on the pull request.
+
+on:
+  workflow_run:
+    workflows: ["GnuTests"]
+    types:
+      - completed
+
+permissions: {}
+jobs:
+  post-comment:
+    permissions:
+      actions: read  #  to list workflow run artifacts
+      pull-requests: write  #  to comment on the pr
+
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: 'Download artifact'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // List all artifacts from GnuTests
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: ${{ github.event.workflow_run.id }},
+            });
+
+            // Download the "comment" artifact, which contains a PR number (NR) and result.txt
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "comment"
+            })[0];
+
+            if (!matchArtifact) {
+              console.log('No comment artifact found');
+              return;
+            }
+
+            var download = await github.rest.actions.downloadArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: matchArtifact.id,
+              archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
+      - run: unzip comment.zip || echo "Failed to unzip comment artifact"
+
+      - name: 'Comment on PR'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            var fs = require('fs');
+
+            // Check if files exist
+            if (!fs.existsSync('./NR')) {
+              console.log('No NR file found, skipping comment');
+              return;
+            }
+            if (!fs.existsSync('./result.txt')) {
+              console.log('No result.txt file found, skipping comment');
+              return;
+            }
+
+            var issue_number = Number(fs.readFileSync('./NR'));
+            var content = fs.readFileSync('./result.txt');
+
+            if (content.toString().trim().length > 7) { // 7 because we have backquote + \n
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue_number,
+                body: 'GNU awk testsuite comparison:\n```\n' + content + '```'
+              });
+            } else {
+              console.log('Comment content too short, skipping');
+            }

.github/workflows/GnuTests.yml

@@ -0,0 +1,229 @@
+name: GnuTests
+
+# Run the upstream GNU awk (gawk) testsuite against the Rust awk implementation
+# to track and guard byte-for-byte compatibility. See util/run-gnu-testsuite.sh.
+
+on:
+  pull_request:
+  push:
+    branches:
+      - '*'
+
+permissions:
+  contents: write # Publish awk instead of discarding
+
+# End the current execution if there is a new changeset in the PR.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+  TEST_FULL_SUMMARY_FILE: 'awk-gnu-full-result.json'
+
+jobs:
+  native:
+    name: Run GNU awk testsuite
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Checkout code (awk)
+      uses: actions/checkout@v4
+      with:
+        path: 'awk'
+        persist-credentials: false
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: stable
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: "./awk -> target"
+
+    - name: Install GNU awk build dependencies
+      shell: bash
+      run: |
+        ## gawk's ./configure (which generates test/Makefile) needs a C toolchain
+        sudo apt-get update
+        sudo apt-get install -y build-essential autoconf automake gettext bison
+
+    - name: Fetch GNU awk testsuite
+      shell: bash
+      run: |
+        ## Download and extract the upstream GNU awk release tarball
+        mkdir -p gnu.awk
+        cd gnu.awk
+        bash ../awk/util/fetch-gnu.sh
+
+    - name: Build Rust awk binary
+      shell: bash
+      run: |
+        cd 'awk'
+        cargo build --release --config=profile.release.strip=true
+        tar -C target/release -cf - awk | zstd -19 -o ../awk-x86_64-unknown-linux-gnu.tar.zst
+    - name: Publish latest commit
+      uses: softprops/action-gh-release@v3
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      with:
+        tag_name: latest-commit
+        body: |
+          commit: ${{ github.sha }}
+        draft: false
+        prerelease: true
+        files: |
+          awk-x86_64-unknown-linux-gnu.tar.zst
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run GNU awk testsuite
+      shell: bash
+      run: |
+        cd 'awk'
+        export GNU_AWK_DIR="../gnu.awk"
+        ./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true
+
+    - name: Upload full json results
+      uses: actions/upload-artifact@v4
+      with:
+        name: awk-gnu-full-result
+        path: awk/${{ env.TEST_FULL_SUMMARY_FILE }}
+        if-no-files-found: warn
+
+  aggregate:
+    needs: [native]
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: read
+    name: Aggregate GNU test results
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Initialize workflow variables
+      id: vars
+      shell: bash
+      run: |
+        ## VARs setup
+        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+        TEST_SUMMARY_FILE='awk-gnu-result.json'
+        outputs TEST_SUMMARY_FILE
+
+    - name: Checkout code (awk)
+      uses: actions/checkout@v4
+      with:
+        path: 'awk'
+        persist-credentials: false
+
+    - name: Retrieve reference artifacts
+      uses: dawidd6/action-download-artifact@v6
+      continue-on-error: true
+      with:
+        workflow: GnuTests.yml
+        branch: "${{ env.DEFAULT_BRANCH }}"
+        workflow_conclusion: completed
+        path: "reference"
+        if_no_artifact_found: warn
+
+    - name: Download full json results
+      uses: actions/download-artifact@v4
+      with:
+        name: awk-gnu-full-result
+        path: results
+
+    - name: Extract/summarize testing info
+      id: summary
+      shell: bash
+      run: |
+        ## Extract/summarize testing info
+        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+
+        RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
+        if [[ ! -f "$RESULT_FILE" ]]; then
+          echo "::error ::Result file $RESULT_FILE not found"
+          find results -type f || true
+          exit 1
+        fi
+
+        TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE")
+        PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE")
+        FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE")
+        SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE")
+        ERROR=0  # Our format doesn't distinguish errors from failures
+
+        output="GNU awk tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
+        echo "${output}"
+        if [[ "$FAIL" -gt 0 ]]; then
+          echo "::warning ::${output}"
+        fi
+
+        # Build the date-keyed summary consumed by the awk-tracking repo.
+        jq -n \
+              --arg date "$(date --rfc-email)" \
+              --arg sha "$GITHUB_SHA" \
+              --arg total "$TOTAL" \
+              --arg pass "$PASS" \
+              --arg skip "$SKIP" \
+              --arg fail "$FAIL" \
+              --arg error "$ERROR" \
+              '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
+
+        HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
+        outputs HASH TOTAL PASS FAIL SKIP
+
+    - name: Upload SHA1/ID of 'test-summary'
+      uses: actions/upload-artifact@v4
+      with:
+        name: "${{ steps.summary.outputs.HASH }}"
+        path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
+
+    - name: Upload test results summary
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-summary
+        path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
+
+    - name: Compare test failures VS reference
+      shell: bash
+      run: |
+        ## Compare current results against the reference summary from the default branch
+        REF_SUMMARY_FILE='reference/awk-gnu-full-result/${{ env.TEST_FULL_SUMMARY_FILE }}'
+        CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
+        IGNORE_INTERMITTENT="awk/.github/workflows/ignore-intermittent.txt"
+
+        # Set up comment directory for the GnuComment workflow.
+        COMMENT_DIR="reference/comment"
+        mkdir -p ${COMMENT_DIR}
+        echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
+        COMMENT_LOG="${COMMENT_DIR}/result.txt"
+        : > "${COMMENT_LOG}"
+
+        COMPARISON_RESULT=0
+        if test -f "${REF_SUMMARY_FILE}"; then
+          python3 awk/util/compare_test_results.py \
+            --ignore-file "${IGNORE_INTERMITTENT}" \
+            --output "${COMMENT_LOG}" \
+            "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" || COMPARISON_RESULT=$?
+        else
+          echo "::warning ::Skipping test comparison; no prior reference summary at '${REF_SUMMARY_FILE}'."
+        fi
+
+        if [ ${COMPARISON_RESULT} -eq 1 ]; then
+          echo "::error ::Found new non-intermittent test failures"
+          UPLOAD_EXIT=1
+        else
+          echo "::notice ::No new test failures detected"
+          UPLOAD_EXIT=0
+        fi
+        echo "UPLOAD_EXIT=${UPLOAD_EXIT}" >> $GITHUB_ENV
+
+    - name: Upload comparison log (for GnuComment workflow)
+      if: success() || failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: comment
+        path: reference/comment/
+
+    - name: Report test results
+      if: success() || failure()
+      shell: bash
+      run: |
+        echo "::notice ::GNU awk testsuite: TOTAL ${{ steps.summary.outputs.TOTAL }} / PASS ${{ steps.summary.outputs.PASS }} / FAIL ${{ steps.summary.outputs.FAIL }} / SKIP ${{ steps.summary.outputs.SKIP }}"
+        # Fail the job if the comparison found new non-intermittent regressions.
+        exit "${UPLOAD_EXIT:-0}"

.github/workflows/ignore-intermittent.txt

@@ -0,0 +1,7 @@
+# List of intermittent test names to ignore in result comparisons
+# Format: one test name per line, lines starting with # are comments
+#
+# Add test names that are known to be flaky or environment-dependent
+# Example:
+# basic_substitution
+# line_address_test

README.md

@@ -47,6 +47,42 @@ version, that is, 1.95.0 at the time of writing.
 
 Check out https://github.com/uutils/awk/issues/16.
 
+## Testing
+
+### GNU awk (gawk) Compatibility Testing
+
+Track compatibility against GNU awk by running the upstream gawk testsuite
+against our Rust binary. Rather than reimplement gawk's test harness, we drive
+gawk's own (GPL) test Makefile with `make check AWK=<wrapper>`, where the wrapper
+execs our `awk` — the gawk sources are fetched fresh at test time and never
+copied into this repo.
+
+```bash
+# Fetch the gawk testsuite (one-time setup)
+mkdir -p ../gnu.awk && (cd ../gnu.awk && bash ../awk/util/fetch-gnu.sh)
+
+# Run compatibility tests
+./util/run-gnu-testsuite.sh
+
+# Verbose mode shows the diff for each failing test
+./util/run-gnu-testsuite.sh -v
+
+# Generate JSON results for CI
+./util/run-gnu-testsuite.sh --json-output results.json
+```
+
+The harness builds our `awk`, runs gawk's `make check` with a wrapper named
+`gawk`, and classifies each test the way gawk's own `pass-fail` target does: a
+leftover `_<name>` file is a failure, its absence a pass, and tests that never
+run (group-skipped because of missing locales, MPFR, or shared-library support)
+are reported as skipped.
+
+### Unit Tests
+
+```bash
+cargo test --workspace
+```
+
 ## Contributing
 
 To contribute to uutils AWK, please see [CONTRIBUTING](https://github.com/uutils/coreutils/blob/main/CONTRIBUTING.md).