diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..1ca3532 --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,83 @@ +name: GnuComment + +# Post the GNU awk testsuite comparison (produced by the GnuTests workflow) +# as a comment on the pull request. + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} +jobs: + post-comment: + permissions: + actions: read # to list workflow run artifacts + pull-requests: write # to comment on the pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v7 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: 'GNU awk testsuite comparison:\n```\n' + content + '```' + }); + } else { + console.log('Comment content too short, skipping'); + } diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml new file mode 100644 index 0000000..6efe542 --- /dev/null +++ b/.github/workflows/GnuTests.yml @@ -0,0 +1,229 @@ +name: GnuTests + +# Run the upstream GNU awk (gawk) testsuite against the Rust awk implementation +# to track and guard byte-for-byte compatibility. See util/run-gnu-testsuite.sh. + +on: + pull_request: + push: + branches: + - '*' + +permissions: + contents: write # Publish awk instead of discarding + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + TEST_FULL_SUMMARY_FILE: 'awk-gnu-full-result.json' + +jobs: + native: + name: Run GNU awk testsuite + runs-on: ubuntu-24.04 + steps: + - name: Checkout code (awk) + uses: actions/checkout@v4 + with: + path: 'awk' + persist-credentials: false + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./awk -> target" + + - name: Install GNU awk build dependencies + shell: bash + run: | + ## gawk's ./configure (which generates test/Makefile) needs a C toolchain + sudo apt-get update + sudo apt-get install -y build-essential autoconf automake gettext bison + + - name: Fetch GNU awk testsuite + shell: bash + run: | + ## Download and extract the upstream GNU awk release tarball + mkdir -p gnu.awk + cd gnu.awk + bash ../awk/util/fetch-gnu.sh + + - name: Build Rust awk binary + shell: bash + run: | + cd 'awk' + cargo build --release --config=profile.release.strip=true + tar -C target/release -cf - awk | zstd -19 -o ../awk-x86_64-unknown-linux-gnu.tar.zst + - name: Publish latest commit + uses: softprops/action-gh-release@v3 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + with: + tag_name: latest-commit + body: | + commit: ${{ github.sha }} + draft: false + prerelease: true + files: | + awk-x86_64-unknown-linux-gnu.tar.zst + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run GNU awk testsuite + shell: bash + run: | + cd 'awk' + export GNU_AWK_DIR="../gnu.awk" + ./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true + + - name: Upload full json results + uses: actions/upload-artifact@v4 + with: + name: awk-gnu-full-result + path: awk/${{ env.TEST_FULL_SUMMARY_FILE }} + if-no-files-found: warn + + aggregate: + needs: [native] + permissions: + actions: read + contents: read + pull-requests: read + name: Aggregate GNU test results + runs-on: ubuntu-24.04 + steps: + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + TEST_SUMMARY_FILE='awk-gnu-result.json' + outputs TEST_SUMMARY_FILE + + - name: Checkout code (awk) + uses: actions/checkout@v4 + with: + path: 'awk' + persist-credentials: false + + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: GnuTests.yml + branch: "${{ env.DEFAULT_BRANCH }}" + workflow_conclusion: completed + path: "reference" + if_no_artifact_found: warn + + - name: Download full json results + uses: actions/download-artifact@v4 + with: + name: awk-gnu-full-result + path: results + + - name: Extract/summarize testing info + id: summary + shell: bash + run: | + ## Extract/summarize testing info + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + if [[ ! -f "$RESULT_FILE" ]]; then + echo "::error ::Result file $RESULT_FILE not found" + find results -type f || true + exit 1 + fi + + TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE") + PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE") + FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE") + SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE") + ERROR=0 # Our format doesn't distinguish errors from failures + + output="GNU awk tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" + echo "${output}" + if [[ "$FAIL" -gt 0 ]]; then + echo "::warning ::${output}" + fi + + # Build the date-keyed summary consumed by the awk-tracking repo. + jq -n \ + --arg date "$(date --rfc-email)" \ + --arg sha "$GITHUB_SHA" \ + --arg total "$TOTAL" \ + --arg pass "$PASS" \ + --arg skip "$SKIP" \ + --arg fail "$FAIL" \ + --arg error "$ERROR" \ + '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' + + HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) + outputs HASH TOTAL PASS FAIL SKIP + + - name: Upload SHA1/ID of 'test-summary' + uses: actions/upload-artifact@v4 + with: + name: "${{ steps.summary.outputs.HASH }}" + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Upload test results summary + uses: actions/upload-artifact@v4 + with: + name: test-summary + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare current results against the reference summary from the default branch + REF_SUMMARY_FILE='reference/awk-gnu-full-result/${{ env.TEST_FULL_SUMMARY_FILE }}' + CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + IGNORE_INTERMITTENT="awk/.github/workflows/ignore-intermittent.txt" + + # Set up comment directory for the GnuComment workflow. + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + : > "${COMMENT_LOG}" + + COMPARISON_RESULT=0 + if test -f "${REF_SUMMARY_FILE}"; then + python3 awk/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" || COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary at '${REF_SUMMARY_FILE}'." + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "::error ::Found new non-intermittent test failures" + UPLOAD_EXIT=1 + else + echo "::notice ::No new test failures detected" + UPLOAD_EXIT=0 + fi + echo "UPLOAD_EXIT=${UPLOAD_EXIT}" >> $GITHUB_ENV + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: comment + path: reference/comment/ + + - name: Report test results + if: success() || failure() + shell: bash + run: | + echo "::notice ::GNU awk testsuite: TOTAL ${{ steps.summary.outputs.TOTAL }} / PASS ${{ steps.summary.outputs.PASS }} / FAIL ${{ steps.summary.outputs.FAIL }} / SKIP ${{ steps.summary.outputs.SKIP }}" + # Fail the job if the comparison found new non-intermittent regressions. + exit "${UPLOAD_EXIT:-0}" diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt new file mode 100644 index 0000000..7889f3d --- /dev/null +++ b/.github/workflows/ignore-intermittent.txt @@ -0,0 +1,7 @@ +# List of intermittent test names to ignore in result comparisons +# Format: one test name per line, lines starting with # are comments +# +# Add test names that are known to be flaky or environment-dependent +# Example: +# basic_substitution +# line_address_test diff --git a/README.md b/README.md index b45e2c2..8a577dc 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,42 @@ version, that is, 1.95.0 at the time of writing. Check out https://github.com/uutils/awk/issues/16. +## Testing + +### GNU awk (gawk) Compatibility Testing + +Track compatibility against GNU awk by running the upstream gawk testsuite +against our Rust binary. Rather than reimplement gawk's test harness, we drive +gawk's own (GPL) test Makefile with `make check AWK=`, where the wrapper +execs our `awk` — the gawk sources are fetched fresh at test time and never +copied into this repo. + +```bash +# Fetch the gawk testsuite (one-time setup) +mkdir -p ../gnu.awk && (cd ../gnu.awk && bash ../awk/util/fetch-gnu.sh) + +# Run compatibility tests +./util/run-gnu-testsuite.sh + +# Verbose mode shows the diff for each failing test +./util/run-gnu-testsuite.sh -v + +# Generate JSON results for CI +./util/run-gnu-testsuite.sh --json-output results.json +``` + +The harness builds our `awk`, runs gawk's `make check` with a wrapper named +`gawk`, and classifies each test the way gawk's own `pass-fail` target does: a +leftover `_` file is a failure, its absence a pass, and tests that never +run (group-skipped because of missing locales, MPFR, or shared-library support) +are reported as skipped. + +### Unit Tests + +```bash +cargo test --workspace +``` + ## Contributing To contribute to uutils AWK, please see [CONTRIBUTING](https://github.com/uutils/coreutils/blob/main/CONTRIBUTING.md). diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100755 index 0000000..7ceb3ff --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from JSON data.""" + if not json_data or "summary" not in json_data: + return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, [] + + summary = json_data["summary"] + tests = json_data.get("tests", []) + + # Extract failed test names + failed_tests = [] + for test in tests: + if test.get("status") == "FAIL": + failed_tests.append(test.get("name", "unknown")) + + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + # Load ignore list + ignore_set = load_ignore_list(ignore_file) + + # Load JSON files + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int( + reference_summary.get("passed", 0) + ) + fail_diff = int(current_summary.get("failed", 0)) - int( + reference_summary.get("failed", 0) + ) + total_diff = int(current_summary.get("total", 0)) - int( + reference_summary.get("total", 0) + ) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Check if results are identical (no changes) + no_changes = ( + pass_diff == 0 + and fail_diff == 0 + and total_diff == 0 + and not new_failures + and not improvements + ) + + # If no changes, write empty output to prevent comment posting + if no_changes: + with open(output_file, "w") as f: + f.write("") + return 0 + + # Prepare output message + output_lines = [] + + # Show current vs reference numbers for debugging + output_lines.append("Test results comparison:") + output_lines.append( + f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}" + ) + output_lines.append( + f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}" + ) + output_lines.append("") + + # Summary of changes + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + # New failures + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + # Improvements + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + # Write output + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + # Return appropriate exit code + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/util/fetch-gnu.sh b/util/fetch-gnu.sh new file mode 100755 index 0000000..51fece0 --- /dev/null +++ b/util/fetch-gnu.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +# This file is part of the uutils awk package. +# +# For the full copyright and license information, please view the LICENSE +# file that was distributed with this source code. +# +# Download and extract the upstream GNU awk (gawk) release tarball into the +# current directory. Run it from an (empty) directory that will hold the gawk +# tree, e.g.: +# +# mkdir -p ../gnu.awk && (cd ../gnu.awk && bash ../awk/util/fetch-gnu.sh) +# +# The extracted tree ships gawk's own testsuite under test/ (a GPL Makefile.am +# plus the .awk programs, .in inputs and .ok expected outputs). We never copy +# that tree into our repo; util/run-gnu-testsuite.sh drives gawk's own +# `make check` against the Rust awk binary, fetched fresh at test time. +ver="5.3.2" +curl -L "https://ftp.gnu.org/gnu/gawk/gawk-${ver}.tar.xz" | tar --strip-components=1 -xJf - diff --git a/util/run-gnu-testsuite.sh b/util/run-gnu-testsuite.sh new file mode 100755 index 0000000..188e037 --- /dev/null +++ b/util/run-gnu-testsuite.sh @@ -0,0 +1,314 @@ +#!/bin/bash +# This file is part of the uutils awk package. +# +# For the full copyright and license information, please view the LICENSE +# file that was distributed with this source code. +# +# Run the upstream GNU awk (gawk) testsuite against the Rust awk implementation. +# +# Unlike grep/sed, gawk does not ship a gnulib init.sh test framework. Its +# testsuite is a (GPL) make-driven suite: test/Makefile.am + a generated +# Maketests, where each test target runs `$(AWK)` and compares the output +# against a committed `.ok` file, leaving a `_` file behind on +# mismatch. We drive gawk's own Makefile with `make check AWK=`, where +# the wrapper execs our Rust `awk` — the faithful analog of grep injecting its +# binary via PATH. gawk's Makefile is never copied into our repo; it is fetched +# fresh at test time. Classification mirrors gawk's own `pass-fail` target: a +# leftover `_` file is a FAIL, its absence a PASS; tests that never ran +# (group-skipped: locale/MPFR/shared-lib) are SKIP. +# +# Get the GNU awk sources with: +# mkdir -p ../gnu.awk && (cd ../gnu.awk && bash ../awk/util/fetch-gnu.sh) +# +# Usage: ./util/run-gnu-testsuite.sh [options] +# +# Options: +# -h, --help Show this help message +# -v, --verbose Show diagnostics (diffs) for failing tests +# -q, --quiet Only print failures and the final summary +# --json-output FILE Write results to FILE as JSON +# +# Environment variables: +# GNU_AWK_DIR Path to the extracted GNU awk source tree +# (default: ../gnu.awk) +# PER_RUN_TIMEOUT Overall timeout in seconds for `make check` +# (default: 1800) + +# Don't exit on failure since test failures are expected. +set -o pipefail + +# Configuration +RUST_AWK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GNU_AWK_DIR="${GNU_AWK_DIR:-${RUST_AWK_DIR}/../gnu.awk}" +GNU_TESTS_DIR="" +VERBOSE=false +QUIET=false +JSON_OUTPUT_FILE="" +PER_RUN_TIMEOUT="${PER_RUN_TIMEOUT:-1800}" +DETAILED_RESULTS=() + +# Statistics +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 +SKIPPED_TESTS=0 + +usage() { + echo "Usage: $0 [options]" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -v, --verbose Show diagnostics (diffs) for failing tests" + echo " -q, --quiet Only print failures and the final summary" + echo " --json-output FILE Write results to FILE as JSON" + echo + echo "Environment variables:" + echo " GNU_AWK_DIR Path to the extracted GNU awk source tree" + echo " (default: ../gnu.awk)" + echo " PER_RUN_TIMEOUT Overall timeout in seconds (default: 1800)" + echo + echo "Setup:" + echo " mkdir -p ../gnu.awk && (cd ../gnu.awk && bash ../awk/util/fetch-gnu.sh)" +} + +log_info() { [[ "$QUIET" != "true" ]] && echo "[INFO] $1"; return 0; } +log_success() { [[ "$QUIET" != "true" ]] && echo "[PASS] $1"; return 0; } +log_skip() { [[ "$QUIET" != "true" ]] && echo "[SKIP] $1"; return 0; } +log_warning() { echo "[WARN] $1"; } +log_error() { echo "[FAIL] $1"; } + +# Generate JSON output (schema shared with ../grep and ../sed so +# compare_test_results.py works across projects). +generate_json_output() { + cd "$RUST_AWK_DIR" || return + + local timestamp + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local rust_version + rust_version=$(cargo metadata --no-deps --format-version 1 2>/dev/null | jq -r '.packages[0].version // "unknown"') + + local tests_json="[]" + if [[ ${#DETAILED_RESULTS[@]} -gt 0 ]]; then + local temp_file + temp_file=$(mktemp) + printf "%s\n" "${DETAILED_RESULTS[@]}" > "$temp_file" + tests_json=$(jq -s '.' < "$temp_file" 2>/dev/null) || tests_json="[]" + rm -f "$temp_file" + fi + + jq -n \ + --arg timestamp "$timestamp" \ + --argjson total "$TOTAL_TESTS" \ + --argjson passed "$PASSED_TESTS" \ + --argjson failed "$FAILED_TESTS" \ + --argjson skipped "$SKIPPED_TESTS" \ + --argjson duration "$duration" \ + --arg rust_version "$rust_version" \ + --arg gnu_testsuite_dir "$GNU_TESTS_DIR" \ + --argjson tests "$tests_json" \ + '{ + timestamp: $timestamp, + summary: { + total: $total, + passed: $passed, + failed: $failed, + skipped: $skipped, + duration_seconds: $duration + }, + environment: { + rust_awk_version: $rust_version, + gnu_testsuite_dir: $gnu_testsuite_dir + }, + tests: $tests + }' > "$JSON_OUTPUT_FILE" + + log_info "JSON results written to: $JSON_OUTPUT_FILE" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) usage; exit 0 ;; + -v|--verbose) VERBOSE=true; shift ;; + -q|--quiet) QUIET=true; shift ;; + --json-output) JSON_OUTPUT_FILE="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; usage; exit 1 ;; + esac +done + +# Validate environment +if [[ -d "$GNU_AWK_DIR" ]]; then + GNU_AWK_DIR="$(cd "$GNU_AWK_DIR" && pwd)" + GNU_TESTS_DIR="$GNU_AWK_DIR/test" +fi + +if [[ ! -f "$GNU_TESTS_DIR/Makefile.am" ]]; then + log_error "GNU awk testsuite not found at: $GNU_AWK_DIR" + log_error "Fetch it with:" + log_error " mkdir -p ${RUST_AWK_DIR}/../gnu.awk && (cd ${RUST_AWK_DIR}/../gnu.awk && bash ${RUST_AWK_DIR}/util/fetch-gnu.sh)" + exit 1 +fi + +if [[ ! -f "$RUST_AWK_DIR/Cargo.toml" ]]; then + log_error "Not in a Rust project directory: $RUST_AWK_DIR" + exit 1 +fi + +# Build the Rust awk implementation +log_info "Building Rust awk implementation..." +cd "$RUST_AWK_DIR" || exit 1 +if ! cargo build --release --quiet; then + log_error "Failed to build Rust awk implementation" + exit 1 +fi + +RUST_AWK_BIN="$RUST_AWK_DIR/target/release/awk" +if [[ ! -x "$RUST_AWK_BIN" ]]; then + log_error "Built awk binary not found at: $RUST_AWK_BIN" + exit 1 +fi +log_info "Using Rust awk binary: $RUST_AWK_BIN" + +# gawk's test/Makefile is generated by configure. Generate it once and cache it. +if [[ ! -f "$GNU_TESTS_DIR/Makefile" ]]; then + log_info "Configuring GNU awk to generate test/Makefile (one-time)..." + if ! ( cd "$GNU_AWK_DIR" && ./configure >/dev/null 2>&1 ); then + log_error "Failed to configure GNU awk in $GNU_AWK_DIR" + exit 1 + fi +fi +if [[ ! -f "$GNU_TESTS_DIR/Makefile" ]]; then + log_error "test/Makefile still missing after configure: $GNU_TESTS_DIR" + exit 1 +fi + +# Create a temporary wrapper that makes our Rust binary look like `gawk`. +# The wrapper is deliberately named `gawk`: gawk's testsuite assumes it is +# invoked under that name (several tests print ARGV[0]), so naming it `awk` +# would spuriously fail those tests on argv[0] alone. +TEST_WORK_DIR=$(mktemp -d) +trap 'rm -rf "$TEST_WORK_DIR"' EXIT +WRAPPER="$TEST_WORK_DIR/gawk" +cat > "$WRAPPER" <.ok expected-output file. +log_info "Discovering tests from $GNU_TESTS_DIR/*.ok" +declare -A IS_TEST=() +for ok in "$GNU_TESTS_DIR"/*.ok; do + [[ -e "$ok" ]] || continue + name=$(basename "$ok" .ok) + IS_TEST["$name"]=1 +done +log_info "Found ${#IS_TEST[@]} known tests" + +# Clean leftover failure markers from any previous run so our count is accurate. +( cd "$GNU_TESTS_DIR" && rm -f _* ) + +# Drive gawk's own testsuite with our binary. -k keeps going past failures. +RUN_LOG="$TEST_WORK_DIR/make.log" +log_info "Running GNU awk testsuite (this can take a while)..." +start_time=$(date +%s) + +timeout --kill-after=30 "$PER_RUN_TIMEOUT" \ + make -k -C "$GNU_TESTS_DIR" check \ + AWK="$WRAPPER" \ + LC_ALL=C \ + "$RUN_LOG" 2>&1 +make_exit=$? + +end_time=$(date +%s) +duration=$((end_time - start_time)) + +if [[ $make_exit -eq 124 || $make_exit -eq 125 ]]; then + log_warning "make check hit the ${PER_RUN_TIMEOUT}s timeout; results may be partial" +fi + +# Record a test result (for JSON output) +record_result() { + if [[ -n "$JSON_OUTPUT_FILE" ]]; then + DETAILED_RESULTS+=("$(jq -n \ + --arg name "$1" --arg status "$2" --arg error "$3" \ + '{name: $name, status: $status, error: $error}')") + fi +} + +# Attempted tests = test names gawk echoed (one bare name per line) that are in +# our universe. Group-skipped tests never echo, so they fall out as SKIP. +declare -A ATTEMPTED=() +while IFS= read -r line; do + [[ -n "${IS_TEST[$line]:-}" ]] && ATTEMPTED["$line"]=1 +done < "$RUN_LOG" + +# Failed tests = leftover `_` markers (gawk only removes them on a match). +declare -A FAILED=() +for marker in "$GNU_TESTS_DIR"/_*; do + [[ -e "$marker" ]] || continue + name=$(basename "$marker") + name="${name#_}" + [[ -n "${IS_TEST[$name]:-}" ]] && FAILED["$name"]=1 +done + +# Some recipes build their `_` output indirectly (e.g. `sed < prog.out > +# _`); when the awk-under-test never produces the intermediate file, that +# `_` is never created and `cmp` errors with "No such file" instead of +# leaving a mismatch marker. gawk's plain marker count would miss these, so we +# also mine the run log for those errors and count them as failures — otherwise +# a non-functional binary would be credited with spurious passes. +while IFS= read -r name; do + [[ -n "${IS_TEST[$name]:-}" ]] && FAILED["$name"]=1 +done < <(sed -n 's/^cmp: _\([^:]*\): No such file.*/\1/p' "$RUN_LOG") + +# Classify every known test. +for name in $(printf '%s\n' "${!IS_TEST[@]}" | sort); do + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + if [[ -n "${FAILED[$name]:-}" ]]; then + FAILED_TESTS=$((FAILED_TESTS + 1)) + log_error "$name" + if [[ "$VERBOSE" == "true" ]]; then + head -10 "$GNU_TESTS_DIR/_$name" 2>/dev/null | sed 's/^/ | /' + fi + record_result "$name" "FAIL" "Output differs from $name.ok" + elif [[ -n "${ATTEMPTED[$name]:-}" ]]; then + PASSED_TESTS=$((PASSED_TESTS + 1)) + log_success "$name" + record_result "$name" "PASS" "" + else + SKIPPED_TESTS=$((SKIPPED_TESTS + 1)) + log_skip "$name" + record_result "$name" "SKIP" "Not run (group-skipped: locale/MPFR/shared-lib)" + fi +done + +# Tidy up the markers we created in the (shared) GNU tree. +( cd "$GNU_TESTS_DIR" && rm -f _* ) + +# Print summary +echo +echo "=========================================" +echo "GNU awk testsuite results" +echo "=========================================" +echo "Total tests: $TOTAL_TESTS" +echo "Passed: $PASSED_TESTS" +echo "Failed: $FAILED_TESTS" +echo "Skipped: $SKIPPED_TESTS" +echo "Duration: ${duration}s" + +if [[ -n "$JSON_OUTPUT_FILE" ]]; then + generate_json_output +fi + +if [[ $((PASSED_TESTS + FAILED_TESTS)) -gt 0 ]]; then + pass_rate=$(( (PASSED_TESTS * 100) / (PASSED_TESTS + FAILED_TESTS) )) + echo "Pass rate: ${pass_rate}%" +fi + +# Mirror the exit convention of ../grep and ../sed: nonzero if anything failed. +[[ $FAILED_TESTS -eq 0 ]] && exit 0 || exit 1