diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bcff91ca..5c0f76b8 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -2,7 +2,7 @@ ## Project -Zig 0.16.x code intelligence server. Tests live in `src/tests.zig`. Build and test with `zig build test`. +Zig 0.17.0-dev code intelligence server (pinned to `0.17.0-dev.813+2153f8143`). Tests live in `src/test_*.zig`. Build and test with `zig build test`. ## Rules diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index 1bb9bc9d..3b95cffd 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -7,6 +7,12 @@ permissions: contents: read pull-requests: write +env: + ZIG_VERSION: 0.17.0-dev.813+2153f8143 + ZIG_SHA256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 + LEGACY_ZIG_VERSION: 0.16.0 + LEGACY_ZIG_SHA256: 70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 + jobs: bench: runs-on: ubuntu-latest @@ -15,41 +21,61 @@ jobs: with: fetch-depth: 0 - - name: Install Zig + - name: Install pinned Zig toolchains run: | - curl -L https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz -o zig.tar.xz - tar -xf zig.tar.xz - echo "$PWD/zig-x86_64-linux-0.16.0" >> "$GITHUB_PATH" + set -euo pipefail + + head_archive="zig-x86_64-linux-${ZIG_VERSION}.tar.xz" + head_dir="zig-x86_64-linux-${ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/builds/${head_archive}" -o zig-head.tar.xz + echo "${ZIG_SHA256} zig-head.tar.xz" | sha256sum --check + tar -xf zig-head.tar.xz + test "$("$PWD/${head_dir}/zig" version)" = "$ZIG_VERSION" + + legacy_archive="zig-x86_64-linux-${LEGACY_ZIG_VERSION}.tar.xz" + legacy_dir="zig-x86_64-linux-${LEGACY_ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/download/${LEGACY_ZIG_VERSION}/${legacy_archive}" -o zig-legacy.tar.xz + echo "${LEGACY_ZIG_SHA256} zig-legacy.tar.xz" | sha256sum --check + tar -xf zig-legacy.tar.xz + test "$("$PWD/${legacy_dir}/zig" version)" = "$LEGACY_ZIG_VERSION" + + echo "$PWD/${head_dir}" >> "$GITHUB_PATH" - name: Benchmark head run: python3 scripts/run-bench-json.py bench-head.json - - name: Benchmark base + - name: Benchmark base with its declared compiler + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | - git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1 + set -euo pipefail + git fetch origin "$BASE_REF" --depth=1 git worktree add ../codedb-base FETCH_HEAD cd ../codedb-base - if python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json"; then - echo "have_base_json=true" >> "$GITHUB_ENV" - else - echo "have_base_json=false" >> "$GITHUB_ENV" - fi + + base_zig_version=$(sed -n 's/.*\.minimum_zig_version = "\([^"]*\)".*/\1/p' build.zig.zon) + case "$base_zig_version" in + "$ZIG_VERSION") + base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${ZIG_VERSION}" + ;; + "$LEGACY_ZIG_VERSION") + base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${LEGACY_ZIG_VERSION}" + ;; + *) + echo "unsupported base Zig version: ${base_zig_version:-missing}" >&2 + exit 1 + ;; + esac + test "$("$base_zig_dir/zig" version)" = "$base_zig_version" + PATH="$base_zig_dir:$PATH" \ + python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json" - name: Compare - if: env.have_base_json == 'true' run: | python3 scripts/compare-bench.py bench-base.json bench-head.json --threshold-pct 10 --markdown-out bench-report.md - - name: Bootstrap report - if: env.have_base_json != 'true' - run: | - cat > bench-report.md <<'EOF' - ## Benchmark Regression Report - - Skipped strict comparison because the base branch does not yet emit machine-readable benchmark JSON. - This PR introduces the JSON benchmark format that future PRs will compare against. - EOF - - name: Upload artifacts uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 00229911..6e83a8ce 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -12,7 +12,7 @@ permissions: contents: write env: - ZIG_VERSION: 0.16.0 + ZIG_VERSION: 0.17.0-dev.813+2153f8143 jobs: build: @@ -23,25 +23,25 @@ jobs: matrix: include: - runner: macos-15-intel + zig_host: x86_64-macos + zig_sha256: 3938c46ae4bca3c13f423b09503e3ef00bb4b7ef12b8bc1e5122ede366057a5b zig_target: x86_64-macos asset_name: codedb-darwin-x86_64 - zig_archive: zig-x86_64-macos-0.16.0.tar.xz - zig_dir: zig-x86_64-macos-0.16.0 - runner: macos-14 + zig_host: aarch64-macos + zig_sha256: 36673d2513afa4a96c86780648ba504beedd7f0451389091cf9d53e38d5b4840 zig_target: aarch64-macos asset_name: codedb-darwin-arm64 - zig_archive: zig-aarch64-macos-0.16.0.tar.xz - zig_dir: zig-aarch64-macos-0.16.0 - runner: ubuntu-24.04 + zig_host: x86_64-linux + zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 zig_target: x86_64-linux asset_name: codedb-linux-x86_64 - zig_archive: zig-x86_64-linux-0.16.0.tar.xz - zig_dir: zig-x86_64-linux-0.16.0 - runner: ubuntu-24.04 + zig_host: x86_64-linux + zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 zig_target: aarch64-linux asset_name: codedb-linux-arm64 - zig_archive: zig-x86_64-linux-0.16.0.tar.xz - zig_dir: zig-x86_64-linux-0.16.0 env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.event.release.tag_name }} APPLE_CERTIFICATE_P12: ${{ secrets.APPLE_CERTIFICATE_P12 }} @@ -53,13 +53,22 @@ jobs: fetch-depth: 0 ref: ${{ env.RELEASE_TAG }} - - name: Install Zig + - name: Install pinned Zig shell: bash run: | set -euo pipefail - curl -L "https://ziglang.org/download/${ZIG_VERSION}/${{ matrix.zig_archive }}" -o zig.tar.xz + grep -Fq ".minimum_zig_version = \"${ZIG_VERSION}\"" build.zig.zon || { + echo "release tag does not require the workflow's pinned Zig ${ZIG_VERSION}" >&2 + exit 1 + } + archive="zig-${{ matrix.zig_host }}-${ZIG_VERSION}.tar.xz" + zig_dir="zig-${{ matrix.zig_host }}-${ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/builds/${archive}" -o zig.tar.xz + echo "${{ matrix.zig_sha256 }} zig.tar.xz" | shasum -a 256 --check tar -xf zig.tar.xz - echo "$PWD/${{ matrix.zig_dir }}" >> "$GITHUB_PATH" + test "$("$PWD/${zig_dir}/zig" version)" = "$ZIG_VERSION" + echo "$PWD/${zig_dir}" >> "$GITHUB_PATH" - name: Import Apple signing certificate if: runner.os == 'macOS' && env.APPLE_CERTIFICATE_P12 != '' diff --git a/CHANGELOG.md b/CHANGELOG.md index 04e537aa..a1e7ae37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,94 @@ # Changelog +## 0.2.5829 - 2026-07-11 + +The release toolchain moves to pinned Zig `0.17.0-dev.813+2153f8143` without +changing codedb's retrieval semantics, while a focused performance pass makes +user-facing search faster than the previous Zig 0.16.0 build. The migration, +performance work, and retrieval-parity requirements are tracked in #673. + +### Zig 0.17 migration + +- **Compiler and release jobs are pinned to the exact tested development + snapshot.** `build.zig.zon` and binary-release CI select and assert + `0.17.0-dev.813+2153f8143`; transition benchmark CI selects each revision's + verified 0.16/0.17 compiler instead of silently skipping the base comparison. + Release downloads are verified before use. The build graph uses Zig 0.17 + passthrough/lazy-path APIs, and removed + language/library APIs were migrated without changing command behavior. +- **Published targets remain covered.** Native, Windows, and freestanding/WASM + paths use target-appropriate atomics and platform guards. Normal POSIX and + Windows argv are borrowed for process lifetime rather than copied again by + the command parser. +- **Dependencies are reproducible.** The Zig-0.17-compatible `nanoregex` source + is vendored with its license; the small protocol-neutral JSON helpers formerly + supplied by `mcp-zig` now live in `src/mcp_json.zig`, avoiding an otherwise + incompatible server-framework dependency. +- **Migration details are documented.** `docs/zig-0.17-migration.md` records the + removed APIs, dependency strategy, target-specific fixes, verification flow, + and the exact codedb file/subsystem inventory. + +### Faster than Zig 0.16, with unchanged retrieval + +Same-machine A/B used native macOS arm64 `ReleaseFast` binaries, clean +`428d8df` on Zig 0.16.0 as the baseline, and an immutable 631-file +`git archive` corpus. Direct tests ran 20 alternating A/B pairs with 200 +iterations per query. CLI tests used 5 warmups and 40 measured Hyperfine runs, +isolated homes/caches, and daemon/telemetry disabled. + +- **Cold single-token CLI search: 127.0ms → 105.7ms** — **1.20× faster** + (−16.8%). Warm snapshot-backed CLI search: 117.2ms → 102.8ms — **1.14× + faster** (−12.3%). Hyperfine reported outliers in both sets; the Zig 0.17 + build also had materially lower variance. +- **Exact-word lookup geometric mean: 3.217× faster.** Representative medians: + `config` 6607.5ns → 2452.5ns, `error` 12792.5ns → 3227.5ns, `request` + 6245ns → 2345ns, and `response` 10860ns → 2885ns. Pair-sorted postings now + compact adjacent duplicate `(doc_id, line)` hits linearly; malformed, legacy, + or fragmented postings retain full hash deduplication and first-occurrence + order through a detected fallback. +- **Tier-0 content retrieval avoids a per-query document map** when posting doc + IDs are nondecreasing. Any ordering break selects the previous direct-slot or + hash-map path, preserving defensive behavior for incrementally fragmented + indexes. Ranking weights, tier order, result caps, and tie-breakers are + unchanged. +- **Cold trigram construction reuses one directory handle per worker** and rolls + both raw and normalized byte windows, reducing overlapping loads and ASCII + normalization from roughly four per byte to one. Whitespace skipping, + `loc_mask`, `next_mask`, case folding, and final-trigram boundaries are + unchanged. +- **Release startup avoids unnecessary work.** Optimized builds execute directly + on the process stack instead of creating and immediately joining a worker + thread; Debug retains its 64MB-stack trampoline. Borrowed argv parsing removes + per-argument allocation/copy/free while retaining Windows bootstrap lifetime. + Completed word-index searches also retain their existing shared lock instead + of unlocking and immediately reacquiring it. + +No scoring, ranking, tokenization, parser, filtering, or result-rendering +formula changed in this pass. Direct benchmark hit counts are identical, and +parity tests cover sorted and malformed word postings, grouped and fragmented +Tier-0 aggregation, sequential versus parallel indexing, persisted/mmap index +round-trips, and lazy word-index rebuilds. + +The overall direct-query geometric mean is **1.405× faster**; search-only and +symbol groups are effectively flat to slightly faster (1.007× and 1.013×). +The remaining known gap is the generic full parse/serial-commit initial scan: +166ms → 169ms median (**1.8% slower**). Production cold single-token search uses +the optimized trigram scan path above. Closing the generic gap requires a +larger parser/commit ownership or pipeline redesign and is intentionally not +mixed into this behavior-preserving release. + +### Verification + +- uncached full Zig suite: 23/23 build steps; 892/896 tests passed, 4 platform skips +- focused suites include 181/181 index, 132/132 explore, 113/113 search, + and 152/156 MCP tests (4 platform skips) +- MCP E2E: 20/20 passed across roots handshake, explicit-root, no-roots, and + inline-argument scenarios +- immutable-corpus direct-query hit counts matched the Zig 0.16.0 baseline +- `git diff --check` and modified-file lint passed + + ## 0.2.5828 - 2026-07-05 Windows warm-daemon parity, a 2× faster `codedb_context`, OSTree (Fedora diff --git a/README.md b/README.md index 6b36a070..4eaf2a37 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Release License - Zig 0.16 + Zig 0.17.0-dev Alpha Ask DeepWiki
@@ -121,6 +121,7 @@ This replaces the `codedb` binary with the latest GitHub Release and keeps your - **[CLI reference](docs/cli.md)** — every command, every flag - **[Architecture](docs/architecture.md)** — engine internals, index layout - **[Benchmarks](docs/benchmarks.md)** — micro-benchmarks + agentic-eval results vs codegraph, FTS5, lean-ctx +- **[Zig 0.17.0-dev migration guide](docs/zig-0.17-migration.md)** — repeatable zigup workflow and API change recipes | Platform | Binary | Signed | |----------|--------|--------| @@ -454,7 +455,7 @@ rm -f codedb.snapshot # remove snapshot from current project only ## 🔨 Building from Source -**Requirements:** Zig 0.16+ +**Requirements:** Zig `0.17.0-dev.813+2153f8143` (the exact tested development snapshot). See the [migration guide](docs/zig-0.17-migration.md) for reproducible zigup setup. ```bash git clone https://github.com/justrach/codedb.git diff --git a/build.zig b/build.zig index f962257e..908ddddf 100644 --- a/build.zig +++ b/build.zig @@ -33,10 +33,6 @@ pub fn build(b: *std.Build) void { }), }); - // ── mcp-zig dependency ── - const mcp_dep = b.dependency("mcp_zig", .{}); - exe.root_module.addImport("mcp", mcp_dep.module("mcp")); - // ── nanoregex dependency ── const nanoregex_dep = b.dependency("nanoregex", .{}); exe.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); @@ -58,16 +54,15 @@ pub fn build(b: *std.Build) void { "--timestamp", "-s", identity, - b.getInstallPath(.bin, "codedb"), }); - codesign.step.dependOn(&install_exe.step); - b.getInstallStep().dependOn(&codesign.step); + codesign.addFileArg(exe.getEmittedBin()); + install_exe.step.dependOn(&codesign.step); } } const run_cmd = b.addRunArtifact(exe); run_cmd.step.dependOn(b.getInstallStep()); - if (b.args) |args| run_cmd.addArgs(args); + run_cmd.addPassthruArgs(); const run_step = b.step("run", "Run codedb daemon"); run_step.dependOn(&run_cmd.step); @@ -76,16 +71,16 @@ pub fn build(b: *std.Build) void { const test_filter = b.option([]const u8, "test-filter", "Only run tests whose name contains this substring"); const test_step = b.step("test", "Run all tests"); - const test_files = [_]struct { name: []const u8, path: []const u8, needs_mcp: bool, needs_nanoregex: bool }{ - .{ .name = "test-core", .path = "src/test_core.zig", .needs_mcp = false, .needs_nanoregex = false }, - .{ .name = "test-explore", .path = "src/test_explore.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-index", .path = "src/test_index.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-parser", .path = "src/test_parser.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-search", .path = "src/test_search.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-snapshot", .path = "src/test_snapshot.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-mcp", .path = "src/test_mcp.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-query", .path = "src/test_query.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-bench", .path = "src/test_bench.zig", .needs_mcp = false, .needs_nanoregex = true }, + const test_files = [_]struct { name: []const u8, path: []const u8, needs_nanoregex: bool }{ + .{ .name = "test-core", .path = "src/test_core.zig", .needs_nanoregex = false }, + .{ .name = "test-explore", .path = "src/test_explore.zig", .needs_nanoregex = true }, + .{ .name = "test-index", .path = "src/test_index.zig", .needs_nanoregex = true }, + .{ .name = "test-parser", .path = "src/test_parser.zig", .needs_nanoregex = true }, + .{ .name = "test-search", .path = "src/test_search.zig", .needs_nanoregex = true }, + .{ .name = "test-snapshot", .path = "src/test_snapshot.zig", .needs_nanoregex = true }, + .{ .name = "test-mcp", .path = "src/test_mcp.zig", .needs_nanoregex = true }, + .{ .name = "test-query", .path = "src/test_query.zig", .needs_nanoregex = true }, + .{ .name = "test-bench", .path = "src/test_bench.zig", .needs_nanoregex = true }, }; for (test_files) |tf| { @@ -97,7 +92,6 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - if (tf.needs_mcp) t.root_module.addImport("mcp", mcp_dep.module("mcp")); if (tf.needs_nanoregex) t.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); if (test_filter) |f| { const filters = b.allocator.alloc([]const u8, 1) catch @panic("oom"); @@ -111,7 +105,6 @@ pub fn build(b: *std.Build) void { individual_step.dependOn(&run.step); } - // ── Library tests (verify the module root compiles) ── const lib_tests = b.addTest(.{ .root_module = b.createModule(.{ @@ -135,7 +128,6 @@ pub fn build(b: *std.Build) void { adversarial_tests.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); test_step.dependOn(&b.addRunArtifact(adversarial_tests).step); - // ── Benchmarks ── const bench = b.addExecutable(.{ .name = "bench", @@ -147,9 +139,8 @@ pub fn build(b: *std.Build) void { }), }); const bench_run = b.addRunArtifact(bench); - bench.root_module.addImport("mcp", mcp_dep.module("mcp")); bench.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); - if (b.args) |args| bench_run.addArgs(args); + bench_run.addPassthruArgs(); const bench_step = b.step("bench", "Run benchmarks"); bench_step.dependOn(&bench_run.step); @@ -163,10 +154,9 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - bench_edge.root_module.addImport("mcp", mcp_dep.module("mcp")); bench_edge.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); const bench_edge_run = b.addRunArtifact(bench_edge); - if (b.args) |args| bench_edge_run.addArgs(args); + bench_edge_run.addPassthruArgs(); const bench_edge_step = b.step("bench-edge", "Run edge-case benchmarks (synthetic pathological corpus)"); bench_edge_step.dependOn(&bench_edge_run.step); @@ -180,10 +170,9 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - benchmark.root_module.addImport("mcp", mcp_dep.module("mcp")); benchmark.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); const benchmark_run = b.addRunArtifact(benchmark); - if (b.args) |args| benchmark_run.addArgs(args); + benchmark_run.addPassthruArgs(); const benchmark_step = b.step("benchmark", "Run repo benchmark (use -- --root /path/to/repo)"); benchmark_step.dependOn(&benchmark_run.step); diff --git a/build.zig.zon b/build.zig.zon index e19fabdb..63178473 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,20 +1,16 @@ .{ .name = .codedb2, .fingerprint = 0x6e18d96ca2a31757, - .version = "0.2.5826", - .minimum_zig_version = "0.16.0", + .version = "0.2.5829", + .minimum_zig_version = "0.17.0-dev.813+2153f8143", .dependencies = .{ - .mcp_zig = .{ - .url = "https://github.com/justrach/mcp-zig/archive/refs/heads/feature/7-zig-0-16-0-migration.tar.gz", - .hash = "mcp_zig-0.2.0-_PilzNJkAQADzH2t3vqpd_nl_W0ta-gDaumXKttuPyBy", - }, .nanoregex = .{ - .url = "git+https://github.com/justrach/nanoregex#736b46703454d5f37d3e46164fc91354386bb29c", - .hash = "nanoregex-0.0.1-EdkhcVDrAgB3eslIGGP7RK60162q26bVDA9GH44_iKnt", + .path = "vendor/nanoregex", }, }, .paths = .{ "src", + "vendor/nanoregex", "build.zig", "build.zig.zon", }, diff --git a/docs/agents.md b/docs/agents.md index ecbd3fdb..2b542f89 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -2,7 +2,7 @@ ## Project -Zig 0.16.x code intelligence server. Tests live in the split `src/test_*.zig` files — one binary per area (see `build.zig`). Build and test with `zig build test`; run a single binary with e.g. `zig build test-index`. +Zig 0.17.0-dev code intelligence server (pinned to `0.17.0-dev.813+2153f8143`). Tests live in the split `src/test_*.zig` files — one binary per area (see `build.zig`). Build and test with `zig build test`; run a single binary with e.g. `zig build test-index`. ## Rules diff --git a/docs/zig-0.17-migration.md b/docs/zig-0.17-migration.md new file mode 100644 index 00000000..2c79b80d --- /dev/null +++ b/docs/zig-0.17-migration.md @@ -0,0 +1,448 @@ +# Migrating a Zig project to 0.17.0-dev with zigup + +This guide records the migration process used for codedb and is intended to be reusable in other Zig repositories. + +The tested compiler is: + +```text +0.17.0-dev.813+2153f8143 +``` + +Zig development snapshots are moving targets. Pin the full version (including the build metadata) in CI and local setup while migrating. Upgrade to a newer snapshot deliberately, rerun the full matrix, and update the pin only after it passes. + +## 1. Establish a baseline + +Before changing compilers: + +```bash +git status --short +zig version +zig build +zig build test +``` + +Record existing failures and preserve unrelated working-tree changes. If the project has benchmarks, capture the benchmark-critical results before migration so regressions can be compared against a real baseline. + +## 2. Install and select the compiler with zigup + +This guide uses [`marler8997/zigup`](https://github.com/marler8997/zigup), not an unrelated tool with a similar name. On a clean machine, install a pinned zigup release from its [release assets](https://github.com/marler8997/zigup/releases), verify the downloaded asset against the checksum recorded by your organization, put the binary on `PATH`, and confirm `zigup help` works. Avoid piping a remote installer directly into a shell. + +Install/select the exact compiler used by codedb: + +```bash +zigup fetch 0.17.0-dev.813+2153f8143 +zigup default 0.17.0-dev.813+2153f8143 +zigup keep 0.17.0-dev.813+2153f8143 +zig version +``` + +`zigup default` displays the selected version when called without an argument. `zigup keep` is optional; it only prevents a later `zigup clean` from deleting the pinned compiler. Do not use `zigup --version`: zigup treats positional input as a requested Zig version rather than exposing that conventional flag. + +For exploratory work, `zigup master` selects the latest development compiler. Do not use a moving `master` selection in CI or release automation. + +In CI, download the exact host archive from `https://ziglang.org/builds/`, verify a repository-pinned SHA-256 before extraction, and assert that `zig version` equals the pin. A versioned URL alone does not protect a privileged release job from replaced or compromised bytes. + +## 3. Pin the package minimum + +Update `build.zig.zon`: + +```zig +.{ + // ... + .minimum_zig_version = "0.17.0-dev.813+2153f8143", +} +``` + +A full development build pin makes the tested compiler explicit. If a library intentionally supports a wider range of 0.17 snapshots, lower the minimum only after testing the oldest claimed snapshot. + +## 4. Migrate the build graph first + +Run `zig build` and fix `build.zig` errors before source errors. Zig configures dependency build scripts too, so an incompatible transitive `build.zig` can stop the build before your own source is compiled. + +### Forward arguments with `addPassthruArgs` + +The `Build.args` field was removed. + +Before: + +```zig +const run = b.addRunArtifact(exe); +if (b.args) |args| run.addArgs(args); +``` + +After: + +```zig +const run = b.addRunArtifact(exe); +run.addPassthruArgs(); +``` + +This preserves commands such as: + +```bash +zig build run -- --help +``` + +Apply the same change to benchmark and utility run steps. + +### Use lazy paths instead of `getInstallPath` + +`b.getInstallPath` was removed. Prefer artifact and lazy-path APIs so the build graph can track dependencies. + +For a command that operates on the compiled executable: + +```zig +const command = b.addSystemCommand(&.{ "tool", "--flag" }); +command.addFileArg(exe.getEmittedBin()); +install_artifact.step.dependOn(&command.step); +``` + +For codesigning, codedb signs the emitted artifact and makes installation depend on the signing step. Avoid constructing `zig-out` paths manually because `--prefix` and cross-target installation can change them. + +## 5. Replace removed repetition syntax + +The `value ** count` array/string repetition syntax is gone. Use `@splat` with a result type supplied by the destination. + +### Arrays + +Before: + +```zig +var flags: [256]bool = [_]bool{false} ** 256; +var matrix: [256][256]u64 = .{.{0} ** 256} ** 256; +``` + +After: + +```zig +var flags: [256]bool = @splat(false); +var matrix: [256][256]u64 = @splat(@splat(0)); +``` + +### Repeated byte data + +Before: + +```zig +const payload = "x" ** 300; +consume(payload); +``` + +After: + +```zig +const payload: [300]u8 = @splat('x'); +consume(&payload); +``` + +Zig 0.17 no longer implicitly coerces an array value to a slice in this case; use `&payload` when the callee expects `[]const u8`. + +### Inline fixed buffers + +Give `@splat` an explicit array type when no destination type is available: + +```zig +try writer.writeAll(&@as([40]u8, @splat(0))); +``` + +A useful first-pass search is: + +```bash +grep -R -n ' \*\* ' src --include='*.zig' +``` + +Review matches rather than replacing blindly: glob documentation and comments legitimately contain `**`. + +## 6. Replace `bufPrintZ` + +`std.fmt.bufPrintZ` became `std.fmt.bufPrintSentinel`: + +```zig +const path_z = try std.fmt.bufPrintSentinel(&buf, "{s}", .{path}, 0); +``` + +For a large codebase, a compatibility helper keeps the migration focused: + +```zig +pub fn bufPrintZ( + buf: []u8, + comptime fmt: []const u8, + args: anytype, +) std.fmt.BufPrintError![:0]u8 { + return std.fmt.bufPrintSentinel(buf, fmt, args, 0); +} +``` + +Call sites can then move to the helper without changing their return types. + +## 7. Stop calling unstable I/O vtable fields directly + +The direct `io.vtable.netRead` field is no longer available. Use the public stream operation: + +Before: + +```zig +var iov: [1][]u8 = .{dest}; +const n = try io.vtable.netRead(io.userdata, stream.socket.handle, &iov); +``` + +After: + +```zig +var iov: [1][]u8 = .{dest}; +const n = try stream.read(io, &iov); +``` + +The public method routes through `Io.operate` and remains compatible with the selected backend. Treat direct vtable access as an implementation detail unless the standard library explicitly documents it as public API. + +## 8. Update allocating writers and freestanding targets + +The unmanaged `ArrayList(u8).writer(allocator)` helper is no longer available. Use `std.Io.Writer.Allocating` when the output is fundamentally a byte stream: + +```zig +var out: std.Io.Writer.Allocating = .init(allocator); +defer out.deinit(); +const writer = &out.writer; +try writer.print("value={d}", .{value}); +const bytes = try out.toOwnedSlice(); +``` + +`toOwnedSlice()` resets the allocating writer, so the deferred `deinit()` is safe after ownership transfers. + +Compile every freestanding/WASM target separately. A native build can hide target-specific problems: + +- baseline `wasm32` supports atomic integers only up to 32 bits in this toolchain; use 32-bit counters/generations when wraparound is acceptable, or explicitly choose and validate a WebAssembly threads/atomics target; +- guard filesystem, subprocess, environment, mmap, clock, and pthread paths with `builtin.os.tag == .freestanding`; +- provide no-op locks only where the target is intentionally single-threaded; +- avoid reaching `std.posix` or libc declarations from exported WASM call graphs. + +Use a larger reference trace to locate an unexpected host-only dependency: + +```bash +zig build wasm -freference-trace=20 +``` + +## 9. Audit every dependency + +A project is not migrated if a clean checkout still depends on a package whose build script or source requires an older compiler. + +For each dependency: + +1. Run its own tests under the pinned compiler. +2. Prefer an upstream 0.17-compatible release or immutable commit. +3. If no compatible release exists, temporarily vendor or fork it. +4. Preserve its license and source history/attribution. +5. Point `build.zig.zon` at the reproducible source, not a patched local cache. +6. Add the vendored directory to the root package's `.paths` if releases must include it. + +Example local dependency: + +```zig +.dependencies = .{ + .some_library = .{ + .path = "vendor/some-library", + }, +}, +.paths = .{ + "src", + "vendor/some-library", + "build.zig", + "build.zig.zon", +}, +``` + +Never treat edits inside `.zig-cache`, a global package cache, or an ignored dependency cache as the final fix. They make only one workstation pass. + +For codedb, `nanoregex` is vendored until its upstream package is 0.17-compatible. The previous `mcp-zig` dependency supplied only small protocol-neutral JSON helpers and a root value type, so those helpers were moved into codedb instead of retaining a second server-framework dependency. + +## 10. Format narrowly while iterating + +During the error-fixing loop, format only touched files: + +```bash +zig fmt build.zig build.zig.zon src/file_you_changed.zig +``` + +A new Zig formatter can produce repository-wide canonical-format changes. Run `zig fmt src` only when that churn is intentional, and review the diff separately from semantic migration changes. + +## 11. Verify in layers + +Use a narrow-to-wide loop: + +```bash +# Build graph and main executable +zig build + +# Individual project steps while iterating +zig build test-core +zig build test-mcp + +# Full unit suite +zig build test + +# Optional targets your project publishes +zig build wasm +zig build -Doptimize=ReleaseFast +``` + +For codedb, MCP changes also require: + +```bash +zig build test +python3 scripts/e2e_mcp_test.py \ + --binary zig-out/bin/codedb \ + --project "$PWD" +``` + +The E2E suite checks root handshakes, normal explicit-root startup, and no-roots-client behavior. Run target/platform CI after local verification, especially Windows and cross-compiled release targets whose code may be skipped on the host. + +## 12. Diagnose nested build failures + +Tests that invoke `zig build` may report only that a child process exited nonzero. Run the nested command directly to expose the compiler diagnostic: + +```bash +zig build +zig build test-mcp +``` + +Also check for stale assumptions in test helpers, CI images, release workflows, editor configuration, and documentation. Search for the old version string and removed API names: + +```bash +grep -R -n '0\.16\|b\.args\|getInstallPath\|bufPrintZ\|vtable\.netRead' \ + --include='*.zig' --include='*.zon' --include='*.md' . +``` + +## Completion checklist + +- [ ] Exact Zig development snapshot selected and recorded +- [ ] `build.zig.zon` minimum updated +- [ ] Build graph compiles without deprecated/removed APIs +- [ ] Source repetition syntax migrated to `@splat` +- [ ] Sentinel formatting calls migrated +- [ ] Public I/O operations replace direct vtable access +- [ ] Every dependency builds from a clean, reproducible source +- [ ] `zig build` passes +- [ ] `zig build test` passes +- [ ] Optional targets and release modes pass +- [ ] Project-specific E2E tests pass +- [ ] CI, README, and contributor setup reference the pinned compiler +- [ ] Diff reviewed for unrelated formatter churn and pre-existing work preserved + +## codedb 0.2.5829 implementation record + +This section records the concrete release changes behind the reusable guidance +above. It is intentionally explicit so a future compiler upgrade can distinguish +compatibility edits from retrieval-sensitive performance work. + +### Toolchain, build, and release automation + +| Area | Files | Change | +|---|---|---| +| Compiler/package pin | `build.zig.zon`, `README.md`, `.github/copilot-instructions.md`, `docs/agents.md` | Require the exact tested `0.17.0-dev.813+2153f8143` snapshot. | +| Build graph | `build.zig` | Replace removed argument and install-path APIs, keep benchmark argument forwarding, and preserve codesign/install dependencies through emitted lazy paths. | +| CI and releases | `.github/workflows/bench-regression.yml`, `.github/workflows/release-binaries.yml` | Download verified pinned compilers and assert `zig version`. Release targets use the 0.17 snapshot; transition benchmarks select 0.16 or 0.17 from each revision's `minimum_zig_version` so the cross-compiler regression gate cannot silently skip its base. | +| Dependencies | `build.zig.zon`, `vendor/nanoregex/`, `src/mcp_json.zig` | Vendor the compatible regex implementation and internalize the small JSON protocol helpers formerly taken from `mcp-zig`. | + +### Runtime compatibility inventory + +- `src/cio.zig` adapts sentinel formatting and Zig I/O entry points while + retaining POSIX/Windows behavior. +- `src/main.zig`, `src/cli_proxy.zig`, `src/commands.zig`, `src/server.zig`, and + `src/mcp.zig` adapt argv and stream handling. Parsed arguments borrow the + process-lifetime bootstrap argv; they are not retained beyond that lifetime. +- `src/explore.zig`, `src/hot_cache.zig`, `src/codegraph.zig`, + `src/snapshot.zig`, and `src/telemetry.zig` use target-compatible atomics, + writers, and host-operation guards. Freestanding/WASM builds do not enter + filesystem, process, telemetry, or mmap paths. +- Removed repetition syntax was converted to typed `@splat` expressions across + runtime and tests. Large fixtures that only encoded compiler-stress behavior + were reduced or removed where Zig 0.17 no longer accepts the old construct; + retrieval assertions were retained in focused suites. +- `src/wasm.zig` uses a 32-bit generation on baseline `wasm32` and keeps native + 64-bit generations elsewhere, preserving cache-invalidation behavior within + each target's supported atomic widths. + +### Performance changes and invariants + +1. **Exact-word deduplication (`src/index.zig`).** Valid posting lists are sorted + by `(doc_id, line_num)`, so `searchDeduped` writes directly into one + caller-owned allocation and removes adjacent duplicates. It detects any + ordering break before returning and reruns the complete list through the old + hash-dedup contract. The fallback preserves first-occurrence ordering for + malformed, legacy, and incrementally fragmented data. +2. **Tier-0 document aggregation (`src/explore.zig`).** A nondecreasing posting + list guarantees one contiguous run per document, allowing direct result + construction without a slots array or hash map. Nonmonotonic lists use the + defensive pre-migration aggregation path. Scores, line selection, ranking, + and caps are not changed. +3. **Word-index lock lifetime (`src/explore.zig`).** The common completed-index + path keeps its original shared lock. Only an incomplete index releases the + lock for exclusive rebuild and reacquires it before lookup. +4. **Release startup (`src/main.zig`).** Debug keeps the 64MB worker-stack + trampoline. Optimized builds run the measured approximately 190KB frame on + the process stack, removing worker creation/join overhead. Mainstream native + process stacks have ample headroom; constrained-stack targets should retain + or reintroduce the trampoline after target-specific validation. +5. **Worker directory reuse (`src/watcher.zig`).** Initial-scan and trigram-shard + workers open the project root once and pass the directory handle through file + reads instead of reopening it per file. Every worker closes its own handle; + root-open failures follow the existing null-result/error publication paths. +6. **Rolling trigram extraction (`src/watcher.zig`).** Shared extraction rolls + `c0..c3` and normalized `n0..n3`, loading and normalizing each incoming byte + once. It preserves whitespace-triple suppression, case folding, location and + following-character masks, and the no-next-byte boundary on the final + trigram. +7. **Persistence invariants (`src/index.zig`).** Trigram posting insertion keeps + doc IDs sorted after ID reuse, and disk remapping follows ascending in-memory + IDs. Streamed writes preserve mmap merge/binary-search ordering requirements. + +### Reproducible A/B result + +Baseline and candidate: + +- baseline: clean `428d8df`, Zig 0.16.0, native arm64 `ReleaseFast`; +- candidate: Zig 0.17 migration plus the optimizations above, compiler + `0.17.0-dev.813+2153f8143`, native arm64 `ReleaseFast`; +- corpus: immutable 631-file `git archive` of the baseline, outside the working + repository; +- direct queries: 20 alternating A/B pairs, 200 iterations per query; +- CLI: Hyperfine, 5 warmups, 40 measured runs, isolated `HOME` and cache, + `CODEDB_ALLOW_TEMP=1`, `CODEDB_NO_CLI_DAEMON=1`, and + `CODEDB_NO_TELEMETRY=1`. + +| Layer | Zig 0.16.0 | Zig 0.17 optimized | Result | +|---|---:|---:|---:| +| Cold single-token CLI | 127.0 ± 26.0ms | 105.7 ± 10.7ms | 1.20× faster | +| Warm snapshot CLI | 117.2 ± 27.8ms | 102.8 ± 13.3ms | 1.14× faster | +| Exact-word geometric mean | — | — | 3.217× faster | +| All direct queries geometric mean | — | — | 1.405× faster | +| Search-only geometric mean | — | — | 1.007× faster | +| Symbol geometric mean | — | — | 1.013× faster | +| Generic full initial scan, median | 166ms | 169ms | 1.8% slower | + +Hyperfine identified outliers in both CLI samples, so means are reported with +standard deviation and should not be interpreted as deterministic single-run +latency. Direct-query hit counts were asserted equal. The generic initial-scan +gap is dominated by parse plus serial commit; changing that ownership/pipeline +is a separate, higher-risk project rather than a compatibility migration. + +### Retrieval-parity gates + +The release requires all of the following before tagging: + +- exact ordered output parity between grouped Tier-0 postings and semantically + equivalent fragmented postings that force the defensive fallback; +- duplicate suppression for naturally built and malformed/non-adjacent word + postings, including first-occurrence order; +- exact trigram candidate and posting-mask parity between canonical extraction + and worker-sharded rolling extraction, including 3-byte input, whitespace, + mixed case, location-bit rollover, and final `next_mask` boundaries; +- sequential/parallel initial scan and word-shard parity; +- heap, persisted, and mmap trigram candidate parity after file removal/doc-ID + reuse; +- lazy word-index rebuild parity with an eagerly complete index; +- the full Zig test suite and MCP E2E scenarios from the project instructions. + +These are retrieval-quality checks, not only crash or hit-count checks: ordered +paths, line numbers/text, scores where applicable, candidate sets, and mask +metadata must remain equal. diff --git a/npm/package.json b/npm/package.json index b89bd269..78a11d5d 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "codedeebee", - "version": "0.2.5828", + "version": "0.2.5829", "description": "Zig code intelligence MCP server — npx launcher for the codedb native binary", "license": "MIT", "author": "justrach", diff --git a/src/adversarial_tests.zig b/src/adversarial_tests.zig index 988cf474..2bc4258a 100644 --- a/src/adversarial_tests.zig +++ b/src/adversarial_tests.zig @@ -196,7 +196,7 @@ test "adversarial: extractSparseNgrams on all-same-character content" { try testing.expect(result.len > 0); // Verify every byte position is covered by at least one n-gram - var covered: [19]bool = .{false} ** 19; + var covered: [19]bool = @splat(false); for (result) |ng| { for (ng.pos..ng.pos + ng.len) |p| { if (p < 19) covered[p] = true; @@ -649,7 +649,7 @@ test "adversarial: setFrequencyTable changes pairWeight output" { const before = pairWeight('a', 'b'); // Create a custom table where 'a','b' has a very different weight - var custom: [256][256]u16 = .{.{0x5000} ** 256} ** 256; + var custom: [256][256]u16 = @splat(@splat(0x5000)); custom['a']['b'] = 0x0100; // very low setFrequencyTable(&custom); defer resetFrequencyTable(); diff --git a/src/bench.zig b/src/bench.zig index 11317b40..f5b6b355 100644 --- a/src/bench.zig +++ b/src/bench.zig @@ -192,7 +192,6 @@ fn copyCorpus(io: std.Io, allocator: std.mem.Allocator, repo_root: []const u8, t } } - fn makeTempCorpusDir(io: std.Io, buf: *[std.fs.max_path_bytes]u8) ![]const u8 { const base = cio.posixGetenv("TMPDIR") orelse "/tmp"; const ns = cio.nanoTimestamp(); diff --git a/src/cio.zig b/src/cio.zig index a54e3902..41c713bb 100644 --- a/src/cio.zig +++ b/src/cio.zig @@ -9,6 +9,7 @@ const std = @import("std"); const builtin = @import("builtin"); const is_windows = builtin.os.tag == .windows; +const is_freestanding = builtin.os.tag == .freestanding; // POSIX libc byte-I/O on integer fds. On Windows the CRT spells the same // primitives with a leading underscore; the cross-platform wrappers below pick @@ -60,7 +61,7 @@ fn getenv(name: [*:0]const u8) ?[*:0]const u8 { /// a write error instead of killing the process. No-op on Windows — there is no /// SIGPIPE; a broken pipe is reported through the write call's return value. pub fn ignoreSigpipe() void { - if (is_windows) { + if (is_windows or is_freestanding) { return; } else { var act: std.posix.Sigaction = .{ @@ -80,7 +81,7 @@ pub fn ignoreSigpipe() void { /// malfunctions. Called once at cli-daemon startup. On Windows the equivalent /// detachment is requested at spawn time (DETACHED_PROCESS), so this is a no-op. pub fn detachFromTerminal() void { - if (is_windows) { + if (is_windows or is_freestanding) { return; } else { _ = std.c.setsid(); @@ -97,6 +98,11 @@ pub fn detachFromTerminal() void { // ── Stdio ──────────────────────────────────────────────────────────────── +/// Zig 0.17 renamed std.fmt.bufPrintZ to bufPrintSentinel. +pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) std.fmt.BufPrintError![:0]u8 { + return std.fmt.bufPrintSentinel(buf, fmt, args, 0); +} + pub const File = struct { handle: c_int, @@ -174,6 +180,12 @@ pub const Mutex = if (is_windows) struct { pub fn tryLock(self: *Mutex) bool { return winsync.TryAcquireSRWLockExclusive(&self.inner) != 0; } +} else if (is_freestanding) struct { + pub fn lock(_: *Mutex) void {} + pub fn unlock(_: *Mutex) void {} + pub fn tryLock(_: *Mutex) bool { + return true; + } } else struct { inner: std.c.pthread_mutex_t = .{}, @@ -209,6 +221,17 @@ pub const RwLock = if (is_windows) struct { pub fn tryLockShared(self: *RwLock) bool { return winsync.TryAcquireSRWLockShared(&self.inner) != 0; } +} else if (is_freestanding) struct { + pub fn lock(_: *RwLock) void {} + pub fn unlock(_: *RwLock) void {} + pub fn lockShared(_: *RwLock) void {} + pub fn unlockShared(_: *RwLock) void {} + pub fn tryLock(_: *RwLock) bool { + return true; + } + pub fn tryLockShared(_: *RwLock) bool { + return true; + } } else struct { inner: std.c.pthread_rwlock_t = .{}, @@ -236,6 +259,7 @@ pub const RwLock = if (is_windows) struct { /// Wall-clock nanoseconds since the Unix epoch. pub fn nanoTimestamp() i128 { + if (is_freestanding) return 0; if (is_windows) { var ft: winsync.FILETIME = undefined; winsync.GetSystemTimePreciseAsFileTime(&ft); @@ -255,6 +279,7 @@ pub fn milliTimestamp() i64 { /// Monotonic tick source for Timer (raw counter units, not nanoseconds). fn monoTicks() u64 { + if (is_freestanding) return 0; if (is_windows) { var ctr: i64 = undefined; _ = winsync.QueryPerformanceCounter(&ctr); @@ -305,8 +330,10 @@ pub fn randU64() u64 { const now: u128 = @bitCast(nanoTimestamp()); const ns: u64 = @truncate(now); const sec: u64 = @truncate(now / 1_000_000_000); - const tid: u64 = @intCast(std.Thread.getCurrentId()); - const pid: u64 = if (is_windows) + const tid: u64 = if (is_freestanding) 0 else @intCast(std.Thread.getCurrentId()); + const pid: u64 = if (is_freestanding) + 0 + else if (is_windows) @intCast(winsync.GetCurrentProcessId()) else @intCast(std.c.getpid()); @@ -321,16 +348,18 @@ pub fn randU64() u64 { } pub fn currentProcessId() u32 { + if (is_freestanding) return 0; if (is_windows) return @intCast(winsync.GetCurrentProcessId()); return @intCast(std.c.getpid()); } pub fn userId() usize { - if (is_windows) return 0; + if (is_windows or is_freestanding) return 0; return @intCast(std.c.getuid()); } pub fn sleepMs(ms: u64) void { + if (is_freestanding) return; if (is_windows) { winsync.Sleep(@intCast(@min(ms, @as(u64, std.math.maxInt(u32))))); } else { @@ -363,6 +392,7 @@ pub fn readFd(fd: c_int, buf: []u8) isize { } pub fn posixGetenv(name: []const u8) ?[]const u8 { + if (is_freestanding) return null; var buf: [256]u8 = undefined; if (name.len >= buf.len) return null; @memcpy(buf[0..name.len], name); @@ -990,6 +1020,7 @@ pub fn mmapReadonly(handle: anytype, len: usize) MmapError![]align(std.heap.page /// Release a view previously returned by mmapReadonly. pub fn munmap(data: []align(std.heap.page_size_min) const u8) void { + if (is_freestanding) return; if (is_windows) { _ = winmap.UnmapViewOfFile(data.ptr); } else { diff --git a/src/cli_proxy.zig b/src/cli_proxy.zig index d0713079..45e5fa9e 100644 --- a/src/cli_proxy.zig +++ b/src/cli_proxy.zig @@ -154,7 +154,7 @@ fn cliPipeMetadataPath(allocator: std.mem.Allocator, data_dir: []const u8) ![]u8 } fn cliRandomPipeName(buf: []u8) ?[:0]const u8 { - return std.fmt.bufPrintZ(buf, "\\\\.\\pipe\\codedb-{d}-{x:0>16}-{x:0>16}", .{ + return cio.bufPrintZ(buf, "\\\\.\\pipe\\codedb-{d}-{x:0>16}-{x:0>16}", .{ cio.currentProcessId(), secureRandomU64(), secureRandomU64(), @@ -450,7 +450,7 @@ const DaemonLock = if (builtin.os.tag == .windows) win.HANDLE else c_int; pub fn daemonLockTryAcquire(data_dir: []const u8) ?DaemonLock { var buf: [std.fs.max_path_bytes]u8 = undefined; - const p = std.fmt.bufPrintZ(&buf, "{s}/cli-daemon.lock", .{data_dir}) catch return null; + const p = cio.bufPrintZ(&buf, "{s}/cli-daemon.lock", .{data_dir}) catch return null; if (builtin.os.tag == .windows) { const path_w = windowsPathZ(std.heap.page_allocator, p) orelse return null; defer std.heap.page_allocator.free(path_w); @@ -556,7 +556,7 @@ fn cliSendYieldRequest(sa: SockAddr) void { /// Returns null if `shutdown` is set while waiting. pub fn cliAcquireListener(sock_path: []const u8, retry: bool, retry_interval_ms: u64, shutdown: *std.atomic.Value(bool)) ?c_int { var z_buf: [128]u8 = undefined; - const sock_path_z = std.fmt.bufPrintZ(&z_buf, "{s}", .{sock_path}) catch return null; + const sock_path_z = cio.bufPrintZ(&z_buf, "{s}", .{sock_path}) catch return null; const sa = cliFillSockaddr(sock_path) orelse return null; var yield_sent = false; while (true) { @@ -706,7 +706,7 @@ fn cliDaemonListenPosix(io: std.Io, allocator: std.mem.Allocator, explorer: *Exp return; }; var path_z_buf: [128]u8 = undefined; - const sock_path_z = std.fmt.bufPrintZ(&path_z_buf, "{s}", .{sock_path}) catch { + const sock_path_z = cio.bufPrintZ(&path_z_buf, "{s}", .{sock_path}) catch { shutdown.store(true, .release); return; }; diff --git a/src/codegraph.zig b/src/codegraph.zig index 687692a1..fc93e0a7 100644 --- a/src/codegraph.zig +++ b/src/codegraph.zig @@ -36,13 +36,13 @@ pub const FuncInput = struct { /// languages); over-filtering a rare real call only loses an additive boost. fn isCallKeyword(name: []const u8) bool { const kws = [_][]const u8{ - "if", "else", "for", "while", "switch", "return", "catch", - "try", "defer", "errdefer", "and", "or", "orelse", "sizeof", - "typeof", "do", "case", "when", "match", "with", "in", - "not", "is", "await", "yield", "throw", "new", "delete", - "fn", "func", "function", "def", "class", "struct", "enum", - "union", "const", "var", "let", "static", "assert", "where", - "select", "from", "foreach", "using", "unless", "until", "elif", + "if", "else", "for", "while", "switch", "return", "catch", + "try", "defer", "errdefer", "and", "or", "orelse", "sizeof", + "typeof", "do", "case", "when", "match", "with", "in", + "not", "is", "await", "yield", "throw", "new", "delete", + "fn", "func", "function", "def", "class", "struct", "enum", + "union", "const", "var", "let", "static", "assert", "where", + "select", "from", "foreach", "using", "unless", "until", "elif", }; for (kws) |kw| if (std.mem.eql(u8, name, kw)) return true; return false; diff --git a/src/commands.zig b/src/commands.zig index 8a9244bf..84d31484 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -323,7 +323,7 @@ pub fn runCliDaemon(ctx: *RunCtx) !void { var sock_buf: [128]u8 = undefined; if (cliSocketPath(&sock_buf, abs_root)) |sock_path| { var sock_z_buf: [128]u8 = undefined; - if (std.fmt.bufPrintZ(&sock_z_buf, "{s}", .{sock_path})) |sock_z| { + if (cio.bufPrintZ(&sock_z_buf, "{s}", .{sock_path})) |sock_z| { _ = std.c.unlink(sock_z.ptr); } else |_| {} } diff --git a/src/explore.zig b/src/explore.zig index 5ef95ee9..8e017169 100644 --- a/src/explore.zig +++ b/src/explore.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const ContentCache = @import("hot_cache.zig").ContentCache; const nanoregex = @import("nanoregex"); const cio = @import("cio.zig"); @@ -12,6 +13,8 @@ const SparseNgramIndex = idx.SparseNgramIndex; const codegraph = @import("codegraph.zig"); const git = @import("git.zig"); +const SearchGeneration = if (builtin.cpu.arch == .wasm32) u32 else u64; + /// Fast hash context for u32-keyed maps on hot paths (ranked search aggregation). /// Zig's AutoHashMap runs the 4 key bytes through Wyhash even for an integer key; /// std.hash.int is a couple of multiply/shift/xor ops with full avalanche. Pure @@ -1351,7 +1354,7 @@ pub const Explorer = struct { plain_render_cache: PlainRenderCache, /// Bumped (atomically — searches run under the SHARED lock) by every /// mutation that can change search results; see bumpSearchGen callers. - search_gen: std.atomic.Value(u64) = std.atomic.Value(u64).init(0), + search_gen: std.atomic.Value(SearchGeneration) = std.atomic.Value(SearchGeneration).init(0), /// Buffers adopted from snapshot loads (the raw outline_state section). /// Restored FileOutlines borrow their import/symbol strings as slices into /// these (see FileOutline.borrows_strings), so they must outlive every @@ -2515,6 +2518,7 @@ pub const Explorer = struct { if (self.contents.get(path)) |cached| { return .{ .data = cached, .owned = false, .allocator = allocator }; } + if (builtin.os.tag == .freestanding) return null; const io = self.io orelse return null; const dir = self.root_dir orelse std.Io.Dir.cwd(); const data = dir.readFileAlloc(io, path, allocator, .limited(64 * 1024 * 1024)) catch return null; @@ -3497,7 +3501,24 @@ pub const Explorer = struct { const SLOT_NONE = std.math.maxInt(u32); const SLOT_INVALID = SLOT_NONE - 1; const ndocs = self.word_index.id_to_path.items.len; - const use_slots = ndocs > 0 and (ndocs <= 4096 or (ndocs <= 65536 and word_hits.len >= 512)); + + // Fresh and persisted indexes keep postings in nondecreasing doc-id + // order, which guarantees one contiguous run per document. That is + // the overwhelmingly common read path, so avoid allocating and + // populating a dedup table for it. Incremental swap-removal from an + // older/mutated index can fragment a document into multiple runs; + // detect any ordering break and retain the defensive table below. + var postings_grouped = true; + var previous_doc = word_hits[0].doc_id; + for (word_hits[1..]) |hit| { + if (hit.doc_id < previous_doc) { + postings_grouped = false; + break; + } + previous_doc = hit.doc_id; + } + + const use_slots = !postings_grouped and ndocs > 0 and (ndocs <= 4096 or (ndocs <= 65536 and word_hits.len >= 512)); var slots: []u32 = &.{}; defer if (slots.len > 0) allocator.free(slots); var idx_by_doc = std.AutoHashMap(u32, u32).init(allocator); @@ -3505,7 +3526,7 @@ pub const Explorer = struct { if (use_slots) { slots = try allocator.alloc(u32, ndocs); @memset(slots, SLOT_NONE); - } else { + } else if (!postings_grouped) { idx_by_doc.ensureTotalCapacity(@intCast(@min(word_hits.len, 1024))) catch {}; } @@ -3523,6 +3544,7 @@ pub const Explorer = struct { defer run_start = run_end; var cur: u32 = blk: { + if (postings_grouped) break :blk SLOT_NONE; if (use_slots) { if (doc_id >= ndocs) break :blk SLOT_INVALID; break :blk slots[doc_id]; @@ -3569,10 +3591,12 @@ pub const Explorer = struct { cur = SLOT_INVALID; }; } - if (use_slots) { - if (doc_id < ndocs) slots[doc_id] = cur; - } else { - idx_by_doc.put(doc_id, cur) catch {}; + if (!postings_grouped) { + if (use_slots) { + if (doc_id < ndocs) slots[doc_id] = cur; + } else { + idx_by_doc.put(doc_id, cur) catch {}; + } } } if (cur != SLOT_INVALID) { @@ -4451,6 +4475,7 @@ pub const Explorer = struct { /// Caps query at 256 bytes, results at 50 entries, file at 10 MB /// (rotates by truncate-clobber). fn appendRerankTrace(self: *const Explorer, query: []const u8, results: []const SearchResult) void { + if (builtin.os.tag == .freestanding) return; const path = self.rerank_trace_path orelse return; const io_inst = self.io orelse return; @@ -4671,6 +4696,7 @@ pub const Explorer = struct { /// git repo — the attempted flag stops re-shelling. Mirrors /// ensureCallGraph: call while holding at least a shared lock on `mu`. fn ensureCoChange(self: *Explorer) void { + if (builtin.os.tag == .freestanding) return; if (self.co_change != null or self.co_change_attempted) return; self.cochange_build_mu.lock(); defer self.cochange_build_mu.unlock(); @@ -5342,15 +5368,18 @@ pub const Explorer = struct { /// Search for a word using the inverted word index. O(1) lookup. pub fn searchWord(self: *Explorer, word: []const u8, allocator: std.mem.Allocator) ![]const idx.WordHit { self.mu.lockShared(); + var shared_locked = true; + defer if (shared_locked) self.mu.unlockShared(); + const needs_rebuild = !self.word_index_complete and (self.contents.len() > 0 or (self.io != null and self.root_dir != null)); - self.mu.unlockShared(); if (needs_rebuild) { + self.mu.unlockShared(); + shared_locked = false; try self.rebuildWordIndex(); + self.mu.lockShared(); + shared_locked = true; } - - self.mu.lockShared(); - defer self.mu.unlockShared(); // #569: a multi-word query can never be a single index token — fall back // to per-token matching so phrase-shaped queries don't silently dead-end. if (std.mem.indexOfScalar(u8, word, ' ') != null) { @@ -7529,7 +7558,7 @@ pub fn regexMatch(haystack: []const u8, pattern: []const u8) bool { /// the SIMD anchor inside indexOfCaseInsensitive — lower is rarer. Anchor /// choice never affects correctness, only how often candidates verify. const code_char_freq: [256]u8 = blk: { - var t = [_]u8{3} ** 256; + var t: [256]u8 = @splat(3); const ranks = "zqjxkvbywgpfmucdlhrsnioate"; for (ranks, 0..) |c, i| t[c] = @intCast(i + 4); for ('0'..'9' + 1) |c| t[c] = 5; @@ -8599,7 +8628,7 @@ fn fuzzyFilenameStart(path: []const u8) usize { // chars can be rejected in O(path) without the DP — and would fail the LCS check // anyway, so no result changes. pub fn fuzzyPresenceReject(query: []const u8, path: []const u8) bool { - var present = [_]bool{false} ** 256; + var present: [256]bool = @splat(false); for (path) |c| present[toLowerByte(c)] = true; var hits: usize = 0; for (query) |c| { @@ -8729,9 +8758,9 @@ pub fn fuzzyScoreBatch(query: []const u8, paths: []const []const u8, out_best: * const BoolV = @Vector(FZ_LANES, bool); const count = paths.len; - var lens: [FZ_LANES]usize = .{0} ** FZ_LANES; - var len_arr: [FZ_LANES]u32 = .{0} ** FZ_LANES; - var fname_arr: [FZ_LANES]u32 = .{0} ** FZ_LANES; + var lens: [FZ_LANES]usize = @splat(0); + var len_arr: [FZ_LANES]u32 = @splat(0); + var fname_arr: [FZ_LANES]u32 = @splat(0); var max_len: usize = 0; for (0..count) |l| { lens[l] = paths[l].len; @@ -8748,9 +8777,9 @@ pub fn fuzzyScoreBatch(query: []const u8, paths: []const []const u8, out_best: * var po_t: [FUZZY_MAX_PATH]U8V = undefined; var wb_t: [FUZZY_MAX_PATH]BoolV = undefined; for (0..max_len) |j| { - var lo: [FZ_LANES]u8 = .{0} ** FZ_LANES; - var orig: [FZ_LANES]u8 = .{0} ** FZ_LANES; - var wbf: [FZ_LANES]bool = .{false} ** FZ_LANES; + var lo: [FZ_LANES]u8 = @splat(0); + var orig: [FZ_LANES]u8 = @splat(0); + var wbf: [FZ_LANES]bool = @splat(false); for (0..count) |l| { if (j < lens[l]) { const c = paths[l][j]; diff --git a/src/hot_cache.zig b/src/hot_cache.zig index 5010b14c..cac1c128 100644 --- a/src/hot_cache.zig +++ b/src/hot_cache.zig @@ -1,4 +1,7 @@ const std = @import("std"); +const builtin = @import("builtin"); + +const AtomicCounter = if (builtin.cpu.arch == .wasm32) u32 else u64; /// Fixed-capacity CLOCK eviction cache for file contents. /// Keys are always owned (duped on put, freed on eviction/remove/clear/deinit). @@ -13,9 +16,9 @@ pub const ContentCache = struct { capacity: u32, count_: u32, allocator: std.mem.Allocator, - hits_: std.atomic.Value(u64), - misses_: std.atomic.Value(u64), - evictions_: std.atomic.Value(u64), + hits_: std.atomic.Value(AtomicCounter), + misses_: std.atomic.Value(AtomicCounter), + evictions_: std.atomic.Value(AtomicCounter), /// Total bytes of cache-owned values (borrowed values are exempt). owned_bytes: usize, /// CLOCK hand for the global budget sweep (window eviction has its own). @@ -74,9 +77,9 @@ pub const ContentCache = struct { .capacity = capacity, .count_ = 0, .allocator = allocator, - .hits_ = std.atomic.Value(u64).init(0), - .misses_ = std.atomic.Value(u64).init(0), - .evictions_ = std.atomic.Value(u64).init(0), + .hits_ = std.atomic.Value(AtomicCounter).init(0), + .misses_ = std.atomic.Value(AtomicCounter).init(0), + .evictions_ = std.atomic.Value(AtomicCounter).init(0), .owned_bytes = 0, .sweep_hand = 0, .byte_budget = DEFAULT_BYTE_BUDGET, @@ -93,9 +96,9 @@ pub const ContentCache = struct { .capacity = capacity, .count_ = 0, .allocator = allocator, - .hits_ = std.atomic.Value(u64).init(0), - .misses_ = std.atomic.Value(u64).init(0), - .evictions_ = std.atomic.Value(u64).init(0), + .hits_ = std.atomic.Value(AtomicCounter).init(0), + .misses_ = std.atomic.Value(AtomicCounter).init(0), + .evictions_ = std.atomic.Value(AtomicCounter).init(0), .owned_bytes = 0, .sweep_hand = 0, .byte_budget = DEFAULT_BYTE_BUDGET, @@ -473,13 +476,13 @@ test "ContentCache: byte budget evicts owned values until the new value fits" { cache.byte_budget = 1000; cache.max_entry_bytes = 1000; - const v300 = "x" ** 300; - try cache.put("a", v300); - try cache.put("b", v300); - try cache.put("c", v300); + const v300: [300]u8 = @splat('x'); + try cache.put("a", &v300); + try cache.put("b", &v300); + try cache.put("c", &v300); try std.testing.expectEqual(@as(usize, 900), cache.stats().owned_bytes); - try cache.put("d", v300); + try cache.put("d", &v300); const s = cache.stats(); try std.testing.expect(s.owned_bytes <= 1000); try std.testing.expect(s.evictions >= 1); @@ -494,8 +497,8 @@ test "ContentCache: per-entry ceiling refuses oversized values and drops the sta try cache.put("k", "small"); try std.testing.expect(cache.get("k") != null); - const big = "y" ** 200; - try cache.put("k", big); + const big: [200]u8 = @splat('y'); + try cache.put("k", &big); try std.testing.expect(cache.get("k") == null); try std.testing.expectEqual(@as(usize, 0), cache.stats().owned_bytes); } @@ -505,8 +508,8 @@ test "ContentCache: borrowed values are exempt from the byte budget" { defer cache.deinit(); cache.byte_budget = 10; - const big = "z" ** 1000; - try cache.putBorrowed("snap", big); + const big: [1000]u8 = @splat('z'); + try cache.putBorrowed("snap", &big); try std.testing.expect(cache.get("snap") != null); try std.testing.expectEqual(@as(usize, 0), cache.stats().owned_bytes); try std.testing.expectEqual(@as(u64, 0), cache.stats().evictions); diff --git a/src/index.zig b/src/index.zig index c7e3632b..15016513 100644 --- a/src/index.zig +++ b/src/index.zig @@ -517,34 +517,46 @@ pub const WordIndex = struct { self.mmap_data = null; } - /// Look up hits, returning results allocated by the caller. - /// Deduplicates by (path, line_num). + /// Look up hits, returning results allocated by the caller and deduplicated + /// by (doc_id, line_num). Valid posting lists are pair-sorted, so their only + /// possible duplicates are adjacent and compact in one allocation without a + /// hash table. Retain a hash fallback for malformed, legacy, or incrementally + /// swap-removed lists whose ordering has been broken. pub fn searchDeduped(self: *WordIndex, word: []const u8, allocator: std.mem.Allocator) ![]const WordHit { const hits = self.search(word); - if (hits.len == 0) return try allocator.alloc(WordHit, 0); - if (hits.len == 1) { - var out = try allocator.alloc(WordHit, 1); - out[0] = hits[0]; - return out; + var result = try allocator.alloc(WordHit, hits.len); + errdefer allocator.free(result); + + var result_len: usize = 0; + var sorted = true; + for (hits) |hit| { + if (result_len > 0) { + const previous = result[result_len - 1]; + if (hit.doc_id < previous.doc_id or + (hit.doc_id == previous.doc_id and hit.line_num < previous.line_num)) + { + sorted = false; + break; + } + if (hit.doc_id == previous.doc_id and hit.line_num == previous.line_num) continue; + } + result[result_len] = hit; + result_len += 1; } + if (sorted) return try allocator.realloc(result, result_len); const DedupKey = struct { doc_id: u32, line_num: u32 }; var seen = std.AutoHashMap(DedupKey, void).init(allocator); defer seen.deinit(); try seen.ensureTotalCapacity(@intCast(hits.len)); - - var result: std.ArrayList(WordHit) = .empty; - errdefer result.deinit(allocator); - try result.ensureTotalCapacity(allocator, hits.len); - + result_len = 0; for (hits) |hit| { - const key = DedupKey{ .doc_id = hit.doc_id, .line_num = hit.line_num }; - const gop = try seen.getOrPut(key); - if (!gop.found_existing) { - result.appendAssumeCapacity(hit); - } + const gop = try seen.getOrPut(.{ .doc_id = hit.doc_id, .line_num = hit.line_num }); + if (gop.found_existing) continue; + result[result_len] = hit; + result_len += 1; } - return result.toOwnedSlice(allocator); + return try allocator.realloc(result, result_len); } /// Collect all hits for index keys that begin with `prefix_raw` (normalized internally to @@ -700,7 +712,7 @@ pub const WordIndex = struct { var file_buf: [256 * 1024]u8 = undefined; var writer = file.writer(io, &file_buf); - var header_buf: [51]u8 = [_]u8{0} ** 51; + var header_buf: [51]u8 = @splat(0); @memcpy(header_buf[0..4], &DISK_MAGIC); std.mem.writeInt(u16, header_buf[4..6], DISK_FORMAT_VERSION, .little); std.mem.writeInt(u32, header_buf[6..10], @intCast(file_table.items.len), .little); @@ -1401,11 +1413,12 @@ pub const TrigramIndex = struct { if (!idx_gop.found_existing) { idx_gop.value_ptr.* = .{ .path_to_id = &self.path_to_id }; } - try idx_gop.value_ptr.items.append(self.allocator, .{ - .doc_id = doc_id, - .next_mask = mask.next_mask, - .loc_mask = mask.loc_mask, - }); + // removeFile can free and then reuse a lower doc_id, so append + // would violate PostingList's sorted-doc-ID invariant. Mmap-backed + // persistence relies on that ordering for its merge/binary searches. + const posting = try idx_gop.value_ptr.getOrAddPosting(self.allocator, doc_id); + posting.next_mask = mask.next_mask; + posting.loc_mask = mask.loc_mask; try tri_list.append(self.allocator, tri); } try self.file_trigrams.put(path, tri_list); @@ -1717,40 +1730,27 @@ pub const TrigramIndex = struct { }; /// Write the current in-memory index to disk in a two-file format. - /// Files are written atomically (write to tmp, then rename). + /// Each file is replaced via tmp + rename; publishing the pair is not transactional. pub fn writeToDisk(self: *TrigramIndex, io: std.Io, dir_path: []const u8, git_head: ?[40]u8) !void { - // Step 1: Build file table from path_to_id (reuse existing doc IDs for consistency) + // Step 1: Build the file table in ascending in-memory doc_id order. Posting + // lists are sorted by doc_id, so this makes their remapped disk file_ids + // sorted too — an invariant required by the mmap merge and binary searches. + // Retain the existing tracked-file selection while avoiding an intermediate + // path→disk-ID map and its second lookup pass. var file_table: std.ArrayList([]const u8) = .empty; defer file_table.deinit(self.allocator); - var disk_path_to_id = std.StringHashMap(u32).init(self.allocator); - defer disk_path_to_id.deinit(); - - if (self.file_trigrams.count() > 0) { - var ft_iter = self.file_trigrams.keyIterator(); - while (ft_iter.next()) |path_ptr| { - const id: u32 = @intCast(file_table.items.len); - try file_table.append(self.allocator, path_ptr.*); - try disk_path_to_id.put(path_ptr.*, id); - } - } else { - for (self.id_to_path.items) |path| { - if (path.len == 0) continue; - const id: u32 = @intCast(file_table.items.len); - try file_table.append(self.allocator, path); - try disk_path_to_id.put(path, id); - } - } - - // Precompute doc_id -> disk file_id ONCE (one path hash per doc), so the - // Step-3 posting loop does an array load instead of hashing the full path - // string per posting (tens of millions of times on a large repo). Mirrors - // the WordIndex doc_to_disk fix (#475). const doc_to_disk = try self.allocator.alloc(u32, self.id_to_path.items.len); defer self.allocator.free(doc_to_disk); @memset(doc_to_disk, std.math.maxInt(u32)); + const use_tracked_files = self.file_trigrams.count() > 0; for (self.id_to_path.items, 0..) |path, doc_idx| { if (path.len == 0) continue; - doc_to_disk[doc_idx] = disk_path_to_id.get(path) orelse std.math.maxInt(u32); + if (use_tracked_files and !self.file_trigrams.contains(path)) continue; + if (path.len > std.math.maxInt(u16)) return error.NameTooLong; + if (file_table.items.len >= std.math.maxInt(u32)) return error.IndexTooLarge; + const disk_id: u32 = @intCast(file_table.items.len); + try file_table.append(self.allocator, path); + doc_to_disk[doc_idx] = disk_id; } const file_count: u32 = @intCast(file_table.items.len); @@ -1770,28 +1770,46 @@ pub const TrigramIndex = struct { } }.lt); - // Step 3: Stream postings to the temporary file while retaining only - // the compact lookup table. Materializing every DiskPosting first can - // briefly double the memory required by a large cold-built index and - // then makes the allocator copy the whole postings blob again on grow. - var lookup_entries: std.ArrayList(LookupEntry) = .empty; - defer lookup_entries.deinit(self.allocator); + // Step 3: The sorted trigram list determines the lookup header count. + // Stream both payloads in fixed batches so auxiliary persistence memory + // remains bounded by the two batches rather than index cardinality. + if (trigrams_sorted.items.len > std.math.maxInt(u32)) return error.IndexTooLarge; + const lookup_entry_count: u32 = @intCast(trigrams_sorted.items.len); - // Step 4: Write postings file atomically (random suffix prevents collisions) + // Step 4: Write postings and lookup files to random-suffixed temporaries. const post_rand: u64 = cio.randU64(); const postings_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.postings.{x}.tmp", .{ dir_path, post_rand }); defer self.allocator.free(postings_tmp); const postings_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.postings", .{dir_path}); defer self.allocator.free(postings_final); + const lk_rand: u64 = cio.randU64(); + const lookup_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup.{x}.tmp", .{ dir_path, lk_rand }); + defer self.allocator.free(lookup_tmp); + const lookup_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup", .{dir_path}); + defer self.allocator.free(lookup_final); { - const file = try std.Io.Dir.cwd().createFile(io, postings_tmp, .{}); - defer file.close(io); + const postings_file = try std.Io.Dir.cwd().createFile(io, postings_tmp, .{}); + defer postings_file.close(io); + const lookup_file = try std.Io.Dir.cwd().createFile(io, lookup_tmp, .{}); + defer lookup_file.close(io); var pw_buf: [256 * 1024]u8 = undefined; - var pw = file.writer(io, &pw_buf); + var pw = postings_file.writer(io, &pw_buf); + var lw_buf: [64 * 1024]u8 = undefined; + var lw = lookup_file.writer(io, &lw_buf); - // Header v3: magic(4) + version(2) + file_count(4) + head_len(1) + head(40) = 51 bytes + // Lookup header: magic(4) + version(2) + pad(2) + entry_count(4) = 12 bytes + try lw.interface.writeAll(&LOOKUP_MAGIC); + var lookup_version_buf: [2]u8 = undefined; + std.mem.writeInt(u16, &lookup_version_buf, FORMAT_VERSION, .little); + try lw.interface.writeAll(&lookup_version_buf); + try lw.interface.writeAll(&[_]u8{ 0, 0 }); + var lookup_count_buf: [4]u8 = undefined; + std.mem.writeInt(u32, &lookup_count_buf, lookup_entry_count, .little); + try lw.interface.writeAll(&lookup_count_buf); + + // Postings header v3: magic(4) + version(2) + file_count(4) + head_len(1) + head(40) = 51 bytes try pw.interface.writeAll(&POSTINGS_MAGIC); var ver_buf: [2]u8 = undefined; std.mem.writeInt(u16, &ver_buf, FORMAT_VERSION, .little); @@ -1805,7 +1823,7 @@ pub const TrigramIndex = struct { try pw.interface.writeAll(&head); } else { try pw.interface.writeAll(&.{0}); - try pw.interface.writeAll(&([_]u8{0} ** 40)); + try pw.interface.writeAll(&@as([40]u8, @splat(0))); } // File table: for each file, path_len(u16) + path bytes @@ -1820,10 +1838,12 @@ pub const TrigramIndex = struct { // too so the inner loop does not dispatch one virtual write per // posting. The batch is fixed-size, keeping peak extra memory flat. var posting_batch: [4096]DiskPosting = undefined; - var batch_len: usize = 0; + var posting_batch_len: usize = 0; + var lookup_batch: [4096]LookupEntry = undefined; + var lookup_batch_len: usize = 0; var postings_len: usize = 0; for (trigrams_sorted.items) |tri| { - const posting_list = self.index.getPtr(tri) orelse continue; + const posting_list = self.index.getPtr(tri) orelse unreachable; const offset: u32 = @intCast(postings_len); var count: u32 = 0; for (posting_list.items.items) |p| { @@ -1835,62 +1855,40 @@ pub const TrigramIndex = struct { // Reject instead of wrapping once its representable range // is full; a corrupt index would otherwise be persisted. if (postings_len >= std.math.maxInt(u32) or count >= std.math.maxInt(u32)) return error.IndexTooLarge; - posting_batch[batch_len] = .{ + posting_batch[posting_batch_len] = .{ .file_id = fid, .next_mask = p.next_mask, .loc_mask = p.loc_mask, }; - batch_len += 1; + posting_batch_len += 1; postings_len += 1; count += 1; - if (batch_len == posting_batch.len) { - try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..batch_len])); - batch_len = 0; + if (posting_batch_len == posting_batch.len) { + try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..posting_batch_len])); + posting_batch_len = 0; } } - try lookup_entries.append(self.allocator, .{ + lookup_batch[lookup_batch_len] = .{ .trigram = @as(u32, tri), .offset = offset, .count = count, - }); + }; + lookup_batch_len += 1; + if (lookup_batch_len == lookup_batch.len) { + try lw.interface.writeAll(std.mem.sliceAsBytes(lookup_batch[0..lookup_batch_len])); + lookup_batch_len = 0; + } } - if (batch_len > 0) { - try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..batch_len])); + if (posting_batch_len > 0) { + try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..posting_batch_len])); + } + if (lookup_batch_len > 0) { + try lw.interface.writeAll(std.mem.sliceAsBytes(lookup_batch[0..lookup_batch_len])); } try pw.interface.flush(); - } - try std.Io.Dir.cwd().rename(postings_tmp, std.Io.Dir.cwd(), postings_final, io); - - // Step 5: Write lookup file atomically (random suffix prevents collisions) - const lk_rand: u64 = cio.randU64(); - const lookup_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup.{x}.tmp", .{ dir_path, lk_rand }); - defer self.allocator.free(lookup_tmp); - const lookup_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup", .{dir_path}); - defer self.allocator.free(lookup_final); - - { - const file = try std.Io.Dir.cwd().createFile(io, lookup_tmp, .{}); - defer file.close(io); - - var lw_buf: [64 * 1024]u8 = undefined; - var lw = file.writer(io, &lw_buf); - - // Header: magic(4) + version(2) + pad(2) + entry_count(4) = 12 bytes - try lw.interface.writeAll(&LOOKUP_MAGIC); - var ver_buf2: [2]u8 = undefined; - std.mem.writeInt(u16, &ver_buf2, FORMAT_VERSION, .little); - try lw.interface.writeAll(&ver_buf2); - var pad_buf: [2]u8 = .{ 0, 0 }; - try lw.interface.writeAll(&pad_buf); - var ec_buf: [4]u8 = undefined; - std.mem.writeInt(u32, &ec_buf, @intCast(lookup_entries.items.len), .little); - try lw.interface.writeAll(&ec_buf); - - // Entries (already aligned at 12 bytes each) - const entry_bytes = std.mem.sliceAsBytes(lookup_entries.items); - try lw.interface.writeAll(entry_bytes); try lw.interface.flush(); } + try std.Io.Dir.cwd().rename(postings_tmp, std.Io.Dir.cwd(), postings_final, io); try std.Io.Dir.cwd().rename(lookup_tmp, std.Io.Dir.cwd(), lookup_final, io); } @@ -3146,7 +3144,7 @@ pub const MAX_NGRAM_LEN: usize = 16; /// Rare pairs → HIGH weight (they become n-gram boundaries). /// All unspecified pairs default to 0xFE00 (rare = high weight). pub const default_pair_freq: [256][256]u16 = blk: { - var table: [256][256]u16 = .{.{0xFE00} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0xFE00)); // English bigrams (lowercase) — common in identifiers and prose table['t']['h'] = 0x1000; table['h']['e'] = 0x1000; @@ -3244,7 +3242,7 @@ pub fn resetFrequencyTable() void { /// Build a per-project frequency table by counting byte-pair occurrences in /// `content`, then inverting counts to weights (common → low, rare → high). pub fn buildFrequencyTable(content: []const u8) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); if (content.len >= 2) { for (0..content.len - 1) |i| { counts[content[i]][content[i + 1]] += 1; @@ -3257,7 +3255,7 @@ pub fn buildFrequencyTable(content: []const u8) [256][256]u16 { /// Zero extra memory — counts pairs within each slice, skipping cross-slice /// boundaries (negligible loss for large corpora). pub fn buildFrequencyTableFromSlices(slices: []const []const u8) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); for (slices) |content| { if (content.len < 2) continue; for (0..content.len - 1) |i| { @@ -3270,7 +3268,7 @@ pub fn buildFrequencyTableFromSlices(slices: []const []const u8) [256][256]u16 { /// Build a frequency table by streaming over a StringHashMap of content. /// Iterates file-by-file — no concatenation, zero extra memory. pub fn buildFrequencyTableFromMap(contents: *ContentCache) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); var iter = contents.iterator(); while (iter.next()) |entry| { const content = entry.value_ptr.*; @@ -3288,7 +3286,7 @@ pub fn buildFrequencyTableFromMap(contents: *ContentCache) [256][256]u16 { pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: std.mem.Allocator, worker_count: usize) ![256][256]u16 { const n_files = contents.count(); if (n_files == 0) { - const zero: [256][256]u64 = .{.{0} ** 256} ** 256; + const zero: [256][256]u64 = @splat(@splat(0)); return finishFrequencyTable(&zero); } @@ -3301,7 +3299,7 @@ pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: st if (content.len >= 2) slices.appendAssumeCapacity(content); } if (slices.items.len == 0) { - const zero: [256][256]u64 = .{.{0} ** 256} ** 256; + const zero: [256][256]u64 = @splat(@splat(0)); return finishFrequencyTable(&zero); } @@ -3332,7 +3330,7 @@ pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: st for (threads[0..spawned]) |t| t.join(); // Sum worker tables. - var merged: Counts = .{.{0} ** 256} ** 256; + var merged: Counts = @splat(@splat(0)); for (worker_counts) |*wc| { for (0..256) |a| { for (0..256) |b| { @@ -3359,7 +3357,7 @@ fn finishFrequencyTable(counts: *const [256][256]u64) [256][256]u16 { } } // Invert: count 0 → 0xFE00 (rare, high); max_count → 0x1000 (common, low). - var table: [256][256]u16 = .{.{0xFE00} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0xFE00)); for (0..256) |a| { for (0..256) |b| { const c = counts[a][b]; diff --git a/src/linter.zig b/src/linter.zig index 918997ec..332101c5 100644 --- a/src/linter.zig +++ b/src/linter.zig @@ -528,8 +528,8 @@ pub const DiagnosticsCache = struct { mu: cio.Mutex = .{}, alloc: std.mem.Allocator, - entries: [MAX]?Entry = .{null} ** MAX, - pending: [MAX]?[]u8 = .{null} ** MAX, + entries: [MAX]?Entry = @splat(null), + pending: [MAX]?[]u8 = @splat(null), inflight: usize = 0, pub fn init(alloc: std.mem.Allocator) DiagnosticsCache { diff --git a/src/main.zig b/src/main.zig index 3e3fee82..555e29e0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -39,11 +39,11 @@ const mcpRootIsImplicitCwd = cli_args.mcpRootIsImplicitCwd; const mcpRootAcceptsEnv = cli_args.mcpRootAcceptsEnv; const isHelpRequest = cli_args.isHelpRequest; -/// The real entry point. In Debug builds, Zig may merge all command-branch +/// The real entry point. In Debug builds, Zig may merge all command-branch /// stack frames into one producing a frame that overflows the default OS stack, -/// so we trampoline through a thread with an explicit 64 MB stack. -/// In optimised builds the merged frame is ~190 KB, so 8 MB is ample and -/// avoids triggering Rosetta 2's 64 MB stack allocation bug on x86_64-macos. +/// so those builds trampoline through a thread with an explicit 64 MB stack. +/// Optimized builds have a ~190 KB frame and run directly on the process stack, +/// avoiding an OS thread create/join on every CLI invocation. /// /// #504: must have a non-error-union return type. A Zig binary with /// `pub fn main(...) !void` ad-hoc-signed and run via Rosetta (or, in the @@ -59,7 +59,11 @@ pub fn main(init: std.process.Init.Minimal) void { const argv = cio.bootstrapArgs(init.args); cio.setProcessArgs(argv); if (handleFastPath(argv)) return; - mainTrampoline() catch |err| { + if (builtin.mode != .Debug) { + mainInner(argv); + return; + } + mainTrampoline(argv) catch |err| { // Surface the failure on stderr so users see something even if the // worker thread crashes during startup. var buf: [256]u8 = undefined; @@ -70,9 +74,8 @@ pub fn main(init: std.process.Init.Minimal) void { }; } -fn mainTrampoline() !void { - const stack_size: usize = if (builtin.mode == .Debug) 64 * 1024 * 1024 else 8 * 1024 * 1024; - const thread = try std.Thread.spawn(.{ .stack_size = stack_size }, mainInner, .{}); +fn mainTrampoline(argv: []const [*:0]const u8) !void { + const thread = try std.Thread.spawn(.{ .stack_size = 64 * 1024 * 1024 }, mainInner, .{argv}); thread.join(); } @@ -107,8 +110,8 @@ fn handleFastPath(argv: []const [*:0]const u8) bool { return false; } -fn mainInner() void { - mainImpl() catch |err| { +fn mainInner(argv: []const [*:0]const u8) void { + mainImpl(argv) catch |err| { std.debug.print("fatal: {s}\n", .{@errorName(err)}); std.process.exit(1); }; @@ -127,7 +130,7 @@ const loadUserConfig = bootstrap.loadUserConfig; const getDataDir = bootstrap.getDataDir; const commands = @import("commands.zig"); -fn mainImpl() !void { +fn mainImpl(argv: []const [*:0]const u8) !void { // Use c_allocator (libc malloc) — better page reclamation than GPA const allocator = std.heap.c_allocator; cio.ignoreSigpipe(); @@ -145,9 +148,6 @@ fn mainImpl() !void { var out = Out{ .file = stdout, .alloc = allocator }; defer out.flush(); - const raw_args = try cio.argsAlloc(allocator); - defer cio.argsFree(allocator, raw_args); - // Extract --config-file= / --config-file before positional // arg parsing so a leading `--config-file=X` isn't misread as the root. // See #101, #102. @@ -156,15 +156,15 @@ fn mainImpl() !void { const args = blk: { var filtered: std.ArrayList([]const u8) = .empty; errdefer filtered.deinit(allocator); - try filtered.append(allocator, raw_args[0]); + try filtered.append(allocator, std.mem.span(argv[0])); var i: usize = 1; - while (i < raw_args.len) : (i += 1) { - const a = raw_args[i]; + while (i < argv.len) : (i += 1) { + const a = std.mem.span(argv[i]); if (std.mem.startsWith(u8, a, "--config-file=")) { explicit_config = a["--config-file=".len..]; continue; - } else if (std.mem.eql(u8, a, "--config-file") and i + 1 < raw_args.len) { - explicit_config = raw_args[i + 1]; + } else if (std.mem.eql(u8, a, "--config-file") and i + 1 < argv.len) { + explicit_config = std.mem.span(argv[i + 1]); i += 1; continue; } else if (std.mem.eql(u8, a, "--no-telemetry")) { diff --git a/src/mcp.zig b/src/mcp.zig index d4a852be..3fd7b038 100644 --- a/src/mcp.zig +++ b/src/mcp.zig @@ -2,13 +2,15 @@ const cio = @import("cio.zig"); // // Exposes codedb's exploration + edit engine as MCP tools. -// Uses mcp-zig for protocol utilities; adds roots support for workspace awareness. +// Implements the JSON-RPC transport directly and adds roots support for workspace awareness. const std = @import("std"); const testing = std.testing; -const mcp_lib = @import("mcp"); -const mcpj = mcp_lib.json; -pub const Root = mcp_lib.mcp.Root; +const mcpj = @import("mcp_json.zig"); +pub const Root = struct { + uri: []u8, + name: []u8, +}; const Store = @import("store.zig").Store; const explore_mod = @import("explore.zig"); const Explorer = explore_mod.Explorer; @@ -329,7 +331,7 @@ const ProjectCache = struct { return .{ .mu = .{}, .alloc = alloc_, - .entries = [_]?*Entry{null} ** MAX_CACHED, + .entries = @splat(null), .default_path = default_path_, .default_snapshot_cache = .{}, .default_deps_cache = .{}, @@ -857,7 +859,7 @@ pub const ConvergenceGovernor = struct { pub const HISTORY = 8; // ring-buffer window of recent calls pub const WARN_AT = 3; // same signature this many times in the window -> nudge - sigs: [HISTORY]u64 = [_]u64{0} ** HISTORY, + sigs: [HISTORY]u64 = @splat(0), head: usize = 0, /// Record a call signature and return how many times it has occurred within diff --git a/src/mcp_json.zig b/src/mcp_json.zig new file mode 100644 index 00000000..13c04f74 --- /dev/null +++ b/src/mcp_json.zig @@ -0,0 +1,97 @@ +//! Small JSON-RPC helpers used by codedb's MCP transport. +//! +//! These were formerly supplied by mcp-zig. Keeping the protocol-neutral +//! helpers local avoids coupling codedb's build to a second server framework. + +const std = @import("std"); + +pub const MAX_LINE = 1024 * 1024; + +/// Read one newline-terminated message from a buffered reader. +/// The caller owns the returned slice. +pub fn readLineBuf(alloc: std.mem.Allocator, reader: *std.Io.Reader) ?[]u8 { + var line: std.ArrayList(u8) = .empty; + while (true) { + const byte = reader.takeByte() catch { + if (line.items.len == 0) { + line.deinit(alloc); + return null; + } + return line.toOwnedSlice(alloc) catch null; + }; + if (byte == '\n') return line.toOwnedSlice(alloc) catch null; + line.append(alloc, byte) catch { + line.deinit(alloc); + return null; + }; + if (line.items.len > MAX_LINE) { + line.deinit(alloc); + return null; + } + } +} + +pub fn getStr(obj: *const std.json.ObjectMap, key: []const u8) ?[]const u8 { + return switch (obj.get(key) orelse return null) { + .string => |s| s, + else => null, + }; +} + +pub fn getInt(obj: *const std.json.ObjectMap, key: []const u8) ?i64 { + return switch (obj.get(key) orelse return null) { + .integer => |n| n, + else => null, + }; +} + +pub fn getBool(obj: *const std.json.ObjectMap, key: []const u8) bool { + return switch (obj.get(key) orelse return false) { + .bool => |b| b, + else => false, + }; +} + +pub fn eql(a: []const u8, b: []const u8) bool { + return std.mem.eql(u8, a, b); +} + +/// Append a JSON-escaped string without adding surrounding quotes. +pub fn writeEscaped(alloc: std.mem.Allocator, out: *std.ArrayList(u8), s: []const u8) void { + const Vec = @Vector(16, u8); + var i: usize = 0; + + while (i < s.len) { + const start = i; + while (i + 16 <= s.len) { + const chunk: Vec = s[i..][0..16].*; + const lo = chunk < @as(Vec, @splat(@as(u8, 0x20))); + const dq = chunk == @as(Vec, @splat(@as(u8, '"'))); + const bs = chunk == @as(Vec, @splat(@as(u8, '\\'))); + if (@reduce(.Or, lo | dq | bs)) break; + i += 16; + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c < 0x20 or c == '"' or c == '\\') break; + } + + if (i > start) out.appendSlice(alloc, s[start..i]) catch return; + if (i >= s.len) break; + + const c = s[i]; + switch (c) { + '"' => out.appendSlice(alloc, "\\\"") catch return, + '\\' => out.appendSlice(alloc, "\\\\") catch return, + '\n' => out.appendSlice(alloc, "\\n") catch return, + '\r' => out.appendSlice(alloc, "\\r") catch return, + '\t' => out.appendSlice(alloc, "\\t") catch return, + else => { + const hex = "0123456789abcdef"; + const escaped = [6]u8{ '\\', 'u', '0', '0', hex[c >> 4], hex[c & 0x0f] }; + out.appendSlice(alloc, &escaped) catch return; + }, + } + i += 1; + } +} diff --git a/src/release_info.zig b/src/release_info.zig index 79197512..0a6e3bb1 100644 --- a/src/release_info.zig +++ b/src/release_info.zig @@ -1 +1 @@ -pub const semver = "0.2.5828"; +pub const semver = "0.2.5829"; diff --git a/src/server.zig b/src/server.zig index d262ffed..9531150e 100644 --- a/src/server.zig +++ b/src/server.zig @@ -687,13 +687,11 @@ fn handleConnection( // ── Transport helpers ─────────────────────────────────────────── /// Read some bytes from the TCP stream into `dest`. Returns 0 on clean EOF. -/// This is the direct-vtable analogue of the old `conn.stream.read`: no -/// buffered reader state, so each call issues one `netRead` syscall. +/// This uses the low-level stream API without allocating buffered reader state. fn readSome(io: std.Io, stream: std.Io.net.Stream, dest: []u8) !usize { if (dest.len == 0) return 0; var iov: [1][]u8 = .{dest}; - const n = try io.vtable.netRead(io.userdata, stream.socket.handle, &iov); - return n; + return stream.read(io, &iov); } // ── Response helpers ──────────────────────────────────────────── diff --git a/src/snapshot.zig b/src/snapshot.zig index 14a88098..8039aba9 100644 --- a/src/snapshot.zig +++ b/src/snapshot.zig @@ -354,7 +354,7 @@ pub fn writeSnapshot( if (git_head) |head| { try fw.writeAll(&head); } else { - try fw.writeAll(&([_]u8{0x00} ** 40)); + try fw.writeAll(&@as([40]u8, @splat(0x00))); } var sc_buf: [4]u8 = undefined; diff --git a/src/telemetry.zig b/src/telemetry.zig index af80234d..3fc2e8c6 100644 --- a/src/telemetry.zig +++ b/src/telemetry.zig @@ -37,7 +37,7 @@ pub const Event = struct { pub const Kind = union(enum) { tool_call: struct { - tool: [32]u8 = .{0} ** 32, + tool: [32]u8 = @splat(0), tool_len: u8 = 0, latency_ns: i128, err: bool, @@ -339,9 +339,9 @@ pub const Telemetry = struct { }, .search_breakdown => |sb| { try w.print(",\"event_type\":\"search_breakdown\",\"tier0_ns\":{d},\"tier05_ns\":{d},\"tier1_ns\":{d},\"tier2_ns\":{d},\"tier3_ns\":{d},\"tier4_ns\":{d},\"tier5_ns\":{d},\"rerank_ns\":{d},\"tier_reached\":{d},\"candidates\":{d},\"results\":{d}", .{ - sb.tier0_ns, sb.tier05_ns, sb.tier1_ns, - sb.tier2_ns, sb.tier3_ns, sb.tier4_ns, - sb.tier5_ns, sb.rerank_ns, sb.tier_reached, + sb.tier0_ns, sb.tier05_ns, sb.tier1_ns, + sb.tier2_ns, sb.tier3_ns, sb.tier4_ns, + sb.tier5_ns, sb.rerank_ns, sb.tier_reached, sb.candidate_count, sb.result_count, }); }, diff --git a/src/test_core.zig b/src/test_core.zig index a9bc2a6d..a46f8784 100644 --- a/src/test_core.zig +++ b/src/test_core.zig @@ -1048,7 +1048,7 @@ test "issue-584: ContentCache probe-window — overflow inserts, holes, and dupl var bufs: [5][16]u8 = undefined; var lens: [5]usize = undefined; var nkeys: usize = 0; - var counts = [_]u8{0} ** 64; + var counts: [64]u8 = @splat(0); var pick: u32 = 0; var i: usize = 0; while (i < 4096) : (i += 1) { diff --git a/src/test_explore.zig b/src/test_explore.zig index f3e63b76..9314a192 100644 --- a/src/test_explore.zig +++ b/src/test_explore.zig @@ -139,7 +139,6 @@ const watcher = @import("watcher.zig"); const git_mod = @import("git.zig"); const snapshot_mod = @import("snapshot.zig"); - test "word tokenizer" { var tok = WordTokenizer{ .buf = "pub fn main() !void {" }; const w1 = tok.next().?; @@ -153,7 +152,6 @@ test "word tokenizer" { try testing.expect(tok.next() == null); } - test "word index: index and search" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -175,7 +173,6 @@ test "word index: index and search" { try testing.expect(const_hits[0].line_num == 2); } - test "word index: re-index clears old entries" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -188,7 +185,6 @@ test "word index: re-index clears old entries" { try testing.expect(wi.search("new_func").len > 0); } - test "word index: removeFile" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -200,7 +196,6 @@ test "word index: removeFile" { try testing.expect(wi.search("hello").len == 0); } - test "word index: deduped search" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -213,7 +208,6 @@ test "word index: deduped search" { try testing.expect(hits.len == 1); } - test "explorer: sparse ngram index integrated into searchContent" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -227,7 +221,6 @@ test "explorer: sparse ngram index integrated into searchContent" { try testing.expectEqualStrings("src/alpha.zig", results[0].path); } - test "explorer: searchContent finds query embedded in longer identifier" { // Verify that searchContent correctly finds files whose content contains // the query string. The sparse index (sliding-window) and trigram index @@ -248,7 +241,6 @@ test "explorer: searchContent finds query embedded in longer identifier" { try testing.expect(found); } - test "explorer: packed outline clone owns one string backing" { const expected_mask = explore.FileOutline.nameLenBit("packedSymbol".len); var packed_outline = blk: { @@ -317,7 +309,6 @@ test "explorer: index file and get outline" { try testing.expect(outline.symbols.items.len == 3); } - test "explorer: findSymbol" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -331,7 +322,6 @@ test "explorer: findSymbol" { try testing.expectEqualStrings("a.zig", result.?.path); } - test "explorer: findAllSymbols returns multiple" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -345,7 +335,6 @@ test "explorer: findAllSymbols returns multiple" { try testing.expect(results.len == 2); } - test "explorer: searchContent with trigram acceleration" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -368,7 +357,6 @@ test "explorer: searchContent with trigram acceleration" { try testing.expect(results[0].line_num == 1); } - test "explorer: searchWord via inverted index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -382,6 +370,52 @@ test "explorer: searchWord via inverted index" { try testing.expectEqualStrings("math.zig", explorer.word_index.hitPath(hits[0])); } +test "explorer: searchWord lazy rebuild matches eager index" { + var eager = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer eager.deinit(); + var lazy = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer lazy.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "src/a.zig", .content = "pub fn alpha() void {}\n// alpha again\n" }, + .{ .path = "src/b.zig", .content = "const alpha = 1;\n" }, + }; + for (fixtures) |fixture| { + try eager.indexFile(fixture.path, fixture.content); + try lazy.indexFile(fixture.path, fixture.content); + } + lazy.markWordIndexIncomplete(false); + + const expected = try eager.searchWord("alpha", testing.allocator); + defer testing.allocator.free(expected); + const rebuilt = try lazy.searchWord("alpha", testing.allocator); + defer testing.allocator.free(rebuilt); + try testing.expectEqual(expected.len, rebuilt.len); + // Rebuild walks the content hash map, so internal doc IDs and posting order + // are intentionally unspecified. Compare the complete semantic hit set. + for (expected) |a| { + var found = false; + for (rebuilt) |b| { + if (a.line_num == b.line_num and + std.mem.eql(u8, eager.word_index.hitPath(a), lazy.word_index.hitPath(b))) + { + found = true; + break; + } + } + try testing.expect(found); + } + + // The second call takes the optimized already-complete shared-lock path and + // must preserve the rebuilt index's exact sequence. + const warm = try lazy.searchWord("alpha", testing.allocator); + defer testing.allocator.free(warm); + try testing.expectEqual(rebuilt.len, warm.len); + for (rebuilt, warm) |a, b| { + try testing.expectEqual(a.doc_id, b.doc_id); + try testing.expectEqual(a.line_num, b.line_num); + } +} test "explorer: removeFile cleans up everything" { var arena = std.heap.ArenaAllocator.init(testing.allocator); @@ -397,7 +431,6 @@ test "explorer: removeFile cleans up everything" { try testing.expect((try explorer.findSymbol("doStuff", testing.allocator)) == null); } - test "explorer: python parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -415,7 +448,6 @@ test "explorer: python parser" { try testing.expect(outline.symbols.items.len == 3); // import, class, def } - test "explorer: typescript parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -432,7 +464,6 @@ test "explorer: typescript parser" { try testing.expect(outline.symbols.items.len >= 3); } - test "explorer: reindex OOM keeps prior outline reachable" { // Use a real allocator for the explorer so the first indexFile always succeeds. // We can't use FailingAllocator for the whole explorer because deinit would crash. @@ -475,7 +506,6 @@ test "explorer: reindex OOM keeps prior outline reachable" { try testing.expect(new_results.len == 1); } - test "explorer: getOutline clone OOM preserves source outline" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -510,7 +540,6 @@ test "explorer: getOutline clone OOM preserves source outline" { try testing.expect(induced_oom); } - test "explorer: outline copy survives source removal" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -526,7 +555,6 @@ test "explorer: outline copy survives source removal" { try testing.expect(outline.symbols.items.len > 0); } - test "explorer: removeFile frees owned map key" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -544,7 +572,6 @@ test "explorer: removeFile frees owned map key" { try testing.expect(explorer.dep_graph.count() == 0); } - test "word index: removeFile prunes empty buckets" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -558,7 +585,6 @@ test "word index: removeFile prunes empty buckets" { try testing.expect(wi.search("uniqueWordOnlyHere").len == 0); } - test "extractLines: basic range with line numbers" { const content = "line1\nline2\nline3\nline4\nline5"; const result = try extractLines(content, 2, 4, true, false, .unknown, testing.allocator); @@ -570,7 +596,6 @@ test "extractLines: basic range with line numbers" { try testing.expect(std.mem.indexOf(u8, result, "line5") == null); } - test "extractLines: start beyond file returns empty" { const content = "line1\nline2"; const result = try extractLines(content, 10, 20, true, false, .unknown, testing.allocator); @@ -578,7 +603,6 @@ test "extractLines: start beyond file returns empty" { try testing.expect(result.len == 0); } - test "extractLines: compact skips comments and blanks" { const content = "fn main() void {}\n// this is a comment\n\n return 0;\n}"; const result = try extractLines(content, 1, 5, false, true, .zig, testing.allocator); @@ -589,7 +613,6 @@ test "extractLines: compact skips comments and blanks" { try testing.expect(std.mem.indexOf(u8, result, "return 0") != null); } - test "isCommentOrBlank: detects language-specific comments" { try testing.expect(isCommentOrBlank(" // zig comment", .zig)); try testing.expect(isCommentOrBlank(" # python comment", .python)); @@ -603,7 +626,6 @@ test "isCommentOrBlank: detects language-specific comments" { try testing.expect(!isCommentOrBlank("// comment", .unknown)); } - test "explorer: getSymbolBody returns source lines" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -620,7 +642,6 @@ test "explorer: getSymbolBody returns source lines" { } } - test "explorer: getSymbolBody returns null for unknown file" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -630,7 +651,6 @@ test "explorer: getSymbolBody returns null for unknown file" { try testing.expect(body == null); } - test "explorer: searchContentWithScope annotates results" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -657,7 +677,6 @@ test "explorer: searchContentWithScope annotates results" { try testing.expectEqualStrings("handleAuth", results[0].scope_name.?); } - test "explorer: searchContentWithScope no scope for standalone line" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -680,7 +699,6 @@ test "explorer: searchContentWithScope no scope for standalone line" { try testing.expect(results[0].scope_name == null); } - test "content hash: Wyhash produces consistent hash" { const content = "pub fn main() void {}"; const hash1 = std.hash.Wyhash.hash(0, content); @@ -691,7 +709,6 @@ test "content hash: Wyhash produces consistent hash" { try testing.expect(hash1 != hash3); } - test "detectLanguage: public access and correct detection" { try testing.expect(explore.detectLanguage("src/main.zig") == .zig); try testing.expect(explore.detectLanguage("app.py") == .python); @@ -699,7 +716,6 @@ test "detectLanguage: public access and correct detection" { try testing.expect(explore.detectLanguage("style.css") == .css); } - test "extractLines: without line numbers" { const content = "alpha\nbeta\ngamma"; const result = try extractLines(content, 1, 3, false, false, .unknown, testing.allocator); @@ -707,7 +723,6 @@ test "extractLines: without line numbers" { try testing.expectEqualStrings("alpha\nbeta\ngamma\n", result); } - test "extractLines: start only reads to EOF" { const content = "a\nb\nc\nd\ne"; const result = try extractLines(content, 3, std.math.maxInt(u32), true, false, .unknown, testing.allocator); @@ -719,7 +734,6 @@ test "extractLines: start only reads to EOF" { try testing.expect(std.mem.indexOf(u8, result, "| b") == null); } - test "extractLines: end beyond file clamps to EOF" { const content = "x\ny\nz"; const result = try extractLines(content, 2, 999, true, false, .unknown, testing.allocator); @@ -730,7 +744,6 @@ test "extractLines: end beyond file clamps to EOF" { try testing.expect(std.mem.count(u8, result, "\n") == 2); } - test "extractLines: single line range (start == end)" { const content = "one\ntwo\nthree"; const result = try extractLines(content, 2, 2, true, false, .unknown, testing.allocator); @@ -739,7 +752,6 @@ test "extractLines: single line range (start == end)" { try testing.expect(std.mem.count(u8, result, "\n") == 1); } - test "extractLines: empty content returns single empty line" { const result = try extractLines("", 1, 10, true, false, .unknown, testing.allocator); defer testing.allocator.free(result); @@ -747,7 +759,6 @@ test "extractLines: empty content returns single empty line" { try testing.expect(result.len > 0); } - test "extractLines: compact with Python comments" { const content = "# comment\nimport os\n\ndef hello():\n # inline comment\n print('hi')"; const result = try extractLines(content, 1, 6, false, true, .python, testing.allocator); @@ -759,7 +770,6 @@ test "extractLines: compact with Python comments" { try testing.expect(std.mem.indexOf(u8, result, "print('hi')") != null); } - test "extractLines: compact with JS/TS comments" { const content = "// header\nconst x = 1;\n/* block */\n* star line\nexport default x;"; const result = try extractLines(content, 1, 5, false, true, .typescript, testing.allocator); @@ -771,26 +781,22 @@ test "extractLines: compact with JS/TS comments" { try testing.expect(std.mem.indexOf(u8, result, "export default x;") != null); } - test "isCommentOrBlank: rust double-slash" { try testing.expect(isCommentOrBlank(" // rust comment", .rust)); try testing.expect(!isCommentOrBlank(" let x = 1;", .rust)); } - test "isCommentOrBlank: go double-slash" { try testing.expect(isCommentOrBlank(" // go comment", .go_lang)); try testing.expect(!isCommentOrBlank(" func main() {", .go_lang)); } - test "isCommentOrBlank: dart comments" { try testing.expect(isCommentOrBlank(" // dart comment", .dart)); try testing.expect(isCommentOrBlank(" /* dart block comment */", .dart)); try testing.expect(!isCommentOrBlank(" class WidgetBuilder {}", .dart)); } - test "isCommentOrBlank: cpp block and line comments" { try testing.expect(isCommentOrBlank(" // cpp line comment", .cpp)); try testing.expect(isCommentOrBlank(" /* cpp block comment */", .cpp)); @@ -798,7 +804,6 @@ test "isCommentOrBlank: cpp block and line comments" { try testing.expect(!isCommentOrBlank(" int x = 0;", .cpp)); } - test "isCommentOrBlank: detected extension language comments" { try testing.expect(isCommentOrBlank(" // java line comment", .java)); try testing.expect(isCommentOrBlank(" // kotlin line comment", .kotlin)); @@ -817,21 +822,18 @@ test "isCommentOrBlank: detected extension language comments" { try testing.expect(!isCommentOrBlank(" SELECT * FROM users;", .sql)); } - test "isCommentOrBlank: tabs and mixed whitespace" { try testing.expect(isCommentOrBlank("\t\t// tabbed comment", .zig)); try testing.expect(isCommentOrBlank(" \t \t ", .zig)); try testing.expect(isCommentOrBlank("\t", .python)); } - test "isCommentOrBlank: markdown and json never strip" { try testing.expect(!isCommentOrBlank("# heading", .markdown)); try testing.expect(!isCommentOrBlank("// not a comment in json", .json)); try testing.expect(!isCommentOrBlank("# not a comment in yaml", .yaml)); } - test "explorer: getSymbolBody multi-line range" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -853,7 +855,6 @@ test "explorer: getSymbolBody multi-line range" { } } - test "explorer: getSymbolBody range beyond file length" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -870,7 +871,6 @@ test "explorer: getSymbolBody range beyond file length" { } } - test "explorer: searchContentWithScope across multiple files" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -896,7 +896,6 @@ test "explorer: searchContentWithScope across multiple files" { } } - test "explorer: searchContentWithScope respects max_results" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -917,7 +916,6 @@ test "explorer: searchContentWithScope respects max_results" { try testing.expect(results.len == 2); } - test "explorer: searchContentWithScope no results for missing query" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -938,7 +936,6 @@ test "explorer: searchContentWithScope no results for missing query" { try testing.expect(results.len == 0); } - test "content hash: format as hex string" { const content = "hello world"; const hash = std.hash.Wyhash.hash(0, content); @@ -954,14 +951,12 @@ test "content hash: format as hex string" { try testing.expectEqualStrings(hex, hex2); } - test "content hash: empty content hashes consistently" { const h1 = std.hash.Wyhash.hash(0, ""); const h2 = std.hash.Wyhash.hash(0, ""); try testing.expect(h1 == h2); } - test "detectLanguage: all supported extensions" { try testing.expect(explore.detectLanguage("main.zig") == .zig); try testing.expect(explore.detectLanguage("lib.c") == .c); @@ -1003,7 +998,6 @@ test "detectLanguage: all supported extensions" { try testing.expect(explore.detectLanguage("no_ext") == .unknown); } - test "explorer: getSymbolBody with line number format" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1024,7 +1018,6 @@ test "explorer: getSymbolBody with line number format" { } } - test "extractLines: compact preserves brace-only lines" { const content = "fn main() void {\n // comment\n doWork();\n}"; const result = try extractLines(content, 1, 4, false, true, .zig, testing.allocator); @@ -1035,7 +1028,6 @@ test "extractLines: compact preserves brace-only lines" { try testing.expect(std.mem.indexOf(u8, result, "// comment") == null); } - test "extractLines: compact on all-comment file returns empty" { const content = "// comment 1\n// comment 2\n// comment 3"; const result = try extractLines(content, 1, 3, false, true, .zig, testing.allocator); @@ -1043,7 +1035,6 @@ test "extractLines: compact on all-comment file returns empty" { try testing.expect(result.len == 0); } - test "explorer: searchContentRegex end-to-end" { var explorer_inst = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_inst.deinit(); @@ -1073,7 +1064,6 @@ test "explorer: searchContentRegex end-to-end" { try testing.expect(found2); } - test "explorer: searchContentRegex no match" { var explorer_inst = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_inst.deinit(); @@ -1086,7 +1076,6 @@ test "explorer: searchContentRegex no match" { try testing.expectEqual(@as(usize, 0), results.len); } - test "git: getGitHead returns 40-char hex SHA in a git repo" { if (builtin.os.tag == .windows) return error.SkipZigTest; @@ -1100,14 +1089,12 @@ test "git: getGitHead returns 40-char hex SHA in a git repo" { } } - test "git: getGitHead returns null for non-git directory" { // /tmp is not a git repo const head = try git_mod.getGitHead("/tmp", testing.allocator); try testing.expect(head == null); } - test "thread-safe: concurrent TrigramIndex.candidates() with per-thread allocators" { var ti = TrigramIndex.init(testing.allocator); defer ti.deinit(); @@ -1139,7 +1126,6 @@ test "thread-safe: concurrent TrigramIndex.candidates() with per-thread allocato try testing.expectEqual(@as(u32, 0), ctx.errors.load(.monotonic)); } - test "thread-safe: concurrent SparseNgramIndex.candidates() with per-thread allocators" { var sni = SparseNgramIndex.init(testing.allocator); defer sni.deinit(); @@ -1169,7 +1155,6 @@ test "thread-safe: concurrent SparseNgramIndex.candidates() with per-thread allo for (threads) |t| t.join(); } - test "issue-43: trigram_index swap in scanBg races with concurrent MCP queries" { // Regression: the scanBg disk-load path must serialize trigram_index swaps // with readers by taking exp.mu.lock() before replacing the index. @@ -1200,7 +1185,6 @@ test "issue-43: trigram_index swap in scanBg races with concurrent MCP queries" try testing.expect(!raced); } - test "issue-116: getGitHead returns valid SHA for git repos" { const git = @import("git.zig"); @@ -1215,7 +1199,6 @@ test "issue-116: getGitHead returns valid SHA for git repos" { } } - test "issue-224: codedb_symbol body=true returns full body — line_end populated" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1245,7 +1228,6 @@ test "issue-224: codedb_symbol body=true returns full body — line_end populate try testing.expect(std.mem.indexOf(u8, body, "return a + b;") != null); } - test "issue-224: Python def line_end covers full body" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1268,14 +1250,12 @@ test "issue-224: Python def line_end covers full body" { try testing.expectEqual(@as(u32, 3), sym.line_end); } - test "issue-108: detectLanguage handles .tf and .tfvars" { try testing.expectEqual(Language.hcl, explore.detectLanguage("main.tf")); try testing.expectEqual(Language.hcl, explore.detectLanguage("prod.tfvars")); try testing.expectEqual(Language.hcl, explore.detectLanguage("config.hcl")); } - test "issue-215: detectLanguage handles .r and .R" { try testing.expectEqual(Language.r, explore.detectLanguage("script.r")); try testing.expectEqual(Language.r, explore.detectLanguage("analysis.R")); @@ -1286,7 +1266,6 @@ test "issue-532: detectLanguage handles .res and .resi" { try testing.expectEqualStrings("rescript", @tagName(explore.detectLanguage("src/User.resi"))); } - test "dep-graph: reverse index gives O(1) imported_by lookup" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1320,7 +1299,6 @@ test "dep-graph: reverse index gives O(1) imported_by lookup" { try testing.expectEqualStrings("main.zig", imported_by2[0]); } - test "dep-graph: setDeps removes old reverse edges" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1359,7 +1337,6 @@ test "dep-graph: setDeps removes old reverse edges" { try testing.expectEqual(@as(usize, 1), utils_deps.len); } - test "dep-graph: transitive dependents via BFS" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1395,7 +1372,6 @@ test "dep-graph: transitive dependents via BFS" { try testing.expectEqualStrings("store.zig", shallow[0]); } - test "dep-graph: transitive dependencies (forward BFS)" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1430,7 +1406,6 @@ test "dep-graph: transitive dependencies (forward BFS)" { try testing.expectEqual(@as(usize, 2), deps_shallow.len); } - test "dep-graph: remove cleans forward and reverse edges" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1459,7 +1434,6 @@ test "dep-graph: remove cleans forward and reverse edges" { try testing.expectEqualStrings("server.zig", imported_by[0]); } - test "dep-graph: cycle does not cause infinite BFS" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1495,7 +1469,6 @@ test "dep-graph: cycle does not cause infinite BFS" { try testing.expectEqual(@as(usize, 2), fwd.len); } - test "dep-graph: Explorer integration — getImportedBy uses reverse index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1513,7 +1486,6 @@ test "dep-graph: Explorer integration — getImportedBy uses reverse index" { try testing.expectEqual(@as(usize, 2), deps.len); } - test "dep-graph: Explorer transitive dependents" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1532,7 +1504,6 @@ test "dep-graph: Explorer transitive dependents" { try testing.expectEqual(@as(usize, 2), blast.len); } - test "issue-445: dep-graph dedupes multi-aliased forward imports" { // A file that imports the same dep under multiple aliases // const idx = @import("index.zig"); @@ -1564,7 +1535,6 @@ test "issue-445: dep-graph dedupes multi-aliased forward imports" { try testing.expectEqualStrings("index.zig", fwd[0]); } - test "symbol-index: O(1) findSymbol via symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1597,7 +1567,6 @@ test "symbol-index: O(1) findSymbol via symbol_index" { try testing.expectEqual(@as(usize, 2), all.len); } - test "symbol-index: removeFile cleans symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1616,7 +1585,6 @@ test "symbol-index: removeFile cleans symbol_index" { try testing.expect(after == null); } - test "symbol-index: re-index updates symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1641,7 +1609,6 @@ test "symbol-index: re-index updates symbol_index" { if (r3.?.symbol.detail) |d| testing.allocator.free(d); } - test "word-index: splitIdentifier snake_case" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1657,7 +1624,6 @@ test "word-index: splitIdentifier snake_case" { try testing.expectEqualStrings("put", out.items[2]); } - test "word-index: splitIdentifier camelCase" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1672,7 +1638,6 @@ test "word-index: splitIdentifier camelCase" { try testing.expectEqualStrings("token", out.items[1]); } - test "word-index: splitIdentifier acronym (HTTPHandler)" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1687,7 +1652,6 @@ test "word-index: splitIdentifier acronym (HTTPHandler)" { try testing.expectEqualStrings("handler", out.items[1]); } - test "word-index: splitIdentifier simple word emits itself" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1701,7 +1665,6 @@ test "word-index: splitIdentifier simple word emits itself" { try testing.expectEqualStrings("handler", out.items[0]); } - test "word-index: sub-token search finds camelCase components" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1735,7 +1698,6 @@ test "word-index: sub-token search finds camelCase components" { try testing.expectEqualStrings("b.zig", r2[0].path); } - test "word-index: sub-token search finds snake_case components" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1766,7 +1728,6 @@ test "word-index: sub-token search finds snake_case components" { try testing.expect(r2.len >= 1); } - test "word-index: case-insensitive lookup finds exact identifiers" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1786,7 +1747,6 @@ test "word-index: case-insensitive lookup finds exact identifiers" { try testing.expectEqual(@as(usize, 1), r1.len); } - test "word-index: searchPrefix finds extensions of a prefix" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1801,7 +1761,6 @@ test "word-index: searchPrefix finds extensions of a prefix" { try testing.expect(hits.len >= 1); } - test "word-index: searchPrefix skips exact match (Tier 0 responsibility)" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1822,7 +1781,6 @@ test "word-index: searchPrefix skips exact match (Tier 0 responsibility)" { try testing.expect(hits_prefix.len >= 1); } - test "word-index: searchPrefix respects max_results cap" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1843,7 +1801,6 @@ test "word-index: searchPrefix respects max_results cap" { try testing.expect(hits.len > 0); } - test "integration: Tier 0.5 prefix expansion finds partial identifier" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1864,7 +1821,6 @@ test "integration: Tier 0.5 prefix expansion finds partial identifier" { try testing.expect(results.len >= 1); } - test "issue-389: FilteredWalker yields symlinked source files" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -1900,7 +1856,6 @@ test "issue-389: FilteredWalker yields symlinked source files" { try testing.expect(explorer.contents.contains("src/alias.zig")); } - test "issue-405: FilteredWalker walks directory symlinks safely (cycle + escape)" { if (builtin.os.tag == .windows) return error.SkipZigTest; @@ -1967,7 +1922,6 @@ test "issue-405: FilteredWalker walks directory symlinks safely (cycle + escape) } } - test "issue-405: cleanupStaleTmpFiles deletes in-flight sibling tmp files" { // BUG: snapshot.zig:cleanupStaleTmpFiles deletes ANY file matching // `*.tmp` in the snapshot directory with no age guard. @@ -2025,7 +1979,6 @@ test "issue-405: cleanupStaleTmpFiles deletes in-flight sibling tmp files" { }; } - test "issue-409: snapshot .env prefix filter wrongly excludes .envoy/.environment files" { // BUG: snapshot.zig:isSensitivePath uses // if (basename.len >= 4 and std.mem.eql(u8, basename[0..4], ".env")) return true; @@ -2075,7 +2028,6 @@ test "issue-409: snapshot .env prefix filter wrongly excludes .envoy/.environmen try testing.expect(exp2.outlines.contains(".envoy.json")); } - test "issue-208: content cache evicts cold entries under pressure" { const ContentCache = @import("hot_cache.zig").ContentCache; const cap = 50; @@ -2119,8 +2071,6 @@ test "issue-208: content cache evicts cold entries under pressure" { try testing.expect(s.evictions > 0); } - - test "explorer: renderSkeleton elides bodies, keeps signatures" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -2154,7 +2104,6 @@ test "explorer: renderSkeleton elides bodies, keeps signatures" { try testing.expect(std.mem.indexOf(u8, s, "codedb_outline sk.zig") != null); } - test "explorer: multi-line signature gets correct line_end (findBraceEnd paren-awareness)" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); diff --git a/src/test_index.zig b/src/test_index.zig index c21482b2..5cf0aa87 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -391,7 +391,7 @@ test "frequency table: little-endian byte order on disk" { const dir_path_len = try tmp_dir.dir.realPathFile(io, ".", &dir_buf); const dir_path = dir_buf[0..dir_path_len]; - var table: [256][256]u16 = .{.{0} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0)); table[0][0] = 0x1234; // little-endian on disk: 0x34, 0x12 table[0][1] = 0xABCD; // little-endian on disk: 0xCD, 0xAB try writeFrequencyTable(io, &table, dir_path); @@ -416,7 +416,7 @@ test "frequency table: little-endian byte order on disk" { test "setFrequencyTable / resetFrequencyTable: pairWeight output changes" { // Build a table where 'th' is rare (high weight) — opposite of default. - var custom: [256][256]u16 = .{.{0x1000} ** 256} ** 256; // all common + var custom: [256][256]u16 = @splat(@splat(0x1000)); // all common custom['q']['x'] = 0xFE00; // make 'qx' rare const before_th = pairWeight('t', 'h'); @@ -541,10 +541,64 @@ test "watcher: queue event copies path bytes" { try testing.expect(event.seq == 99); } +test "word index: posting construction suppresses duplicate line hits" { + var wi = WordIndex.init(testing.allocator); + defer wi.deinit(); + + // Repeated source tokens and identifier splitting can present the same + // normalized term more than once, but each (document, line) is one hit. + try wi.indexFile("a.zig", "alpha alpha alpha_value alpha\nalpha\n"); + try wi.indexFile("b.zig", "alpha alpha\n"); + + const raw = wi.search("alpha"); + try testing.expectEqual(@as(usize, 3), raw.len); + try testing.expectEqual(@as(u32, 1), raw[0].line_num); + try testing.expectEqual(@as(u32, 2), raw[1].line_num); + try testing.expectEqual(@as(u32, 1), raw[2].line_num); + + const owned = try wi.searchDeduped("ALPHA", testing.allocator); + defer testing.allocator.free(owned); + try testing.expectEqual(raw.len, owned.len); + for (raw, owned) |expected, actual| { + try testing.expectEqual(expected.doc_id, actual.doc_id); + try testing.expectEqual(expected.line_num, actual.line_num); + } +} + +test "word index: searchDeduped handles non-adjacent malformed postings" { + var wi = WordIndex.init(testing.allocator); + defer wi.deinit(); + + try wi.indexFile("a.zig", "alpha\n"); + try wi.indexFile("b.zig", "alpha\n"); + const alpha = wi.index.getPtr("alpha") orelse return error.TestUnexpectedResult; + // Simulate a legacy/corrupt unsorted list with a repeated first posting. + try alpha.append(testing.allocator, alpha.items[0]); + + const hits = try wi.searchDeduped("alpha", testing.allocator); + defer testing.allocator.free(hits); + try testing.expectEqual(@as(usize, 2), hits.len); + try testing.expectEqualStrings("a.zig", wi.hitPath(hits[0])); + try testing.expectEqualStrings("b.zig", wi.hitPath(hits[1])); +} + test "watcher: parallel initial scan matches sequential results" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); + const previous_load_workers = if (cio.posixGetenv("CODEDB_LOAD_WORKERS")) |value| + try testing.allocator.dupe(u8, value) + else + null; + defer { + if (previous_load_workers) |value| { + cio.posixSetenv("CODEDB_LOAD_WORKERS", value); + testing.allocator.free(value); + } else { + cio.posixUnsetenv("CODEDB_LOAD_WORKERS"); + } + } + try tmp_dir.dir.createDirPath(io, "src/nested"); try tmp_dir.dir.writeFile(io, .{ .sub_path = "src/main.zig", .data = "const std = @import(\"std\");\npub fn alpha() void {}\n// TODO: keep me\n" }); try tmp_dir.dir.writeFile(io, .{ .sub_path = "src/nested/util.py", .data = "def beta():\n return 42\n# TODO later\n" }); @@ -559,6 +613,7 @@ test "watcher: parallel initial scan matches sequential results" { var explorer_seq = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_seq.deinit(); explorer_seq.setRoot(io, root); + cio.posixSetenv("CODEDB_LOAD_WORKERS", "1"); try watcher.initialScanWithWorkerCount(io, &store_seq, &explorer_seq, root, testing.allocator, false, 1); var store_par = Store.init(testing.allocator); @@ -566,6 +621,7 @@ test "watcher: parallel initial scan matches sequential results" { var explorer_par = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_par.deinit(); explorer_par.setRoot(io, root); + cio.posixSetenv("CODEDB_LOAD_WORKERS", "4"); try watcher.initialScanWithWorkerCount(io, &store_par, &explorer_par, root, testing.allocator, false, 4); const tree_seq = try explorer_seq.getTree(testing.allocator, false); @@ -581,6 +637,60 @@ test "watcher: parallel initial scan matches sequential results" { try testing.expectEqual(seq_hits.len, par_hits.len); try testing.expectEqual(explorer_seq.outlines.count(), explorer_par.outlines.count()); + + // Parallel stat workers only fill disjoint entry slots; snapshots are + // committed serially afterward, preserving discovery-order sequence IDs. + for ([_][]const u8{ "src/main.zig", "src/nested/util.py", "README.md" }) |path| { + const seq_version = store_seq.getLatest(path) orelse return error.TestUnexpectedResult; + const par_version = store_par.getLatest(path) orelse return error.TestUnexpectedResult; + try testing.expectEqual(seq_version.seq, par_version.seq); + try testing.expectEqual(seq_version.size, par_version.size); + } +} + +test "watcher: rolling trigram shards match canonical masks exactly" { + var source = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer source.deinit(); + var canonical = TrigramIndex.init(testing.allocator); + defer canonical.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "edge/empty.txt", .content = "" }, + .{ .path = "edge/one.txt", .content = "A" }, + .{ .path = "edge/two.txt", .content = "Ab" }, + .{ .path = "edge/three.txt", .content = "AbC" }, + .{ .path = "edge/whitespace.txt", .content = " \t\nAbCdEf" }, + .{ .path = "edge/rollover.txt", .content = "aaaaaaaaaaaaZ" }, + .{ .path = "edge/mixed.txt", .content = "MiXeD-case final" }, + }; + for (fixtures) |fixture| { + try source.indexFile(fixture.path, fixture.content); + try canonical.indexFile(fixture.path, fixture.content); + } + + const sharded = try watcher.buildTrigramsFromCache( + &source.contents, + testing.allocator, + testing.allocator, + 4, + ); + defer { + sharded.deinit(); + testing.allocator.destroy(sharded); + } + + try testing.expectEqual(canonical.index.count(), sharded.index.count()); + var tri_iter = canonical.index.iterator(); + while (tri_iter.next()) |entry| { + const actual_postings = sharded.index.get(entry.key_ptr.*) orelse return error.TestUnexpectedResult; + try testing.expectEqual(entry.value_ptr.count(), actual_postings.count()); + for (entry.value_ptr.items.items) |expected_posting| { + const path = canonical.id_to_path.items[expected_posting.doc_id]; + const actual_mask = actual_postings.get(path) orelse return error.TestUnexpectedResult; + try testing.expectEqual(expected_posting.loc_mask, actual_mask.loc_mask); + try testing.expectEqual(expected_posting.next_mask, actual_mask.next_mask); + } + } } test "watcher: parallel word-index shards match sequential (skip_file_words)" { @@ -2030,6 +2140,145 @@ test "disk index: streamed postings cross a batch boundary" { try testing.expectEqual(@as(usize, 4097), candidates.?.len); } +test "disk index: streamed lookup crosses a batch boundary" { + const alloc = testing.allocator; + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + var ti = TrigramIndex.init(alloc); + ti.owns_paths = true; + defer ti.deinit(); + + // 4,097 distinct trigrams forces one full lookup batch and a one-entry + // tail. Check entries before, at, and after the boundary with both readers. + for (0..4097) |i| { + const content = [3]u8{ + alphabet[i % alphabet.len], + alphabet[(i / alphabet.len) % alphabet.len], + alphabet[(i / (alphabet.len * alphabet.len)) % alphabet.len], + }; + const path = try std.fmt.allocPrint(alloc, "src/lookup-{d}.zig", .{i}); + errdefer alloc.free(path); + try ti.indexFile(path, &content); + alloc.free(path); + } + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + const loaded = TrigramIndex.readFromDisk(io, dir_path, alloc); + try testing.expect(loaded != null); + var heap_index = loaded.?; + defer heap_index.deinit(); + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + + for ([_]usize{ 0, 4095, 4096 }) |i| { + const query = [3]u8{ + alphabet[i % alphabet.len], + alphabet[(i / alphabet.len) % alphabet.len], + alphabet[(i / (alphabet.len * alphabet.len)) % alphabet.len], + }; + const expected_path = try std.fmt.allocPrint(alloc, "src/lookup-{d}.zig", .{i}); + defer alloc.free(expected_path); + + const heap_candidates = heap_index.candidates(&query, alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 1), heap_candidates.?.len); + try testing.expectEqualStrings(expected_path, heap_candidates.?[0]); + + const mmap_candidates = mmap_index.candidates(&query, alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 1), mmap_candidates.?.len); + try testing.expectEqualStrings(expected_path, mmap_candidates.?[0]); + } +} + +test "disk index: mmap candidate parity survives persisted doc ID remapping" { + const alloc = testing.allocator; + var ti = TrigramIndex.init(alloc); + defer ti.deinit(); + + // Each short file contributes one distinct query trigram; only the final + // file contains the whole query. This catches disk file IDs that do not + // preserve the sorted posting-list order required by mmap intersections. + try ti.indexFile("src/abc.zig", "abc"); + try ti.indexFile("src/bcd.zig", "bcd"); + try ti.indexFile("src/cde.zig", "cde"); + try ti.indexFile("src/def.zig", "def"); + try ti.indexFile("src/efg.zig", "efg"); + try ti.indexFile("src/fgh.zig", "fgh"); + try ti.indexFile("src/ghi.zig", "ghi"); + try ti.indexFile("src/full.zig", "abcdefghi"); + + const heap_candidates = ti.candidates("abcdefghi", alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 1), heap_candidates.?.len); + try testing.expectEqualStrings("src/full.zig", heap_candidates.?[0]); + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + const mmap_candidates = mmap_index.candidates("abcdefghi", alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 1), mmap_candidates.?.len); + try testing.expectEqualStrings(heap_candidates.?[0], mmap_candidates.?[0]); +} + +test "trigram index: insertExtracted preserves mmap posting order after doc ID reuse" { + const alloc = testing.allocator; + var ti = TrigramIndex.init(alloc); + defer ti.deinit(); + + var extracted = TrigramIndex.extractTrigrams("abcdefghi", alloc); + defer extracted.deinit(); + try ti.insertExtracted("src/a.zig", &extracted); + try ti.insertExtracted("src/b.zig", &extracted); + try ti.insertExtracted("src/c.zig", &extracted); + ti.removeFile("src/a.zig"); + try ti.insertExtracted("src/a.zig", &extracted); + + const heap_candidates = ti.candidates("abcdefghi", alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 3), heap_candidates.?.len); + try testing.expectEqualStrings("src/a.zig", heap_candidates.?[0]); + try testing.expectEqualStrings("src/b.zig", heap_candidates.?[1]); + try testing.expectEqualStrings("src/c.zig", heap_candidates.?[2]); + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + const mmap_candidates = mmap_index.candidates("abcdefghi", alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 3), mmap_candidates.?.len); + try testing.expectEqualStrings("src/a.zig", mmap_candidates.?[0]); + try testing.expectEqualStrings("src/b.zig", mmap_candidates.?[1]); + try testing.expectEqualStrings("src/c.zig", mmap_candidates.?[2]); +} + test "disk index: readFromDisk returns null for missing files" { const loaded = TrigramIndex.readFromDisk(io, "/tmp/codedb_nonexistent_dir_12345", testing.allocator); try testing.expect(loaded == null); diff --git a/src/test_mcp.zig b/src/test_mcp.zig index e0aea984..f84ca1df 100644 --- a/src/test_mcp.zig +++ b/src/test_mcp.zig @@ -29,10 +29,62 @@ const cli_proxy_mod = @import("cli_proxy.zig"); const bootstrap_mod = @import("bootstrap.zig"); const background_mod = @import("background.zig"); const commands_mod = @import("commands.zig"); +const mcp_json = @import("mcp_json.zig"); comptime { _ = @import("config.zig"); } +test "mcp json: line reader preserves newline and EOF framing" { + var reader: std.Io.Reader = .fixed("first\nlast"); + + const first = mcp_json.readLineBuf(testing.allocator, &reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(first); + try testing.expectEqualStrings("first", first); + + const last = mcp_json.readLineBuf(testing.allocator, &reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(last); + try testing.expectEqualStrings("last", last); + try testing.expect(mcp_json.readLineBuf(testing.allocator, &reader) == null); +} + +test "mcp json: line reader enforces maximum message size" { + const exact = try testing.allocator.alloc(u8, mcp_json.MAX_LINE); + defer testing.allocator.free(exact); + @memset(exact, 'x'); + var exact_reader: std.Io.Reader = .fixed(exact); + const accepted = mcp_json.readLineBuf(testing.allocator, &exact_reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(accepted); + try testing.expectEqual(exact.len, accepted.len); + + const oversized = try testing.allocator.alloc(u8, mcp_json.MAX_LINE + 1); + defer testing.allocator.free(oversized); + @memset(oversized, 'y'); + var oversized_reader: std.Io.Reader = .fixed(oversized); + try testing.expect(mcp_json.readLineBuf(testing.allocator, &oversized_reader) == null); +} + +test "mcp json: typed fields and escaping preserve protocol behavior" { + const parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, + \\{"name":"codedb","count":7,"enabled":true,"other":null} + , .{}); + defer parsed.deinit(); + const object = &parsed.value.object; + try testing.expectEqualStrings("codedb", mcp_json.getStr(object, "name").?); + try testing.expectEqual(@as(i64, 7), mcp_json.getInt(object, "count").?); + try testing.expect(mcp_json.getBool(object, "enabled")); + try testing.expect(mcp_json.getStr(object, "count") == null); + try testing.expect(mcp_json.getInt(object, "name") == null); + try testing.expect(!mcp_json.getBool(object, "missing")); + + var escaped: std.ArrayList(u8) = .empty; + defer escaped.deinit(testing.allocator); + mcp_json.writeEscaped(testing.allocator, &escaped, "plain \"quote\" \\ slash\n\r\t\x00\x08\x0c\x1f"); + try testing.expectEqualStrings( + "plain \\\"quote\\\" \\\\ slash\\n\\r\\t\\u0000\\u0008\\u000c\\u001f", + escaped.items, + ); +} + fn zigExe() []const u8 { return "zig"; } @@ -3002,7 +3054,7 @@ test "issue-619: serve/mcp daemon reclaims the cli socket after the owner exits" var pbuf: [128]u8 = undefined; const sock_path = cli_proxy_mod.cliSocketPath(&pbuf, abs_root).?; var zbuf: [128]u8 = undefined; - const sock_z = try std.fmt.bufPrintZ(&zbuf, "{s}", .{sock_path}); + const sock_z = try cio.bufPrintZ(&zbuf, "{s}", .{sock_path}); _ = std.c.unlink(sock_z.ptr); defer _ = std.c.unlink(sock_z.ptr); @@ -3097,7 +3149,7 @@ const CliHandoverOwnerSim = struct { // socket up entirely rather than waiting for an idle timeout. _ = std.c.close(self.fd); var zbuf: [128]u8 = undefined; - if (std.fmt.bufPrintZ(&zbuf, "{s}", .{self.sock_path})) |z| { + if (cio.bufPrintZ(&zbuf, "{s}", .{self.sock_path})) |z| { _ = std.c.unlink(z.ptr); } else |_| {} self.sd.store(true, .release); @@ -3112,7 +3164,7 @@ test "issue-619: serve/mcp daemon signals a live owner to yield instead of waiti var pbuf: [128]u8 = undefined; const sock_path = cli_proxy_mod.cliSocketPath(&pbuf, abs_root).?; var zbuf: [128]u8 = undefined; - const sock_z = try std.fmt.bufPrintZ(&zbuf, "{s}", .{sock_path}); + const sock_z = try cio.bufPrintZ(&zbuf, "{s}", .{sock_path}); _ = std.c.unlink(sock_z.ptr); defer _ = std.c.unlink(sock_z.ptr); diff --git a/src/test_parser.zig b/src/test_parser.zig index 5cf5c740..d24c78ba 100644 --- a/src/test_parser.zig +++ b/src/test_parser.zig @@ -9,7 +9,6 @@ const SymbolKind = explore.SymbolKind; const DependencyGraph = explore.DependencyGraph; const Store = @import("store.zig").Store; - fn expectOutlineSymbol(outline: *const explore.FileOutline, name: []const u8, kind: SymbolKind) !void { for (outline.symbols.items) |sym| { if (std.mem.eql(u8, sym.name, name) and sym.kind == kind) return; @@ -17,7 +16,6 @@ fn expectOutlineSymbol(outline: *const explore.FileOutline, name: []const u8, ki return error.TestUnexpectedResult; } - fn expectOutlineImport(outline: *const explore.FileOutline, import_path: []const u8) !void { for (outline.imports.items) |imp| { if (std.mem.eql(u8, imp, import_path)) return; @@ -25,7 +23,6 @@ fn expectOutlineImport(outline: *const explore.FileOutline, import_path: []const return error.TestUnexpectedResult; } - test "issue-1: multi-line TS/JS import paths captured for dep graph" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -240,7 +237,6 @@ test "issue-301: Dart / Flutter parser" { try testing.expect(std.mem.indexOf(u8, tree, "home_screen.dart dart") != null); } - test "issue-php-1: PHP class definition herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -268,7 +264,6 @@ test "issue-php-1: PHP class definition herkend" { try testing.expect(found); } - test "issue-php-2: PHP methode binnen class herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -298,7 +293,6 @@ test "issue-php-2: PHP methode binnen class herkend" { try testing.expectEqual(@as(usize, 2), method_count); } - test "issue-php-3: PHP top-level functie herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -325,7 +319,6 @@ test "issue-php-3: PHP top-level functie herkend" { try testing.expectEqual(@as(usize, 2), fn_count); } - test "issue-php-4: PHP interface herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -352,7 +345,6 @@ test "issue-php-4: PHP interface herkend" { try testing.expect(found); } - test "issue-php-5: PHP trait herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -381,7 +373,6 @@ test "issue-php-5: PHP trait herkend" { try testing.expect(found); } - test "issue-php-6: PHP use-import omgezet naar pad in dep_graph" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -401,7 +392,6 @@ test "issue-php-6: PHP use-import omgezet naar pad in dep_graph" { try testing.expectEqualStrings("illuminate/Support/Facades/DB.php", outline.imports.items[1]); } - test "issue-php-7: PHP commentaarregels worden overgeslagen" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -425,7 +415,6 @@ test "issue-php-7: PHP commentaarregels worden overgeslagen" { try testing.expect(outline.symbols.items[0].kind == .class_def); } - test "issue-php-8: PHP function after class is top-level, not method" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -458,7 +447,6 @@ test "issue-php-8: PHP function after class is top-level, not method" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-9: PHP 8.1 enum herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -487,7 +475,6 @@ test "issue-php-9: PHP 8.1 enum herkend" { try testing.expect(found_method); } - test "issue-php-10: PHP grouped use-statement parsed into individual imports" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -509,7 +496,6 @@ test "issue-php-10: PHP grouped use-statement parsed into individual imports" { try testing.expectEqualStrings("app/Models/Role.php", outline.imports.items[2]); } - test "issue-php-11: PHP readonly class herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -535,7 +521,6 @@ test "issue-php-11: PHP readonly class herkend" { try testing.expect(found); } - test "issue-php-12: PHP class and public constants herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -561,7 +546,6 @@ test "issue-php-12: PHP class and public constants herkend" { try testing.expectEqual(@as(usize, 3), constant_count); } - test "issue-php-13: PHP nested braces in methods do not break class tracking" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -603,7 +587,6 @@ test "issue-php-13: PHP nested braces in methods do not break class tracking" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-14: PHP multi-line block comments do not produce symbols" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -640,7 +623,6 @@ test "issue-php-14: PHP multi-line block comments do not produce symbols" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-15: PHP use-as alias stripped from import path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -658,7 +640,6 @@ test "issue-php-15: PHP use-as alias stripped from import path" { try testing.expectEqualStrings("app/Models/User.php", outline.imports.items[0]); } - test "issue-php-16: PHP escaped quotes do not end string mode" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -696,7 +677,6 @@ test "issue-php-16: PHP escaped quotes do not end string mode" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-17: PHP code after block comment terminator is parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -722,7 +702,6 @@ test "issue-php-17: PHP code after block comment terminator is parsed" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-18: PHP use-as alias case-insensitive" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -743,7 +722,6 @@ test "issue-php-18: PHP use-as alias case-insensitive" { try testing.expectEqualStrings("app/Services/Logger.php", outline.imports.items[2]); } - test "issue-111: Python triple-quote docstrings not parsed as code" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -768,7 +746,6 @@ test "issue-111: Python triple-quote docstrings not parsed as code" { try testing.expect(func_count == 1); } - test "issue-112: Python import-as alias stripped from dep path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -785,7 +762,6 @@ test "issue-112: Python import-as alias stripped from dep path" { try testing.expect(deps.len == 1); } - test "issue-113: TypeScript block comments not parsed as code" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -807,7 +783,6 @@ test "issue-113: TypeScript block comments not parsed as code" { try testing.expect(func_count == 1); } - test "issue-114: TypeScript import-as alias does not affect dep path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -823,7 +798,6 @@ test "issue-114: TypeScript import-as alias does not affect dep path" { try testing.expectEqualStrings("./mod", outline.imports.items[0]); } - test "issue-151: Go func and type definitions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -867,7 +841,6 @@ test "issue-151: Go func and type definitions" { try testing.expect(interface_count == 1); // Handler } - test "issue-151: Ruby class, module, and def" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -903,7 +876,6 @@ test "issue-151: Ruby class, module, and def" { try testing.expect(outline.imports.items.len == 2); // json + ./helpers } - test "issue-151: Ruby =begin/=end comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -929,7 +901,6 @@ test "issue-151: Ruby =begin/=end comments skipped" { try testing.expect(func_count == 1); // only real_method } - test "issue-151: Go block comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -953,7 +924,6 @@ test "issue-151: Go block comments skipped" { try testing.expect(func_count == 1); // only realFunc } - test "issue-301: Dart block comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -980,7 +950,6 @@ test "issue-301: Dart block comments skipped" { try testing.expectEqual(@as(usize, 0), func_count); } - test "issue-179: block comment does not produce phantom symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1002,7 +971,6 @@ test "issue-179: block comment does not produce phantom symbols" { try testing.expect(!found_fake); } - test "issue-179: code after single-line /* */ comment is parsed" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1021,7 +989,6 @@ test "issue-179: code after single-line /* */ comment is parsed" { try testing.expect(found); } - test "issue-179: Python docstring with text does not leak symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1069,7 +1036,6 @@ test "issue-518: Python class is labeled class_def, not struct_def" { try expectOutlineSymbol(&outline, "Widget", .class_def); } - test "issue-108: HCL resource block parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1086,7 +1052,6 @@ test "issue-108: HCL resource block parsed" { try testing.expectEqual(SymbolKind.struct_def, results[0].symbol.kind); } - test "issue-108: HCL variable and output parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1110,7 +1075,6 @@ test "issue-108: HCL variable and output parsed" { try testing.expectEqual(SymbolKind.constant, outs[0].symbol.kind); } - test "issue-108: HCL module and provider parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1132,7 +1096,6 @@ test "issue-108: HCL module and provider parsed" { try testing.expect(mods.len == 1); } - test "issue-108: HCL comment lines skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1148,7 +1111,6 @@ test "issue-108: HCL comment lines skipped" { try testing.expect(results.len == 1); } - test "issue-215: R function assignment parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1165,7 +1127,6 @@ test "issue-215: R function assignment parsed" { try testing.expectEqual(SymbolKind.function, results[0].symbol.kind); } - test "issue-215: R library import parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1179,7 +1140,6 @@ test "issue-215: R library import parsed" { try testing.expectEqual(@as(usize, 2), outline.imports.items.len); } - test "issue-215: R setClass parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1198,7 +1158,6 @@ test "issue-215: R setClass parsed" { try testing.expect(a2.len == 1); } - test "issue-319: C parser extracts includes macros types and functions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1276,7 +1235,6 @@ test "issue-319: C parser extracts includes macros types and functions" { try testing.expectEqual(SymbolKind.function, alloc_item[0].symbol.kind); } - test "issue-319: C parser avoids comments strings prototypes and macro calls" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1323,7 +1281,6 @@ test "issue-319: C parser avoids comments strings prototypes and macro calls" { try testing.expectEqual(@as(usize, 0), handler.len); } - test "issue-321: common detected extensions produce outlines" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1639,7 +1596,6 @@ test "issue-321: common detected extensions produce outlines" { try testing.expectEqual(@as(usize, 1), r0.len); } - test "issue-179: Python inline docstring does not leak symbols" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1661,7 +1617,6 @@ test "issue-179: Python inline docstring does not leak symbols" { try testing.expectEqual(@as(usize, 0), fake.len); } - test "issue-179: Python multi-line docstring with def inside" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1687,7 +1642,6 @@ test "issue-179: Python multi-line docstring with def inside" { try testing.expectEqual(@as(usize, 0), inner.len); } - test "issue-331: C parser does not index indented call sites as functions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1726,7 +1680,6 @@ test "issue-331: C parser does not index indented call sites as functions" { try testing.expect(found_real); } - test "issue-331: C parser finds nginx-style split-line definitions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1760,7 +1713,6 @@ test "issue-331: C parser finds nginx-style split-line definitions" { try testing.expect(found_create); } - test "issue-392: Swift parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1872,7 +1824,6 @@ test "issue-532: ReScript parser" { try expectOutlineImport(&outline, "Belt"); } - // ─── audit (2026-06-09): latent-issue sweep — parser/deps fixes ─── // src/explore.zig parseDelimitedImport — Kotlin/Swift `import X as Y` stored "X as Y" as @@ -1945,7 +1896,6 @@ test "audit: Python comma import records all modules" { try expectOutlineImport(&outline, "beta.py"); } - // renderImportedBy (the MCP codedb_deps reverse path) kept the unconditional // basename fallback when getImportedBy gained the ambiguity guard — the same // bare import is still attributed to every same-basename file through the diff --git a/src/test_query.zig b/src/test_query.zig index 87545cb6..9e87b056 100644 --- a/src/test_query.zig +++ b/src/test_query.zig @@ -13,12 +13,10 @@ const Language = explore.Language; const SymbolKind = explore.SymbolKind; const mcp_mod = @import("mcp.zig"); - const fuzzyScore = @import("explore.zig").fuzzyScore; const AgentRegistry = @import("agent.zig").AgentRegistry; const edit_mod = @import("edit.zig"); - test "issue-360: edit rejects mismatched if_hash and leaves file untouched" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -53,7 +51,6 @@ test "issue-360: edit rejects mismatched if_hash and leaves file untouched" { try testing.expectEqualStrings(original, after_bytes); } - test "issue-360: edit response reports hex hash matching codedb_read" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -87,7 +84,6 @@ test "issue-360: edit response reports hex hash matching codedb_read" { try testing.expectEqual(expected_hash, result.new_hash); } - test "issue-360: edit dry_run returns diff preview and leaves file untouched" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -134,7 +130,6 @@ test "issue-360: edit dry_run returns diff preview and leaves file untouched" { try testing.expect(std.mem.indexOf(u8, preview, "+BETA") != null); } - test "issue-163: fuzzy exact match scores highest" { const exact = fuzzyScore("main.zig", "src/main.zig"); const partial = fuzzyScore("main.zig", "src/main_helper.zig"); @@ -143,21 +138,18 @@ test "issue-163: fuzzy exact match scores highest" { try testing.expect(exact.? > partial.?); } - test "issue-163: fuzzy subsequence match works" { const score = fuzzyScore("authmid", "src/auth_middleware.py"); try testing.expect(score != null); try testing.expect(score.? > 0); } - test "issue-163: fuzzy typo-tolerant (missing char)" { // "auth_midlware" missing the 'd' in middleware — should still match via subsequence const score = fuzzyScore("auth_midlware", "src/auth_middleware.py"); try testing.expect(score != null); } - test "issue-163: fuzzy word boundary bonus" { // "auth" at word boundary should score higher than "auth" buried in a word const boundary = fuzzyScore("auth", "src/auth_handler.py"); @@ -167,7 +159,6 @@ test "issue-163: fuzzy word boundary bonus" { try testing.expect(boundary.? > buried.?); } - test "issue-163: fuzzy filename ranks above directory" { // "test" in filename portion should score higher than "test" only in directory const in_name = fuzzyScore("test", "src/test_auth.py"); @@ -177,7 +168,6 @@ test "issue-163: fuzzy filename ranks above directory" { try testing.expect(in_name.? > in_dir.?); } - test "issue-163: fuzzy no match returns null" { const score = fuzzyScore("zzzzxyz", "src/main.zig"); try testing.expect(score == null); @@ -196,7 +186,6 @@ test "issue-518: fuzzy find has a subsequence floor — garbage queries return n try testing.expect(fuzzyScore("auth_midlware", "src/auth_middleware.py") != null); } - test "issue-163: fuzzyFindFiles via Explorer" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -214,7 +203,6 @@ test "issue-163: fuzzyFindFiles via Explorer" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_middleware") != null); } - test "issue-163: multi-part query matches both parts" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -231,7 +219,6 @@ test "issue-163: multi-part query matches both parts" { try testing.expect(std.mem.indexOf(u8, results[0].path, "middleware") != null); } - test "issue-163: extension constraint filters results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -250,7 +237,6 @@ test "issue-163: extension constraint filters results" { } } - test "issue-163: special entry point files get bonus" { const score_main = fuzzyScore("main", "src/main.zig"); const score_regular = fuzzyScore("main", "src/maintain.zig"); @@ -260,7 +246,6 @@ test "issue-163: special entry point files get bonus" { try testing.expect(score_main.? > score_regular.?); } - test "issue-163: transpositions handled by Smith-Waterman" { // These all failed with the old subsequence matcher try testing.expect(fuzzyScore("mpc", "src/mcp.zig") != null); @@ -269,7 +254,6 @@ test "issue-163: transpositions handled by Smith-Waterman" { try testing.expect(fuzzyScore("indxe", "src/index.zig") != null); } - test "issue-168: query pipeline find → limit produces file set" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -295,7 +279,6 @@ test "issue-168: query pipeline find → limit produces file set" { try testing.expect(found_handler); } - test "issue-168: query pipeline search returns matching lines" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -316,7 +299,6 @@ test "issue-168: query pipeline search returns matching lines" { try testing.expect(std.mem.indexOf(u8, results[0].path, "main.zig") != null); } - test "issue-168: query pipeline filter by extension" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -335,7 +317,6 @@ test "issue-168: query pipeline filter by extension" { } } - test "issue-168: query pipeline outline returns symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -347,7 +328,6 @@ test "issue-168: query pipeline outline returns symbols" { try testing.expect(outline.symbols.items.len >= 2); } - test "issue-168: query pipeline chained find → filter narrows results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -368,7 +348,6 @@ test "issue-168: query pipeline chained find → filter narrows results" { try testing.expect(py_only.len < all.len); // filtered set is smaller } - test "issue-168: query pipeline handles empty results gracefully" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -381,7 +360,6 @@ test "issue-168: query pipeline handles empty results gracefully" { try testing.expectEqual(@as(usize, 0), results.len); } - test "issue-168: recall — find + filter preserves only matching extension" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -403,7 +381,6 @@ test "issue-168: recall — find + filter preserves only matching extension" { for (py) |r| try testing.expect(std.mem.endsWith(u8, r.path, ".py")); } - test "issue-168: recall — search finds content across multiple files" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -436,7 +413,6 @@ test "issue-168: recall — search finds content across multiple files" { try testing.expect(!found_c); } - test "issue-168: recall — fuzzy find ranks exact matches highest" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -457,7 +433,6 @@ test "issue-168: recall — fuzzy find ranks exact matches highest" { } } - test "issue-168: recall — multi-part query intersection" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -475,7 +450,6 @@ test "issue-168: recall — multi-part query intersection" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_controller") != null); } - test "issue-168: recall — transposition tolerance in pipeline" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -492,7 +466,6 @@ test "issue-168: recall — transposition tolerance in pipeline" { try testing.expect(std.mem.indexOf(u8, results[0].path, "middleware") != null); } - test "auto-retry: delimiter stripping finds results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -506,7 +479,6 @@ test "auto-retry: delimiter stripping finds results" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_middleware") != null); } - test "per-file truncation: max 5 matches per file in output" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -533,7 +505,6 @@ test "per-file truncation: max 5 matches per file in output" { try testing.expect(results.len >= 10); } - test "issue-359: globPaths matches files by glob pattern" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -570,7 +541,6 @@ test "issue-359: globPaths matches files by glob pattern" { } } - test "issue-359: lsDir returns immediate children with file metadata" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -624,7 +594,6 @@ test "issue-359: lsDir returns immediate children with file metadata" { try testing.expectEqual(@as(usize, 2), file_count); } - test "issue-359: mcp.globMatch backtracks across **/* boundary" { // Pipeline filter (codedb_query) calls mcp.globMatch on each path. The // iterative version forgot the outer ** position when it entered the @@ -641,7 +610,6 @@ test "issue-359: mcp.globMatch backtracks across **/* boundary" { try testing.expect(!mcp_mod.globMatch("docs/*.md", "src/mcp.zig")); } - test "issue-511: glob supports brace alternatives" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -662,7 +630,6 @@ test "issue-511: glob supports brace alternatives" { try testing.expect(!mcp_mod.globMatch("src/{mcp,explore}.zig", "src/main.zig")); } - test "issue-359: globPaths recall — every matching path survives at every depth" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -712,7 +679,6 @@ test "issue-359: globPaths recall — every matching path survives at every dept try testing.expect(!mcp_mod.globMatch("src/**/*.zig", "tests/h.zig")); } - test "issue-359/360: retrieval recall — search/word/symbol/fuzzy/glob/deps all return ground truth" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -854,7 +820,6 @@ test "issue-359/360: retrieval recall — search/word/symbol/fuzzy/glob/deps all } } - test "issue-356-1: codedb_query returns partial results when a step fails" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -895,7 +860,6 @@ test "issue-356-1: codedb_query returns partial results when a step fails" { try testing.expect(std.mem.indexOf(u8, out.items, "failed_at: 1") != null); } - test "issue-356-2: codedb_outline suggests fuzzy alternatives for non-indexed paths" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -930,7 +894,6 @@ test "issue-356-2: codedb_outline suggests fuzzy alternatives for non-indexed pa try testing.expect(std.mem.indexOf(u8, out.items, "src/main.zig") != null); } - test "issue-356-3: codedb_query surfaces received keys on missing-arg errors" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -965,7 +928,6 @@ test "issue-356-3: codedb_query surfaces received keys on missing-arg errors" { try testing.expect(std.mem.indexOf(u8, out.items, "q") != null); } - test "issue-356-p2: codedb_outline missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -995,7 +957,6 @@ test "issue-356-p2: codedb_outline missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "file_path") != null); } - test "issue-356-p2: codedb_symbol missing name surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1025,7 +986,6 @@ test "issue-356-p2: codedb_symbol missing name surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "symbol") != null); } - test "issue-356-p2: codedb_search missing query surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1054,7 +1014,6 @@ test "issue-356-p2: codedb_search missing query surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_word missing word surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1083,7 +1042,6 @@ test "issue-356-p2: codedb_word missing word surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_read missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1112,7 +1070,6 @@ test "issue-356-p2: codedb_read missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_deps missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1141,7 +1098,6 @@ test "issue-356-p2: codedb_deps missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p3: codedb_query emits per-stage summary tail on success" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1180,7 +1136,6 @@ test "issue-356-p3: codedb_query emits per-stage summary tail on success" { try testing.expect(std.mem.indexOf(u8, out.items, "1: sort") != null); } - test "issue-356-p3: codedb_outline includes actionable hint when parser fails" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1216,7 +1171,6 @@ test "issue-356-p3: codedb_outline includes actionable hint when parser fails" { try testing.expect(std.mem.indexOf(u8, out.items, "codedb_index") != null); } - test "issue-356-p3: codedb_read appends fuzzy suggestions when path is unreadable" { const tmp_io = testing.io; var tmp = testing.tmpDir(.{}); @@ -1264,7 +1218,6 @@ test "issue-356-p3: codedb_read appends fuzzy suggestions when path is unreadabl try testing.expect(std.mem.indexOf(u8, out.items, "src/main.zig") != null); } - test "issue-558: codedb_query filter must filter (or error) — never silently pass every file" { // Live repro: [{find},{filter,"pattern":"src/index.zig"},{limit,n:3}] // returned all 50 find results — filter only reads 'ext'/'glob', so an diff --git a/src/test_search.zig b/src/test_search.zig index bf4fa13e..6cc370d6 100644 --- a/src/test_search.zig +++ b/src/test_search.zig @@ -507,6 +507,61 @@ test "issue-393: BM25 ranking surfaces high-density file before single-mention f } } +test "searchContent Tier-0 grouped postings match fragmented fallback exactly" { + var grouped = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer grouped.deinit(); + var fragmented = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer fragmented.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "src/a.zig", .content = "const needle = 1;\n// needle second occurrence\n" }, + .{ .path = "docs/b.md", .content = "needle documentation\n" }, + .{ .path = "src/c.zig", .content = "fn needle() void {}\n" }, + .{ .path = "src/noise.zig", .content = "const unrelated = true;\n" }, + }; + for (fixtures) |fixture| { + try grouped.indexFile(fixture.path, fixture.content); + try fragmented.indexFile(fixture.path, fixture.content); + } + + // Natural postings are grouped by document: A1, A2, B1, C1. Reorder the + // equivalent control list to A1, B1, A2, C1 so searchContent must take its + // defensive slots/hash aggregation path. No semantic hit is added or lost. + const postings = fragmented.word_index.index.getPtr("needle") orelse return error.TestUnexpectedResult; + try testing.expectEqual(@as(usize, 4), postings.items.len); + const a_second = postings.items[1]; + postings.items[1] = postings.items[2]; + postings.items[2] = a_second; + try testing.expect(postings.items[1].doc_id > postings.items[2].doc_id); + + // A cap of two forces both paths to make the same ordering/truncation + // decision rather than merely comparing an unbounded candidate set. + const expected = try grouped.searchContent("needle", testing.allocator, 2); + defer { + for (expected) |result| { + testing.allocator.free(result.path); + testing.allocator.free(result.line_text); + } + testing.allocator.free(expected); + } + const actual = try fragmented.searchContent("needle", testing.allocator, 2); + defer { + for (actual) |result| { + testing.allocator.free(result.path); + testing.allocator.free(result.line_text); + } + testing.allocator.free(actual); + } + + try testing.expectEqual(expected.len, actual.len); + for (expected, actual) |a, b| { + try testing.expectEqualStrings(a.path, b.path); + try testing.expectEqual(a.line_num, b.line_num); + try testing.expectEqualStrings(a.line_text, b.line_text); + try testing.expectEqual(a.score, b.score); + } +} + test "issue-400: BM25 ranks both-terms file above single-term files" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1907,8 +1962,7 @@ test "def-first: renderPlainSearch surfaces the definition line before mentions // A comment mentioning `gadget` appears BEFORE its definition; several call // sites follow. Line-order rendering would show the line-1 comment first. - try explorer.indexFile("src/g.zig", - "// gadget helper\nconst a = gadget;\nconst b = gadget;\npub fn gadget() void {}\ngadget();\ngadget();\n"); + try explorer.indexFile("src/g.zig", "// gadget helper\nconst a = gadget;\nconst b = gadget;\npub fn gadget() void {}\ngadget();\ngadget();\n"); var out: std.ArrayList(u8) = .empty; defer out.deinit(testing.allocator); diff --git a/src/test_snapshot.zig b/src/test_snapshot.zig index 434a4e8e..891c16d5 100644 --- a/src/test_snapshot.zig +++ b/src/test_snapshot.zig @@ -17,7 +17,6 @@ const git_mod = @import("git.zig"); const AgentRegistry = @import("agent.zig").AgentRegistry; const edit_mod = @import("edit.zig"); - test "issue-625: in-tree snapshot is added to .git/info/exclude" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -110,7 +109,6 @@ test "issue-35: edits immediately update explorer and snapshot output" { try testing.expect(std.mem.indexOf(u8, after_snap, "oldName") == null); } - test "snapshot_json: snapshot builds and is valid JSON" { // Explorer uses arena for internal data var arena = std.heap.ArenaAllocator.init(testing.allocator); @@ -148,7 +146,6 @@ test "snapshot_json: snapshot builds and is valid JSON" { try testing.expect(symbol_index.contains("version")); } - test "issue-44: snapshot stale after working tree changes cause stale query results" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -205,7 +202,6 @@ test "issue-44: snapshot stale after working tree changes cause stale query resu try testing.expect(results.len == 1); } - test "issue-46: empty-repo snapshot rejected on load" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -378,7 +374,6 @@ test "snapshot: CONTENT_HASHES records the correct per-file hash (order-aligned) } } - test "issue-220: snapshot fast load restores outlines and lazily rebuilds word index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -535,7 +530,7 @@ test "snapshot: parallel freshness load re-indexes changed files, restores the r try testing.expect(snapshot_mod.loadSnapshot(io, snap_path, &exp2, &store, arena2.allocator())); try testing.expectEqual(total, exp2.outlines.count()); - var is_changed = [_]bool{false} ** total; + var is_changed: [total]bool = @splat(false); for (changed) |i| is_changed[i] = true; // Changed files carry fresh content (changed branch); the rest keep snapshot @@ -559,7 +554,6 @@ test "snapshot: parallel freshness load re-indexes changed files, restores the r } } - test "snapshot: writer streams uncached file contents for large repos" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -617,7 +611,6 @@ test "snapshot: writer streams uncached file contents for large repos" { try testing.expect(loaded.wordIndexIsComplete()); } - test "issue-220: partial word index state rebuilds before search" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -660,7 +653,6 @@ test "issue-220: partial word index state rebuilds before search" { try testing.expect(exp2.wordIndexNeedsPersist()); } - test "issue-220: word index persistence tracking skips redundant rewrites" { var exp = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer exp.deinit(); @@ -685,7 +677,6 @@ test "issue-220: word index persistence tracking skips redundant rewrites" { try testing.expect(!exp.wordIndexNeedsPersist()); } - test "issue-45: snapshot written in non-git directory cannot be loaded" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -717,7 +708,6 @@ test "issue-45: snapshot written in non-git directory cannot be loaded" { try testing.expect(snap_head == null); } - test "issue-47: concurrent snapshot writes from parallel instances corrupt file" { // BUG: Two codedb instances indexing the same repo write codedb.snapshot // concurrently with no file locking. The second writer can overwrite a @@ -787,7 +777,6 @@ test "issue-47: concurrent snapshot writes from parallel instances corrupt file" try testing.expect(loaded); } - test "issue-42: scan thread is joined before allocator-backed state is freed" { var gpa = std.heap.DebugAllocator(.{}){}; const allocator = gpa.allocator(); @@ -819,7 +808,6 @@ test "issue-42: scan thread is joined before allocator-backed state is freed" { _ = gpa.deinit(); } - test "issue-40: truncated snapshot silently loads partial data" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -860,7 +848,6 @@ test "issue-40: truncated snapshot silently loads partial data" { try testing.expect(!loaded); } - test "issue-41: snapshot not validated against repo identity allows cross-project loading" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -889,7 +876,6 @@ test "issue-41: snapshot not validated against repo identity allows cross-projec try testing.expect(!loaded); } - test "snapshot: symbol detail longer than 4096 bytes survives round-trip" { // Regression for readSectionString rejecting names/details > 4096 bytes. // Before the fix max_len was 4096; any detail longer than that triggered @@ -936,7 +922,6 @@ test "snapshot: symbol detail longer than 4096 bytes survives round-trip" { try testing.expect(results.len >= 1); } - test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { // Regression for the codedb 0.2.56 writer u16 overflow bug: when OUTLINE_STATE // contains a detail that overflows u16 the section cursor de-syncs, making @@ -970,7 +955,7 @@ test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { const ols = sections.get(@intFromEnum(snapshot_mod.SectionId.outline_state)) orelse return; const f = try std.Io.Dir.cwd().openFile(io, snap_path, .{ .mode = .read_write }); defer f.close(io); - try f.writePositionalAll(io, &([_]u8{0xFF} ** 16), ols.offset); + try f.writePositionalAll(io, &@as([16]u8, @splat(0xFF)), ols.offset); } var exp2 = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); @@ -988,7 +973,6 @@ test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { try testing.expect(results.len >= 1); } - test "issue-379: snapshot loader returns true with zero outlines for empty-explorer snapshot" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1017,7 +1001,6 @@ test "issue-379: snapshot loader returns true with zero outlines for empty-explo } } - test "issue-528: isSensitivePath parity between snapshot.zig and watcher.zig" { // The secret/credential filter is duplicated (snapshot persistence vs live // indexing). The #528 audit flagged a possible divergence; the two are @@ -1026,20 +1009,20 @@ test "issue-528: isSensitivePath parity between snapshot.zig and watcher.zig" { // one path but not the other. const cases = [_][]const u8{ // secrets — both copies must block - ".env", ".env.local", ".env.production", - ".env.development", ".env.staging", ".env.test", - ".dev.vars", ".npmrc", ".pypirc", - ".netrc", "credentials.json", "service-account.json", - "secrets.json", "secrets.yaml", "secrets.yml", - "id_rsa", "id_ed25519", "server.key", - "cert.pem", "keystore.jks", "identity.pfx", - "bundle.p12", "config/.env.local", "a/b/secrets.yaml", + ".env", ".env.local", ".env.production", + ".env.development", ".env.staging", ".env.test", + ".dev.vars", ".npmrc", ".pypirc", + ".netrc", "credentials.json", "service-account.json", + "secrets.json", "secrets.yaml", "secrets.yml", + "id_rsa", "id_ed25519", "server.key", + "cert.pem", "keystore.jks", "identity.pfx", + "bundle.p12", "config/.env.local", "a/b/secrets.yaml", "deep/nested/.ssh/known_hosts", ".gnupg/secring.gpg", "x/.aws/credentials", // non-secrets — both copies must allow (esp. the .env-prefix edge cases) - ".envoy.json", ".environment", ".envrc", - ".envconfig.yaml", "main.zig", "src/server.zig", - "README.md", "package.json", "id_rsa.pub", - "envvars.ts", "Makefile", "Dockerfile", + ".envoy.json", ".environment", ".envrc", + ".envconfig.yaml", "main.zig", "src/server.zig", + "README.md", "package.json", "id_rsa.pub", + "envvars.ts", "Makefile", "Dockerfile", }; for (cases) |p| { try testing.expectEqual(watcher.isSensitivePath(p), snapshot_mod.isSensitivePath(p)); @@ -1271,7 +1254,6 @@ test "issue-539b: search recall ranks a relevant restored file above quota (inde try testing.expect(found_canonical); } - test "issue-564: snapshot fast-load defers the symbol index until first symbol use" { // Pre-#564 the fast-load eagerly rebuilt the global symbol index for every // restored file (~symbols-per-file map inserts and their heap) even though @@ -1320,7 +1302,7 @@ test "audit: long import specifier round-trips on fast-restore (borrow path pres var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); var exp = Explorer.init(arena.allocator(), Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); - const long_imp = "a" ** 5000; // > 4096 (old read cap), <= 65535 (write cap) + const long_imp: [5000]u8 = @splat('a'); // > 4096 (old read cap), <= 65535 (write cap) const src = "package main\nimport \"" ++ long_imp ++ "\"\nfunc mainFn() {}\n"; try exp.indexFile("probe/main.go", src); const pre = exp.outlines.get("probe/main.go") orelse return error.MissingOutline; diff --git a/src/wasm.zig b/src/wasm.zig index 29e4009f..52ecd334 100644 --- a/src/wasm.zig +++ b/src/wasm.zig @@ -96,8 +96,9 @@ export fn wasm_get_outline( defer outline.deinit(); // Serialize to JSON with proper escaping - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; for (outline.symbols.items, 0..) |sym, i| { if (i > 0) writer.writeByte(',') catch return null; @@ -115,7 +116,7 @@ export fn wasm_get_outline( } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -142,8 +143,9 @@ export fn wasm_search( } // Serialize to JSON - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; for (results, 0..) |r, i| { if (i > 0) writer.writeByte(',') catch return null; @@ -171,7 +173,7 @@ export fn wasm_search( } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -180,8 +182,9 @@ export fn wasm_search( export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { const exp = getExplorer(); - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; var first = true; var iter = exp.outlines.iterator(); @@ -201,7 +204,7 @@ export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -210,8 +213,9 @@ export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { export fn wasm_get_all_outlines(out_len_ptr: *usize) ?[*]u8 { const exp = getExplorer(); - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('{') catch return null; var first = true; var iter = exp.outlines.iterator(); @@ -239,7 +243,7 @@ export fn wasm_get_all_outlines(out_len_ptr: *usize) ?[*]u8 { } writer.writeByte('}') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } diff --git a/src/watcher.zig b/src/watcher.zig index cac08b49..d570fb9c 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -44,7 +44,7 @@ pub const FsEvent = struct { pub const EventQueue = struct { const CAPACITY = 4096; - events: [CAPACITY]?FsEvent = [_]?FsEvent{null} ** CAPACITY, + events: [CAPACITY]?FsEvent = @splat(null), head: usize = 0, tail: usize = 0, mu: cio.Mutex = .{}, @@ -85,6 +85,7 @@ const FileMap = std.StringHashMap(FileState); const InitialScanEntry = struct { path: []u8, size: u64, + stat_succeeded: bool, skip_trigram: bool, }; @@ -507,20 +508,15 @@ pub fn trigramFileCap() usize { return std.fmt.parseInt(usize, raw, 10) catch 15_000; } -// statScanForInitial: helper matching loadSnapshotFast's freshnessScan pattern. -// Paths collected serially first (in caller), then stats fanned via chunks to -// workers; side-effect is stat + recordSnapshot + storing the size back into the -// entry. Per-stat errors are swallowed like the ref; record failures set shared -// err_out and stop the worker. -fn statScanForInitial(io: std.Io, store: *Store, dir: std.Io.Dir, recs: []InitialScanEntry, err_out: *?anyerror) void { +// Paths are collected serially first, then stat calls fan out across disjoint +// chunks. Workers only publish size/stat_succeeded into their own entries; the +// caller records snapshots serially in discovery order after every join. That +// keeps sequence assignment deterministic and avoids shared worker error state. +fn statScanForInitial(io: std.Io, dir: std.Io.Dir, recs: []InitialScanEntry) void { for (recs) |*record| { - if (err_out.* != null) return; const ds = dir.statFile(io, record.path, .{}) catch continue; record.size = ds.size; - _ = store.recordSnapshot(record.path, ds.size, 0) catch |e| { - err_out.* = e; - return; - }; + record.stat_succeeded = true; } } @@ -540,14 +536,14 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat try entries.append(allocator, .{ .path = try allocator.dupe(u8, entry.path), .size = 0, + .stat_succeeded = false, .skip_trigram = false, // set after parallel stat fan }); } - // Stats fanned out to workers (exact pattern from loadSnapshotFast: Pass A serial collect, - // Pass B chunk+spawn to statScanForInitial on disjoint slices; swallows per-stat errors, - // propagates record error, uses env/256/4 idiom, static chunks, join). + // Stats fan out across disjoint slices; per-stat errors are swallowed. After + // joining, snapshots are recorded serially so sequence order remains stable + // and record errors propagate without cross-thread error sharing. if (entries.items.len > 0) { - var build_err: ?anyerror = null; const want_workers = blk: { if (cio.posixGetenv("CODEDB_LOAD_WORKERS")) |raw| { const parsed = std.fmt.parseInt(usize, raw, 10) catch 0; @@ -558,10 +554,17 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat break :blk @min(@as(usize, @intCast(cpu_count)), 4); }; const n_workers = @max(@as(usize, 1), @min(want_workers, entries.items.len)); - if (n_workers <= 1) { - statScanForInitial(io, store, dir, entries.items, &build_err); - } else if (allocator.alloc(std.Thread, n_workers)) |threads| { + stat_fan: { + if (n_workers <= 1) { + statScanForInitial(io, dir, entries.items); + break :stat_fan; + } + const threads = allocator.alloc(std.Thread, n_workers) catch { + statScanForInitial(io, dir, entries.items); + break :stat_fan; + }; defer allocator.free(threads); + const chunk = entries.items.len / n_workers; const rem = entries.items.len % n_workers; var off: usize = 0; @@ -573,22 +576,23 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat off += chunk + extra; const recs = entries.items[start..off]; if (spawn_failed) { - statScanForInitial(io, store, dir, recs, &build_err); + statScanForInitial(io, dir, recs); continue; } - if (std.Thread.spawn(.{}, statScanForInitial, .{ io, store, dir, recs, &build_err })) |t| { + if (std.Thread.spawn(.{}, statScanForInitial, .{ io, dir, recs })) |t| { threads[spawned] = t; spawned += 1; } else |_| { - statScanForInitial(io, store, dir, recs, &build_err); + statScanForInitial(io, dir, recs); spawn_failed = true; } } for (threads[0..spawned]) |t| t.join(); - } else |_| { - statScanForInitial(io, store, dir, entries.items, &build_err); } - if (build_err) |e| return e; + for (entries.items) |entry| { + if (!entry.stat_succeeded) continue; + _ = try store.recordSnapshot(entry.path, entry.size, 0); + } } // Set skip_trigram using serial count (preserves original cap semantics on discovery order). var file_count: usize = 0; @@ -601,14 +605,12 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat fn parseInitialScanEntry( io: std.Io, - root: []const u8, + dir: std.Io.Dir, entry: InitialScanEntry, content_alloc: std.mem.Allocator, parse_alloc: std.mem.Allocator, ) !?ParsedScanFile { if (shouldSkipFile(entry.path)) return null; - const dir = try std.Io.Dir.cwd().openDir(io, root, .{}); - defer dir.close(io); const content = (try readIndexableFile(io, dir, entry.path, content_alloc, entry.size, true)) orelse return null; errdefer content_alloc.free(content); // Threshold for including a file in the trigram index. Bumped from 64KB to @@ -629,13 +631,13 @@ fn parseInitialScanEntry( fn parseInitialScanWorkerEntry( io: std.Io, results: *WorkerParsedResults, - root: []const u8, + dir: std.Io.Dir, entry: InitialScanEntry, scratch_alloc: std.mem.Allocator, word_shard: ?*WordIndex, ) ?PersistedScanFile { const persist = results.arena.allocator(); - const parsed = parseInitialScanEntry(io, root, entry, persist, scratch_alloc) catch return null; + const parsed = parseInitialScanEntry(io, dir, entry, persist, scratch_alloc) catch return null; if (parsed == null) return null; var file = parsed.?; @@ -661,6 +663,12 @@ fn parseInitialScanWorkerEntry( } fn initialScanWorker(io: std.Io, results: *WorkerParsedResults, root: []const u8, entries: []const InitialScanEntry, word_shard: ?*WordIndex, trigram_shard: ?*TrigramIndex) void { + const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch { + for (0..entries.len) |index| results.publish(io, index, null); + return; + }; + defer dir.close(io); + // Read content directly into the persistent result arena, while allocating // transient parser state in a per-file scratch arena that is reset between // files. The final outline uses packed libc-owned storage so the main thread @@ -676,7 +684,7 @@ fn initialScanWorker(io: std.Io, results: *WorkerParsedResults, root: []const u8 defer scratch.deinit(); for (entries, 0..) |entry, index| { _ = scratch.reset(.retain_capacity); - const file = parseInitialScanWorkerEntry(io, results, root, entry, scratch.allocator(), word_shard); + const file = parseInitialScanWorkerEntry(io, results, dir, entry, scratch.allocator(), word_shard); if (file) |f| { if (trigram_shard) |ts| { if (!f.skip_trigram and f.content.len <= max_trigram_file_bytes) { @@ -724,7 +732,7 @@ pub fn initialScanWithWorkerCount(io: std.Io, store: *Store, explorer: *Explorer { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); - const parsed = try parseInitialScanEntry(io, root, entry, arena.allocator(), arena.allocator()); + const parsed = try parseInitialScanEntry(io, dir, entry, arena.allocator(), arena.allocator()); if (parsed) |file| { try explorer.commitParsedFileOwnedOutline(file.path, file.content, file.outline, true, file.skip_trigram); } @@ -843,42 +851,68 @@ pub fn initialScanWithWorkerCount(io: std.Io, store: *Store, explorer: *Explorer } } +fn extractTrigramMasks( + content: []const u8, + local: *std.AutoHashMap(@import("index.zig").Trigram, @import("index.zig").PostingMask), +) !void { + if (content.len < 3) return; + const index_m = @import("index.zig"); + var c0 = content[0]; + var c1 = content[1]; + var c2 = content[2]; + var n0 = index_m.normalizeChar(c0); + var n1 = index_m.normalizeChar(c1); + var n2 = index_m.normalizeChar(c2); + + for (0..content.len - 2) |i| { + const has_next = i + 3 < content.len; + const c3 = if (has_next) content[i + 3] else 0; + const n3 = if (has_next) index_m.normalizeChar(c3) else 0; + if (!((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and + (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and + (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r'))) + { + const tri = index_m.packTrigram(n0, n1, n2); + const gop = try local.getOrPut(tri); + if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; + gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i & 7); + if (has_next) { + gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(n3 & 7); + } + } + c0 = c1; + c1 = c2; + c2 = c3; + n0 = n1; + n1 = n2; + n2 = n3; + } +} + // Build a private trigram shard from a chunk of InitialScanEntry items — reads // each file, extracts trigrams, and inserts postings directly, all on the worker. // Used by the skip_outlines fast path in initialScanWithTrigrams. fn readAndBuildTrigramShardWorker(io: std.Io, shard: *TrigramIndex, root: []const u8, entries: []const InitialScanEntry, build_error: *?anyerror) void { + const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch |err| { + build_error.* = err; + return; + }; + defer dir.close(io); + const index_m = @import("index.zig"); var local = std.AutoHashMap(index_m.Trigram, index_m.PostingMask).init(std.heap.c_allocator); defer local.deinit(); local.ensureTotalCapacity(4096) catch {}; for (entries) |entry| { if (shouldSkipFile(entry.path)) continue; - const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch continue; - defer dir.close(io); const content = (readIndexableFile(io, dir, entry.path, std.heap.c_allocator, entry.size, false) catch continue) orelse continue; defer std.heap.c_allocator.free(content); if (content.len > max_trigram_file_bytes) continue; local.clearRetainingCapacity(); - if (content.len >= 3) { - for (0..content.len - 2) |i| { - const c0 = content[i]; - const c1 = content[i + 1]; - const c2 = content[i + 2]; - if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and - (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and - (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; - const tri = index_m.packTrigram(index_m.normalizeChar(c0), index_m.normalizeChar(c1), index_m.normalizeChar(c2)); - const gop = local.getOrPut(tri) catch |err| { - build_error.* = err; - return; - }; - if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; - gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); - if (i + 3 < content.len) { - gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(index_m.normalizeChar(content[i + 3]) % 8); - } - } - } + extractTrigramMasks(content, &local) catch |err| { + build_error.* = err; + return; + }; shard.insertBulkMapNew(entry.path, &local) catch |err| { build_error.* = err; return; @@ -898,26 +932,10 @@ fn cachedTrigramBuildWorker(shard: *TrigramIndex, entries: []const CachedEntry, local.ensureTotalCapacity(4096) catch {}; for (entries) |entry| { local.clearRetainingCapacity(); - if (entry.content.len >= 3) { - for (0..entry.content.len - 2) |i| { - const c0 = entry.content[i]; - const c1 = entry.content[i + 1]; - const c2 = entry.content[i + 2]; - if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and - (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and - (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; - const tri = index_m.packTrigram(index_m.normalizeChar(c0), index_m.normalizeChar(c1), index_m.normalizeChar(c2)); - const gop = local.getOrPut(tri) catch |err| { - build_error.* = err; - return; - }; - if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; - gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); - if (i + 3 < entry.content.len) { - gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(index_m.normalizeChar(entry.content[i + 3]) % 8); - } - } - } + extractTrigramMasks(entry.content, &local) catch |err| { + build_error.* = err; + return; + }; shard.insertBulkMapNew(entry.path, &local) catch |err| { build_error.* = err; return; @@ -1070,7 +1088,7 @@ pub fn initialScanWithTrigrams( for (entries.items) |entry| { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); - const parsed = parseInitialScanEntry(io, root, entry, arena.allocator(), arena.allocator()) catch null; + const parsed = parseInitialScanEntry(io, dir, entry, arena.allocator(), arena.allocator()) catch null; if (parsed) |file| { if (!skip_outlines) { explorer.commitParsedFileOwnedOutline(file.path, file.content, file.outline, true, true) catch continue; diff --git a/vendor/nanoregex/.gitignore b/vendor/nanoregex/.gitignore new file mode 100644 index 00000000..93cfddff --- /dev/null +++ b/vendor/nanoregex/.gitignore @@ -0,0 +1,12 @@ +# Zig build artifacts +.zig-cache/ +zig-cache/ +zig-out/ + +# macOS +.DS_Store + +# Editor +*.swp +.vscode/ +.idea/ diff --git a/vendor/nanoregex/LICENSE b/vendor/nanoregex/LICENSE new file mode 100644 index 00000000..7b09d8bb --- /dev/null +++ b/vendor/nanoregex/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rach Pradhan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/nanoregex/README.md b/vendor/nanoregex/README.md new file mode 100644 index 00000000..14830ed4 --- /dev/null +++ b/vendor/nanoregex/README.md @@ -0,0 +1,136 @@ +# nanoregex + +Small, fast, pure-Zig regex engine. Built to be a drop-in replacement for `zig-regex 0.1.1` with substantially better performance on real workloads. + +3,200 lines of Zig. No FFI. No external dependencies. 27/27 parity fixtures green against Python's `re` module. + +## Benchmarks + +Search across a 142 KB Zig source file, 200 iterations, ReleaseFast. + +| Pattern | Python `re` | **nanoregex** | Winner | +|---|---|---|---| +| pure literal `compileAllocFlags` | 0.067ms | **0.037ms** | nanoregex 1.8× | +| literal-prefix `compileAllocFlags\([a-z]+` | 0.061ms | **0.037ms** | nanoregex 1.65× | +| `fn [A-Za-z]+\(.*alloc` | **0.061ms** | 0.092ms | python 1.5× | +| `\d+` (1307 matches) | 0.991ms | **0.455ms** | nanoregex 2.2× | +| `[a-z]+` (17711 matches) | 1.654ms | **0.594ms** | nanoregex 2.8× | +| alt `foo\|bar\|baz` | 0.731ms | **0.423ms** | nanoregex 1.7× | +| IPv4-ish `\d+\.\d+\.\d+\.\d+` | 0.904ms | **0.416ms** | nanoregex 2.2× | + +8 of 8 head-to-head non-anchored patterns won. Versus `zig-regex 0.1.1` on a pattern that triggers catastrophic backtracking, nanoregex is ~5000× faster (43 seconds → 8 milliseconds). + +## Architecture + +Layered, with five dispatch tiers that compose at compile time: + +``` +parser.zig pattern bytes → AST +ast.zig AST node tagged union, arena-owned +nfa.zig AST → Thompson NFA +exec.zig Pike-VM simulation (always-correct fallback) +dfa.zig Lazy subset-construction DFA (perf path) +minterm.zig Byte-class compression for the DFA's transition table +prefilter.zig Literal-prefix / required-substring extraction +root.zig Public API + dispatch +``` + +`findAll` and `search` route to the cheapest engine that can correctly handle a given pattern: + +1. **Pure-literal pattern** → `std.mem.indexOf` loop (memmem) +2. **Required-literal absent** → return empty (no engine work at all) +3. **Literal-prefix + DFA-eligible** → `indexOfPos` to candidate starts, DFA at each hit +4. **DFA-eligible** → plain lazy DFA +5. **Otherwise** → Pike VM + +DFA-eligible means: no capture groups, no anchors (`^`, `$`, `\b`), no lazy quantifiers, not case-insensitive, and the on-demand DFA stays under the 4096-state budget. Everything that doesn't fit those rules takes the Pike-VM path, which is linear-time and correct on every input. + +Bytes are folded to **minterm classes** before indexing the DFA's transition table. A pattern with `[a-z]+` reduces 256 bytes to 2 classes (in-set, out-of-set), shrinking the per-state row from 1 KB to 8 bytes and letting the whole transition table live in L1 cache. + +## API + +Mirrors `zig-regex 0.1.1` enough that most callers can switch by changing one path in `build.zig`: + +```zig +const nanoregex = @import("nanoregex"); + +var r = try nanoregex.Regex.compile(allocator, "(\\w+)@(\\w+)"); +defer r.deinit(); + +const matches = try r.findAll(allocator, "alice@example bob@host"); +defer { + for (matches) |*m| @constCast(m).deinit(allocator); + allocator.free(matches); +} +for (matches) |m| { + std.debug.print("{d}..{d}\n", .{ m.span.start, m.span.end }); +} +``` + +Methods take `*Regex` (mutable) rather than `*const Regex` because the lazy DFA fills its transition table on the fly. The first `findAll` call on a fresh `Regex` warms the cache; subsequent calls are pure table lookups. + +Compile flags: + +```zig +try nanoregex.Regex.compileWithFlags(alloc, pattern, .{ + .case_insensitive = false, + .multiline = true, // grep-like default — `^`/`$` match line edges + .dot_all = false, +}); +``` + +Backreference expansion in `replaceAll` (`\1`, `\2`, ...): + +```zig +const out = try r.replaceAll(alloc, "alice@example", "\\2/\\1"); +// → "example/alice" +``` + +## Supported syntax (v1) + +- Literals, `.`, character classes `[abc]` / `[^abc]` / `[a-z]` +- Shorthand `\d \D \w \W \s \S` +- Quantifiers `? * + {n} {n,m}` — greedy and lazy (`*?`, `+?`, `??`, `{n,m}?`) +- Groups `(foo)` capturing, `(?:foo)` non-capturing +- Alternation `foo|bar` +- Anchors `^ $ \b \B \A \z` +- Flags: case-insensitive, multiline, dot-all + +**Not yet supported**: backreferences in *patterns* (`\1` inside the regex itself), lookaround `(?=...)`/`(?!...)`, inline flag groups `(?i)...`, named groups `(?P...)`, Unicode property classes. Patterns using these features parse OK if the syntax shape is recognised, but matching may diverge — fall back to a richer engine if you need them. + +## Build + +```bash +zig build install -Doptimize=ReleaseFast +# → zig-out/bin/nanoregex_probe (parity test CLI) +# → zig-out/bin/nanoregex_bench (single-file benchmark) +``` + +## Tests + +Tests are split into narrow per-module steps so the inner loop stays tight: + +```bash +zig build test-ast # 3 tests +zig build test-parser # parser + ast tests +zig build test-nfa # nfa + parser + ast +zig build test-exec # Pike VM tests +zig build test-prefilter # literal-extraction tests +zig build test-minterm # byte-class compression +zig build test-dfa # DFA construction + matching +zig build test-root # public API +zig build parity # Python re parity (requires python3) +zig build test-all # everything, explicit and opt-in +``` + +Add `-Dtest-filter='substring'` to any step to narrow further. + +## Why it exists + +This was extracted from the [zigrepper](https://github.com/justrach/zigrepper) toolchain, where `zig-regex 0.1.1`'s backtracking engine was making `zigrep --regex` take 43 seconds on patterns like `compileAllocFlags\([a-z]+` against a directory tree. After this engine landed, the same query finished in 0.43 seconds end-to-end. + +Inspired by Russ Cox's writing on regex implementation, RE2's lazy DFA, and the [RE# / Resharp blog post](https://iev.ee/blog/resharp-how-we-built-the-fastest-regex-in-fsharp/) which laid out minterm compression and several other optimizations cleanly. + +## License + +MIT diff --git a/vendor/nanoregex/UPSTREAM.md b/vendor/nanoregex/UPSTREAM.md new file mode 100644 index 00000000..b5478ac9 --- /dev/null +++ b/vendor/nanoregex/UPSTREAM.md @@ -0,0 +1,17 @@ +# Vendored source provenance + +This directory is vendored from [`justrach/nanoregex`](https://github.com/justrach/nanoregex) at commit: + +```text +736b46703454d5f37d3e46164fc91354386bb29c +``` + +It was the dependency previously pinned by codedb's `build.zig.zon`. The local changes are limited to Zig `0.17.0-dev.813+2153f8143` compatibility: removed repetition syntax was replaced with typed `@splat`, the package minimum was updated, and the touched build/source files were canonicalized by that Zig snapshot's formatter (format-only whitespace/layout changes). Keep the upstream MIT `LICENSE` file when updating this copy. + +To verify after an update: + +```bash +cd vendor/nanoregex +zig build test-all +zig build parity +``` diff --git a/vendor/nanoregex/build.zig b/vendor/nanoregex/build.zig new file mode 100644 index 00000000..769b5dd6 --- /dev/null +++ b/vendor/nanoregex/build.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // Pass-through filter that the user can layer on top of any named step + // to narrow further: zig build test-dfa -Dtest-filter='alternation' + const user_filter = b.option([]const u8, "test-filter", "Narrow test name filter (substring)"); + + const nanoregex_mod = b.addModule("nanoregex", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + + const probe = b.addExecutable(.{ + .name = "nanoregex_probe", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/probe.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(probe); + + const bench = b.addExecutable(.{ + .name = "nanoregex_bench", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/bench.zig"), + .target = target, + .optimize = .ReleaseFast, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(bench); + + const bench_comptime = b.addExecutable(.{ + .name = "nanoregex_bench_comptime", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/bench_comptime.zig"), + .target = target, + .optimize = .ReleaseFast, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(bench_comptime); + + // ───────────────────────────────────────────────────────────────── + // Per-module test steps. + // + // The user's preferred iteration loop is to run ONE narrow named step + // at a time, so each module has its own step that compiles a small + // test binary scoped to that source file (+ its imports). There is + // NO aggregate `test` step — `test-all` is explicit and opt-in. + // + // To narrow further: zig build test-dfa -Dtest-filter='alternation' + // ───────────────────────────────────────────────────────────────── + + _ = addTestStep(b, "test-ast", "src/ast.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser", "src/parser.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa", "src/nfa.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-exec", "src/exec.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-prefilter", "src/prefilter.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-dfa", "src/dfa.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-minterm", "src/minterm.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-root", "src/root.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-comptime", "src/comptime.zig", &.{}, user_filter, target, optimize); + + // ── Pre-baked filtered shortcuts ── + // Each named step runs only tests whose name contains one of the + // listed substrings (Zig's --test-filter is OR-of-substrings). + // Compile is cached and shared with the parent module's step. + + _ = addTestStep(b, "test-parser-core", "src/parser.zig", &.{ "literal", "concat", "alternation" }, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-quant", "src/parser.zig", &.{"quantifier"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-class", "src/parser.zig", &.{"class"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-group", "src/parser.zig", &.{"group"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-error", "src/parser.zig", &.{"errors"}, user_filter, target, optimize); + + _ = addTestStep(b, "test-nfa-basic", "src/nfa.zig", &.{ "literal", "concat" }, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-quant", "src/nfa.zig", &.{ "star", "plus", "question", "counted" }, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-alt", "src/nfa.zig", &.{"alt"}, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-group", "src/nfa.zig", &.{"group"}, user_filter, target, optimize); + + _ = addTestStep(b, "test-exec-basic", "src/exec.zig", &.{ "literal", "no match" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-quant", "src/exec.zig", &.{ "greedy", "lazy", "counted", "optional" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-class", "src/exec.zig", &.{ "class", "digit", "word" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-group", "src/exec.zig", &.{"group"}, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-anchor", "src/exec.zig", &.{ "anchor", "boundary" }, user_filter, target, optimize); + + _ = addTestStep(b, "test-dfa-rejects", "src/dfa.zig", &.{"rejects"}, user_filter, target, optimize); + _ = addTestStep(b, "test-dfa-match", "src/dfa.zig", &.{ "literal", "plus", "alt", "class", "wildcard", "longest" }, user_filter, target, optimize); + + _ = addTestStep(b, "test-prefilter-full", "src/prefilter.zig", &.{"full literal"}, user_filter, target, optimize); + _ = addTestStep(b, "test-prefilter-required", "src/prefilter.zig", &.{"required literal"}, user_filter, target, optimize); + + // ───────────────────────────────────────────────────────────────── + // Aggregate sweeps — explicit, opt-in. Run these AFTER you're done + // iterating, not in the inner loop. + // ───────────────────────────────────────────────────────────────── + + const ast_all = addTestStep(b, "_test-ast-all", "src/ast.zig", &.{}, user_filter, target, optimize); + const parser_all = addTestStep(b, "_test-parser-all", "src/parser.zig", &.{}, user_filter, target, optimize); + const nfa_all = addTestStep(b, "_test-nfa-all", "src/nfa.zig", &.{}, user_filter, target, optimize); + const exec_all = addTestStep(b, "_test-exec-all", "src/exec.zig", &.{}, user_filter, target, optimize); + const prefilter_all = addTestStep(b, "_test-prefilter-all", "src/prefilter.zig", &.{}, user_filter, target, optimize); + const dfa_all = addTestStep(b, "_test-dfa-all", "src/dfa.zig", &.{}, user_filter, target, optimize); + const root_all = addTestStep(b, "_test-root-all", "src/root.zig", &.{}, user_filter, target, optimize); + const comptime_all = addTestStep(b, "_test-comptime-all", "src/comptime.zig", &.{}, user_filter, target, optimize); + + const test_all = b.step("test-all", "Run ALL unit tests across every module (slow — use named steps in the inner loop)"); + test_all.dependOn(ast_all); + test_all.dependOn(parser_all); + test_all.dependOn(nfa_all); + test_all.dependOn(exec_all); + test_all.dependOn(prefilter_all); + test_all.dependOn(dfa_all); + test_all.dependOn(root_all); + test_all.dependOn(comptime_all); + + // ── Parity vs Python re (separate; never auto-runs) ── + const parity_cmd = b.addSystemCommand(&.{"bash"}); + parity_cmd.addFileArg(b.path("tests/parity/run.sh")); + parity_cmd.addFileArg(probe.getEmittedBin()); + parity_cmd.addDirectoryArg(b.path("tests/parity/fixtures")); + const parity_step = b.step("parity", "Run Python-re parity tests (separate; opt-in)"); + parity_step.dependOn(&parity_cmd.step); +} + +/// Build one focused test step. `filters` is OR'd substring matching +/// (test runs iff its name contains at least one of the filters, or all +/// pass when the list is empty). The user-level -Dtest-filter is appended +/// so a named step can be narrowed further from the command line. +fn addTestStep( + b: *std.Build, + step_name: []const u8, + root_path: []const u8, + base_filters: []const []const u8, + user_filter: ?[]const u8, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +) *std.Build.Step { + var filter_list = std.ArrayList([]const u8).empty; + filter_list.appendSlice(b.allocator, base_filters) catch @panic("OOM"); + if (user_filter) |f| filter_list.append(b.allocator, f) catch @panic("OOM"); + + const test_mod = b.createModule(.{ + .root_source_file = b.path(root_path), + .target = target, + .optimize = optimize, + }); + const test_exe = b.addTest(.{ + .root_module = test_mod, + .filters = filter_list.toOwnedSlice(b.allocator) catch @panic("OOM"), + }); + const run_step = b.addRunArtifact(test_exe); + const step = b.step(step_name, b.fmt("Run tests in {s}", .{root_path})); + step.dependOn(&run_step.step); + return step; +} diff --git a/vendor/nanoregex/build.zig.zon b/vendor/nanoregex/build.zig.zon new file mode 100644 index 00000000..31960386 --- /dev/null +++ b/vendor/nanoregex/build.zig.zon @@ -0,0 +1,8 @@ +.{ + .name = .nanoregex, + .version = "0.0.1", + .fingerprint = 0xc8e46b5d7121d911, + .minimum_zig_version = "0.17.0-dev.813+2153f8143", + .dependencies = .{}, + .paths = .{""}, +} diff --git a/vendor/nanoregex/src/ast.zig b/vendor/nanoregex/src/ast.zig new file mode 100644 index 00000000..6740c8b1 --- /dev/null +++ b/vendor/nanoregex/src/ast.zig @@ -0,0 +1,186 @@ +//! Regex AST. Built by parser.zig, consumed by nfa.zig (later). +//! +//! All nodes are arena-owned by the parent Regex. The arena is freed in one +//! shot on Regex.deinit, so individual nodes never need their own deinit. +//! Child references use `*const Node` rather than slices to keep the union +//! tag size predictable and to avoid sub-allocations for unary nodes. + +const std = @import("std"); + +pub const Node = union(enum) { + /// A single literal byte. Case-folding (when flags.case_insensitive) is + /// handled at match time so the AST stays case-preserving — useful for + /// reporting in error messages. + literal: u8, + + /// `.` — matches any byte except `\n`, or any byte at all when the + /// pattern was compiled with `flags.dot_all`. The matcher consults the + /// flag; the AST node itself is flag-agnostic. + dot, + + /// `[abc]`, `[^abc]`, `[a-z]`, plus the shorthands `\d \D \w \W \s \S` + /// which the parser desugars into a Class node with the right bitmap. + class: *const Class, + + /// Zero-width anchors: `^ $ \b \B \A \z`. + anchor: Anchor, + + /// A concatenation `abc` — match each sub-node in order. + concat: []const *const Node, + + /// Alternation `a|b|c` — match any one branch (left to right, first + /// wins under leftmost-first semantics, matching Python re). + alt: []const *const Node, + + /// Quantified sub-pattern: `a*`, `a+`, `a?`, `a{n,m}`. + repeat: *const Repeat, + + /// `(foo)` or `(?:foo)`. The group's `index` is 0 for non-capturing, + /// 1..N for capturing groups in left-paren order. Index 0 (the whole + /// match) is implicit at the top level — not a Group node. + group: *const Group, +}; + +pub const Anchor = enum { + /// `^` — start of input, or start of any line in multiline mode. + line_start, + /// `$` — end of input, or end of any line in multiline mode. + line_end, + /// `\b` — boundary between a word char (`[A-Za-z0-9_]`) and not. + word_boundary, + /// `\B` — anywhere `\b` doesn't match. + non_word_boundary, + /// `\A` — start of input (ignores multiline flag). + string_start, + /// `\z` — end of input (ignores multiline flag). + string_end, +}; + +/// Character class represented as a 256-bit bitmap. `bitmap[byte/8] & +/// (1 << (byte%8))` is set iff the byte is included. Negation is folded +/// into the bitmap at parse time so the matcher does a single bit test. +pub const Class = struct { + bitmap: [32]u8, + + pub fn empty() Class { + return .{ .bitmap = @splat(0) }; + } + + pub fn set(self: *Class, byte: u8) void { + self.bitmap[byte / 8] |= @as(u8, 1) << @intCast(byte % 8); + } + + pub fn setRange(self: *Class, lo: u8, hi: u8) void { + var b: usize = lo; + while (b <= hi) : (b += 1) { + self.set(@intCast(b)); + if (b == 0xff) break; + } + } + + pub fn contains(self: *const Class, byte: u8) bool { + return (self.bitmap[byte / 8] >> @intCast(byte % 8)) & 1 != 0; + } + + pub fn negate(self: *Class) void { + for (&self.bitmap) |*b| b.* = ~b.*; + } +}; + +pub const Repeat = struct { + sub: *const Node, + min: u32, + /// `std.math.maxInt(u32)` represents unbounded (`*` and `+`). + max: u32, + /// True for `*`/`+`/`?`/`{n,m}`, false for the lazy `??`/`*?`/`+?`/`{n,m}?` + /// variants. Greedy is the Python re default. + greedy: bool, +}; + +pub const Group = struct { + sub: *const Node, + /// 0 = non-capturing; ≥1 = capture index in left-paren declaration order. + index: u32, + capturing: bool, +}; + +// ── Test helpers ── + +/// Pretty-print an AST for debugging and tests. Indents to make tree shape +/// visible. The format is stable enough to assert against in tests. +/// Pretty-print an AST for debugging and tests. Indents to make tree shape +/// visible. The format is stable enough to assert against in tests. +/// Writes into an ArrayList(u8) rather than a std.io.Writer because Zig 0.16 +/// reworked the writer interface and we don't want to chase the new shape +/// from a leaf debug helper. +pub fn debugWrite(node: *const Node, buf: *std.ArrayList(u8), alloc: std.mem.Allocator, indent: u32) error{OutOfMemory}!void { + var i: u32 = 0; + while (i < indent) : (i += 1) try buf.appendSlice(alloc, " "); + var tmp: [128]u8 = undefined; + switch (node.*) { + .literal => |c| { + const line = std.fmt.bufPrint(&tmp, "literal '{c}'\n", .{c}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .dot => try buf.appendSlice(alloc, "dot\n"), + .anchor => |a| { + const line = std.fmt.bufPrint(&tmp, "anchor {s}\n", .{@tagName(a)}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .class => |c| { + var popcnt: u32 = 0; + for (c.bitmap) |b| popcnt += @popCount(b); + const line = std.fmt.bufPrint(&tmp, "class [{d} bytes]\n", .{popcnt}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .concat => |children| { + try buf.appendSlice(alloc, "concat\n"); + for (children) |child| try debugWrite(child, buf, alloc, indent + 1); + }, + .alt => |children| { + try buf.appendSlice(alloc, "alt\n"); + for (children) |child| try debugWrite(child, buf, alloc, indent + 1); + }, + .repeat => |r| { + const line = std.fmt.bufPrint(&tmp, "repeat min={d} max={d} greedy={}\n", .{ r.min, r.max, r.greedy }) catch unreachable; + try buf.appendSlice(alloc, line); + try debugWrite(r.sub, buf, alloc, indent + 1); + }, + .group => |g| { + const line = std.fmt.bufPrint(&tmp, "group #{d} cap={}\n", .{ g.index, g.capturing }) catch unreachable; + try buf.appendSlice(alloc, line); + try debugWrite(g.sub, buf, alloc, indent + 1); + }, + } +} + +test "class bitmap set/contains" { + var c = Class.empty(); + c.set('a'); + c.set('z'); + try std.testing.expect(c.contains('a')); + try std.testing.expect(c.contains('z')); + try std.testing.expect(!c.contains('b')); + try std.testing.expect(!c.contains('y')); +} + +test "class range" { + var c = Class.empty(); + c.setRange('a', 'd'); + try std.testing.expect(c.contains('a')); + try std.testing.expect(c.contains('b')); + try std.testing.expect(c.contains('c')); + try std.testing.expect(c.contains('d')); + try std.testing.expect(!c.contains('e')); + try std.testing.expect(!c.contains('`')); +} + +test "class negate" { + var c = Class.empty(); + c.setRange('a', 'z'); + c.negate(); + try std.testing.expect(!c.contains('a')); + try std.testing.expect(!c.contains('z')); + try std.testing.expect(c.contains('A')); + try std.testing.expect(c.contains('0')); +} diff --git a/vendor/nanoregex/src/bench.zig b/vendor/nanoregex/src/bench.zig new file mode 100644 index 00000000..e9715b4f --- /dev/null +++ b/vendor/nanoregex/src/bench.zig @@ -0,0 +1,145 @@ +//! Single-file regex benchmark. +//! +//! Wire shape: nanoregex_bench [iters] +//! +//! Reads `path` into memory once, then runs `r.findAll` against it `iters` +//! times (default 20). Prints mean per-iteration time + match count so the +//! comparison script can diff us against python re and zig-regex on the +//! exact same input. +//! +//! We measure findAll only — not parse/compile — because the latter is a +//! one-shot cost the user pays once per CLI invocation, while findAll +//! dominates real workloads (walks across thousands of files). + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; + +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |a| try args_list.append(alloc, a); + const args = args_list.items; + if (args.len < 3) { + writeAll(2, "usage: nanoregex_bench [iters]\n"); + std.process.exit(2); + } + + const pattern = args[1]; + const path = args[2]; + const iters: usize = if (args.len >= 4) + std.fmt.parseInt(usize, args[3], 10) catch 20 + else + 20; + + // Read the whole file. Using libc fopen+fread to avoid the std.fs API + // churn in 0.16 — this binary is throwaway so the simplest path wins. + const data = readFile(alloc, path) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "read error: {s}\n", .{@errorName(err)}) catch "read error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer alloc.free(data); + + var r = nanoregex.Regex.compile(alloc, pattern) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "parse error: {s}\n", .{@errorName(err)}) catch "parse error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer r.deinit(); + + var total_ns: u128 = 0; + var match_count: usize = 0; + + // One untimed warm-up so JIT-like effects don't bias the first sample. + { + const ms = r.findAll(alloc, data) catch { + writeAll(2, "engine error during warm-up\n"); + std.process.exit(1); + }; + match_count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var iter: usize = 0; + while (iter < iters) : (iter += 1) { + const start_ns = nowNs(); + const ms = r.findAll(alloc, data) catch { + writeAll(2, "engine error in timed loop\n"); + std.process.exit(1); + }; + const end_ns = nowNs(); + total_ns += @intCast(end_ns - start_ns); + match_count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + const mean_ms: f64 = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)) / 1_000_000.0; + + var out_buf: [256]u8 = undefined; + const line = std.fmt.bufPrint(&out_buf, "nanoregex: matches={d} mean={d:.3}ms ({d}KB, {d} iters)\n", .{ + match_count, + mean_ms, + data.len / 1024, + iters, + }) catch return; + writeAll(1, line); +} + +const Timespec = extern struct { tv_sec: i64, tv_nsec: i64 }; +extern "c" fn clock_gettime(clk: c_int, ts: *Timespec) c_int; +const CLOCK_MONOTONIC: c_int = 6; + +fn nowNs() i128 { + var ts: Timespec = .{ .tv_sec = 0, .tv_nsec = 0 }; + _ = clock_gettime(CLOCK_MONOTONIC, &ts); + return @as(i128, ts.tv_sec) * 1_000_000_000 + ts.tv_nsec; +} + +extern "c" fn fopen(path: [*:0]const u8, mode: [*:0]const u8) ?*anyopaque; +extern "c" fn fclose(stream: *anyopaque) c_int; +extern "c" fn fread(ptr: [*]u8, size: usize, n: usize, stream: *anyopaque) usize; +extern "c" fn fseek(stream: *anyopaque, offset: c_long, whence: c_int) c_int; +extern "c" fn ftell(stream: *anyopaque) c_long; +const SEEK_END: c_int = 2; +const SEEK_SET: c_int = 0; + +fn readFile(alloc: std.mem.Allocator, path: []const u8) ![]u8 { + var path_buf: [4096]u8 = undefined; + if (path.len >= path_buf.len) return error.PathTooLong; + @memcpy(path_buf[0..path.len], path); + path_buf[path.len] = 0; + const path_z: [*:0]const u8 = @ptrCast(&path_buf); + + const f = fopen(path_z, "rb") orelse return error.OpenFailed; + defer _ = fclose(f); + + if (fseek(f, 0, SEEK_END) != 0) return error.SeekFailed; + const size_raw = ftell(f); + if (size_raw < 0) return error.SizeFailed; + const size: usize = @intCast(size_raw); + if (fseek(f, 0, SEEK_SET) != 0) return error.SeekFailed; + + const buf = try alloc.alloc(u8, size); + errdefer alloc.free(buf); + const n = fread(buf.ptr, 1, size, f); + if (n != size) return error.ReadShort; + return buf; +} diff --git a/vendor/nanoregex/src/bench_comptime.zig b/vendor/nanoregex/src/bench_comptime.zig new file mode 100644 index 00000000..15b2be1e --- /dev/null +++ b/vendor/nanoregex/src/bench_comptime.zig @@ -0,0 +1,248 @@ +//! bench_comptime.zig — head-to-head: comptime-specialised vs runtime Regex. +//! +//! Wire shape: nanoregex_bench_comptime [iters] +//! +//! Reads `path` once, then runs three fixed patterns through both code paths: +//! +//! 1. "hello" — pure-literal (comptime: needle constant; runtime: memmem) +//! 2. "\\d+" — single-class (comptime: .rodata table; runtime: dfa+table) +//! 3. "[a-z]+" — single-class (comptime: .rodata table; runtime: dfa+table) +//! +//! For each pattern we report: +//! comptime mean, runtime mean, and the ratio (runtime/comptime ≥ 1 = comptime wins). +//! +//! We also accept the bench file path so the same 142 KB fixture used for +//! parity tests can be passed in as the haystack. + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +const Timespec = extern struct { tv_sec: i64, tv_nsec: i64 }; +extern "c" fn clock_gettime(clk: c_int, ts: *Timespec) c_int; +const CLOCK_MONOTONIC: c_int = 6; +fn nowNs() i128 { + var ts: Timespec = .{ .tv_sec = 0, .tv_nsec = 0 }; + _ = clock_gettime(CLOCK_MONOTONIC, &ts); + return @as(i128, ts.tv_sec) * 1_000_000_000 + ts.tv_nsec; +} + +extern "c" fn fopen(path: [*:0]const u8, mode: [*:0]const u8) ?*anyopaque; +extern "c" fn fclose(stream: *anyopaque) c_int; +extern "c" fn fread(ptr: [*]u8, size: usize, n: usize, stream: *anyopaque) usize; +extern "c" fn fseek(stream: *anyopaque, offset: c_long, whence: c_int) c_int; +extern "c" fn ftell(stream: *anyopaque) c_long; +const SEEK_END: c_int = 2; +const SEEK_SET: c_int = 0; + +fn readFile(alloc: std.mem.Allocator, path: []const u8) ![]u8 { + var path_buf: [4096]u8 = undefined; + if (path.len >= path_buf.len) return error.PathTooLong; + @memcpy(path_buf[0..path.len], path); + path_buf[path.len] = 0; + const path_z: [*:0]const u8 = @ptrCast(&path_buf); + const f = fopen(path_z, "rb") orelse return error.OpenFailed; + defer _ = fclose(f); + if (fseek(f, 0, SEEK_END) != 0) return error.SeekFailed; + const size_raw = ftell(f); + if (size_raw < 0) return error.SizeFailed; + const size: usize = @intCast(size_raw); + if (fseek(f, 0, SEEK_SET) != 0) return error.SeekFailed; + const buf = try alloc.alloc(u8, size); + errdefer alloc.free(buf); + const n = fread(buf.ptr, 1, size, f); + if (n != size) return error.ReadShort; + return buf; +} + +/// Run `findAll` `iters` times and return (mean_ns, match_count). +fn benchRuntime(alloc: std.mem.Allocator, pattern: []const u8, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + var r = try nanoregex.Regex.compile(alloc, pattern); + defer r.deinit(); + + // Warm-up. + var count: usize = 0; + { + const ms = try r.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try r.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeLiteral(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("hello"); + + // Warm-up. + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeDigits(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("\\d+"); + + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeLower(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("[a-z]+"); + + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |a| try args_list.append(alloc, a); + const args = args_list.items; + + if (args.len < 2) { + writeAll(2, "usage: nanoregex_bench_comptime [iters]\n"); + std.process.exit(2); + } + + const path = args[1]; + const iters: usize = if (args.len >= 3) + std.fmt.parseInt(usize, args[2], 10) catch 50 + else + 50; + + const data = readFile(alloc, path) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "read error: {s}\n", .{@errorName(err)}) catch "read error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer alloc.free(data); + + var hdr: [128]u8 = undefined; + const hdr_line = std.fmt.bufPrint(&hdr, "nanoregex_bench_comptime file={s} size={d}KB iters={d}\n", .{ + path, data.len / 1024, iters, + }) catch "header\n"; + writeAll(1, hdr_line); + writeAll(1, "pattern comptime(µs) runtime(µs) ratio(rt/ct)\n"); + writeAll(1, "──────────────────────────────────────────────────────────\n"); + + // Pattern 1: pure literal "hello" + { + const ct = try benchComptimeLiteral(alloc, data, iters); + const rt = try benchRuntime(alloc, "hello", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"hello\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } + + // Pattern 2: \d+ + { + const ct = try benchComptimeDigits(alloc, data, iters); + const rt = try benchRuntime(alloc, "\\d+", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"\\d+\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } + + // Pattern 3: [a-z]+ + { + const ct = try benchComptimeLower(alloc, data, iters); + const rt = try benchRuntime(alloc, "[a-z]+", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"[a-z]+\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } +} diff --git a/vendor/nanoregex/src/comptime.zig b/vendor/nanoregex/src/comptime.zig new file mode 100644 index 00000000..166a3c92 --- /dev/null +++ b/vendor/nanoregex/src/comptime.zig @@ -0,0 +1,546 @@ +//! comptime.zig — comptime-specialised regex matchers. +//! +//! `compileComptime(pattern)` returns a `ComptimeRegex(pattern)` whose +//! internals are fully resolved at compile time for two pattern shapes: +//! +//! 1. Pure literal — no metacharacters → tight `std.mem.indexOf` loop +//! with the needle baked as a `*const [N]u8` constant. +//! +//! 2. Single-class quantifier — `\d+`, `[a-z]+`, `\w{2,}`, etc. → a +//! 256-element `[256]bool` membership table baked into `.rodata` at +//! compile time; the hot loop is a single `table[b]` lookup per byte. +//! +//! Any other pattern shape falls back to the runtime `Regex.compile` path. +//! The fallback is wrapped in the same API surface so callers don't need to +//! branch. +//! +//! # Usage +//! +//! const cx = comptime nanoregex.compileComptime("\\d+"); +//! const matches = try cx.findAll(allocator, haystack); +//! defer { for (matches) |*m| @constCast(m).deinit(allocator); allocator.free(matches); } +//! +//! Note: `cx` is a value type (no heap, no deinit needed). The `findAll` and +//! `search` methods do allocate for the returned Match slices. + +const std = @import("std"); +const root = @import("root.zig"); + +pub const Span = root.Span; +pub const Match = root.Match; + +// ── Pattern classification (all comptime) ────────────────────────────────── + +/// Classify a pattern string into one of three shapes that we can +/// specialize at compile time. +const PatternKind = enum { + /// Pattern has no metacharacters — every byte is literal. + pure_literal, + /// Pattern is exactly `CLASS+` or `CLASS{n,}` where CLASS is one of: + /// `\d`, `\D`, `\w`, `\W`, `\s`, `\S`, or `[...]`. + /// We bake the 256-bool membership table into `.rodata`. + single_class_plus, + /// Everything else — delegate to the runtime engine. + runtime_fallback, +}; + +/// Returns true iff `c` is a regex metacharacter that may alter meaning. +fn isMeta(c: u8) bool { + return switch (c) { + '.', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '^', '$', '\\' => true, + else => false, + }; +} + +/// Classify `pattern` at comptime. +fn classify(comptime pattern: []const u8) PatternKind { + if (pattern.len == 0) return .pure_literal; + + // Check for pure literal: no metacharacters at all. + var has_meta = false; + for (pattern) |c| { + if (isMeta(c)) { + has_meta = true; + break; + } + } + if (!has_meta) return .pure_literal; + + // Check for single-class-plus: CLASS+ or CLASS{n,} + // CLASS is one of the shorthand escapes \d \D \w \W \s \S, or a + // bracket expression [...]. We do a lightweight hand-rolled parse + // here (no allocator, comptime-safe). + return classifyAsSingleClass(pattern); +} + +/// Lightweight comptime check: is `pattern` exactly `CLASS+` or `CLASS{n,}`? +fn classifyAsSingleClass(comptime pattern: []const u8) PatternKind { + if (pattern.len < 2) return .runtime_fallback; + + // Unwrap optional non-capturing group `(?:...)` at the very outside. + const inner: []const u8 = blk: { + if (pattern.len > 4 and + pattern[0] == '(' and pattern[1] == '?' and pattern[2] == ':' and + pattern[pattern.len - 1] == ')') + { + break :blk pattern[3 .. pattern.len - 1]; + } + break :blk pattern; + }; + + // Determine the class span and the quantifier that follows it. + var class_end: usize = 0; + if (inner[0] == '\\') { + // Shorthand escape: \d \D \w \W \s \S + if (inner.len < 2) return .runtime_fallback; + switch (inner[1]) { + 'd', 'D', 'w', 'W', 's', 'S' => class_end = 2, + else => return .runtime_fallback, + } + } else if (inner[0] == '[') { + // Bracket expression — scan forward for the closing `]`. + // Handle `[^...]` negation and `[]...]` (literal `]` first). + var i: usize = 1; + if (i < inner.len and inner[i] == '^') i += 1; + if (i < inner.len and inner[i] == ']') i += 1; // literal `]` at start + while (i < inner.len and inner[i] != ']') i += 1; + if (i >= inner.len) return .runtime_fallback; + class_end = i + 1; // include the `]` + } else { + return .runtime_fallback; + } + + // What follows the class? + const quant = inner[class_end..]; + // Accept: `+`, `{n,}`, `{n,m}` where n >= 1 + if (quant.len == 0) return .runtime_fallback; + if (quant[0] == '+') { + if (quant.len == 1) return .single_class_plus; + return .runtime_fallback; + } + if (quant[0] == '{') { + // Minimal parse: must have a digit, then `,`, optionally digits, then `}`. + var i: usize = 1; + if (i >= quant.len or quant[i] < '0' or quant[i] > '9') return .runtime_fallback; + // Read the min value; reject min == 0. + var min_val: usize = 0; + while (i < quant.len and quant[i] >= '0' and quant[i] <= '9') { + min_val = min_val * 10 + (quant[i] - '0'); + i += 1; + } + if (min_val == 0) return .runtime_fallback; + if (i >= quant.len or quant[i] != ',') return .runtime_fallback; + i += 1; // skip ',' + // Optional upper bound digits. + while (i < quant.len and quant[i] >= '0' and quant[i] <= '9') i += 1; + if (i >= quant.len or quant[i] != '}') return .runtime_fallback; + if (i + 1 == quant.len) return .single_class_plus; + return .runtime_fallback; + } + return .runtime_fallback; +} + +// ── Comptime membership table generation ────────────────────────────────── + +/// Build a `[256]bool` membership table for the CLASS at the front of +/// `pattern` (same syntax as `classifyAsSingleClass` recognises). +/// Called at comptime so the array lives in `.rodata`. +fn buildClassTable(comptime pattern: []const u8) [256]bool { + // Strip outer (?:...) group if present. + const inner: []const u8 = blk: { + if (pattern.len > 4 and + pattern[0] == '(' and pattern[1] == '?' and pattern[2] == ':' and + pattern[pattern.len - 1] == ')') + { + break :blk pattern[3 .. pattern.len - 1]; + } + break :blk pattern; + }; + + var table: [256]bool = @splat(false); + + if (inner[0] == '\\') { + // Shorthand escape. + switch (inner[1]) { + 'd' => { + var b: usize = '0'; + while (b <= '9') : (b += 1) table[b] = true; + }, + 'D' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + if (b < '0' or b > '9') table[b] = true; + } + }, + 'w' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_') table[b] = true; + } + }, + 'W' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if (!((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_')) table[b] = true; + } + }, + 's' => { + for ([_]u8{ ' ', '\t', '\n', '\r', 0x0C, 0x0B }) |c| table[c] = true; + }, + 'S' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if (c != ' ' and c != '\t' and c != '\n' and c != '\r' and + c != 0x0C and c != 0x0B) table[b] = true; + } + }, + else => {}, + } + return table; + } + + // Bracket expression `[...]` or `[^...]`. + std.debug.assert(inner[0] == '['); + var i: usize = 1; + var negate = false; + if (i < inner.len and inner[i] == '^') { + negate = true; + i += 1; + } + + // Handle literal `]` as first char inside brackets. + if (i < inner.len and inner[i] == ']') { + table[']'] = true; + i += 1; + } + + while (i < inner.len and inner[i] != ']') { + if (i + 2 < inner.len and inner[i + 1] == '-' and inner[i + 2] != ']') { + // Range a-z. + var lo: usize = inner[i]; + const hi: usize = inner[i + 2]; + while (lo <= hi) : (lo += 1) table[lo] = true; + i += 3; + } else if (inner[i] == '\\' and i + 1 < inner.len) { + // Escape inside bracket. + switch (inner[i + 1]) { + 'd' => { + var b: usize = '0'; + while (b <= '9') : (b += 1) table[b] = true; + }, + 'w' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_') table[b] = true; + } + }, + 's' => { + for ([_]u8{ ' ', '\t', '\n', '\r', 0x0C, 0x0B }) |c| table[c] = true; + }, + 'n' => table['\n'] = true, + 't' => table['\t'] = true, + 'r' => table['\r'] = true, + else => table[inner[i + 1]] = true, + } + i += 2; + } else { + table[inner[i]] = true; + i += 1; + } + } + + if (negate) { + for (&table) |*b| b.* = !b.*; + } + return table; +} + +// ── Core scanner functions ───────────────────────────────────────────────── + +/// Find the first match using a compile-time-known literal needle. +fn literalFirst(comptime needle: []const u8, alloc: std.mem.Allocator, haystack: []const u8) !?Match { + if (needle.len == 0) { + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = 0, .end = 0 }; + return .{ .span = .{ .start = 0, .end = 0 }, .captures = caps }; + } + const idx = std.mem.indexOf(u8, haystack, needle) orelse return null; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = idx, .end = idx + needle.len }; + return .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = caps }; +} + +/// Find all matches using a compile-time-known literal needle. +fn literalAll(comptime needle: []const u8, alloc: std.mem.Allocator, haystack: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + if (needle.len == 0) return try results.toOwnedSlice(alloc); + var pos: usize = 0; + while (pos <= haystack.len) { + const idx = std.mem.indexOfPos(u8, haystack, pos, needle) orelse break; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = idx, .end = idx + needle.len }; + try results.append(alloc, .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = caps }); + pos = idx + needle.len; + if (needle.len == 0) break; // guard against empty-needle infinite loop + } + return try results.toOwnedSlice(alloc); +} + +/// Find the first match using a compile-time membership table. +fn classFirst(comptime table: *const [256]bool, alloc: std.mem.Allocator, input: []const u8) !?Match { + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + return .{ .span = .{ .start = start, .end = i }, .captures = caps }; + } + i += 1; + } + return null; +} + +/// Find all matches using a compile-time membership table. +fn classAll(comptime table: *const [256]bool, alloc: std.mem.Allocator, input: []const u8) ![]Match { + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + try out.append(alloc, .{ .span = .{ .start = start, .end = i }, .captures = caps }); + } else { + i += 1; + } + } + return try out.toOwnedSlice(alloc); +} + +// ── Public API ───────────────────────────────────────────────────────────── + +/// A pattern-specialised matcher. The concrete type (and its `search` / +/// `findAll` implementation) are chosen at compile time based on the +/// pattern's shape. Callers never call `deinit` on the `ComptimeRegex` +/// value itself — there's nothing to free. +pub fn ComptimeRegex(comptime pattern: []const u8) type { + const kind = classify(pattern); + return switch (kind) { + .pure_literal => struct { + const needle: []const u8 = pattern; + + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + return literalFirst(needle, alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + return literalAll(needle, alloc, input); + } + }, + .single_class_plus => struct { + const table: [256]bool = buildClassTable(pattern); + + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + return classFirst(&table, alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + return classAll(&table, alloc, input); + } + }, + .runtime_fallback => struct { + // Thin wrapper: we call Regex.compile at first use (lazy) or + // callers can use this struct as a value and call the runtime path. + // + // Because we can't store a Regex by value here without an + // allocator, the runtime fallback re-compiles on each call. This + // is intentional: the comptime path is only meant for patterns + // that ARE pure-literal or single-class. For everything else the + // normal Regex API should be used. The fallback here exists only + // so callers don't need to branch. + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + var r = try root.Regex.compile(alloc, pattern); + defer r.deinit(); + return r.search(alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + var r = try root.Regex.compile(alloc, pattern); + defer r.deinit(); + return r.findAll(alloc, input); + } + }, + }; +} + +/// Construct a `ComptimeRegex(pattern)` value. Call with a comptime-known +/// string literal; the compiler constant-folds the entire specialisation. +/// +/// const cx = comptime compileComptime("hello"); +/// const m = try cx.findAll(allocator, "say hello world"); +pub fn compileComptime(comptime pattern: []const u8) ComptimeRegex(pattern) { + return .{}; +} + +// ── Tests ────────────────────────────────────────────────────────────────── + +test "comptime: pure-literal findAll matches runtime path" { + const cx = comptime compileComptime("hello"); + const haystack = "say hello world, hello again"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + // Runtime reference + var r = try root.Regex.compile(std.testing.allocator, "hello"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: pure-literal search first match" { + const cx = comptime compileComptime("fox"); + var m = (try cx.search(std.testing.allocator, "the quick brown fox jumps")).?; + defer m.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 16), m.span.start); + try std.testing.expectEqual(@as(usize, 19), m.span.end); +} + +test "comptime: pure-literal no match returns null" { + const cx = comptime compileComptime("xyz"); + const result = try cx.search(std.testing.allocator, "hello world"); + try std.testing.expect(result == null); +} + +test "comptime: \\d+ class findAll matches runtime path" { + const cx = comptime compileComptime("\\d+"); + const haystack = "abc 42 def 1234 xyz"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + var r = try root.Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: [a-z]+ class findAll matches runtime path" { + const cx = comptime compileComptime("[a-z]+"); + const haystack = "Hello World foo BAR baz"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + var r = try root.Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: [A-Z]+ class (uppercase only)" { + const cx = comptime compileComptime("[A-Z]+"); + const haystack = "Hello WORLD foo BAR"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); + try std.testing.expectEqualStrings("H", haystack[ms[0].span.start..ms[0].span.end]); + try std.testing.expectEqualStrings("WORLD", haystack[ms[1].span.start..ms[1].span.end]); + try std.testing.expectEqualStrings("BAR", haystack[ms[2].span.start..ms[2].span.end]); +} + +test "comptime: \\w+ word class" { + const cx = comptime compileComptime("\\w+"); + const haystack = "foo bar_baz 123"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "comptime: runtime fallback compiles and runs" { + // `(abc)+` is a compound pattern — falls back to runtime. + const cx = comptime compileComptime("(abc)+"); + const ms = try cx.findAll(std.testing.allocator, "abcabc def abc"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + // Should find matches regardless of code path. + try std.testing.expect(ms.len >= 1); +} + +test "comptime: classify pure_literal" { + try std.testing.expectEqual(.pure_literal, comptime classify("hello")); + try std.testing.expectEqual(.pure_literal, comptime classify("compileAllocFlags")); + try std.testing.expectEqual(.pure_literal, comptime classify("")); +} + +test "comptime: classify single_class_plus" { + try std.testing.expectEqual(.single_class_plus, comptime classify("\\d+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("[a-z]+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("\\w+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("\\s+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("[0-9]{3,}")); + try std.testing.expectEqual(.single_class_plus, comptime classify("(?:\\d+)")); +} + +test "comptime: classify runtime_fallback" { + try std.testing.expectEqual(.runtime_fallback, comptime classify("(abc)+")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("a.b")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("foo|bar")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("\\d*")); // min=0 skipped +} diff --git a/vendor/nanoregex/src/dfa.zig b/vendor/nanoregex/src/dfa.zig new file mode 100644 index 00000000..29193a80 --- /dev/null +++ b/vendor/nanoregex/src/dfa.zig @@ -0,0 +1,453 @@ +//! Lazy DFA built by subset construction over the Thompson NFA. +//! +//! Each DFA state is a sorted set of NFA state IDs. Transitions are computed +//! on demand: when (state, byte) is first encountered we run `step`, hash +//! the resulting NFA-state set, look it up (or create a new DFA state for +//! it), and cache the edge. Subsequent bytes through the same transition +//! are a single indexed lookup. +//! +//! Scope for v1: +//! - No capture-group tracking (caller must check `nfa.n_groups == 0`). +//! - No anchors (caller must check the AST for `Anchor` nodes; if present, +//! fall back to the Pike VM). +//! - Bounded state count (MAX_STATES). On overflow we surface an error +//! and the caller falls back to the Pike VM. +//! +//! The forward-only driver below handles unanchored search by trying each +//! starting position. Per-character cost is one table lookup, so this is +//! O(n) for cases where matches don't overlap heavily. + +const std = @import("std"); +const ast = @import("ast.zig"); +const nfa = @import("nfa.zig"); +const minterm = @import("minterm.zig"); + +pub const DfaStateId = u32; +pub const DEAD: DfaStateId = std.math.maxInt(DfaStateId); +const UNCOMPUTED: DfaStateId = std.math.maxInt(DfaStateId) - 1; + +const MAX_STATES: u32 = 4096; + +pub const Error = error{ OutOfMemory, TooManyStates, HasCaptures, HasAnchors, HasLazyQuantifier }; + +/// Knobs the runtime needs the DFA to bake in at construction time — +/// flags whose meaning isn't visible from the NFA alone. (case_insensitive +/// is intentionally absent: v1 falls back to the Pike VM when CI is set.) +pub const BuildOptions = struct { + /// Forwarded from the public Flags.dot_all. When true, `.` matches `\n`. + dot_all: bool = false, +}; + +/// Set of NFA states reached after a particular byte sequence. Sorted, +/// deduplicated; the byte representation is the hash-map key. +const NfaSet = struct { + ids: []const nfa.StateId, + accepts: bool, +}; + +/// Hash-map context that compares NfaSet identity by content of `ids`. +/// We key on the raw byte slice of the sorted ids — same length, same +/// bytes, same set. +const SetMapCtx = struct { + pub fn hash(_: SetMapCtx, key: []const u8) u64 { + return std.hash.Wyhash.hash(0, key); + } + pub fn eql(_: SetMapCtx, a: []const u8, b: []const u8) bool { + if (a.len != b.len) return false; + return std.mem.eql(u8, a, b); + } +}; + +const SetMap = std.HashMap([]const u8, DfaStateId, SetMapCtx, std.hash_map.default_max_load_percentage); + +pub const Dfa = struct { + /// Arena for state sets and transition rows. Lives until deinit. + arena: *std.heap.ArenaAllocator, + parent_alloc: std.mem.Allocator, + nfa_ref: *const nfa.Nfa, + + states: std.ArrayList(NfaSet), + /// Flat 2-D transition table indexed by `state * minterm.n_classes + class_id`. + /// Far smaller than 256-per-row when the pattern's atomic predicates + /// partition the alphabet into a handful of equivalence classes — + /// `[a-z]+` ends up with 2 classes, a 128× shrink. + transitions: []DfaStateId, + /// Resolves byte → class id. Built once from the AST at compile time. + minterm: minterm.Table, + set_to_id: SetMap, + + start: DfaStateId, + /// Whether `.` should match `\n`. Threaded in from the public Flags + /// at compile time so the inner-loop test stays branch-cheap. + dot_all: bool, + + pub fn fromNfa(alloc: std.mem.Allocator, n: *const nfa.Nfa, root: *const ast.Node, opts: BuildOptions) Error!Dfa { + if (n.n_groups != 0) return Error.HasCaptures; + if (containsAnchor(root)) return Error.HasAnchors; + // Lazy quantifiers need leftmost-shortest semantics, which a + // plain subset-construction DFA cannot express — it always picks + // leftmost-longest. Bail and let the Pike VM handle the pattern. + if (containsLazy(root)) return Error.HasLazyQuantifier; + + const arena = try alloc.create(std.heap.ArenaAllocator); + errdefer alloc.destroy(arena); + arena.* = std.heap.ArenaAllocator.init(alloc); + errdefer arena.deinit(); + const aa = arena.allocator(); + + // Compute byte equivalence classes from the pattern so the + // transition table can be indexed by class instead of raw byte. + // Typical pattern → 4-20 classes, so the row shrinks 12-64× and + // fits in L1 instead of L2. + const mt = try minterm.build(aa, root, opts.dot_all); + + const transitions = try aa.alloc(DfaStateId, @as(usize, MAX_STATES) * mt.n_classes); + @memset(transitions, UNCOMPUTED); + + var dfa: Dfa = .{ + .arena = arena, + .parent_alloc = alloc, + .nfa_ref = n, + .states = .empty, + .transitions = transitions, + .minterm = mt, + .set_to_id = SetMap.init(aa), + .start = 0, + .dot_all = opts.dot_all, + }; + + // Seed: the start DFA state is the epsilon-closure of {nfa.start}. + const seed = try epsilonClosure(aa, n, &.{n.start}); + dfa.start = try dfa.internState(seed); + + return dfa; + } + + pub fn deinit(self: *Dfa) void { + self.arena.deinit(); + self.parent_alloc.destroy(self.arena); + self.* = undefined; + } + + /// Insert a state set into the DFA, returning either a fresh id or the + /// existing one. Sorts the input slice in place before hashing. + fn internState(self: *Dfa, ids: []const nfa.StateId) Error!DfaStateId { + // Sort + dedupe — caller is allowed to pass an unsorted set. + const dup = try self.arena.allocator().dupe(nfa.StateId, ids); + std.mem.sort(nfa.StateId, dup, {}, comptime std.sort.asc(nfa.StateId)); + const deduped = uniqueSorted(dup); + + const key_bytes = std.mem.sliceAsBytes(deduped); + if (self.set_to_id.get(key_bytes)) |existing| return existing; + + if (self.states.items.len >= MAX_STATES) return Error.TooManyStates; + + var accepts = false; + for (deduped) |id| if (id == self.nfa_ref.accept) { + accepts = true; + break; + }; + + const id: DfaStateId = @intCast(self.states.items.len); + try self.states.append(self.arena.allocator(), .{ .ids = deduped, .accepts = accepts }); + try self.set_to_id.put(key_bytes, id); + return id; + } + + /// Public wrapper around the inlined hot-path transition. Mostly used + /// by tests; the matching loop in `matchAt` reads `transitions` and + /// `byte_to_class` directly to skip the function-call overhead. + pub fn transition(self: *Dfa, state: DfaStateId, byte: u8) Error!DfaStateId { + const class_id: usize = self.minterm.byte_to_class[byte]; + const idx = @as(usize, state) * self.minterm.n_classes + class_id; + const cached = self.transitions[idx]; + if (cached != UNCOMPUTED) return cached; + return try self.computeAndCacheTransition(state, class_id); + } + + /// Slow path: compute the successor set for `(state, class_id)`, intern + /// it as a new DFA state if needed, and cache the edge. Called from the + /// hot loops only when the transition is missing. + fn computeAndCacheTransition(self: *Dfa, state: DfaStateId, class_id: usize) Error!DfaStateId { + const idx = @as(usize, state) * self.minterm.n_classes + class_id; + const rep_byte = self.minterm.representatives[class_id]; + + const cur = self.states.items[state]; + var next_ids: std.ArrayList(nfa.StateId) = .empty; + defer next_ids.deinit(self.arena.allocator()); + + for (cur.ids) |sid| { + const ns = self.nfa_ref.states[sid]; + const matched = switch (ns.consume) { + .byte => |b| b == rep_byte, + .any => self.dot_all or rep_byte != '\n', + .class => |cls| cls.contains(rep_byte), + .epsilon, .anchor, .group_start, .group_end => false, + }; + if (matched) { + if (ns.out1) |o| try next_ids.append(self.arena.allocator(), o); + } + } + + if (next_ids.items.len == 0) { + self.transitions[idx] = DEAD; + return DEAD; + } + + const closure = try epsilonClosure(self.arena.allocator(), self.nfa_ref, next_ids.items); + const next_id = try self.internState(closure); + self.transitions[idx] = next_id; + return next_id; + } + + /// Anchored match starting at `start` in `input`. Returns the end index + /// of the longest accepted run, or null if no match. + /// + /// The hot loop reads the byte → class table and the transition table + /// directly. The slow-path branch (`UNCOMPUTED`) is hoisted out so the + /// fast path is a tight series of array reads + one compare. After + /// warmup the slow path is essentially never taken, so this trades + /// one predicted-not-taken branch for skipping a function frame and + /// the redundant idx recomputation that the older `transition` call + /// did inside the loop. + pub fn matchAt(self: *Dfa, input: []const u8, start: usize) Error!?usize { + var cur: DfaStateId = self.start; + var longest: ?usize = if (self.states.items[cur].accepts) start else null; + const byte_to_class = &self.minterm.byte_to_class; + const n_classes: usize = self.minterm.n_classes; + const transitions = self.transitions; + + var i: usize = start; + while (i < input.len) : (i += 1) { + const class_id: usize = byte_to_class[input[i]]; + const idx = @as(usize, cur) * n_classes + class_id; + var next = transitions[idx]; + if (next == UNCOMPUTED) { + next = try self.computeAndCacheTransition(cur, class_id); + } + if (next == DEAD) break; + cur = next; + if (self.states.items[cur].accepts) longest = i + 1; + } + return longest; + } + + /// Find every non-overlapping match span in `input`. Tries each + /// starting position; on a hit, skips past the match end. Zero-width + /// matches advance one byte so we don't loop. + pub fn findAll(self: *Dfa, alloc: std.mem.Allocator, input: []const u8) Error![]Span { + var out: std.ArrayList(Span) = .empty; + errdefer out.deinit(alloc); + + var p: usize = 0; + while (p <= input.len) { + const end_opt = try self.matchAt(input, p); + if (end_opt) |end| { + try out.append(alloc, .{ .start = p, .end = end }); + p = if (end > p) end else p + 1; + } else { + p += 1; + } + } + return try out.toOwnedSlice(alloc); + } +}; + +pub const Span = struct { start: usize, end: usize }; + +// ── Helpers ── + +/// Compute the epsilon-closure of `seeds`: every NFA state reachable from +/// the seeds via zero-width transitions (epsilon, group_start, group_end — +/// not anchor, since we caller-fail when anchors are present). +fn epsilonClosure(alloc: std.mem.Allocator, n: *const nfa.Nfa, seeds: []const nfa.StateId) Error![]nfa.StateId { + var stack: std.ArrayList(nfa.StateId) = .empty; + defer stack.deinit(alloc); + var seen = try alloc.alloc(bool, n.states.len); + defer alloc.free(seen); + @memset(seen, false); + + var out: std.ArrayList(nfa.StateId) = .empty; + errdefer out.deinit(alloc); + + for (seeds) |s| { + if (!seen[s]) { + seen[s] = true; + try stack.append(alloc, s); + } + } + + while (stack.pop()) |sid| { + try out.append(alloc, sid); + if (sid == n.accept) continue; + const ns = n.states[sid]; + switch (ns.consume) { + .epsilon, .group_start, .group_end => { + if (ns.out1) |o| if (!seen[o]) { + seen[o] = true; + try stack.append(alloc, o); + }; + if (ns.out2) |o| if (!seen[o]) { + seen[o] = true; + try stack.append(alloc, o); + }; + }, + // Consuming and anchor states don't contribute to the closure — + // they're already terminal for this iteration. + else => {}, + } + } + + return try out.toOwnedSlice(alloc); +} + +fn uniqueSorted(sorted: []nfa.StateId) []nfa.StateId { + if (sorted.len == 0) return sorted; + var w: usize = 1; + var i: usize = 1; + while (i < sorted.len) : (i += 1) { + if (sorted[i] != sorted[w - 1]) { + sorted[w] = sorted[i]; + w += 1; + } + } + return sorted[0..w]; +} + +fn containsAnchor(node: *const ast.Node) bool { + return switch (node.*) { + .anchor => true, + .literal, .dot, .class => false, + .concat => |children| for (children) |c| { + if (containsAnchor(c)) break true; + } else false, + .alt => |children| for (children) |c| { + if (containsAnchor(c)) break true; + } else false, + .repeat => |r| containsAnchor(r.sub), + .group => |g| containsAnchor(g.sub), + }; +} + +/// True iff any quantifier in the AST is lazy (`*?`/`+?`/`??`/`{n,m}?`). +/// The DFA always yields leftmost-longest matches, which contradicts +/// lazy semantics — we caller-fail so the Pike VM runs instead. +fn containsLazy(node: *const ast.Node) bool { + return switch (node.*) { + .literal, .dot, .class, .anchor => false, + .concat => |children| for (children) |c| { + if (containsLazy(c)) break true; + } else false, + .alt => |children| for (children) |c| { + if (containsLazy(c)) break true; + } else false, + .repeat => |r| !r.greedy or containsLazy(r.sub), + .group => |g| containsLazy(g.sub), + }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildDfa(arena: *std.heap.ArenaAllocator, pattern: []const u8) !Dfa { + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + // Heap-allocate the Nfa on the arena so its address is stable for the + // returned Dfa's `nfa_ref`. An earlier version did `&local_const` + // which was a dangling stack pointer the moment buildDfa returned, + // and every test that actually exercised the DFA either crashed or + // returned bogus values from torn-over stack memory. + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + return try Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{}); +} + +test "dfa: literal pattern matches via findAll" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "abc"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "the abc and abc again"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); + try std.testing.expectEqual(@as(usize, 4), spans[0].start); + try std.testing.expectEqual(@as(usize, 7), spans[0].end); +} + +test "dfa: greedy plus consumes longest run" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "\\d+"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "abc 42 def 1234 xyz"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); + // "42" — two digits. + try std.testing.expectEqual(@as(usize, 2), spans[0].end - spans[0].start); + // "1234" — four digits. The earlier expectation of `6` was a typo. + try std.testing.expectEqual(@as(usize, 4), spans[1].end - spans[1].start); +} + +test "dfa: alternation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "cat|dog|bird"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "the cat saw a dog and a bird"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 3), spans.len); +} + +test "dfa: class quantifier" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "[a-z]+"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "Hello World"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); +} + +test "dfa: rejects capture patterns" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "(abc)"); + const root = try p.parseRoot(); + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + try std.testing.expectError(Error.HasCaptures, Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{})); +} + +test "dfa: rejects anchor patterns" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "^foo"); + const root = try p.parseRoot(); + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + try std.testing.expectError(Error.HasAnchors, Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{})); +} + +test "dfa: dot wildcard" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "a.b"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "axb ayb azb"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 3), spans.len); +} + +test "dfa: longest match wins on greedy" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "a*"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "aaa"); + defer std.testing.allocator.free(spans); + // a* should match "aaa" once at position 0, then zero-width at position 3. + try std.testing.expect(spans.len >= 1); + try std.testing.expectEqual(@as(usize, 0), spans[0].start); + try std.testing.expectEqual(@as(usize, 3), spans[0].end); +} diff --git a/vendor/nanoregex/src/exec.zig b/vendor/nanoregex/src/exec.zig new file mode 100644 index 00000000..0a2f6b60 --- /dev/null +++ b/vendor/nanoregex/src/exec.zig @@ -0,0 +1,469 @@ +//! Pike-VM NFA simulator. +//! +//! Classic two-list simulation: at each input position we hold a `clist` of +//! threads parked at consuming states, then step them on input[pos] into a +//! `nlist`. Zero-width states (epsilon, anchor, group_*) are walked inside +//! `addThread` so they never appear in the active lists — only consuming +//! states (byte / any / class) and the accept state do. +//! +//! Leftmost-first semantics: threads are added by DFS following out1 before +//! out2, so the priority order matches the AST's left-to-right reading. +//! Inside one input position the first thread to reach `accept` wins; lower- +//! priority threads at the same position are stopped from advancing. +//! +//! Captures are per-thread arrays of `?Span`. When a thread crosses a +//! group_start / group_end state, we dupe the array first so siblings +//! don't see each other's updates. This is the simplest correct shape; +//! a copy-on-write or generation-tagged store is the obvious v2 win. + +const std = @import("std"); +const ast = @import("ast.zig"); +const nfa = @import("nfa.zig"); + +pub const Flags = struct { + case_insensitive: bool = false, + /// `^` / `$` match line boundaries (default). When false they only match + /// input start / end. + multiline: bool = true, + /// `.` matches `\n`. Default false. + dot_all: bool = false, +}; + +pub const Span = struct { start: usize, end: usize }; + +pub const MatchResult = struct { + span: Span, + /// Slot 0 is the whole match (equal to `span`). Slots 1..n are capture + /// groups by declaration order. A null entry means the group didn't + /// participate in the match. + captures: []const ?Span, + + pub fn deinit(self: *MatchResult, alloc: std.mem.Allocator) void { + alloc.free(self.captures); + self.* = undefined; + } +}; + +const Thread = struct { + pc: nfa.StateId, + captures: []?Span, +}; + +/// Sparse-set thread list. Generation counter avoids per-step clears. +const ThreadList = struct { + threads: std.ArrayList(Thread), + seen_gen: []u32, + cur_gen: u32, + + fn init(alloc: std.mem.Allocator, n_states: usize) !ThreadList { + const seen = try alloc.alloc(u32, n_states); + @memset(seen, 0); + return .{ + .threads = .empty, + .seen_gen = seen, + .cur_gen = 1, + }; + } + + fn deinit(self: *ThreadList, alloc: std.mem.Allocator) void { + self.threads.deinit(alloc); + alloc.free(self.seen_gen); + } + + /// True iff the state was not yet in this generation. Marks it seen. + fn markIfNew(self: *ThreadList, pc: nfa.StateId) bool { + if (self.seen_gen[pc] == self.cur_gen) return false; + self.seen_gen[pc] = self.cur_gen; + return true; + } + + fn clear(self: *ThreadList, alloc: std.mem.Allocator) void { + self.cur_gen += 1; + self.threads.clearRetainingCapacity(); + _ = alloc; + } +}; + +pub const ExecError = error{OutOfMemory}; + +pub const Vm = struct { + alloc: std.mem.Allocator, + automaton: *const nfa.Nfa, + flags: Flags, + /// Total capture-array length: index 0 = whole match, 1..n_groups = explicit. + cap_len: usize, + + pub fn init(alloc: std.mem.Allocator, automaton: *const nfa.Nfa, flags: Flags) Vm { + return .{ + .alloc = alloc, + .automaton = automaton, + .flags = flags, + .cap_len = @as(usize, automaton.n_groups) + 1, + }; + } + + /// Find a single match starting at or after position 0 (whichever + /// position succeeds first, leftmost-first within that). Returns null + /// when nothing in the input matches. + pub fn search(self: *Vm, input: []const u8) ExecError!?MatchResult { + var start: usize = 0; + while (start <= input.len) : (start += 1) { + if (try self.matchAt(input, start)) |m| return m; + } + return null; + } + + /// Find all non-overlapping matches, leftmost-first. Caller owns the + /// returned slice and each MatchResult's captures. + pub fn findAll(self: *Vm, input: []const u8) ExecError![]MatchResult { + var results: std.ArrayList(MatchResult) = .empty; + errdefer { + for (results.items) |*m| m.deinit(self.alloc); + results.deinit(self.alloc); + } + + var pos: usize = 0; + while (pos <= input.len) { + const m_opt = try self.matchAt(input, pos); + if (m_opt) |m| { + try results.append(self.alloc, m); + // Advance past the match. Zero-width match (start == end) + // must still advance one byte or we'd loop forever. + pos = if (m.span.end > pos) m.span.end else pos + 1; + } else { + pos += 1; + } + } + return results.toOwnedSlice(self.alloc); + } + + /// Try to match the pattern against `input` starting exactly at `start`. + /// Returns the longest match the engine finds via leftmost-first + /// exploration, or null. Threads / captures are allocated from + /// `self.alloc` and the returned MatchResult owns its capture slice. + /// Try to match the pattern against `input` starting exactly at `start`. + /// Returns the longest leftmost-first match the engine finds, or null. + /// Captures in the returned MatchResult are owned by `self.alloc`; the + /// per-attempt scratch arena dies at end-of-scope. + fn matchAt(self: *Vm, input: []const u8, start: usize) ExecError!?MatchResult { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const aa = arena.allocator(); + + var clist = try ThreadList.init(aa, self.automaton.states.len); + var nlist = try ThreadList.init(aa, self.automaton.states.len); + + const initial_caps = try aa.alloc(?Span, self.cap_len); + @memset(initial_caps, null); + initial_caps[0] = .{ .start = start, .end = start }; + + try self.addThread(&clist, self.automaton.start, start, initial_caps, input, aa); + + var best: ?MatchResult = null; + var pos = start; + while (true) : (pos += 1) { + // Scan clist for accept in priority order. Lower-priority + // threads (after the first accept) are killed for this step; + // higher-priority threads (before it) keep stepping — they + // can still yield a longer leftmost-first match at a later + // position, which beats the recorded one. + var accept_idx: ?usize = null; + for (clist.threads.items, 0..) |t, idx| { + if (t.pc == self.automaton.accept) { + var captures = try self.alloc.alloc(?Span, self.cap_len); + for (t.captures, 0..) |c, i| captures[i] = c; + if (captures[0]) |*c0| c0.end = pos; + // Free the previous best (an older, shorter or + // lower-priority accept) before replacing. + if (best) |*old| old.deinit(self.alloc); + best = .{ + .span = .{ .start = start, .end = pos }, + .captures = captures, + }; + accept_idx = idx; + break; + } + } + + if (pos == input.len) break; + if (clist.threads.items.len == 0) break; + + // Priority cutoff: only threads with index < accept_idx are + // allowed to step. When no accept was found this step, all + // threads step. + const step_limit = accept_idx orelse clist.threads.items.len; + + for (clist.threads.items[0..step_limit]) |t| { + if (t.pc == self.automaton.accept) continue; + const state = self.automaton.states[t.pc]; + switch (state.consume) { + .byte => |b| { + if (self.byteMatches(b, input[pos])) { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + .any => { + if (self.flags.dot_all or input[pos] != '\n') { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + .class => |c| { + if (self.classMatches(c, input[pos])) { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + // Zero-width states never reach an active list — + // addThread walks past them. Reaching here means an + // NFA construction bug. + .epsilon, .anchor, .group_start, .group_end => unreachable, + } + } + + // Nothing advanced this step. If we've recorded a match, ship it. + if (nlist.threads.items.len == 0) break; + + clist.clear(aa); + std.mem.swap(ThreadList, &clist, &nlist); + } + + return best; + } + + /// Walk every zero-width state reachable from `pc` and add any consuming + /// states (or the accept state) into `list`. Captures are duplicated at + /// every group boundary so sibling threads don't observe each other's + /// writes. + fn addThread( + self: *Vm, + list: *ThreadList, + pc: nfa.StateId, + pos: usize, + captures: []?Span, + input: []const u8, + aa: std.mem.Allocator, + ) ExecError!void { + if (!list.markIfNew(pc)) return; + + if (pc == self.automaton.accept) { + try list.threads.append(aa, .{ .pc = pc, .captures = captures }); + return; + } + + const state = self.automaton.states[pc]; + switch (state.consume) { + .epsilon => { + if (state.out1) |o| try self.addThread(list, o, pos, captures, input, aa); + if (state.out2) |o| try self.addThread(list, o, pos, captures, input, aa); + }, + .anchor => |a| { + if (self.anchorMatches(a, input, pos)) { + if (state.out1) |o| try self.addThread(list, o, pos, captures, input, aa); + } + // Anchor fail: thread dies here. + }, + .group_start => |idx| { + const new_caps = try dupeAndSet(aa, captures, idx, .{ .start = pos, .end = pos }); + if (state.out1) |o| try self.addThread(list, o, pos, new_caps, input, aa); + }, + .group_end => |idx| { + const new_caps = try dupeAndSetEnd(aa, captures, idx, pos); + if (state.out1) |o| try self.addThread(list, o, pos, new_caps, input, aa); + }, + .byte, .any, .class => { + try list.threads.append(aa, .{ .pc = pc, .captures = captures }); + }, + } + } + + // ── Predicates ── + + fn byteMatches(self: *const Vm, expected: u8, actual: u8) bool { + if (!self.flags.case_insensitive) return expected == actual; + return toLower(expected) == toLower(actual); + } + + fn classMatches(self: *const Vm, cls: *const ast.Class, actual: u8) bool { + if (cls.contains(actual)) return true; + if (self.flags.case_insensitive) { + const swapped = if (actual >= 'A' and actual <= 'Z') + actual + 32 + else if (actual >= 'a' and actual <= 'z') + actual - 32 + else + actual; + if (swapped != actual and cls.contains(swapped)) return true; + } + return false; + } + + fn anchorMatches(self: *const Vm, a: ast.Anchor, input: []const u8, pos: usize) bool { + return switch (a) { + .string_start => pos == 0, + .string_end => pos == input.len, + .line_start => pos == 0 or (self.flags.multiline and pos > 0 and input[pos - 1] == '\n'), + .line_end => pos == input.len or (self.flags.multiline and pos < input.len and input[pos] == '\n'), + .word_boundary => isAtWordBoundary(input, pos), + .non_word_boundary => !isAtWordBoundary(input, pos), + }; + } +}; + +fn dupeAndSet(alloc: std.mem.Allocator, captures: []?Span, idx: u32, value: Span) ![]?Span { + const out = try alloc.alloc(?Span, captures.len); + @memcpy(out, captures); + if (idx < out.len) out[idx] = value; + return out; +} + +fn dupeAndSetEnd(alloc: std.mem.Allocator, captures: []?Span, idx: u32, end: usize) ![]?Span { + const out = try alloc.alloc(?Span, captures.len); + @memcpy(out, captures); + if (idx < out.len) { + if (out[idx]) |*span| { + span.end = end; + } else { + // group_end without a matching group_start — shouldn't happen + // with our NFA construction, but treat as zero-width if it does. + out[idx] = .{ .start = end, .end = end }; + } + } + return out; +} + +fn toLower(c: u8) u8 { + return if (c >= 'A' and c <= 'Z') c + 32 else c; +} + +fn isWordChar(c: u8) bool { + return (c >= 'a' and c <= 'z') or + (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or + c == '_'; +} + +fn isAtWordBoundary(input: []const u8, pos: usize) bool { + const left_is_word = pos > 0 and isWordChar(input[pos - 1]); + const right_is_word = pos < input.len and isWordChar(input[pos]); + return left_is_word != right_is_word; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn runFindAll(alloc: std.mem.Allocator, pattern: []const u8, input: []const u8) ![]MatchResult { + return runFindAllFlags(alloc, pattern, input, .{}); +} + +fn runFindAllFlags(alloc: std.mem.Allocator, pattern: []const u8, input: []const u8, flags: Flags) ![]MatchResult { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const automaton = try nfa.build(arena.allocator(), root, p.n_groups); + var vm = Vm.init(alloc, &automaton, flags); + return try vm.findAll(input); +} + +fn freeMatches(alloc: std.mem.Allocator, matches: []MatchResult) void { + for (matches) |*m| m.deinit(alloc); + alloc.free(matches); +} + +test "exec: literal match" { + const ms = try runFindAll(std.testing.allocator, "abc", "the abc and abc again"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 4), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 7), ms[0].span.end); + try std.testing.expectEqual(@as(usize, 12), ms[1].span.start); +} + +test "exec: no match" { + const ms = try runFindAll(std.testing.allocator, "xyz", "the abc"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 0), ms.len); +} + +test "exec: dot star greedy" { + const ms = try runFindAll(std.testing.allocator, "a.*b", "axxxb yyy"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 1), ms.len); + try std.testing.expectEqual(@as(usize, 0), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: dot star lazy" { + const ms = try runFindAll(std.testing.allocator, "a.*?b", "axxxbxxxb"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 1), ms.len); + // Lazy stops at the first 'b'. + try std.testing.expectEqual(@as(usize, 0), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: char class" { + const ms = try runFindAll(std.testing.allocator, "[a-z]+", "Hello World"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 1), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: alternation prefers leftmost" { + const ms = try runFindAll(std.testing.allocator, "cat|dog|bird", "the dog saw a cat and a bird"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "exec: anchors line start" { + const flags = Flags{ .multiline = true }; + const ms = try runFindAllFlags(std.testing.allocator, "^foo", "foo\nbar foo\nfoo bar", flags); + defer freeMatches(std.testing.allocator, ms); + // multiline: ^foo matches at offset 0 and at offset 8 (after \n). + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "exec: counted quantifier" { + const ms = try runFindAll(std.testing.allocator, "a{2,3}", "a aa aaa aaaa"); + defer freeMatches(std.testing.allocator, ms); + // Single 'a' doesn't match (need >=2). "aa" matches. "aaa" matches. + // "aaaa" matches as "aaa" + "a" (the trailing 'a' alone doesn't qualify), + // so we get exactly: aa, aaa, aaa. + try std.testing.expectEqual(@as(usize, 3), ms.len); + try std.testing.expectEqual(@as(usize, 2), ms[0].span.end - ms[0].span.start); + try std.testing.expectEqual(@as(usize, 3), ms[1].span.end - ms[1].span.start); + try std.testing.expectEqual(@as(usize, 3), ms[2].span.end - ms[2].span.start); +} + +test "exec: capturing group" { + const ms = try runFindAll(std.testing.allocator, "(\\w+)@(\\w+)", "alice@example bob@host"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + // First match: alice@example, groups: alice, example + try std.testing.expect(ms[0].captures[1] != null); + try std.testing.expectEqual(@as(usize, 0), ms[0].captures[1].?.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].captures[1].?.end); + try std.testing.expectEqual(@as(usize, 6), ms[0].captures[2].?.start); + try std.testing.expectEqual(@as(usize, 13), ms[0].captures[2].?.end); +} + +test "exec: case-insensitive flag" { + const flags = Flags{ .case_insensitive = true }; + const ms = try runFindAllFlags(std.testing.allocator, "hello", "HELLO Hello hello", flags); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "exec: word boundary" { + const ms = try runFindAll(std.testing.allocator, "\\bcat\\b", "the cat sat on a catnap"); + defer freeMatches(std.testing.allocator, ms); + // 'cat' alone matches, 'catnap' doesn't (no boundary after 'cat'). + try std.testing.expectEqual(@as(usize, 1), ms.len); +} + +test "exec: digit shorthand" { + const ms = try runFindAll(std.testing.allocator, "\\d+", "abc 42 def 1234 xyz"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); +} diff --git a/vendor/nanoregex/src/minterm.zig b/vendor/nanoregex/src/minterm.zig new file mode 100644 index 00000000..3f0164c2 --- /dev/null +++ b/vendor/nanoregex/src/minterm.zig @@ -0,0 +1,191 @@ +//! Byte-class compression for the DFA's transition table. +//! +//! Two bytes are EQUIVALENT for a given pattern when every atomic predicate +//! in that pattern (literal-byte tests, character classes, `.`) agrees on +//! them. Equivalent bytes drive the DFA to the same next state, so we can +//! collapse them into a single "minterm" class and index transitions by +//! class id instead of raw byte. +//! +//! Concretely: a pattern with `[a-z]+` has two classes — {a..z} and +//! everything else — so the DFA's transition row shrinks from 256 entries +//! to 2. A pattern with several distinct literals plus a class typically +//! has 10-20 classes. Resharp reported a 7× speedup from this alone, +//! mostly because the smaller table fits in L1. +//! +//! Build cost is one O(256 × n_predicates) pass at compile time. We cap +//! n_predicates at 64 (fits a u64 signature); patterns with more atomic +//! predicates than that fall back to an identity (byte == class) table, +//! which makes minterm a no-op and we still match correctly. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const Table = struct { + /// byte → class_id (0..n_classes-1). For n_classes == 256 this is the + /// identity mapping and minterm acts as a no-op. + byte_to_class: [256]u8, + /// Number of distinct classes. 1..256. + n_classes: u16, + /// class_id → arbitrary byte that lives in that class. Used when the + /// DFA's `move` step needs a concrete byte to feed into per-NFA-state + /// predicate tests. + representatives: [256]u8, +}; + +pub const Error = error{OutOfMemory}; + +pub fn build(arena: std.mem.Allocator, root: *const ast.Node, dot_all: bool) Error!Table { + var preds: std.ArrayList(Predicate) = .empty; + defer preds.deinit(arena); + try collectPredicates(root, arena, &preds); + + // Pattern with no consuming atoms — only anchors / empty. Nothing to + // distinguish; collapse the alphabet to a single class. + if (preds.items.len == 0) return singleClass(); + // Too many predicates for our 64-bit signature; bail to identity. The + // matcher still works, the minterm is just a pass-through. + if (preds.items.len > 64) return identity(); + + var sigs: [256]u64 = undefined; + for (0..256) |b| { + var sig: u64 = 0; + for (preds.items, 0..) |pred, i| { + if (matches(pred, @intCast(b), dot_all)) { + sig |= @as(u64, 1) << @intCast(i); + } + } + sigs[b] = sig; + } + + var sig_to_class = std.AutoHashMap(u64, u16).init(arena); + defer sig_to_class.deinit(); + + var byte_to_class: [256]u8 = undefined; + var representatives: [256]u8 = undefined; + var n_classes: u16 = 0; + + for (0..256) |b| { + const sig = sigs[b]; + if (sig_to_class.get(sig)) |existing| { + byte_to_class[b] = @intCast(existing); + } else { + const c = n_classes; + try sig_to_class.put(sig, c); + representatives[c] = @intCast(b); + byte_to_class[b] = @intCast(c); + n_classes += 1; + } + } + + return .{ + .byte_to_class = byte_to_class, + .n_classes = n_classes, + .representatives = representatives, + }; +} + +// ── Internals ── + +const Predicate = union(enum) { + byte: u8, + any, + class: *const ast.Class, +}; + +fn collectPredicates(node: *const ast.Node, arena: std.mem.Allocator, out: *std.ArrayList(Predicate)) Error!void { + switch (node.*) { + .literal => |c| try out.append(arena, .{ .byte = c }), + .dot => try out.append(arena, .any), + .class => |cls| try out.append(arena, .{ .class = cls }), + .anchor => {}, // zero-width — doesn't partition bytes + .concat => |children| for (children) |c| try collectPredicates(c, arena, out), + .alt => |children| for (children) |c| try collectPredicates(c, arena, out), + .repeat => |r| try collectPredicates(r.sub, arena, out), + .group => |g| try collectPredicates(g.sub, arena, out), + } +} + +fn matches(p: Predicate, b: u8, dot_all: bool) bool { + return switch (p) { + .byte => |v| v == b, + .any => dot_all or b != '\n', + .class => |cls| cls.contains(b), + }; +} + +fn singleClass() Table { + var bts: [256]u8 = undefined; + @memset(&bts, 0); + var reps: [256]u8 = undefined; + @memset(&reps, 0); + return .{ .byte_to_class = bts, .n_classes = 1, .representatives = reps }; +} + +fn identity() Table { + var bts: [256]u8 = undefined; + var reps: [256]u8 = undefined; + for (0..256) |i| { + bts[i] = @intCast(i); + reps[i] = @intCast(i); + } + return .{ .byte_to_class = bts, .n_classes = 256, .representatives = reps }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildFor(pattern: []const u8, dot_all: bool) !Table { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return try build(std.testing.allocator, root, dot_all); +} + +test "minterm: single class for a-z+" { + const t = try buildFor("[a-z]+", false); + // {a..z} ⇒ one class, everything else ⇒ another. 2 classes. + try std.testing.expectEqual(@as(u16, 2), t.n_classes); + try std.testing.expectEqual(t.byte_to_class['a'], t.byte_to_class['z']); + try std.testing.expect(t.byte_to_class['a'] != t.byte_to_class['A']); +} + +test "minterm: pure-literal pattern splits each distinct byte" { + const t = try buildFor("abc", false); + // 'a', 'b', 'c' each get a class; everything else is one more. 4. + try std.testing.expectEqual(@as(u16, 4), t.n_classes); + try std.testing.expect(t.byte_to_class['a'] != t.byte_to_class['b']); + try std.testing.expect(t.byte_to_class['b'] != t.byte_to_class['c']); +} + +test "minterm: dot collapses everything except newline" { + const t = try buildFor("a.b", false); + // Classes: 'a', 'b', '\n' (because . doesn't match it), and "rest". + try std.testing.expectEqual(@as(u16, 4), t.n_classes); + try std.testing.expect(t.byte_to_class['\n'] != t.byte_to_class['x']); +} + +test "minterm: dot-all collapses newline with the rest" { + const t = try buildFor("a.b", true); + // With dot_all the . matches \n too. So \n joins the "rest" class. + // Classes: 'a', 'b', "rest" (incl '\n'). 3. + try std.testing.expectEqual(@as(u16, 3), t.n_classes); + try std.testing.expectEqual(t.byte_to_class['\n'], t.byte_to_class['x']); +} + +test "minterm: anchors don't partition bytes" { + const t = try buildFor("^abc$", false); + // ^ and $ are zero-width, they don't appear in the predicate list. + // Classes are the same as for "abc": 'a', 'b', 'c', "rest" = 4. + try std.testing.expectEqual(@as(u16, 4), t.n_classes); +} + +test "minterm: representatives are valid bytes in their class" { + const t = try buildFor("[a-z]+", false); + var c: u16 = 0; + while (c < t.n_classes) : (c += 1) { + const rep = t.representatives[c]; + try std.testing.expectEqual(@as(u16, c), @as(u16, t.byte_to_class[rep])); + } +} diff --git a/vendor/nanoregex/src/nfa.zig b/vendor/nanoregex/src/nfa.zig new file mode 100644 index 00000000..a23369e4 --- /dev/null +++ b/vendor/nanoregex/src/nfa.zig @@ -0,0 +1,449 @@ +//! Thompson NFA construction. +//! +//! Each AST node compiles to a Frag: a single entry state plus a list of +//! "dangling" out-edges that the parent patches when concatenating. The +//! final NFA has one start state and one accept state, both well-defined +//! indices into `states`. The graph is arena-allocated alongside the AST. +//! +//! Greedy vs lazy quantifiers differ only in the *order* of out-edges on +//! the fork state — out1 is preferred by the matcher, so greedy puts the +//! sub-fragment on out1 (consume more) and lazy puts the continuation on +//! out1 (consume less). The matcher honours the ordering during simulation. +//! +//! Counted quantifiers `{m,n}` are unfolded inline — m mandatory copies +//! followed by (n-m) optional copies, capped at 1024 unfolds to keep +//! compile time bounded. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const StateId = u32; + +pub const Consume = union(enum) { + /// No input consumed; no observation. Used at forks and merges. + epsilon, + /// Consume one byte; succeeds iff `byte == value`. + byte: u8, + /// Consume one byte; succeeds iff byte matches dot. The matcher consults + /// its `dot_all` flag to decide whether `\n` is included. + any, + /// Consume one byte; succeeds iff `byte ∈ class.bitmap`. + class: *const ast.Class, + /// Zero-width anchor — succeeds iff the current position satisfies it. + anchor: ast.Anchor, + /// No input consumed. Side effect: mark capture group N start at the + /// current position. The matcher updates its capture array. + group_start: u32, + /// No input consumed. Side effect: mark capture group N end. + group_end: u32, +}; + +pub const State = struct { + consume: Consume, + /// Primary out edge. For consuming states (byte/any/class), out1 is + /// followed only when the byte matches. For zero-width states + /// (epsilon/anchor/group_*), out1 is followed unconditionally. + out1: ?StateId, + /// Secondary out — populated only when this is an alt or repeat fork. + /// Both out edges are then epsilon-like. + out2: ?StateId, +}; + +pub const Nfa = struct { + states: []State, + start: StateId, + /// The accept state is an epsilon state with no out edges. Matcher + /// detects acceptance by id, not by Consume tag. + accept: StateId, + n_groups: u32, +}; + +pub const BuildError = error{ OutOfMemory, QuantifierTooLarge }; + +/// Compile an AST into an NFA. Allocates state buffer + dangling-edge +/// scratch from `arena`. The returned slice is also arena-owned. +pub fn build(arena: std.mem.Allocator, root: *const ast.Node, n_groups: u32) BuildError!Nfa { + var b: Builder = .{ .arena = arena, .states = .empty }; + var frag = try b.compile(root); + defer frag.dangles.deinit(arena); + + // All dangling out-edges become inputs to a single accept state. + const accept = try b.newState(.epsilon, null, null); + for (frag.dangles.items) |p| b.patch(p, accept); + + return .{ + .states = try arena.dupe(State, b.states.items), + .start = frag.start, + .accept = accept, + .n_groups = n_groups, + }; +} + +// ── Internal construction state ── + +const PatchSlot = enum { out1, out2 }; + +const PatchPoint = struct { + state: StateId, + slot: PatchSlot, +}; + +const Frag = struct { + start: StateId, + /// Out-edges that haven't been wired to anything yet. The parent + /// node patches them when concatenating with the next fragment. + dangles: std.ArrayList(PatchPoint), +}; + +const Builder = struct { + arena: std.mem.Allocator, + states: std.ArrayList(State), + + fn newState(self: *Builder, consume: Consume, out1: ?StateId, out2: ?StateId) BuildError!StateId { + const id: StateId = @intCast(self.states.items.len); + try self.states.append(self.arena, .{ .consume = consume, .out1 = out1, .out2 = out2 }); + return id; + } + + fn patch(self: *Builder, point: PatchPoint, target: StateId) void { + switch (point.slot) { + .out1 => self.states.items[point.state].out1 = target, + .out2 => self.states.items[point.state].out2 = target, + } + } + + fn singleton(self: *Builder, consume: Consume) BuildError!Frag { + const s = try self.newState(consume, null, null); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ .state = s, .slot = .out1 }); + return .{ .start = s, .dangles = dangles }; + } + + fn epsilonFrag(self: *Builder) BuildError!Frag { + return self.singleton(.epsilon); + } + + fn compile(self: *Builder, node: *const ast.Node) BuildError!Frag { + return switch (node.*) { + .literal => |c| try self.singleton(.{ .byte = c }), + .dot => try self.singleton(.any), + .class => |cls| try self.singleton(.{ .class = cls }), + .anchor => |a| try self.singleton(.{ .anchor = a }), + .concat => |children| try self.compileConcat(children), + .alt => |children| try self.compileAlt(children), + .repeat => |r| try self.compileRepeat(r), + .group => |g| try self.compileGroup(g), + }; + } + + fn compileConcat(self: *Builder, children: []const *const ast.Node) BuildError!Frag { + if (children.len == 0) return try self.epsilonFrag(); + var acc = try self.compile(children[0]); + var i: usize = 1; + while (i < children.len) : (i += 1) { + const next = try self.compile(children[i]); + for (acc.dangles.items) |p| self.patch(p, next.start); + acc.dangles.deinit(self.arena); + acc.dangles = next.dangles; + } + return acc; + } + + fn compileAlt(self: *Builder, children: []const *const ast.Node) BuildError!Frag { + if (children.len == 1) return self.compile(children[0]); + + // Build right-to-left so the leftmost branch is preferred at the + // top-level fork (matches Python re's leftmost-first semantics). + var i: usize = children.len; + i -= 1; + var rest = try self.compile(children[i]); + while (i > 0) { + i -= 1; + var branch = try self.compile(children[i]); + const fork = try self.newState(.epsilon, branch.start, rest.start); + // Merge danglers from both sides. + for (rest.dangles.items) |p| try branch.dangles.append(self.arena, p); + rest.dangles.deinit(self.arena); + rest = .{ .start = fork, .dangles = branch.dangles }; + } + return rest; + } + + fn compileRepeat(self: *Builder, r: *const ast.Repeat) BuildError!Frag { + // Fast paths for the common shapes — *, +, ?. + if (r.min == 0 and r.max == std.math.maxInt(u32)) return try self.compileStar(r.sub, r.greedy); + if (r.min == 1 and r.max == std.math.maxInt(u32)) return try self.compilePlus(r.sub, r.greedy); + if (r.min == 0 and r.max == 1) return try self.compileQuestion(r.sub, r.greedy); + return try self.compileCounted(r); + } + + fn compileStar(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + for (sub.dangles.items) |p| self.patch(p, fork); + sub.dangles.deinit(self.arena); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = fork, .dangles = dangles }; + } + + fn compilePlus(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + for (sub.dangles.items) |p| self.patch(p, fork); + sub.dangles.deinit(self.arena); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = sub.start, .dangles = dangles }; + } + + fn compileQuestion(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + try sub.dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = fork, .dangles = sub.dangles }; + } + + fn compileCounted(self: *Builder, r: *const ast.Repeat) BuildError!Frag { + // Bound unfold size — a pattern like `a{100000}` is almost certainly + // pathological; refuse so compile-time stays sane. + const max_unfold: u32 = 1024; + if (r.min > max_unfold) return BuildError.QuantifierTooLarge; + if (r.max != std.math.maxInt(u32) and r.max > max_unfold) return BuildError.QuantifierTooLarge; + + var head: ?Frag = null; + var idx: u32 = 0; + while (idx < r.min) : (idx += 1) { + const next = try self.compile(r.sub); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, next.start); + h.dangles.deinit(self.arena); + h.dangles = next.dangles; + } else { + head = next; + } + } + + if (r.max == std.math.maxInt(u32)) { + // {m,∞} → m mandatory copies followed by a star tail. + const tail = try self.compileStar(r.sub, r.greedy); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, tail.start); + h.dangles.deinit(self.arena); + h.dangles = tail.dangles; + return h.*; + } + return tail; + } + + // {m,n} → m mandatory + (n-m) optional copies. + const optionals = r.max - r.min; + var i: u32 = 0; + while (i < optionals) : (i += 1) { + const opt = try self.compileQuestion(r.sub, r.greedy); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, opt.start); + h.dangles.deinit(self.arena); + h.dangles = opt.dangles; + } else { + head = opt; + } + } + + if (head) |h| return h; + // Pure {0,0} — match the empty string. + return try self.epsilonFrag(); + } + + fn compileGroup(self: *Builder, g: *const ast.Group) BuildError!Frag { + var sub = try self.compile(g.sub); + if (!g.capturing) return sub; + + // Wrap sub with group_start ... sub ... group_end. + const start = try self.newState(.{ .group_start = g.index }, sub.start, null); + const end = try self.newState(.{ .group_end = g.index }, null, null); + for (sub.dangles.items) |p| self.patch(p, end); + sub.dangles.deinit(self.arena); + + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ .state = end, .slot = .out1 }); + return .{ .start = start, .dangles = dangles }; + } +}; + +// ── Validation helpers (also used by tests) ── + +/// Walk every state and assert that out-edge ids point at real states. +/// Catches construction bugs that would otherwise show up as wrong matches. +pub fn validate(nfa: Nfa) !void { + if (nfa.start >= nfa.states.len) return error.InvalidStartState; + if (nfa.accept >= nfa.states.len) return error.InvalidAcceptState; + for (nfa.states, 0..) |s, i| { + if (s.out1) |o| if (o >= nfa.states.len) { + std.debug.print("state {d} out1={d} out-of-bounds\n", .{ i, o }); + return error.InvalidOutEdge; + }; + if (s.out2) |o| if (o >= nfa.states.len) { + std.debug.print("state {d} out2={d} out-of-bounds\n", .{ i, o }); + return error.InvalidOutEdge; + }; + } +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildFrom(arena: *std.heap.ArenaAllocator, pattern: []const u8) !Nfa { + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return try build(arena.allocator(), root, p.n_groups); +} + +test "nfa: literal compiles to one byte state + accept" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a"); + try validate(nfa); + try std.testing.expectEqual(@as(usize, 2), nfa.states.len); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: concat 'ab' chains two byte states" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "ab"); + try validate(nfa); + try std.testing.expectEqual(@as(usize, 3), nfa.states.len); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); + const second = nfa.states[nfa.start].out1.?; + try std.testing.expectEqual(Consume{ .byte = 'b' }, nfa.states[second].consume); +} + +test "nfa: alt 'a|b' builds a fork" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a|b"); + try validate(nfa); + // Fork (epsilon) + a + b + accept = 4. + try std.testing.expectEqual(@as(usize, 4), nfa.states.len); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + try std.testing.expect(nfa.states[nfa.start].out1 != null); + try std.testing.expect(nfa.states[nfa.start].out2 != null); +} + +test "nfa: star 'a*' builds fork pointing at sub + continuation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a*"); + try validate(nfa); + // a + fork + accept = 3. + try std.testing.expectEqual(@as(usize, 3), nfa.states.len); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + // Greedy: out1 should be the sub. + const sub_id = nfa.states[nfa.start].out1.?; + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[sub_id].consume); +} + +test "nfa: lazy star 'a*?' reverses fork ordering" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a*?"); + try validate(nfa); + // Lazy: out1 is the continuation, out2 is the sub. + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + const sub_id = nfa.states[nfa.start].out2.?; + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[sub_id].consume); +} + +test "nfa: plus 'a+' starts at sub, not fork" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a+"); + try validate(nfa); + // 'a' must be matched at least once — start is the byte state. + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: question 'a?' makes sub optional" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a?"); + try validate(nfa); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); +} + +test "nfa: capturing group wraps sub with group_start / group_end" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "(ab)"); + try validate(nfa); + try std.testing.expectEqual(@as(u32, 1), nfa.n_groups); + switch (nfa.states[nfa.start].consume) { + .group_start => |idx| try std.testing.expectEqual(@as(u32, 1), idx), + else => return error.ExpectedGroupStart, + } +} + +test "nfa: non-capturing group has no group_start/end" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "(?:ab)"); + try validate(nfa); + try std.testing.expectEqual(@as(u32, 0), nfa.n_groups); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: counted {2,3}" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a{2,3}"); + try validate(nfa); + // 2 mandatory + 1 optional + accept = at least 4 states. + try std.testing.expect(nfa.states.len >= 4); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: oversized quantifier errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "a{2000}"); + const root = try p.parseRoot(); + try std.testing.expectError(BuildError.QuantifierTooLarge, build(arena.allocator(), root, p.n_groups)); +} + +test "nfa: anchor compiles to anchor state" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "^abc"); + try validate(nfa); + switch (nfa.states[nfa.start].consume) { + .anchor => |a| try std.testing.expectEqual(ast.Anchor.line_start, a), + else => return error.ExpectedAnchor, + } +} + +test "nfa: validate catches no-bug case" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a(b|c)*d"); + try validate(nfa); +} diff --git a/vendor/nanoregex/src/parser.zig b/vendor/nanoregex/src/parser.zig new file mode 100644 index 00000000..9f1f943b --- /dev/null +++ b/vendor/nanoregex/src/parser.zig @@ -0,0 +1,495 @@ +//! Recursive-descent regex parser. Pattern bytes → AST. +//! +//! Grammar (v1, mirrors the Python re subset we ship as feature parity): +//! +//! regex ::= alt +//! alt ::= concat ('|' concat)* +//! concat ::= atom* +//! atom ::= primary quantifier? +//! primary ::= literal | dot | class | anchor | group +//! group ::= '(' ('?:')? regex ')' +//! class ::= '[' '^'? class_item+ ']' +//! class_item ::= char | char '-' char | escape +//! quantifier ::= ( '?' | '*' | '+' | '{' n (',' m?)? '}' ) '?'? +//! +//! Deferred to v2: backreferences, lookarounds, named groups, inline flags. +//! +//! All AST nodes are allocated in the caller-provided allocator (expected to +//! be a Regex-owned arena). Parse errors return a typed error; the partial +//! AST is freed when the arena drops. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const ParseError = error{ + UnexpectedEnd, + UnbalancedParen, + UnbalancedBracket, + InvalidEscape, + InvalidQuantifier, + NothingToRepeat, + InvalidCharRange, + OutOfMemory, +}; + +pub const Parser = struct { + alloc: std.mem.Allocator, + src: []const u8, + pos: usize = 0, + /// Count of capturing groups seen so far. Incremented on `(` (but not + /// on `(?:`). Used to assign group indices in declaration order. + n_groups: u32 = 0, + + pub fn init(alloc: std.mem.Allocator, pattern: []const u8) Parser { + return .{ .alloc = alloc, .src = pattern }; + } + + pub fn parseRoot(self: *Parser) ParseError!*const ast.Node { + const root = try self.parseAlt(); + if (self.pos < self.src.len) { + // A stray closing `)` would land here. + return ParseError.UnbalancedParen; + } + return root; + } + + // ── alt = concat ('|' concat)* ── + + fn parseAlt(self: *Parser) ParseError!*const ast.Node { + var branches: std.ArrayList(*const ast.Node) = .empty; + defer branches.deinit(self.alloc); + + try branches.append(self.alloc, try self.parseConcat()); + while (self.peek() == '|') { + self.pos += 1; + try branches.append(self.alloc, try self.parseConcat()); + } + if (branches.items.len == 1) return branches.items[0]; + const slice = try self.alloc.dupe(*const ast.Node, branches.items); + return try self.node(.{ .alt = slice }); + } + + // ── concat = atom* ── + + fn parseConcat(self: *Parser) ParseError!*const ast.Node { + var pieces: std.ArrayList(*const ast.Node) = .empty; + defer pieces.deinit(self.alloc); + + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == '|' or c == ')') break; + try pieces.append(self.alloc, try self.parseAtom()); + } + if (pieces.items.len == 0) { + // Empty concat matches the empty string. Represent as an empty + // concat node — the matcher treats it as zero-width success. + const slice = try self.alloc.dupe(*const ast.Node, &.{}); + return try self.node(.{ .concat = slice }); + } + if (pieces.items.len == 1) return pieces.items[0]; + const slice = try self.alloc.dupe(*const ast.Node, pieces.items); + return try self.node(.{ .concat = slice }); + } + + // ── atom = primary quantifier? ── + + fn parseAtom(self: *Parser) ParseError!*const ast.Node { + const primary = try self.parsePrimary(); + return try self.maybeQuantify(primary); + } + + fn parsePrimary(self: *Parser) ParseError!*const ast.Node { + if (self.pos >= self.src.len) return ParseError.UnexpectedEnd; + const c = self.src[self.pos]; + switch (c) { + '.' => { + self.pos += 1; + return try self.node(.dot); + }, + '^' => { + self.pos += 1; + return try self.node(.{ .anchor = .line_start }); + }, + '$' => { + self.pos += 1; + return try self.node(.{ .anchor = .line_end }); + }, + '(' => return try self.parseGroup(), + '[' => return try self.parseClass(), + '\\' => return try self.parseEscape(), + '*', '+', '?', '{' => return ParseError.NothingToRepeat, + ')', '|' => return ParseError.UnexpectedEnd, + else => { + self.pos += 1; + return try self.node(.{ .literal = c }); + }, + } + } + + // ── group = '(' ('?:')? regex ')' ── + + fn parseGroup(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '('); + self.pos += 1; + + var capturing = true; + if (self.pos + 1 < self.src.len and self.src[self.pos] == '?' and self.src[self.pos + 1] == ':') { + capturing = false; + self.pos += 2; + } + + // Reserve the capture index BEFORE recursing so nested groups get + // higher indices, matching Python re's left-paren declaration order. + var index: u32 = 0; + if (capturing) { + self.n_groups += 1; + index = self.n_groups; + } + + const sub = try self.parseAlt(); + + if (self.peek() != ')') return ParseError.UnbalancedParen; + self.pos += 1; + + const g = try self.alloc.create(ast.Group); + g.* = .{ .sub = sub, .index = index, .capturing = capturing }; + return try self.node(.{ .group = g }); + } + + // ── class = '[' '^'? items ']' ── + + fn parseClass(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '['); + self.pos += 1; + + const cls = try self.alloc.create(ast.Class); + cls.* = ast.Class.empty(); + + var negate = false; + if (self.peek() == '^') { + negate = true; + self.pos += 1; + } + + // A `]` as the very first char inside the class is treated as a + // literal `]`, matching Python re's behaviour. Otherwise `]` ends. + var first = true; + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == ']' and !first) break; + first = false; + + const lo = try self.parseClassChar(); + // Range `a-z` only if the `-` is followed by a non-`]` char. + if (self.pos + 1 < self.src.len and self.src[self.pos] == '-' and self.src[self.pos + 1] != ']') { + self.pos += 1; // consume '-' + const hi = try self.parseClassChar(); + if (hi < lo) return ParseError.InvalidCharRange; + cls.setRange(lo, hi); + } else { + cls.set(lo); + } + } + if (self.peek() != ']') return ParseError.UnbalancedBracket; + self.pos += 1; + + if (negate) cls.negate(); + return try self.node(.{ .class = cls }); + } + + fn parseClassChar(self: *Parser) ParseError!u8 { + if (self.pos >= self.src.len) return ParseError.UnbalancedBracket; + const c = self.src[self.pos]; + if (c == '\\') { + self.pos += 1; + if (self.pos >= self.src.len) return ParseError.InvalidEscape; + const e = self.src[self.pos]; + self.pos += 1; + return switch (e) { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '0' => 0, + else => e, + }; + } + self.pos += 1; + return c; + } + + // ── escape = '\' (shorthand | metaliteral) ── + + fn parseEscape(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '\\'); + self.pos += 1; + if (self.pos >= self.src.len) return ParseError.InvalidEscape; + const e = self.src[self.pos]; + self.pos += 1; + return switch (e) { + 'd' => try self.shorthandClass(digitClass()), + 'D' => try self.shorthandClass(negated(digitClass())), + 'w' => try self.shorthandClass(wordClass()), + 'W' => try self.shorthandClass(negated(wordClass())), + 's' => try self.shorthandClass(spaceClass()), + 'S' => try self.shorthandClass(negated(spaceClass())), + 'b' => try self.node(.{ .anchor = .word_boundary }), + 'B' => try self.node(.{ .anchor = .non_word_boundary }), + 'A' => try self.node(.{ .anchor = .string_start }), + 'z' => try self.node(.{ .anchor = .string_end }), + 'n' => try self.node(.{ .literal = '\n' }), + 't' => try self.node(.{ .literal = '\t' }), + 'r' => try self.node(.{ .literal = '\r' }), + '0' => try self.node(.{ .literal = 0 }), + else => try self.node(.{ .literal = e }), + }; + } + + fn shorthandClass(self: *Parser, cls_value: ast.Class) ParseError!*const ast.Node { + const cls = try self.alloc.create(ast.Class); + cls.* = cls_value; + return try self.node(.{ .class = cls }); + } + + fn digitClass() ast.Class { + var c = ast.Class.empty(); + c.setRange('0', '9'); + return c; + } + + fn wordClass() ast.Class { + var c = ast.Class.empty(); + c.setRange('a', 'z'); + c.setRange('A', 'Z'); + c.setRange('0', '9'); + c.set('_'); + return c; + } + + fn spaceClass() ast.Class { + var c = ast.Class.empty(); + c.set(' '); + c.set('\t'); + c.set('\n'); + c.set('\r'); + c.set(0x0b); // \v + c.set(0x0c); // \f + return c; + } + + fn negated(cls_in: ast.Class) ast.Class { + var c = cls_in; + c.negate(); + return c; + } + + // ── quantifier ── + + fn maybeQuantify(self: *Parser, primary: *const ast.Node) ParseError!*const ast.Node { + if (self.pos >= self.src.len) return primary; + const c = self.src[self.pos]; + var min: u32 = 0; + var max: u32 = 0; + switch (c) { + '?' => { + self.pos += 1; + min = 0; + max = 1; + }, + '*' => { + self.pos += 1; + min = 0; + max = std.math.maxInt(u32); + }, + '+' => { + self.pos += 1; + min = 1; + max = std.math.maxInt(u32); + }, + '{' => { + const parsed = try self.parseCountedQuantifier(); + min = parsed.min; + max = parsed.max; + }, + else => return primary, + } + var greedy = true; + if (self.peek() == '?') { + greedy = false; + self.pos += 1; + } + const r = try self.alloc.create(ast.Repeat); + r.* = .{ .sub = primary, .min = min, .max = max, .greedy = greedy }; + return try self.node(.{ .repeat = r }); + } + + fn parseCountedQuantifier(self: *Parser) ParseError!struct { min: u32, max: u32 } { + std.debug.assert(self.src[self.pos] == '{'); + self.pos += 1; + const lo = try self.readNumber(); + var hi = lo; + if (self.peek() == ',') { + self.pos += 1; + if (self.peek() == '}') { + hi = std.math.maxInt(u32); + } else { + hi = try self.readNumber(); + } + } + if (self.peek() != '}') return ParseError.InvalidQuantifier; + self.pos += 1; + if (hi < lo) return ParseError.InvalidQuantifier; + return .{ .min = lo, .max = hi }; + } + + fn readNumber(self: *Parser) ParseError!u32 { + const start = self.pos; + while (self.pos < self.src.len and self.src[self.pos] >= '0' and self.src[self.pos] <= '9') { + self.pos += 1; + } + if (self.pos == start) return ParseError.InvalidQuantifier; + return std.fmt.parseInt(u32, self.src[start..self.pos], 10) catch ParseError.InvalidQuantifier; + } + + // ── Helpers ── + + fn peek(self: *const Parser) ?u8 { + if (self.pos >= self.src.len) return null; + return self.src[self.pos]; + } + + fn node(self: *Parser, value: ast.Node) ParseError!*const ast.Node { + const n = try self.alloc.create(ast.Node); + n.* = value; + return n; + } +}; + +// ── Tests ── + +fn parseToString(alloc: std.mem.Allocator, pattern: []const u8) ![]u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(alloc); + try ast.debugWrite(root, &buf, alloc, 0); + return alloc.dupe(u8, buf.items); +} + +test "parse single literal" { + const out = try parseToString(std.testing.allocator, "a"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("literal 'a'\n", out); +} + +test "parse concat of literals" { + const out = try parseToString(std.testing.allocator, "abc"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse alternation" { + const out = try parseToString(std.testing.allocator, "a|b"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\alt + \\ literal 'a' + \\ literal 'b' + \\ + , out); +} + +test "parse star quantifier" { + const out = try parseToString(std.testing.allocator, "a*"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=0 max=4294967295 greedy=true + \\ literal 'a' + \\ + , out); +} + +test "parse lazy quantifier" { + const out = try parseToString(std.testing.allocator, "a+?"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=1 max=4294967295 greedy=false + \\ literal 'a' + \\ + , out); +} + +test "parse counted quantifier" { + const out = try parseToString(std.testing.allocator, "a{2,5}"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=2 max=5 greedy=true + \\ literal 'a' + \\ + , out); +} + +test "parse char class with range" { + const out = try parseToString(std.testing.allocator, "[a-z]"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("class [26 bytes]\n", out); +} + +test "parse capturing group" { + const out = try parseToString(std.testing.allocator, "(abc)"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\group #1 cap=true + \\ concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse non-capturing group" { + const out = try parseToString(std.testing.allocator, "(?:abc)"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\group #0 cap=false + \\ concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse shorthand class \\d" { + const out = try parseToString(std.testing.allocator, "\\d"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("class [10 bytes]\n", out); +} + +test "parse anchor word_boundary" { + const out = try parseToString(std.testing.allocator, "\\b"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("anchor word_boundary\n", out); +} + +test "parse unbalanced paren errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), "(abc"); + try std.testing.expectError(ParseError.UnbalancedParen, p.parseRoot()); +} + +test "parse nothing-to-repeat errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), "*abc"); + try std.testing.expectError(ParseError.NothingToRepeat, p.parseRoot()); +} diff --git a/vendor/nanoregex/src/prefilter.zig b/vendor/nanoregex/src/prefilter.zig new file mode 100644 index 00000000..ec044744 --- /dev/null +++ b/vendor/nanoregex/src/prefilter.zig @@ -0,0 +1,517 @@ +//! Compile-time pattern analysis for fast-path optimisations. +//! +//! Two analyses, both consumed by `findAll`: +//! +//! 1. `extractFullLiteral` — when the AST is purely literal (literal bytes, +//! optional non-capturing concat/group wrapping, no metacharacters, no +//! capture groups), returns the literal byte sequence. Callers can then +//! bypass the NFA entirely and use `std.mem.indexOf` in a loop — a +//! 50-100x win on patterns like `compileAllocFlags` that came in as +//! "regex" but have no actual regex content. +//! +//! 2. `extractRequiredLiteral` — for genuinely-regex patterns, finds the +//! longest contiguous run of bytes that MUST appear in any successful +//! match. Callers use it as a pre-filter: if the haystack doesn't +//! contain the substring, no match exists and we skip the engine. When +//! the haystack does contain it, we still gate engine work to windows +//! around hits via `findOccurrences`. +//! +//! Both analyses are conservative: when in doubt, they return null/empty so +//! the matcher falls back to the full engine. The unit tests pin down what +//! we extract for each shape. + +const std = @import("std"); +const ast = @import("ast.zig"); + +/// If the AST is purely literal — only `.literal` / `.concat` / non-capturing +/// `.group` nodes — flatten it to a single byte slice. Returns null when any +/// regex feature (dot, class, anchor, alt, repeat, capturing group) is +/// present. The returned slice is arena-allocated. +pub fn extractFullLiteral(arena: std.mem.Allocator, root: *const ast.Node) !?[]const u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(arena); + if (!try collectLiteral(root, arena, &buf)) { + buf.deinit(arena); + return null; + } + if (buf.items.len == 0) { + buf.deinit(arena); + return null; + } + return try buf.toOwnedSlice(arena); +} + +fn collectLiteral(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) !bool { + return switch (node.*) { + .literal => |c| { + try buf.append(arena, c); + return true; + }, + .concat => |children| { + for (children) |child| { + if (!try collectLiteral(child, arena, buf)) return false; + } + return true; + }, + .group => |g| { + // A capturing group changes externally-observable behaviour + // (callers may want span info), so bail out and let the engine + // handle it. Non-capturing groups are pure parens; transparent. + if (g.capturing) return false; + return collectLiteral(g.sub, arena, buf); + }, + .dot, .class, .anchor, .alt, .repeat => false, + }; +} + +/// Find the longest run of unconditionally-required literal bytes inside +/// the pattern. "Required" means every successful match must contain these +/// bytes contiguously. Returns null when no run of length ≥ `min_len` can +/// be extracted; below that threshold the prefilter overhead beats the +/// win. +pub fn extractRequiredLiteral(arena: std.mem.Allocator, root: *const ast.Node, min_len: usize) !?[]const u8 { + var best: std.ArrayList(u8) = .empty; + errdefer best.deinit(arena); + var current: std.ArrayList(u8) = .empty; + defer current.deinit(arena); + + try walkRequired(root, arena, ¤t, &best); + // Final flush in case the longest run ends at the AST tail. + if (current.items.len > best.items.len) { + best.deinit(arena); + best = current; + current = .empty; + } + + if (best.items.len < min_len) { + best.deinit(arena); + return null; + } + return try best.toOwnedSlice(arena); +} + +fn walkRequired( + node: *const ast.Node, + arena: std.mem.Allocator, + current: *std.ArrayList(u8), + best: *std.ArrayList(u8), +) error{OutOfMemory}!void { + switch (node.*) { + .literal => |c| try current.append(arena, c), + .concat => |children| for (children) |child| try walkRequired(child, arena, current, best), + .group => |g| try walkRequired(g.sub, arena, current, best), + .repeat => |r| { + // The sub-pattern is required iff min ≥ 1. When it is, treat the + // first mandatory copy as additional bytes in the current run. + // Past that, the rest can repeat-with-variation so we flush + // and start fresh. + if (r.min >= 1) { + try walkRequired(r.sub, arena, current, best); + } + try flush(current, best, arena); + }, + .dot, .class, .anchor => try flush(current, best, arena), + .alt => { + // We could pick the longest common prefix across branches but + // for v1 we bail conservatively — the run ends here. + try flush(current, best, arena); + }, + } +} + +fn flush(current: *std.ArrayList(u8), best: *std.ArrayList(u8), arena: std.mem.Allocator) !void { + if (current.items.len > best.items.len) { + best.deinit(arena); + best.* = current.*; + current.* = .empty; + } else { + current.clearRetainingCapacity(); + } +} + +/// Extract the contiguous literal byte sequence at the very START of the +/// pattern, if any. Differs from `extractRequiredLiteral` in two ways: +/// - We anchor at the pattern start instead of picking the longest run. +/// - Callers can use the result as a STARTING-POSITION hint: every match +/// must begin where this byte sequence occurs in the haystack. +/// +/// Bails on alternation at the top level (different branches → different +/// possible prefixes; we'd need their common prefix). Returns null when +/// the prefix is shorter than `min_len`, the threshold below which the +/// indexOf-and-resume overhead beats the engine. +pub fn extractLiteralPrefix(arena: std.mem.Allocator, root: *const ast.Node, min_len: usize) !?[]const u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(arena); + _ = collectPrefix(root, arena, &buf) catch |err| switch (err) { + error.OutOfMemory => return err, + }; + if (buf.items.len < min_len) { + buf.deinit(arena); + return null; + } + return try buf.toOwnedSlice(arena); +} + +/// Walk the AST left-to-right adding ONLY contiguous required literal bytes at +/// the start. Returns `false` when the prefix must stop here — at a non-literal +/// node, or after a quantifier. A quantifier ends the contiguous prefix even +/// when it contributes a mandatory copy: bytes past that copy land at a variable +/// offset, so nothing after the repeat may join the prefix. Without this, `hel+o` +/// yielded the prefix `helo`, which wrongly demands the `o` sit immediately after +/// a single `l` — making `search` miss `helllo` entirely. +fn collectPrefix(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) error{OutOfMemory}!bool { + switch (node.*) { + .literal => |c| { + try buf.append(arena, c); + return true; + }, + .concat => |children| { + for (children) |child| { + const cont = try collectPrefix(child, arena, buf); + if (!cont) return false; + } + return true; + }, + .group => |g| return try collectPrefix(g.sub, arena, buf), + .repeat => |r| { + // min ≥ 1 contributes one mandatory copy of the sub-pattern's + // prefix; min == 0 contributes nothing. Either way the contiguous + // prefix ends here — the repeat may match more bytes, pushing + // anything that follows to a variable offset. + if (r.min >= 1) _ = try collectPrefix(r.sub, arena, buf); + return false; + }, + // Class, dot, anchor, and alt all end the prefix. + .class, .dot, .anchor, .alt => return false, + } +} + +/// If the entire AST is a single `class+` or `class{n,}` quantifier (e.g. +/// `\d+`, `[a-z]+`, `\w{2,}`), return the underlying class so callers can +/// dispatch to a specialised SIMD-only scanner: "first byte in class → +/// first byte not in class → emit; repeat." Avoids the DFA entirely and +/// runs at memmem speed. +/// +/// Returns null for any compound shape (concat, alt, group with non-trivial +/// structure, anchors, etc.) — those need the real engine. +pub fn extractSingleClassRepeat(root: *const ast.Node) ?*const ast.Class { + // Allow `(?:class+)` and `(class+)` wrapping but only if the group is + // non-capturing (we don't track group spans on this path). + var n = root; + while (true) { + switch (n.*) { + .group => |g| if (!g.capturing) { + n = g.sub; + continue; + } else return null, + else => break, + } + } + switch (n.*) { + .repeat => |r| { + if (r.min < 1) return null; // min=0 has a zero-width quirk we skip + if (r.max != std.math.maxInt(u32)) return null; + switch (r.sub.*) { + .class => |cls| return cls, + else => return null, + } + }, + else => return null, + } +} + +/// If the entire AST is a pure literal alternation — `foo|bar|baz`, with +/// each branch a concat of `.literal` nodes only — return the branch byte +/// strings. Callers dispatch to a multi-literal scanner that finds the +/// earliest match across all needles via per-needle `indexOfPos`. This is +/// the same niche `aho-corasick`'s Teddy fills for ripgrep. +/// +/// Returns null for any branch that isn't purely literal, or for a single +/// branch (that'd already be caught by `extractFullLiteral`). Caller owns +/// the returned slice via the passed arena. +pub fn extractLiteralAlternation(arena: std.mem.Allocator, root: *const ast.Node) !?[]const []const u8 { + // Unwrap one layer of non-capturing group at the root (very common + // shape from `(?:foo|bar)`). + var n = root; + if (n.* == .group) { + const g = n.group; + if (g.capturing) return null; + n = g.sub; + } + if (n.* != .alt) return null; + const branches = n.alt; + if (branches.len < 2) return null; + + var out: std.ArrayList([]const u8) = .empty; + errdefer { + for (out.items) |b| arena.free(b); + out.deinit(arena); + } + for (branches) |branch| { + var bytes: std.ArrayList(u8) = .empty; + errdefer bytes.deinit(arena); + if (!try collectBranchLiteral(branch, arena, &bytes)) { + bytes.deinit(arena); + for (out.items) |b| arena.free(b); + out.deinit(arena); + return null; + } + if (bytes.items.len == 0) { + // Empty branch — would match at every position, not worth this fast path. + bytes.deinit(arena); + for (out.items) |b| arena.free(b); + out.deinit(arena); + return null; + } + try out.append(arena, try bytes.toOwnedSlice(arena)); + } + return try out.toOwnedSlice(arena); +} + +fn collectBranchLiteral(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) !bool { + return switch (node.*) { + .literal => |c| blk: { + try buf.append(arena, c); + break :blk true; + }, + .concat => |children| blk: { + for (children) |child| { + if (!try collectBranchLiteral(child, arena, buf)) break :blk false; + } + break :blk true; + }, + .group => |g| if (g.capturing) false else collectBranchLiteral(g.sub, arena, buf), + else => false, + }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn fullFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractFullLiteral(arena.allocator(), root); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn requiredFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractRequiredLiteral(arena.allocator(), root, 3); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn prefixFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractLiteralPrefix(arena.allocator(), root, 3); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn singleClassFor(pattern: []const u8) !bool { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return extractSingleClassRepeat(root) != null; +} + +test "full literal: plain identifier" { + const got = (try fullFor(std.testing.allocator, "compileAllocFlags")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags", got); +} + +test "full literal: non-capturing group is transparent" { + const got = (try fullFor(std.testing.allocator, "(?:abc)def")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("abcdef", got); +} + +test "full literal: capturing group blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "(abc)")) == null); +} + +test "full literal: dot blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "a.b")) == null); +} + +test "full literal: class blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "a[bc]")) == null); +} + +test "full literal: alternation blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "abc|def")) == null); +} + +test "full literal: repeat blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "ab+")) == null); +} + +test "required literal: extracts prefix before class" { + const got = (try requiredFor(std.testing.allocator, "compileAllocFlags\\([a-z]+")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags(", got); +} + +test "required literal: picks longest run" { + const got = (try requiredFor(std.testing.allocator, "[a-z]hello\\d+worlds\\s+end")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("worlds", got); +} + +test "required literal: too short returns null" { + try std.testing.expect((try requiredFor(std.testing.allocator, "a.b")) == null); +} + +test "required literal: alternation bails" { + try std.testing.expect((try requiredFor(std.testing.allocator, "foo|bar")) == null); +} + +test "required literal: min=0 quantifier doesn't contribute" { + // 'a*bar' — 'a' is optional, so the required run is 'bar'. + const got = (try requiredFor(std.testing.allocator, "a*barbaz")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("barbaz", got); +} + +test "required literal: min=1 quantifier contributes single copy" { + // 'a+bar' — 'a' is required (at least once), then 'bar'. + // After walking the repeat, we flush — so 'a' alone is the run, but + // 'bar' is longer. Best = 'bar'. + const got = (try requiredFor(std.testing.allocator, "a+barbaz")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("barbaz", got); +} + +test "literal prefix: simple identifier" { + const got = (try prefixFor(std.testing.allocator, "compileAllocFlags\\([a-z]+")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags(", got); +} + +test "literal prefix: class at start bails" { + try std.testing.expect((try prefixFor(std.testing.allocator, "[a-z]+hello")) == null); +} + +test "literal prefix: alternation at top bails" { + try std.testing.expect((try prefixFor(std.testing.allocator, "foo|bar")) == null); +} + +test "literal prefix: under threshold returns null" { + try std.testing.expect((try prefixFor(std.testing.allocator, "ab.c")) == null); +} + +test "literal prefix: stops after + quantifier (regression)" { + // `hel+o` — the contiguous start prefix is `hel`, never `helo`: the `o` + // sits at a variable offset after one-or-more `l`. Demanding `helo` as a + // starting-position hint made `search` miss `helllo` outright. + const got = (try prefixFor(std.testing.allocator, "hel+o")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("hel", got); +} + +test "literal prefix: stops after {n,m} quantifier (regression)" { + const got = (try prefixFor(std.testing.allocator, "hel{1,3}o")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("hel", got); +} + +test "literal prefix: stops at optional byte" { + // `foox?bar` — 'foo' is mandatory, then 'x?' is optional, so prefix is 'foo'. + const got = (try prefixFor(std.testing.allocator, "foox?bar")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("foo", got); +} + +test "single-class-repeat: \\d+" { + try std.testing.expect(try singleClassFor("\\d+")); +} + +test "single-class-repeat: [a-z]+" { + try std.testing.expect(try singleClassFor("[a-z]+")); +} + +test "single-class-repeat: counted lower bound" { + try std.testing.expect(try singleClassFor("[0-9]{3,}")); +} + +test "single-class-repeat: non-capturing group is transparent" { + try std.testing.expect(try singleClassFor("(?:\\w+)")); +} + +test "single-class-repeat: capturing group blocks" { + try std.testing.expect(!(try singleClassFor("(\\w+)"))); +} + +test "single-class-repeat: concat blocks" { + try std.testing.expect(!(try singleClassFor("\\d+abc"))); +} + +test "single-class-repeat: alternation blocks" { + try std.testing.expect(!(try singleClassFor("\\d+|abc"))); +} + +test "single-class-repeat: star (min=0) skipped" { + // min=0 has a zero-width-match quirk — not yet handled by the fast path. + try std.testing.expect(!(try singleClassFor("\\d*"))); +} + +fn altLiteralFor(alloc: std.mem.Allocator, pattern: []const u8) !?[][]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lits = try extractLiteralAlternation(arena.allocator(), root); + if (lits) |bs| { + var owned = try alloc.alloc([]const u8, bs.len); + for (bs, 0..) |b, i| owned[i] = try alloc.dupe(u8, b); + return owned; + } + return null; +} + +test "literal alternation: three branches" { + const got = (try altLiteralFor(std.testing.allocator, "foo|bar|baz")).?; + defer { + for (got) |b| std.testing.allocator.free(b); + std.testing.allocator.free(got); + } + try std.testing.expectEqual(@as(usize, 3), got.len); + try std.testing.expectEqualStrings("foo", got[0]); + try std.testing.expectEqualStrings("bar", got[1]); + try std.testing.expectEqualStrings("baz", got[2]); +} + +test "literal alternation: non-capturing group wraps" { + const got = (try altLiteralFor(std.testing.allocator, "(?:cat|dog|bird)")).?; + defer { + for (got) |b| std.testing.allocator.free(b); + std.testing.allocator.free(got); + } + try std.testing.expectEqual(@as(usize, 3), got.len); +} + +test "literal alternation: bails on non-literal branch" { + try std.testing.expect((try altLiteralFor(std.testing.allocator, "foo|b.r")) == null); +} + +test "literal alternation: bails on capturing group" { + try std.testing.expect((try altLiteralFor(std.testing.allocator, "(foo|bar)")) == null); +} + +test "literal alternation: single branch is null" { + // Only one branch — handled by extractFullLiteral, not this path. + try std.testing.expect((try altLiteralFor(std.testing.allocator, "foo")) == null); +} diff --git a/vendor/nanoregex/src/probe.zig b/vendor/nanoregex/src/probe.zig new file mode 100644 index 00000000..73cbe201 --- /dev/null +++ b/vendor/nanoregex/src/probe.zig @@ -0,0 +1,87 @@ +//! Parity-test probe CLI. +//! +//! Wire shape: nanoregex_probe [flags] +//! Flags string (3rd argv) is any concatenation of `i`/`m`/`s` matching +//! Python re's IGNORECASE / MULTILINE / DOTALL — same encoding used by +//! tests/parity/run.sh. +//! +//! Output: one match per line as `start..end\tmatched_bytes`. Bytes are +//! emitted raw (no escaping) so the harness can diff byte-for-byte against +//! `re.finditer` output formatted the same way. +//! +//! Stdio is via extern libc `write`: Zig 0.16 removed the synchronous +//! stdlib wrappers we used to rely on, and nanoregex links libc anyway. + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; + +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +fn parseFlags(s: []const u8) nanoregex.Flags { + var f: nanoregex.Flags = .{}; + for (s) |c| switch (c) { + 'i' => f.case_insensitive = true, + 'm' => f.multiline = true, + 's' => f.dot_all = true, + else => {}, + }; + return f; +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |arg| try args_list.append(alloc, arg); + const args = args_list.items; + + if (args.len < 2) { + writeAll(2, "usage: nanoregex_probe [] []\n"); + std.process.exit(2); + } + const pattern = args[1]; + const haystack: []const u8 = if (args.len >= 3) args[2] else ""; + const flags = parseFlags(if (args.len >= 4) args[3] else ""); + + var r = nanoregex.Regex.compileWithFlags(alloc, pattern, flags) catch |err| { + var tmp: [128]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "PARSE_ERROR: {s}\n", .{@errorName(err)}) catch "PARSE_ERROR\n"; + writeAll(1, msg); + // Exit 0 — the harness checks output content, not exit code, so + // PARSE_ERROR on both sides should match cleanly. + std.process.exit(0); + }; + defer r.deinit(); + + const matches = r.findAll(alloc, haystack) catch { + writeAll(2, "ENGINE_ERROR\n"); + std.process.exit(1); + }; + defer { + for (matches) |*m| @constCast(m).deinit(alloc); + alloc.free(matches); + } + + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(alloc); + + var line_buf: [256]u8 = undefined; + for (matches) |m| { + const header = std.fmt.bufPrint(&line_buf, "{d}..{d}\t", .{ m.span.start, m.span.end }) catch continue; + try buf.appendSlice(alloc, header); + try buf.appendSlice(alloc, haystack[m.span.start..m.span.end]); + try buf.append(alloc, '\n'); + } + writeAll(1, buf.items); +} diff --git a/vendor/nanoregex/src/root.zig b/vendor/nanoregex/src/root.zig new file mode 100644 index 00000000..089c79f8 --- /dev/null +++ b/vendor/nanoregex/src/root.zig @@ -0,0 +1,676 @@ +//! nanoregex — pure-Zig regex engine with Python-re-compatible semantics. +//! +//! Layered design: +//! 1. parser.zig — pattern bytes → AST +//! 2. ast.zig — AST node tagged union, arena-owned +//! 3. nfa.zig — AST → Thompson NFA +//! 4. exec.zig — Pike-VM NFA simulation (always-correct fallback) +//! 5. prefilter.zig — literal/required-substring extraction (fast path) +//! 6. dfa.zig — Lazy subset-construction DFA (perf path) +//! +//! Dispatch policy in findAll/search: +//! 1. Pure-literal AST + no captures + case-sensitive → memmem loop +//! 2. Required literal absent in haystack → early-return empty +//! 3. DFA eligible (no captures, no anchors, no case-insensitive) → DFA +//! 4. Otherwise → Pike VM +//! +//! The DFA is built eagerly at compile time (when eligible) so the hot loop +//! is a single transition-table lookup per byte. Regex deinit cleans the +//! DFA's arena. + +const std = @import("std"); +pub const ast = @import("ast.zig"); +pub const parser = @import("parser.zig"); +pub const nfa = @import("nfa.zig"); +pub const exec = @import("exec.zig"); +pub const prefilter = @import("prefilter.zig"); +pub const dfa = @import("dfa.zig"); +pub const minterm = @import("minterm.zig"); +const comptime_mod = @import("comptime.zig"); + +/// comptime-specialised matcher factory. For patterns known at compile time, +/// returns a zero-overhead specialised scanner. See `src/comptime.zig`. +pub const compileComptime = comptime_mod.compileComptime; +pub fn ComptimeRegex(comptime pattern: []const u8) type { + return comptime_mod.ComptimeRegex(pattern); +} + +pub const Flags = exec.Flags; +pub const Span = exec.Span; +pub const Match = exec.MatchResult; + +pub const Regex = struct { + /// Backing arena for the AST + NFA + prefilter slices. Lives until deinit(). + arena: *std.heap.ArenaAllocator, + parent_alloc: std.mem.Allocator, + root: *const ast.Node, + /// Heap-allocated on `arena` so the address is stable for `dfa.nfa_ref` + /// and for `exec.Vm.init`. Storing the Nfa by-value here used to give + /// us a dangling pointer when the Regex was returned by value. + automaton: *nfa.Nfa, + flags: Flags, + n_groups: u32, + + /// Non-null iff the pattern is purely literal AND has zero capture + /// groups. Callers bypass the engine entirely and use SIMD `indexOf`. + pure_literal: ?[]const u8, + /// Non-null iff a contiguous substring is required to appear in every + /// match. Used as a coarse pre-filter. + required_literal: ?[]const u8, + /// Non-null iff every match's start position is at an occurrence of + /// this byte sequence. Used to skip directly to candidate start + /// positions via SIMD-accelerated indexOf, then run the DFA only at + /// those hits. + literal_prefix: ?[]const u8, + + /// Non-null iff the entire pattern is a literal alternation (`foo|bar|baz`). + /// At match time we run one `indexOfPos` scan per literal across the + /// haystack, then merge the results — far cheaper than the DFA for + /// patterns made of a few literal alternatives. Owned by `arena`. + literal_alternation: ?[]const []const u8, + + /// Non-null iff the whole pattern is a `class+` / `class{n,}` repeat — + /// the matcher then bypasses the DFA entirely and uses a precomputed + /// 256-byte membership table to extract every contiguous run of + /// class-matching bytes. One load + one branch per input byte; no + /// library call per match. `\d+` and `[a-z]+` are the canonical + /// cases. Lifetime tied to `arena`. + single_class_table: ?*const [256]bool, + + /// Built eagerly when the pattern is DFA-eligible (no captures, no + /// anchors, no case-insensitive). Null otherwise. Mutable because the + /// DFA fills its transition table lazily during matching. + dfa_engine: ?dfa.Dfa, + + pub fn compile(alloc: std.mem.Allocator, pattern: []const u8) !Regex { + return compileWithFlags(alloc, pattern, .{}); + } + + pub fn compileWithFlags(alloc: std.mem.Allocator, pattern: []const u8, flags: Flags) !Regex { + const arena = try alloc.create(std.heap.ArenaAllocator); + errdefer alloc.destroy(arena); + arena.* = std.heap.ArenaAllocator.init(alloc); + errdefer arena.deinit(); + + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + + // Heap-allocate the Nfa on the arena. We point at it from both the + // Regex itself and the Dfa; storing by value would invalidate the + // address once compileWithFlags returns by value. + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + + const pure_lit: ?[]const u8 = if (flags.case_insensitive) + null + else if (p.n_groups != 0) + null + else + try prefilter.extractFullLiteral(arena.allocator(), root); + + const req_lit: ?[]const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractRequiredLiteral(arena.allocator(), root, 3); + + const lit_prefix: ?[]const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractLiteralPrefix(arena.allocator(), root, 3); + + const lit_alt: ?[]const []const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractLiteralAlternation(arena.allocator(), root); + + // Single-class-repeat fast path: precompute a 256-byte membership + // table so the runtime scanner is a tight `if (table[byte])` loop. + const single_class: ?*const [256]bool = blk: { + if (flags.case_insensitive) break :blk null; + const cls = prefilter.extractSingleClassRepeat(root) orelse break :blk null; + const table = try arena.allocator().create([256]bool); + var b: u16 = 0; + while (b < 256) : (b += 1) { + table[b] = cls.contains(@intCast(b)); + } + break :blk table; + }; + + // Try to build a DFA. Falls back to null (=> Pike VM at runtime) + // when the pattern has captures, anchors, or grows the state + // table past the budget. Case-insensitive also skips DFA for v1 — + // adding case-folding to the bitmap test is straightforward but + // not done yet. + var dfa_engine: ?dfa.Dfa = null; + if (!flags.case_insensitive) { + if (dfa.Dfa.fromNfa(alloc, automaton_ptr, root, .{ .dot_all = flags.dot_all })) |built| { + dfa_engine = built; + } else |_| { + // Any DFA build error (HasCaptures, HasAnchors, TooManyStates, + // OOM) falls back silently. The Pike VM handles the same + // patterns correctly, just slower. + dfa_engine = null; + } + } + + return .{ + .arena = arena, + .parent_alloc = alloc, + .root = root, + .automaton = automaton_ptr, + .flags = flags, + .n_groups = p.n_groups, + .pure_literal = pure_lit, + .required_literal = req_lit, + .literal_prefix = lit_prefix, + .literal_alternation = lit_alt, + .single_class_table = single_class, + .dfa_engine = dfa_engine, + }; + } + + pub fn deinit(self: *Regex) void { + if (self.dfa_engine) |*d| d.deinit(); + self.arena.deinit(); + self.parent_alloc.destroy(self.arena); + self.* = undefined; + } + + /// First leftmost match, or null. Caller owns the returned Match. + pub fn search(self: *Regex, alloc: std.mem.Allocator, input: []const u8) !?Match { + if (self.required_literal) |lit| { + if (std.mem.indexOf(u8, input, lit) == null) return null; + } + if (self.pure_literal) |lit| return try literalFirst(alloc, lit, input); + if (self.literal_alternation) |lits| return try multiLiteralFirst(alloc, lits, input); + if (self.single_class_table) |tbl| return try singleClassFirst(alloc, tbl, input); + if (self.literal_prefix) |prefix| if (self.dfa_engine) |*d| + return try dfaFirstWithPrefix(alloc, d, input, prefix); + if (self.dfa_engine) |*d| return try dfaFirst(alloc, d, input); + + var vm = exec.Vm.init(alloc, self.automaton, self.flags); + return try vm.search(input); + } + + /// All non-overlapping matches, leftmost-first. + pub fn findAll(self: *Regex, alloc: std.mem.Allocator, input: []const u8) ![]Match { + if (self.required_literal) |lit| { + if (std.mem.indexOf(u8, input, lit) == null) { + return try alloc.alloc(Match, 0); + } + } + if (self.pure_literal) |lit| return try literalAll(alloc, lit, input); + if (self.literal_alternation) |lits| return try multiLiteralAll(alloc, lits, input); + if (self.single_class_table) |tbl| return try singleClassAll(alloc, tbl, input); + if (self.literal_prefix) |prefix| if (self.dfa_engine) |*d| + return try dfaAllWithPrefix(alloc, d, input, prefix); + if (self.dfa_engine) |*d| return try dfaAll(alloc, d, input); + + var vm = exec.Vm.init(alloc, self.automaton, self.flags); + return try vm.findAll(input); + } + + /// Replace every non-overlapping match. Backreferences (`\N`) honoured. + pub fn replaceAll(self: *Regex, alloc: std.mem.Allocator, input: []const u8, replacement: []const u8) ![]u8 { + const matches = try self.findAll(alloc, input); + defer { + for (matches) |*m| @constCast(m).deinit(alloc); + alloc.free(matches); + } + + var out: std.ArrayList(u8) = .empty; + defer out.deinit(alloc); + + var cursor: usize = 0; + for (matches) |m| { + try out.appendSlice(alloc, input[cursor..m.span.start]); + try appendReplacement(alloc, &out, replacement, m, input); + cursor = m.span.end; + } + try out.appendSlice(alloc, input[cursor..]); + return try out.toOwnedSlice(alloc); + } +}; + +// ── Literal fast paths ── + +fn literalFirst(alloc: std.mem.Allocator, needle: []const u8, haystack: []const u8) !?Match { + if (needle.len == 0) { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = 0, .end = 0 }; + return .{ .span = .{ .start = 0, .end = 0 }, .captures = captures }; + } + const idx = std.mem.indexOf(u8, haystack, needle) orelse return null; + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = idx, .end = idx + needle.len }; + return .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = captures }; +} + +fn literalAll(alloc: std.mem.Allocator, needle: []const u8, haystack: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + if (needle.len == 0) return try results.toOwnedSlice(alloc); + + var pos: usize = 0; + while (pos <= haystack.len) { + const idx = std.mem.indexOfPos(u8, haystack, pos, needle) orelse break; + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = idx, .end = idx + needle.len }; + try results.append(alloc, .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = captures }); + pos = idx + needle.len; + } + return try results.toOwnedSlice(alloc); +} + +// ── DFA wrappers ── +// +// Adapter from `dfa.Dfa`'s span-only output to the public `Match` shape +// (which carries a captures slice). DFA mode has no captures so we emit a +// 1-element captures array containing just the whole-match span. + +fn dfaFirst(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8) !?Match { + var p: usize = 0; + while (p <= input.len) : (p += 1) { + const end_opt = try d.matchAt(input, p); + if (end_opt) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = p, .end = end }; + return .{ .span = .{ .start = p, .end = end }, .captures = captures }; + } + } + return null; +} + +fn dfaAll(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8) ![]Match { + const spans = try d.findAll(alloc, input); + defer alloc.free(spans); + + var results = try alloc.alloc(Match, spans.len); + var built: usize = 0; + errdefer { + for (results[0..built]) |*m| m.deinit(alloc); + alloc.free(results); + } + for (spans) |span| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = span.start, .end = span.end }; + results[built] = .{ .span = .{ .start = span.start, .end = span.end }, .captures = captures }; + built += 1; + } + return results; +} + +/// Like `dfaFirst` but uses `prefix` to skip directly to candidate match +/// starts via `std.mem.indexOfPos`. Far fewer engine invocations for +/// sparse literal-prefixed patterns. +fn dfaFirstWithPrefix(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8, prefix: []const u8) !?Match { + var pos: usize = 0; + while (true) { + const hit = std.mem.indexOfPos(u8, input, pos, prefix) orelse return null; + if (try d.matchAt(input, hit)) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = hit, .end = end }; + return .{ .span = .{ .start = hit, .end = end }, .captures = captures }; + } + // DFA didn't accept at this hit (the prefix matched but the rest + // of the pattern didn't). Advance one byte past this hit and + // resume the indexOf scan. + pos = hit + 1; + } +} + +fn dfaAllWithPrefix(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8, prefix: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + + var pos: usize = 0; + while (true) { + const hit = std.mem.indexOfPos(u8, input, pos, prefix) orelse break; + if (try d.matchAt(input, hit)) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = hit, .end = end }; + try results.append(alloc, .{ + .span = .{ .start = hit, .end = end }, + .captures = captures, + }); + // Skip past the match end. Zero-width match falls back to + // hit+1 so we don't infinite-loop. + pos = if (end > hit) end else hit + 1; + } else { + pos = hit + 1; + } + } + return try results.toOwnedSlice(alloc); +} + +fn appendReplacement( + alloc: std.mem.Allocator, + out: *std.ArrayList(u8), + replacement: []const u8, + m: Match, + input: []const u8, +) !void { + var i: usize = 0; + while (i < replacement.len) { + const c = replacement[i]; + if (c == '\\' and i + 1 < replacement.len) { + const n = replacement[i + 1]; + switch (n) { + '0'...'9' => { + const idx: usize = n - '0'; + if (idx < m.captures.len) { + if (m.captures[idx]) |span| try out.appendSlice(alloc, input[span.start..span.end]); + } + i += 2; + continue; + }, + 'n' => { + try out.append(alloc, '\n'); + i += 2; + continue; + }, + 't' => { + try out.append(alloc, '\t'); + i += 2; + continue; + }, + 'r' => { + try out.append(alloc, '\r'); + i += 2; + continue; + }, + '\\' => { + try out.append(alloc, '\\'); + i += 2; + continue; + }, + else => { + try out.append(alloc, '\\'); + i += 1; + continue; + }, + } + } + try out.append(alloc, c); + i += 1; + } +} + +// ── Single-class-repeat fast paths ── +// +// For `class+` / `class{n,}` patterns the whole match is "consecutive +// bytes in `class`". indexOfAnyPos finds the first matching byte (start); +// indexOfNonePos finds the first non-matching byte (end). Both are +// SIMD-accelerated in Zig stdlib. Two SIMD scans per match — vs the +// DFA's per-byte transition lookup. + +fn singleClassFirst(alloc: std.mem.Allocator, table: *const [256]bool, input: []const u8) !?Match { + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + return .{ .span = .{ .start = start, .end = i }, .captures = caps }; + } + i += 1; + } + return null; +} + +fn singleClassAll(alloc: std.mem.Allocator, table: *const [256]bool, input: []const u8) ![]Match { + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + try out.append(alloc, .{ .span = .{ .start = start, .end = i }, .captures = caps }); + } else { + i += 1; + } + } + return try out.toOwnedSlice(alloc); +} + +test "single-class fast path: \\d+ matches digits only" { + var r = try Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + try std.testing.expect(r.single_class_table != null); + const ms = try r.findAll(std.testing.allocator, "abc 42 def 1234 xyz"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 4), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 6), ms[0].span.end); +} + +test "single-class fast path: [a-z]+ ignores uppercase" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + try std.testing.expect(r.single_class_table != null); + const ms = try r.findAll(std.testing.allocator, "Hello World"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +// ── Multi-literal alternation fast paths ── +// +// For `lit1|lit2|...|litN` patterns we scan the haystack once per literal +// using std.mem.indexOfPos (memchr-backed when the literal is one byte, +// SIMD-accelerated memmem otherwise). On each step we pick the earliest +// candidate across all literals and emit it. The amortised cost is +// O(needles * input.len) but each scan is on Zig stdlib's tight loop — +// faster than a DFA whose hot path costs ~5 ns/byte. ripgrep accomplishes +// the same with Teddy (a SIMD-pshufb-based multi-string algorithm); this +// is the scalar equivalent. + +const LiteralHit = struct { + start: usize, + len: usize, + /// Index of the matching literal in the alternation. Lower wins ties + /// (leftmost-first alternation semantics, matching Python re). + lit_idx: u32, +}; + +fn hitLessThan(_: void, a: LiteralHit, b: LiteralHit) bool { + if (a.start != b.start) return a.start < b.start; + return a.lit_idx < b.lit_idx; +} + +/// Collect every occurrence of every literal across the input in one +/// pass per literal. Each pass is `std.mem.indexOfPos` (memchr/memmem in +/// Zig stdlib). For N literals on H bytes that's O(N · H), then a sort +/// of M total hits at O(M log M). Avoids the O(N · H · M) explosion the +/// naïve "earliest hit per step" scanner showed on dense workloads. +fn collectAllLiteralHits(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) ![]LiteralHit { + var hits: std.ArrayList(LiteralHit) = .empty; + errdefer hits.deinit(alloc); + for (literals, 0..) |lit, idx| { + if (lit.len == 0) continue; + var pos: usize = 0; + while (std.mem.indexOfPos(u8, input, pos, lit)) |found| { + try hits.append(alloc, .{ + .start = found, + .len = lit.len, + .lit_idx = @intCast(idx), + }); + pos = found + 1; + } + } + const slice = try hits.toOwnedSlice(alloc); + std.mem.sort(LiteralHit, slice, {}, hitLessThan); + return slice; +} + +fn multiLiteralFirst(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) !?Match { + // search() only needs the first hit — we can stop as soon as we find + // the leftmost across all literals. Cheaper than the full collect. + var best: ?LiteralHit = null; + for (literals, 0..) |lit, idx| { + if (lit.len == 0) continue; + const hit = std.mem.indexOf(u8, input, lit) orelse continue; + if (best) |b| { + if (hit < b.start or (hit == b.start and idx < b.lit_idx)) { + best = LiteralHit{ .start = hit, .len = lit.len, .lit_idx = @intCast(idx) }; + } + } else { + best = LiteralHit{ .start = hit, .len = lit.len, .lit_idx = @intCast(idx) }; + } + } + const h = best orelse return null; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = h.start, .end = h.start + h.len }; + return .{ .span = .{ .start = h.start, .end = h.start + h.len }, .captures = caps }; +} + +fn multiLiteralAll(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) ![]Match { + const all_hits = try collectAllLiteralHits(alloc, literals, input); + defer alloc.free(all_hits); + + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var cursor: usize = 0; + for (all_hits) |h| { + // Skip hits that overlap with an already-emitted match. With the + // hits sorted by (start, lit_idx), this also collapses the case + // where two literals would have matched at the same position — + // leftmost-first wins. + if (h.start < cursor) continue; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = h.start, .end = h.start + h.len }; + try out.append(alloc, .{ + .span = .{ .start = h.start, .end = h.start + h.len }, + .captures = caps, + }); + cursor = h.start + h.len; + } + return try out.toOwnedSlice(alloc); +} + +test "multi-literal alternation findAll" { + var r = try Regex.compile(std.testing.allocator, "cat|dog|bird"); + defer r.deinit(); + try std.testing.expect(r.literal_alternation != null); + const ms = try r.findAll(std.testing.allocator, "the cat saw a dog and a bird"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "module imports compile" { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(ast); + std.testing.refAllDecls(parser); + std.testing.refAllDecls(nfa); + std.testing.refAllDecls(exec); + std.testing.refAllDecls(prefilter); + std.testing.refAllDecls(dfa); +} + +test "Regex.search basic" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + var m = (try r.search(std.testing.allocator, "Hello World")).?; + defer m.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 1), m.span.start); + try std.testing.expectEqual(@as(usize, 5), m.span.end); +} + +test "Regex.findAll" { + var r = try Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + const ms = try r.findAll(std.testing.allocator, "abc 42 xyz 1234"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "Regex pure literal fast path" { + var r = try Regex.compile(std.testing.allocator, "compileAllocFlags"); + defer r.deinit(); + try std.testing.expect(r.pure_literal != null); + try std.testing.expectEqualStrings("compileAllocFlags", r.pure_literal.?); + const ms = try r.findAll(std.testing.allocator, "abc compileAllocFlags xyz compileAllocFlags"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "Regex DFA engine is built when eligible" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + // [a-z]+ has no captures, no anchors → DFA should be built. + try std.testing.expect(r.dfa_engine != null); +} + +test "Regex falls back to Pike VM with captures" { + var r = try Regex.compile(std.testing.allocator, "(abc)"); + defer r.deinit(); + try std.testing.expect(r.dfa_engine == null); +} + +test "Regex falls back to Pike VM with anchors" { + var r = try Regex.compile(std.testing.allocator, "^foo"); + defer r.deinit(); + try std.testing.expect(r.dfa_engine == null); +} + +test "Regex required-literal pre-filter skips haystack with no candidates" { + var r = try Regex.compile(std.testing.allocator, "hello\\d+"); + defer r.deinit(); + try std.testing.expectEqualStrings("hello", r.required_literal.?); + const ms = try r.findAll(std.testing.allocator, "no candidates anywhere here"); + defer std.testing.allocator.free(ms); + try std.testing.expectEqual(@as(usize, 0), ms.len); +} + +test "Regex search: literal after + quantifier matches a variable repeat (regression)" { + // The literal-prefix fast path used to extract `helo` for `hel+o`, then + // require `helo` as a contiguous start hint — so `helllo` (he + lll + o) + // was missed entirely. The prefix must be `hel`. See prefilter.collectPrefix. + var r = try Regex.compile(std.testing.allocator, "hel+o"); + defer r.deinit(); + if (try r.search(std.testing.allocator, "helllo")) |m| { + @constCast(&m).deinit(std.testing.allocator); + } else { + return error.TestUnexpectedResult; + } + // Negative: zero `l` must still not satisfy `l+`. + try std.testing.expect((try r.search(std.testing.allocator, "heo")) == null); +} + +test "Regex.replaceAll with backreference" { + var r = try Regex.compile(std.testing.allocator, "(\\w+)@(\\w+)"); + defer r.deinit(); + const out = try r.replaceAll(std.testing.allocator, "alice@example bob@host", "\\2/\\1"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("example/alice host/bob", out); +} diff --git a/vendor/nanoregex/tests/parity/fixtures/001_literal.txt b/vendor/nanoregex/tests/parity/fixtures/001_literal.txt new file mode 100644 index 00000000..24a9ec9e --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/001_literal.txt @@ -0,0 +1,3 @@ +abc + +the quick abc jumps over abc lazy abc diff --git a/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt b/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt new file mode 100644 index 00000000..d195989f --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt @@ -0,0 +1,3 @@ +a.*b + +start aXYZb middle a___b end diff --git a/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt b/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt new file mode 100644 index 00000000..e2ff7827 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt @@ -0,0 +1,3 @@ +[a-z]+ + +Hello World 42 diff --git a/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt b/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt new file mode 100644 index 00000000..bb24a3bf --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt @@ -0,0 +1,5 @@ +^foo +m +foo +bar foo +foo bar diff --git a/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt b/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt new file mode 100644 index 00000000..764da967 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt @@ -0,0 +1,3 @@ +cat|dog|bird + +the cat saw a dog and a bird diff --git a/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt b/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt new file mode 100644 index 00000000..9487302b --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt @@ -0,0 +1,3 @@ +\d+ + +version 1.2.345 build 678 diff --git a/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt b/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt new file mode 100644 index 00000000..f9dd7040 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt @@ -0,0 +1,3 @@ +(\w+)@(\w+) + +alice@example bob@host diff --git a/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt b/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt new file mode 100644 index 00000000..a6140ede --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt @@ -0,0 +1,3 @@ +a.*?b + +start aXXXb middle aYYYb end diff --git a/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt b/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt new file mode 100644 index 00000000..d421e363 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt @@ -0,0 +1,3 @@ +<.+?> + + diff --git a/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt b/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt new file mode 100644 index 00000000..58c877e7 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt @@ -0,0 +1,3 @@ +((a)(b))+ + +ababab xyz abab diff --git a/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt b/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt new file mode 100644 index 00000000..7c850910 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt @@ -0,0 +1,3 @@ +\. + +foo.bar baz.qux .end diff --git a/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt b/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt new file mode 100644 index 00000000..4b919c6c --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt @@ -0,0 +1,3 @@ +\bcat\b + +the cat sat on a catnap but a cat is a cat diff --git a/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt b/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt new file mode 100644 index 00000000..0e4a6f58 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt @@ -0,0 +1,6 @@ +foo$ +m +foo +bar foo +foo bar +foo diff --git a/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt b/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt new file mode 100644 index 00000000..3e78a7ff --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt @@ -0,0 +1,3 @@ +hello +i +HELLO Hello hello hElLo diff --git a/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt b/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt new file mode 100644 index 00000000..94be27f4 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt @@ -0,0 +1,6 @@ +a.b +s +aXb +a +b a +b diff --git a/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt b/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt new file mode 100644 index 00000000..3c4f1201 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt @@ -0,0 +1,3 @@ +a{2,4} + +a aa aaa aaaa aaaaa aaaaaa diff --git a/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt b/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt new file mode 100644 index 00000000..b4c3eb6c --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt @@ -0,0 +1,6 @@ +^cat|dog$ +m +cat sees dog +dog sees cat +catnap +bulldog diff --git a/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt b/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt new file mode 100644 index 00000000..ef772ce5 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt @@ -0,0 +1,3 @@ +(a+)+b + +aaaaaaaaaab end diff --git a/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt b/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt new file mode 100644 index 00000000..4bfefaec --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt @@ -0,0 +1,3 @@ +[^aeiou]+ + +hello world diff --git a/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt b/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt new file mode 100644 index 00000000..4b71115e --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt @@ -0,0 +1,3 @@ +(?:foo|bar)+ + +foo barbar foobarfoo xxx baz diff --git a/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt b/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt new file mode 100644 index 00000000..98b3b539 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt @@ -0,0 +1,3 @@ +colou?r + +color and colour are both valid diff --git a/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt b/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt new file mode 100644 index 00000000..ed8924a6 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt @@ -0,0 +1,3 @@ +x{0,3}y + +y xy xxy xxxy xxxxy diff --git a/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt b/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt new file mode 100644 index 00000000..da21f6ba --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt @@ -0,0 +1,3 @@ +[a-zA-Z0-9_.]+@[a-zA-Z0-9.]+ + +contact me at alice@example.com or bob.test@host.io diff --git a/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt b/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt new file mode 100644 index 00000000..118ddb0f --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt @@ -0,0 +1,4 @@ +\Aabc +m +abc +abc again diff --git a/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt b/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt new file mode 100644 index 00000000..75621efa --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt @@ -0,0 +1,5 @@ +fn [A-Za-z_][A-Za-z0-9_]*\( + +pub fn main() void +fn helper(x: i32) i32 +const fn_ptr = foo diff --git a/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt b/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt new file mode 100644 index 00000000..91855391 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt @@ -0,0 +1,3 @@ +a{3,} + +a aa aaa aaaa aaaaaaaa diff --git a/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt b/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt new file mode 100644 index 00000000..ba281a7a --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt @@ -0,0 +1,3 @@ +a* + +bbb diff --git a/vendor/nanoregex/tests/parity/run.sh b/vendor/nanoregex/tests/parity/run.sh new file mode 100755 index 00000000..e308f410 --- /dev/null +++ b/vendor/nanoregex/tests/parity/run.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# parity test harness: zigregex vs python re +# +# Each fixture in $FIXTURES is a 3+ line text file: +# line 1: regex pattern (raw, no quoting) +# line 2: flags (comma-separated subset of: i, m, s — empty line if none) +# line 3+: haystack (joined with \n if multiline) +# +# For each fixture we run python3's re.findall and the zigregex_probe +# binary on the same input. They must produce byte-identical output. +# +# Until exec.zig lands, the matcher is a panic — we tolerate "matcher not +# yet implemented" on the zig side and only verify the python reference +# itself runs. Flip $REQUIRE_MATCH to 1 once the matcher is wired. + +set -uo pipefail + +PROBE="${1:-}" +FIXTURES="${2:-tests/parity/fixtures}" +REQUIRE_MATCH="${REQUIRE_MATCH:-1}" + +if [ -z "$PROBE" ] || [ ! -x "$PROBE" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +if [ ! -d "$FIXTURES" ]; then + echo "fixtures dir not found: $FIXTURES" >&2 + exit 2 +fi + +pass=0 +fail=0 +skip=0 + +for f in "$FIXTURES"/*.txt; do + [ -e "$f" ] || continue + name=$(basename "$f" .txt) + + pattern=$(sed -n '1p' "$f") + flags=$(sed -n '2p' "$f") + haystack=$(sed -n '3,$p' "$f") + + py_out=$( + PATTERN="$pattern" FLAGS="$flags" HAYSTACK="$haystack" \ + python3 - <<'PY' +import os, re, sys +pattern = os.environ["PATTERN"] +flags_str = os.environ.get("FLAGS", "") +haystack = os.environ["HAYSTACK"] +flag_bits = 0 +if "i" in flags_str: flag_bits |= re.IGNORECASE +if "m" in flags_str: flag_bits |= re.MULTILINE +if "s" in flags_str: flag_bits |= re.DOTALL +try: + pat = re.compile(pattern, flag_bits) +except re.error as e: + print(f"PARSE_ERROR: {e}") + sys.exit(0) +for m in pat.finditer(haystack): + print(f"{m.start()}..{m.end()}\t{m.group(0)}") +PY + ) + + if [ "$REQUIRE_MATCH" != "1" ]; then + # Phase 1: just verify the python reference computes a result. + # We don't yet compare against zig output because the matcher + # panics. Once exec.zig lands, set REQUIRE_MATCH=1. + skip=$((skip + 1)) + echo "SKIP $name (python ref: $(echo "$py_out" | wc -l | tr -d ' ') lines)" + continue + fi + + zig_out=$("$PROBE" "$pattern" "$haystack" "$flags" 2>&1) + if [ "$py_out" = "$zig_out" ]; then + pass=$((pass + 1)) + echo "PASS $name" + else + fail=$((fail + 1)) + echo "FAIL $name" + echo " pattern: $pattern" + echo " py: $(echo "$py_out" | head -5)" + echo " zig: $(echo "$zig_out" | head -5)" + fi +done + +echo "" +echo "parity: $pass passed, $fail failed, $skip skipped" +exit $(( fail > 0 ? 1 : 0 ))