Guard the perf work in CI, exactly where it can be exact

.github/workflows/PerfCompare.yml

@@ -0,0 +1,105 @@
+name: PerfCompare
+
+# Measures this branch against the commit it forked from, on one machine, alternating.
+#
+# Allocation is gated exactly and time is gated against the noise the job observes in itself --
+# see src/XTerm.NET.Bench/ComparePr.cs for why those two are held to different standards.
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - src/**
+  workflow_dispatch:
+    inputs:
+      chars:
+        description: >-
+          Work budget per corpus per run, divided by that corpus's known relative cost so each is
+          measured for about the same length of time. More is slower and quieter.
+        default: "100000000"
+      runs:
+        description: Runs of each side.
+        default: "3"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  compare:
+    runs-on: ubuntu-latest
+
+    env:
+      CHARS: ${{ inputs.chars || '100000000' }}
+      RUNS: ${{ inputs.runs || '3' }}
+      OUT: src/XTerm.NET.Bench/bin/Release/net10.0
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: 10.0.x
+
+    # The harness comes from THIS branch and is used against both libraries, so the two sides are
+    # measured by identical code. That is only safe because the ci mode touches nothing but
+    # Terminal, TerminalOptions and Write(string) -- anything newer would fail against the base.
+    - name: Build the harness and this branch's library
+      run: |
+        dotnet build src/XTerm.NET.Bench -c Release --property WarningLevel=0
+        mkdir -p /tmp/libs
+        cp "$OUT/XTerm.NET.dll" /tmp/libs/head.dll
+
+    - name: Build the base library
+      run: |
+        BASE="${{ github.event.pull_request.base.sha || 'HEAD~1' }}"
+        echo "base commit: $BASE"
+        git worktree add /tmp/base "$BASE"
+        dotnet build /tmp/base/src/XTerm.NET -c Release --property WarningLevel=0
+        cp /tmp/base/src/XTerm.NET/bin/Release/net10.0/XTerm.NET.dll /tmp/libs/base.dll
+
+    # Alternating, so a runner that gets slower or faster part way through does so to both sides
+    # rather than to one. Run through `dotnet <dll>` rather than `dotnet run`, which would rebuild
+    # and put the branch's own library straight back over the one being measured.
+    - name: Measure
+      run: |
+        for r in $(seq 1 "$RUNS"); do
+          for side in base head; do
+            cp "/tmp/libs/$side.dll" "$OUT/XTerm.NET.dll"
+            dotnet "$OUT/XTerm.NET.Bench.dll" ci \
+              --chars "$CHARS" --warm-chars $((CHARS / 4)) \
+              --out "/tmp/$side-$r.json"
+          done
+        done
+
+    - name: Compare
+      id: compare
+      run: |
+        cp /tmp/libs/head.dll "$OUT/XTerm.NET.dll"
+        set +e
+        dotnet "$OUT/XTerm.NET.Bench.dll" compare \
+          --base $(ls /tmp/base-*.json) \
+          --head $(ls /tmp/head-*.json) \
+          --out /tmp/report.md
+        echo "status=$?" >> "$GITHUB_OUTPUT"
+        set -e
+        cat /tmp/report.md >> "$GITHUB_STEP_SUMMARY"
+
+    # Best effort. A pull request from a fork gets a read-only token, so this cannot post -- the job
+    # summary above is the copy that always exists.
+    - name: Comment on the pull request
+      if: github.event_name == 'pull_request'
+      continue-on-error: true
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        gh pr comment "${{ github.event.pull_request.number }}" --edit-last --body-file /tmp/report.md \
+          || gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/report.md
+
+    - name: Fail on a regression
+      if: steps.compare.outputs.status != '0'
+      run: |
+        echo "::error::Perf comparison reported a regression. See the job summary."
+        exit 1

docs/perf-results.md

@@ -142,3 +142,55 @@ which is what the `flood`, `unicode`, `width` and `layout` probe modes exist for
   after this port was taken.
 - Kitty graphics and Sixel — absent. The parser recognises APC and DCS and discards
   the payload.
+
+## Keeping it
+
+Two things guard this in CI, held to deliberately different standards.
+
+**The struct layout is a unit test.** `BufferCell` holds no managed references and is 24 bytes.
+Neither is a measurement — they hold or they do not — so they run in the ordinary test job and cost
+nothing. The reference one is the load-bearing guard: a `string` field added back to the cell would
+undo the largest single win here at a stroke, and nothing else in the suite would notice.
+
+**Throughput is a comparison, not a threshold.** `PerfCompare.yml` builds the branch and the commit
+it forked from, then runs one harness against both, alternating. Allocation per character is gated
+exactly, because bytes allocated for a fixed amount of work is a *count*: it does not care what else
+the machine is doing. Time is gated against the spread the job just observed in itself.
+
+That last part is not a preference, it is what the calibration showed. The same build compared
+against itself, on a quiet laptop:
+
+| work per corpus | apparent Δ on scroll-ascii | spread |
+|---|---|---|
+| 60M chars | **+27%** | ±30% |
+| 300M chars | +0.2% | ±1% |
+
+A fixed threshold would have had to sit above 30% to survive the first row, which is far too loose to
+catch anything real — and the first row is also where `scroll-ascii` read 3.9 ns/char against the
+1.7 it actually runs at, because the work was too short to finish warming. So the gate is
+`max(5%, 3 × observed spread)`: a quiet machine earns a tight gate, a busy one raises its own bar
+rather than crying wolf, and anything between the floor and the gate is reported as worth a look
+instead of vanishing.
+
+Each corpus is measured for about the same length of *time*, not over the same number of characters.
+Equal characters sounds fairer and is not: `flood` costs some 28× per character what `scroll-ascii`
+does, so an equal-character budget measures the fast corpora for a twenty-eighth as long and hands
+them all the noise. The first run on a GitHub runner showed exactly that — `scroll-ascii` came back
+at ±16% against ±1–4% for everything else, putting its gate at 49%, which is no gate at all. Dividing
+the budget by a fixed per-corpus cost brought every spread to ±1–3%, and cut the run to a third of
+the time.
+
+Checked against a regression rather than assumed to work: removing the `_placeholderCell` guard from
+`Print` — a real 12% found by hand while merging Kitty — was flagged at +11.2% against a 7.0% gate,
+with the other five corpora silent.
+
+The harness deliberately touches only `Terminal`, `TerminalOptions` and `Write(string)`. That is what
+lets one build of it measure an older library by assembly substitution; anything newer would fail at
+run time and the job could then only ever compare a build against itself. It reports the module
+version id of what it loaded for the same reason — two runs of the same assembly would otherwise
+report a flawless result and mean nothing.
+
+```
+dotnet run --project src/XTerm.NET.Bench -c Release -- ci --out head.json
+dotnet run --project src/XTerm.NET.Bench -c Release -- compare --base a.json b.json --head c.json d.json
+```

src/XTerm.NET.Bench/CiProbe.cs

@@ -0,0 +1,178 @@
+using System.Diagnostics;
+using System.Text.Json;
+using XTerm;
+using XTerm.Options;
+
+namespace XTerm.Bench;
+
+/// <summary>
+/// One measured run of every corpus, as JSON, for a CI job to compare against another run.
+///
+/// <para>Fixed WORK rather than fixed time, unlike <c>alloc</c>. A time-boxed loop measures a
+/// different amount of work on every machine, which makes two runs incomparable on anything but the
+/// derived rates — and the rate is the noisy part. A fixed pass count means the allocation total is
+/// the same measurement on both sides, and the run takes a bounded, predictable time.</para>
+///
+/// <para>This mode deliberately touches only <see cref="Terminal"/>, <see cref="TerminalOptions"/>
+/// and <c>Write(string)</c>. That is what lets a CI job run THIS harness against an OLDER build of
+/// the library by dropping its assembly in: anything newer would fail at run time, and then the
+/// comparison could only ever be same-version.</para>
+/// </summary>
+public static class CiProbe
+{
+    private const int Cols = 240;
+    private const int Rows = 67;
+
+    /// <summary>
+    /// Roughly what each corpus costs per character, relative to <c>unicode</c>.
+    /// </summary>
+    /// <remarks>
+    /// <para>The budget is divided by these, so every corpus is measured for about the same LENGTH
+    /// OF TIME rather than over the same number of characters. Equal characters sounds fairer and is
+    /// not: <c>flood</c> costs about 28x what <c>scroll-ascii</c> does, so an equal-character budget
+    /// measures the fast corpora for a twenty-eighth as long and hands them all the noise. Observed
+    /// on a GitHub runner, <c>scroll-ascii</c> came back with a ±16% spread against ±1-4% for
+    /// everything else, which put its gate at 49% -- no gate at all.</para>
+    /// <para>Constants, and deliberately not measured at run time: both sides of a comparison must
+    /// do identical work, and a figure derived from a warm-up would differ between them. Being wrong
+    /// only makes the run uneven, never incorrect -- every number is reported per character.</para>
+    /// </remarks>
+    private static readonly Dictionary<string, double> RelativeCost = new()
+    {
+        ["scroll-ascii"] = 0.11,
+        ["sgr-churn"] = 0.30,
+        ["truecolor"] = 0.32,
+        ["alt-redraw"] = 0.40,
+        ["unicode"] = 1.00,
+        ["flood"] = 2.96,
+    };
+
+    public static int Run(string outputPath, long targetChars, long warmChars)
+    {
+        var results = new List<CorpusResult>();
+
+        foreach (var spec in CorpusGenerator.Specs)
+            results.Add(Measure(spec.Name, targetChars, warmChars));
+
+        var report = new Report
+        {
+            Runtime = Environment.Version.ToString(),
+            Library = LibraryVersion(),
+            TargetChars = targetChars,
+            Corpora = results
+        };
+
+        var json = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true });
+        File.WriteAllText(outputPath, json);
+
+        Console.WriteLine($"{"corpus",-14} {"ns/char",9} {"bytes/char",11} {"gen0/Mchar",11}");
+        Console.WriteLine(new string('-', 50));
+        foreach (var r in results)
+            Console.WriteLine($"{r.Name,-14} {r.NsPerChar,9:N2} {r.BytesPerChar,11:N2} {r.Gen0PerMchar,11:N2}");
+        Console.WriteLine();
+        Console.WriteLine($"library under test: {report.Library}");
+        Console.WriteLine($"written to {outputPath}");
+        return 0;
+    }
+
+    private static CorpusResult Measure(string corpus, long targetChars, long warmChars)
+    {
+        var (chunks, chars) = Load(corpus);
+        var terminal = new Terminal(new TerminalOptions { Cols = Cols, Rows = Rows });
+
+        // Passes come from a time budget divided by the corpus's known relative cost, so each is
+        // measured for about as long as the others. Still fixed work: the corpus is generated from a
+        // fixed seed and the cost is a constant, so both sides of a comparison run exactly the same
+        // number of passes over exactly the same bytes.
+        var cost = RelativeCost.TryGetValue(corpus, out var known) ? known : 1.0;
+        var passes = (int)Math.Max(1, targetChars / cost / Math.Max(1, chars));
+        var warmup = (int)Math.Max(1, warmChars / cost / Math.Max(1, chars));
+
+        // Warm to let tiered compilation promote the hot methods. Measuring before that measures the
+        // JIT, which is how warming for a fixed count rather than to convergence produced a number
+        // four times off earlier in this project's history.
+        for (var i = 0; i < warmup; i++)
+            foreach (var c in chunks) terminal.Write(c);
+
+        // Collect first, so nothing from the warm-up is counted against the measured passes.
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+
+        var beforeAlloc = GC.GetTotalAllocatedBytes(precise: true);
+        var beforeGen0 = GC.CollectionCount(0);
+        var sw = Stopwatch.StartNew();
+
+        for (var i = 0; i < passes; i++)
+            foreach (var c in chunks) terminal.Write(c);
+
+        sw.Stop();
+        var allocated = GC.GetTotalAllocatedBytes(precise: true) - beforeAlloc;
+        var gen0 = GC.CollectionCount(0) - beforeGen0;
+
+        long charsDone = (long)chars * passes;
+
+        return new CorpusResult
+        {
+            Name = corpus,
+            Chars = charsDone,
+            AllocatedBytes = allocated,
+            Gen0 = gen0,
+            BytesPerChar = (double)allocated / charsDone,
+            Gen0PerMchar = gen0 / (charsDone / 1_000_000.0),
+            NsPerChar = sw.Elapsed.TotalMilliseconds * 1_000_000.0 / charsDone,
+            MibPerSec = charsDone / 1024.0 / 1024.0 / sw.Elapsed.TotalSeconds,
+            Passes = passes
+        };
+    }
+
+    private static (string[] Chunks, int Chars) Load(string corpus)
+    {
+        var dir = Path.Combine(AppContext.BaseDirectory, "corpus");
+        CorpusGenerator.GenerateAll(dir, targetBytes: 400_000, cols: Cols, rows: Rows);
+
+        var text = File.ReadAllText(Path.Combine(dir, corpus + ".vt"));
+        var chunks = new List<string>();
+        for (var i = 0; i < text.Length; i += 4096)
+            chunks.Add(text.Substring(i, Math.Min(4096, text.Length - i)));
+        return (chunks.ToArray(), text.Length);
+    }
+
+    /// <summary>
+    /// Which XTerm.NET actually got loaded, identified by its module version id.
+    /// </summary>
+    /// <remarks>
+    /// The MVID rather than the path or the version, because a CI job compares two builds by
+    /// swapping the assembly into one output directory -- so the path is identical by design and the
+    /// version usually is too. The MVID is regenerated by every compilation, so it is the one field
+    /// that actually answers "are these two different builds". A comparison that measured the same
+    /// build twice would otherwise report a flawless result and mean nothing at all.
+    /// </remarks>
+    private static string LibraryVersion()
+    {
+        var asm = typeof(Terminal).Assembly;
+        var name = asm.GetName();
+        return $"{name.Name} {name.Version} mvid:{asm.ManifestModule.ModuleVersionId}";
+    }
+}
+
+public sealed class Report
+{
+    public string Runtime { get; set; } = "";
+    public string Library { get; set; } = "";
+    public long TargetChars { get; set; }
+    public List<CorpusResult> Corpora { get; set; } = new();
+}
+
+public sealed class CorpusResult
+{
+    public string Name { get; set; } = "";
+    public int Passes { get; set; }
+    public long Chars { get; set; }
+    public long AllocatedBytes { get; set; }
+    public int Gen0 { get; set; }
+    public double BytesPerChar { get; set; }
+    public double Gen0PerMchar { get; set; }
+    public double NsPerChar { get; set; }
+    public double MibPerSec { get; set; }
+}