From 62a6cefed6a97aedc26b0e7d11ff7f9396151b18 Mon Sep 17 00:00:00 2001 From: John Campion Jr Date: Thu, 27 Aug 2026 21:49:35 -0400 Subject: [PATCH 1/2] Search the scrollback without turning it into text Ghostty's most-requested feature until 1.3, and the design turns on one measurement: asking each line for a string and running IndexOf costs 9.7 ms and 16.3 MiB per search over a 10,000-line buffer, while reading codepoints out of the cells costs 3.7 ms and allocates nothing, for the same 25,325 hits. A find box searches per keystroke, so the first of those is roughly 80 MiB to type "error" -- in a library that went to some trouble to stop allocating per character. So BufferSearch walks cells. The walk moves a (row, column) cursor and compares codepoints, which also dissolves the offset-mapping problem: the position IS the cursor, and a match that crosses a wrap simply produces two runs because the cursor changed row part way through. The runs share a MatchId, the same shape as an OSC 8 link that wrapped, and stepping moves by match so a wrapped one is one stop. Logical lines, not physical: a search that stops at the right edge misses exactly the matches on long output, which are the ones worth finding. Width-0 cells are stepped over rather than compared -- the placeholder behind a wide glyph and an orphaned combining mark are not characters, and comparing them would fail a needle against text that reads exactly like it. Results follow the buffer. The ring drops lines as output arrives and a row index that is not adjusted goes stale silently -- the highlight lands somewhere else, and nothing about it looks wrong. Subscribes to Trimmed and shifts, as SelectionManager does, dropping what fell off the top. HitsOnRow answers by binary search over row-ordered hits and hands back a span in place, because the renderer asks once per row per frame and the answer is almost always nothing. The cap (10,000 runs) is announced through Truncated rather than silently applied: a count that quietly stops being true reads as a bug in the search rather than a limit on it. No regular expressions, deliberately. Regex takes a string, so supporting it means materialising the buffer as text -- the 16 MiB this design exists to avoid, paid by every search rather than the ones that asked. Substring, case-insensitive by default, whole-word as an option. One measurement trap found by this change's own test: it first asserted with GC.GetTotalAllocatedBytes, which is process-wide, and xUnit runs test classes in parallel -- so it counted 66 MB of other tests' allocations against the search and failed a design that was allocating nothing. GetAllocatedBytesForCurrentThread cannot be polluted. The bisection that found this took longer than the fix. Thirteen tests: wrap-straddling matches, whole-word across a wrap boundary, ring trimming, the cap admitting it bit, and the zero-allocation claim asserted per thread. 1191 tests pass. Co-Authored-By: Claude Opus 5 --- src/XTerm.NET.Tests/BufferSearchTests.cs | 228 +++++++++++++ src/XTerm.NET/Search/BufferSearch.cs | 400 +++++++++++++++++++++++ src/XTerm.NET/Search/SearchTypes.cs | 53 +++ 3 files changed, 681 insertions(+) create mode 100644 src/XTerm.NET.Tests/BufferSearchTests.cs create mode 100644 src/XTerm.NET/Search/BufferSearch.cs create mode 100644 src/XTerm.NET/Search/SearchTypes.cs diff --git a/src/XTerm.NET.Tests/BufferSearchTests.cs b/src/XTerm.NET.Tests/BufferSearchTests.cs new file mode 100644 index 0000000..52f7924 --- /dev/null +++ b/src/XTerm.NET.Tests/BufferSearchTests.cs @@ -0,0 +1,228 @@ +using System.Diagnostics; +using XTerm.Options; +using XTerm.Search; +using Xunit; + +namespace XTerm.Tests; + +/// +/// Searching the scrollback without turning the scrollback into text. +/// +public class BufferSearchTests +{ + private const string Esc = "\u001b"; + + private static Terminal Fresh(int cols = 20, int rows = 6, int scrollback = 200) + => new(new TerminalOptions { Cols = cols, Rows = rows, Scrollback = scrollback }); + + [Fact] + public void It_finds_a_word_and_says_where() + { + var t = Fresh(); + t.Write("hello world\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(1, search.Find("world")); + + var hit = search.HitsOnRow(t.Buffer.YBase)[0]; + Assert.Equal(6, hit.Column); + Assert.Equal(5, hit.Cols); + } + + [Fact] + public void Case_is_ignored_unless_asked_for() + { + var t = Fresh(); + t.Write("Error and error\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(2, search.Find("ERROR")); + Assert.Equal(1, search.Find("Error", new SearchOptions { CaseSensitive = true })); + } + + [Fact] + public void Whole_word_refuses_a_match_inside_a_longer_one() + { + var t = Fresh(cols: 40); + t.Write("cat concatenate cat\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(3, search.Find("cat")); + Assert.Equal(2, search.Find("cat", new SearchOptions { WholeWord = true })); + } + + /// + /// The case physical-line search misses, and the reason the walk crosses rows at all. + /// + [Fact] + public void A_match_across_a_wrap_is_found_and_comes_back_as_two_runs() + { + var t = Fresh(cols: 10); + t.Write("aaaaaaaaGREENbbb"); // wraps after 10, so GREEN straddles the boundary + + using var search = new BufferSearch(t); + Assert.Equal(2, search.Find("green")); + + var first = search.HitsOnRow(t.Buffer.YBase); + var second = search.HitsOnRow(t.Buffer.YBase + 1); + Assert.Equal(1, first.Length); + Assert.Equal(1, second.Length); + Assert.Equal(first[0].MatchId, second[0].MatchId); + Assert.Equal(5, first[0].Cols + second[0].Cols); + } + + [Fact] + public void Rows_with_nothing_on_them_come_back_empty() + { + var t = Fresh(); + t.Write("match\r\nnothing here\r\n"); + + using var search = new BufferSearch(t); + search.Find("match"); + + Assert.Empty(search.HitsOnRow(t.Buffer.YBase + 1).ToArray()); + } + + [Fact] + public void Stepping_walks_the_matches_and_wraps_round() + { + var t = Fresh(); + t.Write("a\r\na\r\na\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(3, search.Find("a")); + + var rows = new List(); + for (var i = 0; i < 4; i++) + { + Assert.True(search.TryMoveNext(out var hit)); + rows.Add(hit.BufferRow); + } + + Assert.Equal(rows[0], rows[3]); // wrapped back to the first + } + + /// A wrapped match is one stop, not two. + [Fact] + public void Stepping_counts_a_wrapped_match_once() + { + var t = Fresh(cols: 10); + t.Write("aaaaaaaaGREENbbb"); + + using var search = new BufferSearch(t); + search.Find("green"); + + Assert.True(search.TryMoveNext(out var first)); + Assert.True(search.TryMoveNext(out var second)); + Assert.Equal(first.MatchId, second.MatchId); + } + + /// + /// The results move with the buffer, or they point somewhere else while output scrolls past — + /// and nothing about a wrong row looks wrong. + /// + [Fact] + public void Results_follow_the_buffer_when_the_ring_drops_lines() + { + // Something above the needle, so the ring trims from the top without reaching it. + var t = Fresh(rows: 3, scrollback: 10); + for (var i = 0; i < 5; i++) + t.Write($"before {i}\r\n"); + t.Write("needle\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(1, search.Find("needle")); + var before = FindRow(search, t); + + // Past capacity, so the oldest lines go and every row index below them shifts up. + for (var i = 0; i < 10; i++) + t.Write($"after {i}\r\n"); + + var after = FindRow(search, t); + Assert.Equal(1, search.Count); + Assert.True(after < before, + $"the row should have moved up with the line: was {before}, now {after}"); + Assert.Equal("needle", RowText(t, after)); + } + + [Fact] + public void A_result_scrolled_out_of_the_scrollback_is_dropped() + { + var t = Fresh(rows: 3, scrollback: 3); + t.Write("needle\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(1, search.Find("needle")); + + for (var i = 0; i < 40; i++) + t.Write($"filler {i}\r\n"); + + Assert.Equal(0, search.Count); + } + + /// + /// The cap says when it bit. A count that quietly stops being true reads as a bug in the search. + /// + [Fact] + public void An_enormous_result_is_capped_and_admits_it() + { + var t = Fresh(cols: 80, rows: 10, scrollback: 5000); + for (var i = 0; i < 3000; i++) + t.Write(new string('a', 79) + "\r\n"); + + using var search = new BufferSearch(t); + var count = search.Find("a"); + + Assert.True(search.Truncated, "far more matches than the cap"); + Assert.Equal(BufferSearch.MaxHits, count); + } + + [Fact] + public void An_empty_needle_finds_nothing() + { + var t = Fresh(); + t.Write("anything\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(0, search.Find("")); + Assert.False(search.TryMoveNext(out _)); + } + + /// What the whole design is for: a search allocates nothing worth counting. + [Fact] + public void Searching_does_not_allocate_per_line() + { + var t = Fresh(cols: 240, rows: 50, scrollback: 4000); + for (var i = 0; i < 4000; i++) + t.Write("compiling module target cache resolved warning linking\r\n"); + + using var search = new BufferSearch(t); + search.Find("zzz"); // warm, and matching nothing so no hits are stored + + // THIS thread's allocations, not the process's. xUnit runs test classes in parallel, so + // the process-wide counter picks up whatever the suite happens to be doing on other threads + // -- measured at 66 MB once, none of it from here. A per-thread counter cannot be polluted. + var before = GC.GetAllocatedBytesForCurrentThread(); + search.Find("zzz"); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(allocated < 4096, + $"a search over 4,000 lines allocated {allocated:N0} bytes; the string-per-line version cost megabytes"); + } + + private static string RowText(Terminal t, int row) + { + var line = t.Buffer.Lines[row]!; + return string.Concat(Enumerable.Range(0, line.Length).Select(c => line[c].Content)).TrimEnd(); + } + + private static int FindRow(BufferSearch search, Terminal t) + { + for (var i = 0; i < t.Buffer.Lines.Length; i++) + { + if (search.HitsOnRow(i).Length > 0) + return i; + } + return -1; + } +} diff --git a/src/XTerm.NET/Search/BufferSearch.cs b/src/XTerm.NET/Search/BufferSearch.cs new file mode 100644 index 0000000..05ea0a6 --- /dev/null +++ b/src/XTerm.NET/Search/BufferSearch.cs @@ -0,0 +1,400 @@ +using XTerm.Buffer; + +namespace XTerm.Search; + +/// +/// Finds text in the scrollback, without turning the scrollback into text. +/// +/// +/// The obvious implementation asks each line for a string and runs IndexOf over it. +/// Measured on a 10,000-line buffer at 240 columns, that costs 9.7 ms and 16.3 MiB per search; +/// reading codepoints straight out of the cells costs 3.7 ms and allocates nothing, for the same +/// 25,325 hits. A find box searches on every keystroke, so the first of those is roughly 80 MiB to +/// type "error" — in a library that went to some trouble to stop allocating per character. +/// +/// So nothing here builds a string. The walk moves a (row, column) cursor through the buffer +/// and compares codepoints, which also removes the step where a character offset has to be mapped +/// back onto cells: the position IS the cursor, and a match that crosses a wrap simply produces two +/// runs because the cursor changed row part way through. +/// +/// Call this on the thread that owns the terminal. It reads the buffer directly and the +/// emulator is not thread-safe, so a search running beside a write is a race. At 3.7 ms per search +/// that is affordable inline; a much larger scrollback wants debouncing rather than a thread. +/// +public sealed class BufferSearch : IDisposable +{ + /// + /// The most matches kept. + /// + /// + /// A single-letter search over a long scrollback matches almost everything, and every match kept + /// is a struct in a list. The cap bounds that — and says when it bit, + /// because a count that quietly stops being true reads as a bug in the search rather than a + /// limit on it. + /// + public const int MaxHits = 10_000; + + private readonly Terminal _terminal; + private readonly List _hits = new(); + + private string _needle = string.Empty; + private SearchOptions _options; + private int _current = -1; + private bool _disposed; + + public BufferSearch(Terminal terminal) + { + _terminal = terminal ?? throw new ArgumentNullException(nameof(terminal)); + _terminal.Buffer.Trimmed += OnTrimmed; + } + + /// How many runs are held. See . + public int Count => _hits.Count; + + /// Whether the cap was reached and matches beyond it were not kept. + public bool Truncated { get; private set; } + + /// Index of the current match within , or -1 before one is chosen. + public int CurrentIndex => _current; + + /// The term last searched for. + public string Needle => _needle; + + /// + /// Searches the whole buffer, replacing any previous result. + /// + /// The number of runs found. + public int Find(string needle, SearchOptions options = default) + { + _hits.Clear(); + _current = -1; + Truncated = false; + _needle = needle ?? string.Empty; + _options = options; + + if (_needle.Length == 0) + return 0; + + var lines = _terminal.Buffer.Lines; + var matchId = 0; + + // Logical lines, not physical ones: a match on long output usually straddles a wrap, and + // those are the matches worth finding. A run starts at any line not flagged IsWrapped. + for (var row = 0; row < lines.Length; row++) + { + if (lines[row] is null) + continue; + + if (row > 0 && lines[row]!.IsWrapped) + continue; // a continuation; it was searched as part of the run above + + var end = row; + while (end + 1 < lines.Length && lines[end + 1] is { IsWrapped: true }) + end++; + + SearchRun(lines, row, end, ref matchId); + + if (Truncated) + break; + } + + return _hits.Count; + } + + /// + /// The runs on one row, or empty for the rows that have none — which is nearly all of them. + /// + /// + /// A span rather than a list because this is asked once per row per frame, and the answer is + /// usually nothing. Hits are produced in row order, so the row's block is found by binary search + /// and handed back in place. + /// + public ReadOnlySpan HitsOnRow(int bufferRow) + { + var all = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_hits); + + var lo = 0; + var hi = all.Length - 1; + var found = -1; + + while (lo <= hi) + { + var mid = (lo + hi) / 2; + if (all[mid].BufferRow < bufferRow) lo = mid + 1; + else if (all[mid].BufferRow > bufferRow) hi = mid - 1; + else { found = mid; hi = mid - 1; } + } + + if (found < 0) + return ReadOnlySpan.Empty; + + var length = 0; + while (found + length < all.Length && all[found + length].BufferRow == bufferRow) + length++; + + return all.Slice(found, length); + } + + /// Steps to the next match, wrapping round at the end. + public bool TryMoveNext(out SearchHit hit) => TryMove(1, out hit); + + /// Steps to the previous match, wrapping round at the start. + public bool TryMovePrevious(out SearchHit hit) => TryMove(-1, out hit); + + /// Forgets the results, keeping the search usable. + public void Clear() + { + _hits.Clear(); + _current = -1; + _needle = string.Empty; + Truncated = false; + } + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + _terminal.Buffer.Trimmed -= OnTrimmed; + } + + // ---- the walk ------------------------------------------------------------------------ + + private void SearchRun(CircularList lines, int startRow, int endRow, ref int matchId) + { + for (var row = startRow; row <= endRow; row++) + { + var line = lines[row]; + if (line is null) + continue; + + for (var col = 0; col < line.Length; col++) + { + if (!Matches(lines, endRow, row, col, out var afterRow, out var afterCol)) + continue; + + if (_options.WholeWord && !IsWholeWord(lines, startRow, endRow, row, col, afterRow, afterCol)) + continue; + + AddRuns(lines, matchId++, row, col, afterRow, afterCol); + + if (_hits.Count >= MaxHits) + { + Truncated = true; + return; + } + + // Matches do not overlap. Step past this one, which may have ended on a later row. + if (afterRow != row) + { + row = afterRow; + line = lines[row]; + if (line is null) + return; + col = afterCol - 1; + } + else + { + col = afterCol - 1; + } + } + } + } + + /// + /// Whether the needle sits at (row, col), and where it ends if so. + /// + /// + /// Width-0 cells are stepped over rather than compared. There are two kinds and neither is a + /// character of its own: the placeholder behind a wide glyph, and a combining mark that found + /// nothing to attach to. Comparing either would make a needle fail to match text that reads + /// exactly like it. + /// + private bool Matches(CircularList lines, int endRow, int row, int col, + out int afterRow, out int afterCol) + { + afterRow = row; + afterCol = col; + + var r = row; + var c = col; + + for (var i = 0; i < _needle.Length; i++) + { + if (!Advance(lines, endRow, ref r, ref c, skipZeroWidth: i > 0)) + return false; + + var cell = lines[r]![c]; + if (cell.Width == 0) + return false; + + if (!SameCharacter(cell.CodePoint, _needle[i])) + return false; + + afterRow = r; + afterCol = c + 1; + c++; + } + + return true; + } + + /// + /// Puts the cursor on the next cell that carries a character, crossing a wrap if it has to. + /// + private static bool Advance(CircularList lines, int endRow, ref int row, ref int col, + bool skipZeroWidth) + { + while (true) + { + var line = lines[row]; + if (line is null) + return false; + + if (col >= line.Length) + { + if (row >= endRow) + return false; + + row++; + col = 0; + continue; + } + + if (skipZeroWidth && lines[row]![col].Width == 0) + { + col++; + continue; + } + + return true; + } + } + + private bool SameCharacter(int codePoint, char needle) + { + if (codePoint > 0xFFFF) + return false; // outside the BMP; a needle is UTF-16 and cannot name one in a char + + var cell = (char)codePoint; + return _options.CaseSensitive + ? cell == needle + : char.ToUpperInvariant(cell) == char.ToUpperInvariant(needle); + } + + /// + /// Splits a match into one run per row and records them under one id. + /// + private void AddRuns(CircularList lines, int matchId, + int startRow, int startCol, int endRow, int endCol) + { + for (var row = startRow; row <= endRow; row++) + { + var from = row == startRow ? startCol : 0; + var to = row == endRow ? endCol : (lines[row]?.Length ?? from); + + if (to > from) + _hits.Add(new SearchHit(row, from, to - from, matchId)); + } + } + + private bool IsWholeWord(CircularList lines, int startRow, int endRow, + int row, int col, int afterRow, int afterCol) + { + return !IsWordCharacterBefore(lines, startRow, row, col) + && !IsWordCharacterAt(lines, endRow, afterRow, afterCol); + } + + private static bool IsWordCharacterBefore(CircularList lines, int startRow, int row, int col) + { + var r = row; + var c = col - 1; + + while (c < 0) + { + if (r <= startRow) + return false; // start of the logical line; nothing before it + + r--; + c = (lines[r]?.Length ?? 0) - 1; + } + + return c >= 0 && IsWordCharacter(lines[r]?[c].CodePoint ?? 0); + } + + private static bool IsWordCharacterAt(CircularList lines, int endRow, int row, int col) + { + var r = row; + var c = col; + + if (!Advance(lines, endRow, ref r, ref c, skipZeroWidth: false)) + return false; // end of the logical line + + return IsWordCharacter(lines[r]?[c].CodePoint ?? 0); + } + + private static bool IsWordCharacter(int codePoint) + => codePoint == '_' + || (codePoint <= 0xFFFF && char.IsLetterOrDigit((char)codePoint)); + + private bool TryMove(int direction, out SearchHit hit) + { + hit = default; + if (_hits.Count == 0) + return false; + + // Step by MATCH rather than by run, so a match that wrapped is one stop rather than two. + var startId = _current >= 0 ? _hits[_current].MatchId : (direction > 0 ? -1 : int.MaxValue); + var best = -1; + + for (var i = 0; i < _hits.Count; i++) + { + var id = _hits[i].MatchId; + if (direction > 0 ? id > startId : id < startId) + { + best = i; + if (direction > 0) + break; + } + } + + if (best < 0) + best = direction > 0 ? 0 : _hits.Count - 1; // wrap round + + _current = best; + hit = _hits[best]; + return true; + } + + /// + /// Shifts every result up as the ring drops lines off the top, and drops what fell off with them. + /// + /// + /// The same arrangement SelectionManager uses, and for the same reason: a row index that + /// is not adjusted goes stale silently. Nothing about a wrong row looks wrong — the highlight + /// simply lands somewhere else while a build scrolls past. + /// + private void OnTrimmed(int count) + { + if (count <= 0 || _hits.Count == 0) + return; + + var kept = 0; + for (var i = 0; i < _hits.Count; i++) + { + var hit = _hits[i]; + var row = hit.BufferRow - count; + if (row < 0) + continue; + + _hits[kept++] = new SearchHit(row, hit.Column, hit.Cols, hit.MatchId); + } + + var dropped = _hits.Count - kept; + _hits.RemoveRange(kept, dropped); + + if (_current >= 0) + _current = Math.Max(-1, _current - dropped); + } +} diff --git a/src/XTerm.NET/Search/SearchTypes.cs b/src/XTerm.NET/Search/SearchTypes.cs new file mode 100644 index 0000000..aaed78a --- /dev/null +++ b/src/XTerm.NET/Search/SearchTypes.cs @@ -0,0 +1,53 @@ +namespace XTerm.Search; + +/// +/// One contiguous run of matched cells on one row. +/// +/// +/// A match that straddles a wrap is two of these sharing a — the same shape as +/// an OSC 8 link that wrapped, and for the same reason: the thing is one match, but the screen only +/// ever draws it a row at a time. +/// +public readonly struct SearchHit +{ + /// Absolute row in the buffer, so it stays meaningful while the viewport moves. + public readonly int BufferRow; + + /// First column of the run. + public readonly int Column; + + /// How many columns it covers. + public readonly int Cols; + + /// Which match this run belongs to. Equal for the halves of a wrapped match. + public readonly int MatchId; + + public SearchHit(int bufferRow, int column, int cols, int matchId) + { + BufferRow = bufferRow; + Column = column; + Cols = cols; + MatchId = matchId; + } + + /// One past the last column covered. + public int EndColumn => Column + Cols; + + public override string ToString() => $"{BufferRow}:{Column}+{Cols}#{MatchId}"; +} + +/// How to match. +/// +/// No regular expressions, deliberately. Regex takes a string, so supporting it would mean +/// materialising the buffer as text — which is the 16 MiB per search this whole design exists to +/// avoid, and it would be paid by every search rather than only the ones that asked for it. If it is +/// ever added it should materialise one logical line at a time, not the scrollback. +/// +public readonly struct SearchOptions +{ + /// Match case exactly. Off by default, which is what a find box does. + public bool CaseSensitive { get; init; } + + /// Require a non-word character either side of the match. + public bool WholeWord { get; init; } +} From 760fdec0ada2aa1391ee175fa83d68c01b2d6d4e Mon Sep 17 00:00:00 2001 From: John Campion Jr Date: Thu, 27 Aug 2026 23:29:26 -0400 Subject: [PATCH 2/2] Fix the two live findings from Copilot's review The third finding -- SplitLinksOver appending the right fragment -- was fixed in b1fcc00 before this review posted; Copilot reviewed the pre-merge head. The needle is now codepoints, not chars. A cell stores a codepoint, and comparing it to UTF-16 chars one at a time can never match anything outside the BMP: an emoji in the needle is two chars and neither equals the cell. The needle folds once per Find; astral case pairs are left exact, which is honest rather than lossy. A cell holding a multi-codepoint cluster is still compared by its leading codepoint, and the remarks now say why: matching full cluster text means materialising it per cell, the same cost this class exists to avoid and the same reason it offers no regular expressions. The limitation is one-directional -- base-character searches still find cells carrying combining marks. And the whole-word left scan steps past width-0 cells. The placeholder behind a wide glyph is not a character, and treating it as the neighbour let a whole-word match sit flush against a CJK letter. Two regression tests. 1197 pass. Co-Authored-By: Claude Opus 5 --- src/XTerm.NET.Tests/BufferSearchTests.cs | 28 +++++++++ src/XTerm.NET/Search/BufferSearch.cs | 73 ++++++++++++++++++------ 2 files changed, 85 insertions(+), 16 deletions(-) diff --git a/src/XTerm.NET.Tests/BufferSearchTests.cs b/src/XTerm.NET.Tests/BufferSearchTests.cs index 52f7924..95c13bc 100644 --- a/src/XTerm.NET.Tests/BufferSearchTests.cs +++ b/src/XTerm.NET.Tests/BufferSearchTests.cs @@ -188,6 +188,34 @@ public void An_empty_needle_finds_nothing() Assert.False(search.TryMoveNext(out _)); } + /// A needle outside the BMP is one codepoint in a cell, not two chars. + [Fact] + public void An_emoji_needle_matches_the_cell_that_holds_it() + { + var t = Fresh(); + t.Write("go \U0001F600 now\r\n"); + + using var search = new BufferSearch(t); + Assert.Equal(1, search.Find("\U0001F600")); + Assert.Equal(1, search.Find("go \U0001F600 now")); + } + + /// + /// The placeholder behind a wide glyph is not a character, so a whole-word match must not sit + /// flush against the CJK letter that owns it. + /// + [Fact] + public void Whole_word_sees_through_a_wide_glyphs_placeholder() + { + var t = Fresh(cols: 30); + t.Write("\u6F22word \u6F22 word\r\n"); // CJK+word joined, then separated + + using var search = new BufferSearch(t); + var hits = search.Find("word", new SearchOptions { WholeWord = true }); + + Assert.Equal(1, hits); // only the separated one + } + /// What the whole design is for: a search allocates nothing worth counting. [Fact] public void Searching_does_not_allocate_per_line() diff --git a/src/XTerm.NET/Search/BufferSearch.cs b/src/XTerm.NET/Search/BufferSearch.cs index 05ea0a6..45d4a3c 100644 --- a/src/XTerm.NET/Search/BufferSearch.cs +++ b/src/XTerm.NET/Search/BufferSearch.cs @@ -38,6 +38,20 @@ public sealed class BufferSearch : IDisposable private readonly List _hits = new(); private string _needle = string.Empty; + + /// + /// The needle as CODEPOINTS, folded for case. A cell stores a codepoint, and comparing it to + /// UTF-16 chars one at a time can never match anything outside the BMP -- an emoji in the + /// needle is two chars and neither equals the cell. Built once per Find. + /// + /// + /// A cell whose text is a multi-codepoint cluster is compared by its LEADING codepoint, which + /// is all the cell stores inline. Matching the full cluster text would mean materialising it + /// per cell -- the same cost this class exists to avoid, and the same reason regular + /// expressions are not offered. The limitation is one-directional: base-character searches + /// still find cells that carry combining marks. + /// + private int[] _pattern = Array.Empty(); private SearchOptions _options; private int _current = -1; private bool _disposed; @@ -75,6 +89,8 @@ public int Find(string needle, SearchOptions options = default) if (_needle.Length == 0) return 0; + _pattern = ToFoldedCodePoints(_needle, options.CaseSensitive); + var lines = _terminal.Buffer.Lines; var matchId = 0; @@ -220,7 +236,7 @@ private bool Matches(CircularList lines, int endRow, int row, int co var r = row; var c = col; - for (var i = 0; i < _needle.Length; i++) + for (var i = 0; i < _pattern.Length; i++) { if (!Advance(lines, endRow, ref r, ref c, skipZeroWidth: i > 0)) return false; @@ -229,7 +245,7 @@ private bool Matches(CircularList lines, int endRow, int row, int co if (cell.Width == 0) return false; - if (!SameCharacter(cell.CodePoint, _needle[i])) + if (Fold(cell.CodePoint, _options.CaseSensitive) != _pattern[i]) return false; afterRow = r; @@ -272,15 +288,28 @@ private static bool Advance(CircularList lines, int endRow, ref int } } - private bool SameCharacter(int codePoint, char needle) + private static int[] ToFoldedCodePoints(string needle, bool caseSensitive) + { + var points = new List(needle.Length); + for (var i = 0; i < needle.Length; i++) + { + int cp = needle[i]; + if (char.IsHighSurrogate(needle[i]) && i + 1 < needle.Length && char.IsLowSurrogate(needle[i + 1])) + { + cp = char.ConvertToUtf32(needle[i], needle[i + 1]); + i++; + } + points.Add(Fold(cp, caseSensitive)); + } + return points.ToArray(); + } + + private static int Fold(int codePoint, bool caseSensitive) { - if (codePoint > 0xFFFF) - return false; // outside the BMP; a needle is UTF-16 and cannot name one in a char + if (caseSensitive || codePoint > 0xFFFF) + return codePoint; // astral case pairs are rare enough that exact match is honest - var cell = (char)codePoint; - return _options.CaseSensitive - ? cell == needle - : char.ToUpperInvariant(cell) == char.ToUpperInvariant(needle); + return char.ToUpperInvariant((char)codePoint); } /// @@ -311,16 +340,28 @@ private static bool IsWordCharacterBefore(CircularList lines, int st var r = row; var c = col - 1; - while (c < 0) + while (true) { - if (r <= startRow) - return false; // start of the logical line; nothing before it + while (c < 0) + { + if (r <= startRow) + return false; // start of the logical line; nothing before it - r--; - c = (lines[r]?.Length ?? 0) - 1; - } + r--; + c = (lines[r]?.Length ?? 0) - 1; + } - return c >= 0 && IsWordCharacter(lines[r]?[c].CodePoint ?? 0); + // A width-0 cell is not a character -- it is the placeholder behind a wide glyph, and + // treating it as the neighbour would let a whole-word match sit flush against a CJK + // letter. Step past it to the glyph that owns it. + if ((lines[r]?[c].Width ?? 1) == 0) + { + c--; + continue; + } + + return IsWordCharacter(lines[r]?[c].CodePoint ?? 0); + } } private static bool IsWordCharacterAt(CircularList lines, int endRow, int row, int col)