fix(search): split search terms into words at every separator as 7.x did

The query builders strip only their engine's operator characters from a
search term, where 7.x kept letters, numbers, underscores and whitespace
and turned everything else into a word break. On PostgreSQL the leftover
punctuation reaches websearch_to_tsquery(): `foo/bar` parses as a single
file-path token that no document's tsvector (built from the text with
non-word characters replaced) can contain, so the search matched nothing,
and `foo,bar` (also `;`, `?`, `#`, `%`, `=`, `:`) becomes the phrase
`foo <-> bar` instead of `foo | bar`. `foo_bar` is unchanged: 7.x kept
the underscore as well.

The database builders now prepare the term with the 7.x whitelist before
the query builder formats it, so PostgreSQL ORs the words again. MariaDB
and MySQL use the same preparation, which makes their bound term
identical to 7.x instead of relying on InnoDB's tokenizer to discard the
punctuation the builder leaves in (the case 7.x's whitelist was added
for). SQLite already prepares terms with the 7.x whitelist. The term is
still only ever bound, never written into the statement, and the fix
does not wait for a query library release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: abnegate

CHANGELOG.md

@@ -606,6 +606,10 @@ not change anything for an upgrade from 7.x.
     `Exception\Query`.
   - A vector search ordered by a joined attribute pages with a cursor on PostgreSQL.
   - On SQLite, `count()` and `sum()` next to a fulltext `search()` apply the search instead of throwing.
+  - A fulltext `search()` or `notSearch()` term splits into words at every character other than a letter, a number,
+    an underscore or whitespace, as in 7.x. On PostgreSQL `search('title', 'foo/bar')` matched nothing and
+    `search('title', 'foo,bar')` (or `;`, `?`, `#`, `%`, `=`, `:`) matched only `foo` directly followed by `bar`;
+    both again match either word. MariaDB and MySQL bind the same term as 7.x.
   - A `distinct()` read whose select names only joined columns returns each of them once, under its alias, and
     accepts a joined internal attribute (`alias.$id`).
   - A filter on a joined column is checked against the joined collection's attribute as a filter on the main

src/Database/Builder/MariaDB.php

@@ -5,9 +5,10 @@
 use Utopia\Query\Builder\MariaDB as Base;
 
 /**
- * The MariaDB builder, which also compiles filters on their own.
+ * The MariaDB builder, which also compiles filters on their own and prepares search terms as 7.x did.
  */
 class MariaDB extends Base implements Filtering
 {
     use CompilesFilters;
+    use PreparesSearchTerms;
 }

src/Database/Builder/MySQL.php

@@ -5,9 +5,10 @@
 use Utopia\Query\Builder\MySQL as Base;
 
 /**
- * The MySQL builder, which also compiles filters on their own.
+ * The MySQL builder, which also compiles filters on their own and prepares search terms as 7.x did.
  */
 class MySQL extends Base implements Filtering
 {
     use CompilesFilters;
+    use PreparesSearchTerms;
 }

src/Database/Builder/PostgreSQL.php

@@ -9,12 +9,13 @@
 use Utopia\Query\Schema\ColumnType;
 
 /**
- * The PostgreSQL builder, which also compiles filters on their own, and writes a filter on a path
- * into an object attribute only when every key of the path is a plain key.
+ * The PostgreSQL builder, which also compiles filters on their own, prepares search terms as 7.x did, and writes a
+ * filter on a path into an object attribute only when every key of the path is a plain key.
  */
 class PostgreSQL extends Base implements Filtering
 {
     use CompilesFilters;
+    use PreparesSearchTerms;
 
     /**
      * @throws QueryException

src/Database/Builder/PreparesSearchTerms.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Utopia\Database\Builder;
+
+/**
+ * Prepares a fulltext search term as 7.x did: every character other than a letter, a number, an underscore or
+ * whitespace separates words, so `foo/bar` and `foo,bar` search the words `foo` and `bar`. A term wrapped in double
+ * quotes stays an exact search.
+ */
+trait PreparesSearchTerms
+{
+    private const string SEARCH_QUOTE = '"';
+
+    private const string SEARCH_SEPARATORS = '/[^\p{L}\p{N}_\s]/u';
+
+    private const string SEARCH_WHITESPACE = '/\s+/';
+
+    /**
+     * @param  array<mixed>  $values
+     */
+    #[\Override]
+    protected function compileSearchExpr(string $attribute, array $values, bool $not): string
+    {
+        $term = $values[0] ?? '';
+
+        return parent::compileSearchExpr($attribute, [$this->prepareSearchTerm(\is_string($term) ? $term : '')], $not);
+    }
+
+    private function prepareSearchTerm(string $term): string
+    {
+        $exact = \str_starts_with($term, self::SEARCH_QUOTE) && \str_ends_with($term, self::SEARCH_QUOTE);
+
+        $words = \preg_replace(self::SEARCH_SEPARATORS, ' ', $term) ?? '';
+        $words = \trim(\preg_replace(self::SEARCH_WHITESPACE, ' ', $words) ?? '');
+
+        if ($exact && $words !== '') {
+            return self::SEARCH_QUOTE.$words.self::SEARCH_QUOTE;
+        }
+
+        return $words;
+    }
+}

tests/e2e/Adapter/Scopes/DocumentTests.php

@@ -1870,6 +1870,82 @@ public function testFindFulltextAccentedAndSpecialChars(): void
         $this->assertGreaterThanOrEqual(1, count($documents));
     }
 
+    public function testFindFulltextSeparatorsSplitWords(): void
+    {
+        /** @var Database $database */
+        $database = $this->getDatabase();
+
+        if (! $database->getAdapter()->supports(Capability::Fulltext)) {
+            $this->expectNotToPerformAssertions();
+
+            return;
+        }
+
+        $collection = 'full_text_separators';
+        $database->createCollection(new Collection(id: $collection, permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+        ]));
+
+        $this->assertTrue($database->createAttribute($collection, Attribute::string(key: 'text', size: 128, required: true)));
+        $this->assertTrue($database->createIndex($collection, Index::fullText(key: 'text-ft', attributes: ['text'])));
+
+        $texts = [
+            'lunar' => 'lunar',
+            'solar' => 'solar',
+            'space' => 'lunar solar',
+            'slash' => 'lunar/solar',
+            'comma' => 'lunar,solar',
+            'underscore' => 'lunar_solar',
+            'comet' => 'comet',
+            'orbit' => 'orbit',
+        ];
+        foreach ($texts as $id => $text) {
+            $database->createDocument($collection, new Document([
+                '$id' => $id,
+                '$permissions' => [Permission::read(Role::any())],
+                'text' => $text,
+            ]));
+        }
+
+        $eitherWord = ['comma', 'lunar', 'slash', 'solar', 'space'];
+
+        foreach (['lunar/solar', 'lunar,solar'] as $term) {
+            $found = $this->searchedIds($database, $collection, $term);
+
+            foreach ($eitherWord as $id) {
+                $this->assertContains($id, $found, "search('{$term}') finds the document '{$texts[$id]}'");
+            }
+            $this->assertNotContains('comet', $found, "search('{$term}')");
+            $this->assertSame($this->searchedIds($database, $collection, 'lunar solar'), $found, "search('{$term}') searches the words as 'lunar solar' does");
+        }
+
+        $found = $this->searchedIds($database, $collection, 'comet, lunar/solar');
+        foreach (['comet', ...$eitherWord] as $id) {
+            $this->assertContains($id, $found, "search('comet, lunar/solar') finds the document '{$texts[$id]}'");
+        }
+        $this->assertNotContains('orbit', $found);
+        $this->assertSame($this->searchedIds($database, $collection, 'comet lunar solar'), $found);
+
+        $found = $this->searchedIds($database, $collection, 'lunar_solar');
+        $this->assertContains('underscore', $found);
+        $this->assertNotContains('comet', $found);
+    }
+
+    /**
+     * @return list<string>
+     */
+    private function searchedIds(Database $database, string $collection, string $term): array
+    {
+        $ids = \array_map(
+            fn (Document $document): string => $document->getId(),
+            $database->find($collection, [Query::search('text', $term)]),
+        );
+        \sort($ids);
+
+        return $ids;
+    }
+
     public function testFindByID(): void
     {
         $this->initMoviesFixture();

tests/unit/Builder/SearchTermTest.php

@@ -0,0 +1,123 @@
+<?php
+
+namespace Tests\Unit\Builder;
+
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\TestCase;
+use Utopia\Database\Builder\Filtering;
+use Utopia\Database\Builder\MariaDB;
+use Utopia\Database\Builder\MySQL;
+use Utopia\Database\Builder\PostgreSQL;
+use Utopia\Database\Query;
+
+final class SearchTermTest extends TestCase
+{
+    /**
+     * @return array<string, array{string, string}>
+     */
+    public static function postgreSQLTerms(): array
+    {
+        return [
+            'slash separates words' => ['foo/bar', 'foo or bar'],
+            'comma separates words' => ['foo,bar', 'foo or bar'],
+            'semicolon separates words' => ['foo;bar', 'foo or bar'],
+            'percent separates words' => ['foo%bar', 'foo or bar'],
+            'equals separates words' => ['foo=bar', 'foo or bar'],
+            'question mark separates words' => ['foo?bar', 'foo or bar'],
+            'hash separates words' => ['foo#bar', 'foo or bar'],
+            'colon separates words' => ['foo:bar', 'foo or bar'],
+            'pipe separates words' => ['foo|bar', 'foo or bar'],
+            'ampersand separates words' => ['foo&bar', 'foo or bar'],
+            'exclamation mark separates words' => ['foo!bar', 'foo or bar'],
+            'underscore stays inside a word' => ['foo_bar', 'foo_bar'],
+            'mixed separators and spaces' => ['baz, foo/bar;  qux', 'baz or foo or bar or qux'],
+            'operators are dropped' => ['+foo -bar* @3 <baz> ~qux (quux)', 'foo or bar or 3 or baz or qux or quux'],
+            'accented words are kept' => ['@García!', 'García'],
+            'exact phrase keeps its quotes' => ['"foo/bar baz"', '"foo bar baz"'],
+            'unbalanced quote is not exact' => ['"foo/bar', 'foo or bar'],
+        ];
+    }
+
+    /**
+     * @return array<string, array{string, string}>
+     */
+    public static function mySQLTerms(): array
+    {
+        return [
+            'slash separates words' => ['foo/bar', 'foo bar*'],
+            'comma separates words' => ['foo,bar', 'foo bar*'],
+            'period separates words' => ['foo.bar', 'foo bar*'],
+            'apostrophe separates words' => ["foo'bar", 'foo bar*'],
+            'trailing punctuation keeps the prefix match on the word' => ['foo.', 'foo*'],
+            'underscore stays inside a word' => ['foo_bar', 'foo_bar*'],
+            'mixed separators and spaces' => ['baz, foo/bar;  qux', 'baz foo bar qux*'],
+            'non-operator punctuation is dropped' => ['!!!foo...###', 'foo*'],
+            'exact phrase keeps its quotes' => ['"foo/bar baz"', '"foo bar baz"'],
+        ];
+    }
+
+    /**
+     * @return array<string, array{string}>
+     */
+    public static function termsWithoutWords(): array
+    {
+        return [
+            'separators only' => ['/,;%=?#'],
+            'operators only' => ['+-*@<>~()'],
+            'punctuation only' => ['!!!...###'],
+            'quoted punctuation' => ['"/"'],
+            'whitespace only' => ["  \t "],
+        ];
+    }
+
+    #[DataProvider('postgreSQLTerms')]
+    public function testPostgreSQLSearchesEachWordOfATerm(string $term, string $bound): void
+    {
+        $this->assertSame([$bound], $this->bindings(new PostgreSQL(), Query::search('title', $term)));
+        $this->assertSame([$bound], $this->bindings(new PostgreSQL(), Query::notSearch('title', $term)));
+    }
+
+    #[DataProvider('mySQLTerms')]
+    public function testMySQLSearchesEachWordOfATerm(string $term, string $bound): void
+    {
+        $this->assertSame([$bound], $this->bindings(new MySQL(), Query::search('title', $term)));
+        $this->assertSame([$bound], $this->bindings(new MySQL(), Query::notSearch('title', $term)));
+    }
+
+    #[DataProvider('mySQLTerms')]
+    public function testMariaDBSearchesEachWordOfATerm(string $term, string $bound): void
+    {
+        $this->assertSame([$bound], $this->bindings(new MariaDB(), Query::search('title', $term)));
+        $this->assertSame([$bound], $this->bindings(new MariaDB(), Query::notSearch('title', $term)));
+    }
+
+    #[DataProvider('termsWithoutWords')]
+    public function testATermWithoutWordsBindsNothing(string $term): void
+    {
+        foreach ([new PostgreSQL(), new MySQL(), new MariaDB()] as $builder) {
+            $this->assertSame([], $this->bindings($builder, Query::search('title', $term)), $builder::class);
+            $this->assertSame([], $this->bindings($builder, Query::notSearch('title', $term)), $builder::class);
+        }
+    }
+
+    public function testATermIsAlwaysBoundAndNeverWrittenIntoTheCondition(): void
+    {
+        $term = "foo'); DROP TABLE docs; --";
+
+        foreach ([new PostgreSQL(), new MySQL(), new MariaDB()] as $builder) {
+            $condition = $builder->compileFilters([Query::search('title', $term)]);
+
+            $this->assertStringNotContainsString('DROP', $condition->expression, $builder::class);
+            $this->assertStringNotContainsString('foo', $condition->expression, $builder::class);
+            $this->assertCount(1, $condition->bindings, $builder::class);
+        }
+    }
+
+    /**
+     * @return array<mixed>
+     */
+    private function bindings(Filtering $builder, Query $query): array
+    {
+        return $builder->compileFilters([$query])->bindings;
+    }
+}