feat(search): add pluggable lexical backends

README.md

@@ -238,6 +238,80 @@ const expanded = await store.expandQuery("auth flow", { intent: "user login" })
 const results4 = await store.search({ queries: expanded })
 ```
 
+#### Custom Lexical Backends
+
+QMD uses SQLite FTS5 by default, but the lexical search layer is pluggable.
+Custom backends do not replace QMD's store: they return references to documents
+that already exist in QMD's SQLite database, and QMD hydrates those hits before
+snippets, context lookup, RRF fusion, and reranking.
+
+```typescript
+import { createStore, type LexicalSearchBackend } from '@tobilu/qmd'
+
+const lexicalBackend: LexicalSearchBackend = {
+  name: "my-lexical-backend",
+  async search(request) {
+    // request: { query, limit, collectionName, dbPath }
+    return [
+      // score is normalized 0..1, higher is better
+      { filepath: "qmd://docs/auth.md", score: 0.93 },
+    ]
+  },
+}
+
+const store = await createStore({
+  dbPath: "./index.sqlite",
+  config: {
+    collections: {
+      docs: { path: "/path/to/docs", pattern: "**/*.md" },
+    },
+  },
+  lexicalBackend,
+})
+
+const results = await store.searchLex("auth middleware")
+```
+
+For CLI deployments, configure a command backend in `index.yml`. QMD writes a
+JSON request to the command's stdin and expects either a JSON array of hits or an
+object with a `hits` array on stdout:
+
+```yaml
+search:
+  lexicalBackend:
+    type: command
+    name: tantivy
+    command: qmd-lexical-backend-tantivy
+    args: ["search", "--index-dir", "/path/to/tantivy-index"]
+    timeoutMs: 5000
+```
+
+Command request:
+
+```json
+{
+  "query": "auth middleware",
+  "limit": 20,
+  "collectionName": "docs",
+  "dbPath": "/home/user/.cache/qmd/index.sqlite"
+}
+```
+
+Command response:
+
+```json
+{
+  "hits": [
+    { "documentId": 123, "score": 0.96, "rawScore": 42.1 },
+    { "filepath": "qmd://docs/auth.md", "score": 0.93 }
+  ]
+}
+```
+
+Supported hit references are `documentId`, `hash`, `docid`, `filepath`, or
+`collectionName` + `path`. QMD keeps `SearchResult.source === "fts"` for all
+lexical backends and adds `SearchResult.lexicalBackend` for observability.
+
 #### Retrieval
 
 ```typescript
@@ -436,11 +510,12 @@ The SDK requires explicit `dbPath` — no defaults are assumed. This makes it sa
 
 ## Score Normalization & Fusion
 
-### Search Backends
+### Lexical Backends
 
 | Backend | Raw Score | Conversion | Range |
 |---------|-----------|------------|-------|
 | **FTS (BM25)** | SQLite FTS5 BM25 | `Math.abs(score)` | 0 to ~25+ |
+| **Custom lexical** | Backend-defined | Must return normalized `score` | 0.0 to 1.0 |
 | **Vector** | Cosine distance | `1 / (1 + distance)` | 0.0 to 1.0 |
 | **Reranker** | LLM 0-10 rating | `score / 10` | 0.0 to 1.0 |
 

src/cli/qmd.ts

@@ -13,7 +13,6 @@ import {
   homedir,
   resolve,
   enableProductionMode,
-  searchFTS,
   extractSnippet,
   getContextForFile,
   getContextForPath,
@@ -110,6 +109,7 @@ import {
   type CollectionConfig,
   type ModelsConfig,
 } from "../collections.js";
+import { createLexicalSearchBackendFromConfig } from "../lexical-backends.js";
 
 // NOTE: enableProductionMode() is intentionally NOT called at module scope here.
 // Importing this module for its exports (e.g. buildEditorUri, termLink from
@@ -128,19 +128,25 @@ let currentIndexName = "index";
 
 function getStore(): ReturnType<typeof createStore> {
   if (!store) {
-    store = createStore(storeDbPathOverride);
-    // Sync YAML config into SQLite store_collections so store.ts reads from DB
+    let activeModels: ReturnType<typeof ensureModelsConfiguredForCli> | undefined;
+    let config: CollectionConfig | undefined;
     try {
-      const activeModels = ensureModelsConfiguredForCli();
-      const config = loadConfig();
+      activeModels = ensureModelsConfiguredForCli();
+      config = loadConfig();
+    } catch {
+      // Config may not exist yet — that's fine, DB works without it
+    }
+    store = createStore(storeDbPathOverride, {
+      lexicalBackend: createLexicalSearchBackendFromConfig(config?.search?.lexicalBackend),
+    });
+    if (config && activeModels) {
+      // Sync YAML config into SQLite store_collections so store.ts reads from DB
       syncConfigToDb(store.db, config);
       setDefaultLlamaCpp(new LlamaCpp({
         embedModel: activeModels.embed,
         generateModel: activeModels.generate,
         rerankModel: activeModels.rerank,
       }));
-    } catch {
-      // Config may not exist yet — that's fine, DB works without it
     }
   }
   return store;
@@ -2585,8 +2591,9 @@ function parseStructuredQuery(query: string): ParsedStructuredQuery | null {
   return typed.length > 0 ? { searches: typed, intent } : null;
 }
 
-function search(query: string, opts: OutputOptions): void {
-  const db = getDb();
+async function search(query: string, opts: OutputOptions): Promise<void> {
+  const storeInstance = getStore();
+  const db = storeInstance.db;
 
   // Validate collection filter (supports multiple -c flags)
   // Use default collections if none specified
@@ -2596,7 +2603,7 @@ function search(query: string, opts: OutputOptions): void {
   // Use large limit for --all, otherwise fetch more than needed and let outputResults filter
   const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
   const results = filterByCollections(
-    searchFTS(db, query, fetchLimit, singleCollection),
+    await storeInstance.searchLexical(query, fetchLimit, singleCollection),
     collectionNames
   );
 
@@ -4450,7 +4457,7 @@ if (isMain) {
         console.error("Usage: qmd search [options] <query>");
         process.exit(1);
       }
-      search(cli.query, cli.opts);
+      await search(cli.query, cli.opts);
       break;
 
     case "vsearch":

src/collections.ts

@@ -9,6 +9,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join, dirname, resolve } from "path";
 import { qmdHomedir } from "./paths.js";
 import YAML from "yaml";
+import type { LexicalBackendConfig } from "./lexical-backends.js";
 
 // ============================================================================
 // Types
@@ -51,6 +52,10 @@ export interface CollectionConfig {
   editor_uri_template?: string;               // Alias for editor_uri
   collections: Record<string, Collection>;    // Collection name -> config
   models?: ModelsConfig;
+  search?: {
+    /** Lexical lexical backend. Defaults to SQLite FTS5. */
+    lexicalBackend?: LexicalBackendConfig;
+  };
 }
 
 /**

src/index.ts

@@ -80,6 +80,15 @@ import {
   type NamedCollection,
   type ContextMap,
 } from "./collections.js";
+import {
+  createLexicalSearchBackendFromConfig,
+  type LexicalBackendConfig,
+  type LexicalDocumentRef,
+  type LexicalSearchBackend,
+  type LexicalSearchBackendContext,
+  type LexicalSearchHit,
+  type LexicalSearchRequest,
+} from "./lexical-backends.js";
 
 // Re-export types for SDK consumers
 export type {
@@ -103,13 +112,24 @@ export type {
   CollectionConfig,
   NamedCollection,
   ContextMap,
+  LexicalBackendConfig,
+  LexicalDocumentRef,
+  LexicalSearchBackend,
+  LexicalSearchBackendContext,
+  LexicalSearchHit,
+  LexicalSearchRequest,
 };
 
 // Re-export the internal Store type for advanced consumers
 export type { InternalStore };
 
 // Re-export utility functions and types used by frontends
 export { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES };
+export {
+  createExternalCommandLexicalBackend,
+  createLexicalSearchBackendFromConfig,
+  isSqliteFts5LexicalBackendConfig,
+} from "./lexical-backends.js";
 export type { ChunkStrategy } from "./store.js";
 
 // Re-export getDefaultDbPath for CLI/MCP that need the default database location
@@ -205,6 +225,11 @@ export interface StoreOptions {
   configPath?: string;
   /** Inline collection config (mutually exclusive with `configPath`) */
   config?: CollectionConfig;
+  /**
+   * Custom lexical backend for searchLex() and lex parts of search().
+   * Omit to use the built-in SQLite FTS5 backend.
+   */
+  lexicalBackend?: LexicalSearchBackend;
 }
 
 /**
@@ -346,27 +371,32 @@ export async function createStore(options: StoreOptions): Promise<QMDStore> {
     throw new Error("Provide either configPath or config, not both");
   }
 
-  // Create the internal store (opens DB, creates tables)
-  const internal = createStoreInternal(options.dbPath);
-  const db = internal.db;
-
   // Track whether we have a YAML config path for write-through
   const hasYamlConfig = !!options.configPath;
 
-  // Sync config into SQLite store_collections
+  // Load config before creating the internal store so lexical backend selection
+  // can come from either SDK options or YAML config.
   let config: CollectionConfig | undefined;
   if (options.configPath) {
-    // YAML mode: inject config source for write-through, sync to DB
     setConfigSource({ configPath: options.configPath });
     config = loadConfig();
-    syncConfigToDb(db, config);
   } else if (options.config) {
-    // Inline config mode: inject config source for mutations, sync to DB
     setConfigSource({ config: options.config });
     config = options.config;
+  }
+
+  const lexicalBackend = options.lexicalBackend
+    ?? createLexicalSearchBackendFromConfig(config?.search?.lexicalBackend);
+
+  // Create the internal store (opens DB, creates tables)
+  const internal = createStoreInternal(options.dbPath, { lexicalBackend });
+  const db = internal.db;
+
+  // Sync config into SQLite store_collections
+  if (config) {
     syncConfigToDb(db, config);
   }
-  // else: DB-only mode — no external config, use existing store_collections
+  // else: DB-only mode - no external config, use existing store_collections
 
   // Create a per-store LlamaCpp instance — lazy-loads models on first use,
   // auto-unloads after 5 min inactivity to free VRAM.
@@ -421,7 +451,7 @@ export async function createStore(options: StoreOptions): Promise<QMDStore> {
         chunkStrategy: opts.chunkStrategy,
       });
     },
-    searchLex: async (q, opts) => internal.searchFTS(q, opts?.limit, opts?.collection),
+    searchLex: async (q, opts) => internal.searchLexical(q, opts?.limit, opts?.collection),
     searchVector: async (q, opts) => internal.searchVec(q, llm.embedModelName, opts?.limit, opts?.collection),
     expandQuery: async (q, opts) => internal.expandQuery(q, undefined, opts?.intent),
     get: async (pathOrDocid, opts) => internal.findDocument(pathOrDocid, opts),