From 110a6d9688ed38f74effea1a4d78861ede215351 Mon Sep 17 00:00:00 2001 From: EyJunge1 Date: Mon, 20 Jul 2026 15:19:52 +0200 Subject: [PATCH 1/3] feat: migrate persistence to embedded Turso vectors Replace the SQLite and USearch stack with native libSQL vector storage, including safe legacy migration, lifecycle hardening, API authentication, and upgrade coverage. Co-authored-by: Cursor --- .github/workflows/platform-smoke.yml | 9 + README.md | 29 +- bun.lock | 72 +- package.json | 11 +- scripts/verify-libsql-vector.mjs | 30 + src/config.ts | 31 +- src/index.ts | 93 ++- src/services/ai/ai-provider-factory.ts | 2 +- .../ai/providers/anthropic-messages.ts | 18 +- src/services/ai/providers/google-gemini.ts | 30 +- .../ai/providers/openai-chat-completion.ts | 36 +- src/services/ai/providers/openai-responses.ts | 10 +- src/services/ai/session/ai-session-manager.ts | 267 ++++--- src/services/api-handlers.ts | 474 ++++++------- src/services/auto-capture.ts | 14 +- src/services/cleanup-service.ts | 51 +- src/services/client.ts | 219 +++--- src/services/context.ts | 6 +- src/services/deduplication-service.ts | 51 +- src/services/memory-scope.ts | 58 ++ src/services/migration-service.ts | 306 ++++---- src/services/sqlite/connection-manager.ts | 85 --- src/services/sqlite/shard-manager.ts | 328 --------- src/services/sqlite/sqlite-bootstrap.ts | 121 ---- src/services/sqlite/vector-search.ts | 372 ---------- src/services/turso/connection-manager.ts | 126 ++++ src/services/turso/legacy-migrator.ts | 654 ++++++++++++++++++ src/services/turso/lifecycle.ts | 15 + src/services/turso/operation-lock.ts | 66 ++ src/services/turso/ready.ts | 30 + src/services/turso/shard-manager.ts | 543 +++++++++++++++ src/services/turso/turso-db.ts | 62 ++ src/services/{sqlite => turso}/types.ts | 1 + src/services/turso/vector-search.ts | 475 +++++++++++++ src/services/turso/vector-utils.ts | 71 ++ src/services/user-memory-learning.ts | 22 +- src/services/user-profile/profile-context.ts | 4 +- .../user-profile/user-profile-manager.ts | 326 +++++---- .../user-prompt/user-prompt-manager.ts | 442 ++++++------ .../vector-backends/backend-factory.ts | 120 ---- .../vector-backends/exact-scan-backend.ts | 97 --- src/services/vector-backends/types.ts | 46 -- .../vector-backends/usearch-backend.ts | 241 ------- src/services/web-api-auth.ts | 31 + src/services/web-server-worker.ts | 396 ----------- src/services/web-server.ts | 15 +- src/types/usearch.d.ts | 1 - src/web/app.js | 24 +- tests/memory-scope-helper.test.ts | 64 ++ tests/memory-scope.test.ts | 48 +- tests/memory-tool-warmup.test.ts | 64 ++ tests/package-dependencies.test.ts | 10 +- tests/plugin-loader-contract.test.ts | 6 +- tests/profile-write.test.ts | 34 +- tests/sqlite-bootstrap-array-params.test.ts | 64 -- tests/turso-api-scope.test.ts | 78 +++ tests/turso-auxiliary-db-upgrade.test.ts | 72 ++ tests/turso-connection-lifecycle.test.ts | 50 ++ tests/turso-connection-manager.test.ts | 55 ++ tests/turso-dedup-extract.test.ts | 69 ++ tests/turso-exact-fallback.test.ts | 49 ++ tests/turso-legacy-migrator.test.ts | 358 ++++++++++ tests/turso-migrate-dims-preflight.test.ts | 81 +++ tests/turso-ready-gate.test.ts | 41 ++ tests/turso-reembed-migration.test.ts | 180 +++++ tests/turso-shard-recreate.test.ts | 88 +++ tests/turso-shard-rotation.test.ts | 67 ++ tests/turso-vector-search.test.ts | 69 ++ tests/turso-vector-utils.test.ts | 19 + tests/user-prompt-manager-claim.test.ts | 147 ++-- tests/vector-backends/backend-factory.test.ts | 119 ---- .../exact-scan-backend.test.ts | 107 --- .../migration-fallback.test.ts | 92 --- tests/vector-backends/usearch-backend.test.ts | 158 ----- .../vector-search-backend-integration.test.ts | 203 ------ tests/web-api-auth.test.ts | 42 ++ tests/web-server-health.test.ts | 27 + 77 files changed, 5168 insertions(+), 3824 deletions(-) create mode 100644 scripts/verify-libsql-vector.mjs create mode 100644 src/services/memory-scope.ts delete mode 100644 src/services/sqlite/connection-manager.ts delete mode 100644 src/services/sqlite/shard-manager.ts delete mode 100644 src/services/sqlite/sqlite-bootstrap.ts delete mode 100644 src/services/sqlite/vector-search.ts create mode 100644 src/services/turso/connection-manager.ts create mode 100644 src/services/turso/legacy-migrator.ts create mode 100644 src/services/turso/lifecycle.ts create mode 100644 src/services/turso/operation-lock.ts create mode 100644 src/services/turso/ready.ts create mode 100644 src/services/turso/shard-manager.ts create mode 100644 src/services/turso/turso-db.ts rename src/services/{sqlite => turso}/types.ts (97%) create mode 100644 src/services/turso/vector-search.ts create mode 100644 src/services/turso/vector-utils.ts delete mode 100644 src/services/vector-backends/backend-factory.ts delete mode 100644 src/services/vector-backends/exact-scan-backend.ts delete mode 100644 src/services/vector-backends/types.ts delete mode 100644 src/services/vector-backends/usearch-backend.ts create mode 100644 src/services/web-api-auth.ts delete mode 100644 src/services/web-server-worker.ts delete mode 100644 src/types/usearch.d.ts create mode 100644 tests/memory-scope-helper.test.ts create mode 100644 tests/memory-tool-warmup.test.ts delete mode 100644 tests/sqlite-bootstrap-array-params.test.ts create mode 100644 tests/turso-api-scope.test.ts create mode 100644 tests/turso-auxiliary-db-upgrade.test.ts create mode 100644 tests/turso-connection-lifecycle.test.ts create mode 100644 tests/turso-connection-manager.test.ts create mode 100644 tests/turso-dedup-extract.test.ts create mode 100644 tests/turso-exact-fallback.test.ts create mode 100644 tests/turso-legacy-migrator.test.ts create mode 100644 tests/turso-migrate-dims-preflight.test.ts create mode 100644 tests/turso-ready-gate.test.ts create mode 100644 tests/turso-reembed-migration.test.ts create mode 100644 tests/turso-shard-recreate.test.ts create mode 100644 tests/turso-shard-rotation.test.ts create mode 100644 tests/turso-vector-search.test.ts create mode 100644 tests/turso-vector-utils.test.ts delete mode 100644 tests/vector-backends/backend-factory.test.ts delete mode 100644 tests/vector-backends/exact-scan-backend.test.ts delete mode 100644 tests/vector-backends/migration-fallback.test.ts delete mode 100644 tests/vector-backends/usearch-backend.test.ts delete mode 100644 tests/vector-search-backend-integration.test.ts create mode 100644 tests/web-api-auth.test.ts create mode 100644 tests/web-server-health.test.ts diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index fdf4ed8..025902d 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -8,6 +8,7 @@ on: - "tsconfig.json" - "src/**" - "scripts/native-deps-smoke.mjs" + - "scripts/verify-libsql-vector.mjs" - "scripts/smoke-test.mjs" - "scripts/copy-web-assets.mjs" - ".github/workflows/platform-smoke.yml" @@ -39,6 +40,12 @@ jobs: - name: Install source dependencies run: bun install + - name: Typecheck source + run: bun run typecheck + + - name: Run test suite + run: bun test + - name: Build package artifacts run: bun run build @@ -59,6 +66,8 @@ jobs: npm init -y npm install --ignore-scripts "$tarball_path" cp "$GITHUB_WORKSPACE/scripts/native-deps-smoke.mjs" ./native-deps-smoke.mjs + cp "$GITHUB_WORKSPACE/scripts/verify-libsql-vector.mjs" ./verify-libsql-vector.mjs cp "$GITHUB_WORKSPACE/scripts/smoke-test.mjs" ./smoke-test.mjs node ./native-deps-smoke.mjs + node ./verify-libsql-vector.mjs node ./smoke-test.mjs diff --git a/README.md b/README.md index de34c23..2b58ebf 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ A persistent memory system for AI coding agents that enables long-term context r ## Core Features -Local vector database with SQLite + USearch-first vector indexing and ExactScan fallback, persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection. +Local Turso/libSQL database with native vector search, persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection. ## Prerequisites -This plugin uses `USearch` for preferred in-memory vector indexing with automatic ExactScan fallback. No custom SQLite build or browser runtime shim is required. +This plugin uses embedded Turso/libSQL with native vector indexes (`F32_BLOB`, `vector_top_k`). No separate vector database or custom SQLite build is required. **Recommended runtime:** @@ -35,10 +35,24 @@ This plugin uses `USearch` for preferred in-memory vector indexing with automati **Notes:** -- If `USearch` is unavailable or fails at runtime, the plugin automatically falls back to exact vector scanning. -- SQLite remains the source of truth; search indexes are rebuilt from SQLite data when needed. +- Vector embeddings are stored and searched directly in Turso/libSQL; inserts update the vector index automatically. +- Vector search uses libSQL's DiskANN index via `vector_top_k` (approximate nearest neighbors). - Auto-capture and user profile learning require an AI provider that can return structured/tool-call output. Memory search/add/list still work without auto-capture provider configuration. +### Upgrading from legacy SQLite shards + +On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format: + +- Each shard is backed up as `.db.legacy.bak` before rewrite +- Progress is tracked per shard in `.db.turso-migrate.json` +- A global marker `.turso-migrated` is written only after all shards verify successfully +- Do not run multiple OpenCode instances against the same `storagePath` during migration; a lock file (`.turso-migrate.lock`) prevents concurrent migration +- Manual dimension migrations use `.turso-operation.lock`; other plugin processes reject new memory writes until the migration finishes + +If migration is interrupted, the next startup resumes from the backup automatically. + +If a shard becomes incompatible (for example after changing `embeddingDimensions`), writes are blocked and the original database is left untouched. Use the Web UI's re-embed migration to build and verify a replacement before it is swapped into place. The previous shard remains available as `.db.pre-reembed--.bak`. + ## Getting Started Add to your OpenCode configuration at `~/.config/opencode/opencode.json`: @@ -63,6 +77,10 @@ memory({ mode: "list", limit: 10 }); Access the web interface at `http://127.0.0.1:4747` for visual memory browsing and management. +**Network binding security:** Keep `webServerHost` on `127.0.0.1` unless you intentionally expose the UI. Binding to `0.0.0.0` (or any non-loopback host) requires `webServerApiToken`; all `/api/*` requests must then send `Authorization: Bearer ` or `X-Opencode-Mem-Token`. Open the UI with `?apiToken=` so the browser stores and sends it. + +Dimension migrations generate every new embedding first, import them into a temporary indexed shard, verify the row count, and only then replace the original file. Failed migrations leave the source shard untouched. + ## Configuration Essentials Configure at `~/.config/opencode/opencode-mem.jsonc`: @@ -85,6 +103,9 @@ The plugin creates a full commented template at this path on first startup. This }, "webServerEnabled": true, "webServerPort": 4747, + // Required when webServerHost is not 127.0.0.1/localhost: + // "webServerHost": "0.0.0.0", + // "webServerApiToken": "env://OPENCODE_MEM_WEB_TOKEN", "autoCaptureEnabled": true, "autoCaptureLanguage": "auto", diff --git a/bun.lock b/bun.lock index 44c4629..ddd4bf1 100644 --- a/bun.lock +++ b/bun.lock @@ -6,17 +6,17 @@ "name": "opencode-plugin", "dependencies": { "@huggingface/transformers": "^4.2.0", + "@libsql/client": "^0.17.4", "@opencode-ai/plugin": "^1.3.0", "@opencode-ai/sdk": "^1.3.0", "franc-min": "^6.2.0", "iso-639-3": "^3.0.1", - "usearch": "2.26.0", "zod": "^4.3.6", }, "devDependencies": { "@types/bun": "^1.3.8", "husky": "^9.1.7", - "lint-staged": "^16.2.7", + "lint-staged": "^16.4.0", "prettier": "^3.4.2", "typescript": "^5.7.3", }, @@ -84,6 +84,34 @@ "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@libsql/client": ["@libsql/client@0.17.4", "", { "dependencies": { "@libsql/core": "^0.17.4", "@libsql/hrana-client": "^0.10.0", "js-base64": "^3.7.5", "libsql": "^0.5.28", "promise-limit": "^2.7.0" } }, "sha512-lYayFWasDV78A+TjlEhr6ubb3odBV6OHjb+wdp8VQcyWWAEIjuwbCHaraEUS4m4yWoo0BvZo96It4VdzZRmRWw=="], + + "@libsql/core": ["@libsql/core@0.17.4", "", { "dependencies": { "js-base64": "^3.7.5" } }, "sha512-LqF9gIvnJ38nmAH1y/ChizHqDO/MO1wLgA96XrraulEEbqXxLjleSH92YWTolbuJKgPUmGu4aJk9W3UnAcxLOQ=="], + + "@libsql/darwin-arm64": ["@libsql/darwin-arm64@0.5.29", "", { "os": "darwin", "cpu": "arm64" }, "sha512-K+2RIB1OGFPYQbfay48GakLhqf3ArcbHqPFu7EZiaUcRgFcdw8RoltsMyvbj5ix2fY0HV3Q3Ioa/ByvQdaSM0A=="], + + "@libsql/darwin-x64": ["@libsql/darwin-x64@0.5.29", "", { "os": "darwin", "cpu": "x64" }, "sha512-OtT+KFHsKFy1R5FVadr8FJ2Bb1mghtXTyJkxv0trocq7NuHntSki1eUbxpO5ezJesDvBlqFjnWaYYY516QNLhQ=="], + + "@libsql/hrana-client": ["@libsql/hrana-client@0.10.0", "", { "dependencies": { "@libsql/isomorphic-ws": "^0.1.5", "js-base64": "^3.7.5" } }, "sha512-OoA4EMqRAC7kn7V2P6EQqRcpZf2W+AjsNIyCizBg339Tq/aMC7sRnzs3SklderhmQWAqEzvv8A2vhxVmWpkVvw=="], + + "@libsql/isomorphic-ws": ["@libsql/isomorphic-ws@0.1.5", "", { "dependencies": { "@types/ws": "^8.5.4", "ws": "^8.13.0" } }, "sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg=="], + + "@libsql/linux-arm-gnueabihf": ["@libsql/linux-arm-gnueabihf@0.5.29", "", { "os": "linux", "cpu": "arm" }, "sha512-CD4n4zj7SJTHso4nf5cuMoWoMSS7asn5hHygsDuhRl8jjjCTT3yE+xdUvI4J7zsyb53VO5ISh4cwwOtf6k2UhQ=="], + + "@libsql/linux-arm-musleabihf": ["@libsql/linux-arm-musleabihf@0.5.29", "", { "os": "linux", "cpu": "arm" }, "sha512-2Z9qBVpEJV7OeflzIR3+l5yAd4uTOLxklScYTwpZnkm2vDSGlC1PRlueLaufc4EFITkLKXK2MWBpexuNJfMVcg=="], + + "@libsql/linux-arm64-gnu": ["@libsql/linux-arm64-gnu@0.5.29", "", { "os": "linux", "cpu": "arm64" }, "sha512-gURBqaiXIGGwFNEaUj8Ldk7Hps4STtG+31aEidCk5evMMdtsdfL3HPCpvys+ZF/tkOs2MWlRWoSq7SOuCE9k3w=="], + + "@libsql/linux-arm64-musl": ["@libsql/linux-arm64-musl@0.5.29", "", { "os": "linux", "cpu": "arm64" }, "sha512-fwgYZ0H8mUkyVqXZHF3mT/92iIh1N94Owi/f66cPVNsk9BdGKq5gVpoKO+7UxaNzuEH1roJp2QEwsCZMvBLpqg=="], + + "@libsql/linux-x64-gnu": ["@libsql/linux-x64-gnu@0.5.29", "", { "os": "linux", "cpu": "x64" }, "sha512-y14V0vY0nmMC6G0pHeJcEarcnGU2H6cm21ZceRkacWHvQAEhAG0latQkCtoS2njFOXiYIg+JYPfAoWKbi82rkg=="], + + "@libsql/linux-x64-musl": ["@libsql/linux-x64-musl@0.5.29", "", { "os": "linux", "cpu": "x64" }, "sha512-gquqwA/39tH4pFl+J9n3SOMSymjX+6kZ3kWgY3b94nXFTwac9bnFNMffIomgvlFaC4ArVqMnOZD3nuJ3H3VO1w=="], + + "@libsql/win32-x64-msvc": ["@libsql/win32-x64-msvc@0.5.29", "", { "os": "win32", "cpu": "x64" }, "sha512-4/0CvEdhi6+KjMxMaVbFM2n2Z44escBRoEYpR+gZg64DdetzGnYm8mcNLcoySaDJZNaBd6wz5DNdgRmcI4hXcg=="], + + "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="], + "@opencode-ai/plugin": ["@opencode-ai/plugin@1.3.13", "", { "dependencies": { "@opencode-ai/sdk": "1.3.13", "zod": "4.1.8" }, "peerDependencies": { "@opentui/core": ">=0.1.95", "@opentui/solid": ">=0.1.95" }, "optionalPeers": ["@opentui/core", "@opentui/solid"] }, "sha512-zHgtWfdDz8Wu8srE8f8HUtPT9i6c3jTmgQKoFZUZ+RR5CMQF1kAlb1cxeEe9Xm2DRNFVJog9Cv/G1iUHYgXSUQ=="], "@opencode-ai/sdk": ["@opencode-ai/sdk@1.3.13", "", {}, "sha512-/M6HlNnba+xf1EId6qFb2tG0cvq0db3PCQDug1glrf8wYOU57LYNF8WvHX9zoDKPTMv0F+O4pcP/8J+WvDaxHA=="], @@ -112,6 +140,8 @@ "@types/node": ["@types/node@25.0.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA=="], + "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], + "adm-zip": ["adm-zip@0.5.17", "", {}, "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ=="], "ansi-escapes": ["ansi-escapes@7.3.0", "", { "dependencies": { "environment": "^1.0.0" } }, "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="], @@ -120,12 +150,8 @@ "ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], - "bindings": ["bindings@1.5.0", "", { "dependencies": { "file-uri-to-path": "1.0.0" } }, "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ=="], - "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], - "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], "cli-cursor": ["cli-cursor@5.0.0", "", { "dependencies": { "restore-cursor": "^5.0.0" } }, "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw=="], @@ -160,10 +186,6 @@ "eventemitter3": ["eventemitter3@5.0.4", "", {}, "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="], - "file-uri-to-path": ["file-uri-to-path@1.0.0", "", {}, "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="], - - "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], - "flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="], "franc-min": ["franc-min@6.2.0", "", { "dependencies": { "trigram-utils": "^2.0.0" } }, "sha512-1uDIEUSlUZgvJa2AKYR/dmJC66v/PvGQ9mWfI9nOr/kPpMFyvswK0gPXOwpYJYiYD008PpHLkGfG58SPjQJFxw=="], @@ -184,13 +206,15 @@ "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], - "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - "iso-639-3": ["iso-639-3@3.0.1", "", {}, "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ=="], + "js-base64": ["js-base64@3.9.1", "", {}, "sha512-U73qptcvf/HIOauFOmqT3a0mDUp0MYlfd15oqoe9kqZt5XhiXVb+HG09sLvI9PQ9tZIBFS4nlErai8zbWazP0g=="], + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], - "lint-staged": ["lint-staged@16.2.7", "", { "dependencies": { "commander": "^14.0.2", "listr2": "^9.0.5", "micromatch": "^4.0.8", "nano-spawn": "^2.0.0", "pidtree": "^0.6.0", "string-argv": "^0.3.2", "yaml": "^2.8.1" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-lDIj4RnYmK7/kXMya+qJsmkRFkGolciXjrsZ6PC25GdTfWOAWetR0ZbsNXRAj1EHHImRSalc+whZFg56F5DVow=="], + "libsql": ["libsql@0.5.29", "", { "dependencies": { "@neon-rs/load": "^0.0.4", "detect-libc": "2.0.2" }, "optionalDependencies": { "@libsql/darwin-arm64": "0.5.29", "@libsql/darwin-x64": "0.5.29", "@libsql/linux-arm-gnueabihf": "0.5.29", "@libsql/linux-arm-musleabihf": "0.5.29", "@libsql/linux-arm64-gnu": "0.5.29", "@libsql/linux-arm64-musl": "0.5.29", "@libsql/linux-x64-gnu": "0.5.29", "@libsql/linux-x64-musl": "0.5.29", "@libsql/win32-x64-msvc": "0.5.29" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "arm", "x64", "arm64", ] }, "sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg=="], + + "lint-staged": ["lint-staged@16.4.0", "", { "dependencies": { "commander": "^14.0.3", "listr2": "^9.0.5", "picomatch": "^4.0.3", "string-argv": "^0.3.2", "tinyexec": "^1.0.4", "yaml": "^2.8.2" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-lBWt8hujh/Cjysw5GYVmZpFHXDCgZzhrOm8vbcUdobADZNOK/bRshr2kM3DfgrrtR1DQhfupW9gnIXOfiFi+bw=="], "listr2": ["listr2@9.0.5", "", { "dependencies": { "cli-truncate": "^5.0.0", "colorette": "^2.0.20", "eventemitter3": "^5.0.1", "log-update": "^6.1.0", "rfdc": "^1.4.1", "wrap-ansi": "^9.0.0" } }, "sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g=="], @@ -200,18 +224,10 @@ "matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], - "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - "mimic-function": ["mimic-function@5.0.1", "", {}, "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA=="], "n-gram": ["n-gram@2.0.2", "", {}, "sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ=="], - "nano-spawn": ["nano-spawn@2.0.0", "", {}, "sha512-tacvGzUY5o2D8CBh2rrwxyNojUsZNU2zjNTzKQrkgGJQTbGAfArVWXSKMBokBeeg6C7OLRGUEyoFlYbfeWQIqw=="], - - "node-addon-api": ["node-addon-api@8.6.0", "", {}, "sha512-gBVjCaqDlRUk0EwoPNKzIr9KkS9041G/q31IBShPs1Xz6UTA+EXdZADbzqAJQrpDRq71CIMnOP5VMut3SL0z5Q=="], - - "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], - "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], "onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], @@ -222,14 +238,14 @@ "onnxruntime-web": ["onnxruntime-web@1.26.0-dev.20260416-b7804b056c", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="], - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - - "pidtree": ["pidtree@0.6.0", "", { "bin": { "pidtree": "bin/pidtree.js" } }, "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], "prettier": ["prettier@3.7.4", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA=="], + "promise-limit": ["promise-limit@2.7.0", "", {}, "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw=="], + "protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="], "restore-cursor": ["restore-cursor@5.1.0", "", { "dependencies": { "onetime": "^7.0.0", "signal-exit": "^4.1.0" } }, "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA=="], @@ -258,7 +274,7 @@ "strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="], - "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + "tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], "trigram-utils": ["trigram-utils@2.0.1", "", { "dependencies": { "collapse-white-space": "^2.0.0", "n-gram": "^2.0.0" } }, "sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ=="], @@ -270,16 +286,18 @@ "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - "usearch": ["usearch@2.26.0", "", { "dependencies": { "bindings": "^1.5.0", "node-addon-api": "^8.5.0", "node-gyp-build": "^4.8.4" } }, "sha512-Le3KoJh1I+YY9h8bgfJi6G0wgvyRl2+sBFxZFp7LcN2jOkgEcBKiYtyOJgZ9KnfybKlkqYjjXvcmyH4/PG9shw=="], - "wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], + "ws": ["ws@8.21.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw=="], + "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@opencode-ai/plugin/zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], + "libsql/detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], + "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.24.0-dev.20251116-b39e144322", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="], "wrap-ansi/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], diff --git a/package.json b/package.json index f70c68c..99d61a2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "opencode-mem", "version": "2.19.4", - "description": "OpenCode plugin that gives coding agents persistent memory using local vector database", + "description": "OpenCode plugin that gives coding agents persistent memory using local Turso/libSQL vector search", "type": "module", "main": "dist/plugin.js", "types": "dist/index.d.ts", @@ -22,6 +22,7 @@ "scripts": { "build": "rm -rf dist && bunx tsc && bun scripts/copy-web-assets.mjs", "dev": "tsc --watch", + "test": "bun test", "typecheck": "tsc --noEmit", "format": "prettier --write \"src/**/*.{ts,js,css,html}\"", "format:check": "prettier --check \"src/**/*.{ts,js,css,html}\"", @@ -47,13 +48,13 @@ "access": "public" }, "dependencies": { + "@huggingface/transformers": "^4.2.0", + "@libsql/client": "^0.17.4", "@opencode-ai/plugin": "^1.3.0", "@opencode-ai/sdk": "^1.3.0", "franc-min": "^6.2.0", "iso-639-3": "^3.0.1", - "zod": "^4.3.6", - "@huggingface/transformers": "^4.2.0", - "usearch": "2.26.0" + "zod": "^4.3.6" }, "//": "Remove onnxruntime-node override after https://github.com/microsoft/onnxruntime/issues/27961 is resolved", "overrides": { @@ -62,7 +63,7 @@ "devDependencies": { "@types/bun": "^1.3.8", "husky": "^9.1.7", - "lint-staged": "^16.2.7", + "lint-staged": "^16.4.0", "prettier": "^3.4.2", "typescript": "^5.7.3" }, diff --git a/scripts/verify-libsql-vector.mjs b/scripts/verify-libsql-vector.mjs new file mode 100644 index 0000000..eff98d9 --- /dev/null +++ b/scripts/verify-libsql-vector.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createClient } from "@libsql/client"; + +const dbPath = join(tmpdir(), `opencode-mem-libsql-smoke-${process.pid}-${Date.now()}.db`); +const client = createClient({ url: `file:${dbPath}` }); + +try { + await client.batch( + [ + "CREATE TABLE vectors (id INTEGER PRIMARY KEY, vector F32_BLOB(4) NOT NULL)", + "CREATE INDEX vectors_idx ON vectors (libsql_vector_idx(vector, 'metric=cosine'))", + "INSERT INTO vectors (vector) VALUES (vector32('[1,0,0,0]'))", + ], + "write" + ); + const result = await client.execute( + "SELECT id FROM vector_top_k('vectors_idx', vector32('[1,0,0,0]'), 1)" + ); + assert.equal(result.rows.length, 1, "vector_top_k must return the inserted vector"); + assert.equal(Number(result.rows[0]?.id), 1, "vector_top_k must return the expected row"); + console.log("libSQL vector smoke test passed"); +} finally { + client.close(); + for (const suffix of ["", "-shm", "-wal"]) { + rmSync(`${dbPath}${suffix}`, { force: true }); + } +} diff --git a/src/config.ts b/src/config.ts index 79153de..918de51 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,11 +49,11 @@ interface OpenCodeMemConfig { memoryExtraParams?: Record; opencodeProvider?: string; opencodeModel?: string; - vectorBackend?: "usearch-first" | "usearch" | "exact-scan"; aiSessionRetentionDays?: number; webServerEnabled?: boolean; webServerPort?: number; webServerHost?: string; + webServerApiToken?: string; maxVectorsPerShard?: number; autoCleanupEnabled?: boolean; autoCleanupRetentionDays?: number; @@ -109,6 +109,7 @@ const DEFAULTS: Required< | "autoCaptureLanguage" | "userEmailOverride" | "userNameOverride" + | "webServerApiToken" > > & { embeddingApiUrl?: string; @@ -121,10 +122,10 @@ const DEFAULTS: Required< memoryExtraParams?: Record; opencodeProvider?: string; opencodeModel?: string; - vectorBackend?: "usearch-first" | "usearch" | "exact-scan"; autoCaptureLanguage?: string; userEmailOverride?: string; userNameOverride?: string; + webServerApiToken?: string; memory?: { defaultScope?: "project" | "all-projects"; }; @@ -141,7 +142,6 @@ const DEFAULTS: Required< autoCaptureMaxIterations: 5, autoCaptureIterationTimeout: 30000, autoCaptureMaxRetries: 3, - vectorBackend: "usearch-first", aiSessionRetentionDays: 7, webServerEnabled: true, webServerPort: 4747, @@ -255,6 +255,9 @@ const CONFIG_TEMPLATE = `{ // Host address for web UI (use 127.0.0.1 for local only, 0.0.0.0 for network access) "webServerHost": "127.0.0.1", + + // Required when webServerHost is not loopback. Protects /api/* with Bearer / X-Opencode-Mem-Token. + // "webServerApiToken": "env://OPENCODE_MEM_WEB_TOKEN", // ============================================ // Database Settings @@ -537,15 +540,24 @@ function getEmbeddingDimensions(model: string): number { function buildConfig(fileConfig: OpenCodeMemConfig) { const memoryApiKey = resolveSecretValue(fileConfig.memoryApiKey); + const embeddingDimensions = + fileConfig.embeddingDimensions ?? + getEmbeddingDimensions(fileConfig.embeddingModel ?? DEFAULTS.embeddingModel); + + if ( + !Number.isInteger(embeddingDimensions) || + embeddingDimensions <= 0 || + embeddingDimensions > 65536 + ) { + throw new Error(`Invalid embeddingDimensions config: ${embeddingDimensions}`); + } return { storagePath: expandPath(fileConfig.storagePath ?? DEFAULTS.storagePath), userEmailOverride: fileConfig.userEmailOverride, userNameOverride: fileConfig.userNameOverride, embeddingModel: fileConfig.embeddingModel ?? DEFAULTS.embeddingModel, - embeddingDimensions: - fileConfig.embeddingDimensions ?? - getEmbeddingDimensions(fileConfig.embeddingModel ?? DEFAULTS.embeddingModel), + embeddingDimensions, embeddingApiUrl: fileConfig.embeddingApiUrl, embeddingApiKey: fileConfig.embeddingApiUrl ? resolveSecretValue(fileConfig.embeddingApiKey ?? process.env.OPENAI_API_KEY) @@ -580,14 +592,13 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { memoryApiUrl: fileConfig.memoryApiUrl, memoryApiKey, }), - vectorBackend: (fileConfig.vectorBackend ?? "usearch-first") as - | "usearch-first" - | "usearch" - | "exact-scan", aiSessionRetentionDays: fileConfig.aiSessionRetentionDays ?? DEFAULTS.aiSessionRetentionDays, webServerEnabled: fileConfig.webServerEnabled ?? DEFAULTS.webServerEnabled, webServerPort: fileConfig.webServerPort ?? DEFAULTS.webServerPort, webServerHost: fileConfig.webServerHost ?? DEFAULTS.webServerHost, + webServerApiToken: fileConfig.webServerApiToken + ? resolveSecretValue(fileConfig.webServerApiToken) + : undefined, maxVectorsPerShard: fileConfig.maxVectorsPerShard ?? DEFAULTS.maxVectorsPerShard, autoCleanupEnabled: fileConfig.autoCleanupEnabled ?? DEFAULTS.autoCleanupEnabled, autoCleanupRetentionDays: diff --git a/src/index.ts b/src/index.ts index 12bf94b..38753c9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,8 @@ import { performAutoCapture } from "./services/auto-capture.js"; import { performUserProfileLearning } from "./services/user-memory-learning.js"; import { userPromptManager } from "./services/user-prompt/user-prompt-manager.js"; import { startWebServer, WebServer } from "./services/web-server.js"; +import { ensureTursoReady } from "./services/turso/ready.js"; +import { tursoConnectionManager } from "./services/turso/connection-manager.js"; import { isConfigured, CONFIG, initConfig } from "./config.js"; import { log } from "./services/logger.js"; @@ -72,15 +74,13 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { const GLOBAL_PLUGIN_WARMUP_KEY = Symbol.for("opencode-mem.plugin.warmedup"); if (!(globalThis as any)[GLOBAL_PLUGIN_WARMUP_KEY] && isConfigured()) { - // Fire-and-forget: warmup is slow (embedding model load + index rebuild). - // Awaiting it here serializes opencode's plugin loader and starves the TUI, - // which gave the symptom "opencode hangs ~70s then disconnects on startup". + // Fire-and-forget: DB ready + embedding model must not block plugin init. (async () => { try { await memoryClient.warmup(); (globalThis as any)[GLOBAL_PLUGIN_WARMUP_KEY] = true; } catch (error) { - log("Plugin warmup failed", { error: String(error) }); + log("Plugin memory warmup failed", { error: String(error) }); } })(); } @@ -107,11 +107,34 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { } })(); - if (CONFIG.webServerEnabled) { + let tursoReadyForWeb = !isConfigured(); + if (CONFIG.webServerEnabled && isConfigured()) { + try { + await ensureTursoReady(); + tursoReadyForWeb = true; + } catch (error) { + log("Turso ready gate failed before web server start", { error: String(error) }); + if (ctx.client?.tui) { + ctx.client.tui + .showToast({ + body: { + title: "Memory Explorer", + message: "Database migration failed; web UI not started", + variant: "error", + duration: 8000, + }, + }) + .catch(() => {}); + } + } + } + + if (CONFIG.webServerEnabled && tursoReadyForWeb) { startWebServer({ port: CONFIG.webServerPort, host: CONFIG.webServerHost, enabled: CONFIG.webServerEnabled, + apiToken: CONFIG.webServerApiToken, }) .then((server) => { webServer = server; @@ -178,13 +201,16 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { }); } + let cleanedUp = false; const cleanupPlugin = async () => { + if (cleanedUp) return; + cleanedUp = true; if (idleTimeout) { clearTimeout(idleTimeout); idleTimeout = null; } if (webServer) await webServer.stop(); - if (memoryClient) memoryClient.close(); + if (memoryClient) await memoryClient.close(); }; const shutdownHandler = async () => { @@ -198,9 +224,20 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { process.on("SIGINT", shutdownHandler); process.on("SIGTERM", shutdownHandler); + process.on("beforeExit", () => { + if (!cleanedUp) { + void cleanupPlugin(); + } + }); process.on("exit", () => { - if (webServer) webServer.stop().catch(() => {}); - if (memoryClient) memoryClient.close(); + // Best-effort sync close when the host exits without SIGINT/SIGTERM. + if (!cleanedUp) { + try { + tursoConnectionManager.closeAllSync(); + } catch { + // ignore — module may already be torn down + } + } }); return { @@ -220,7 +257,12 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { return; } - userPromptManager.savePrompt(input.sessionID, output.message.id, directory, userMessage); + await userPromptManager.savePrompt( + input.sessionID, + output.message.id, + directory, + userMessage + ); const messagesResponse = await ctx.client.session.messages({ path: { id: input.sessionID }, @@ -276,7 +318,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { }; const userId = tags.user.userEmail || null; - const memoryContext = formatContextForPrompt(userId, projectMemories); + const memoryContext = await formatContextForPrompt(userId, projectMemories); if (memoryContext) { const contextPart: Part = { @@ -310,7 +352,11 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { if (!isConfigured() || CONFIG.opencodeModel !== "inherit") return; try { - userPromptManager.setPromptModel(input.message.id, input.model.providerID, input.model.id); + await userPromptManager.setPromptModel( + input.message.id, + input.model.providerID, + input.model.id + ); } catch (error) { log("chat.params: ERROR", { error: String(error) }); } @@ -346,9 +392,14 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { }); } - const needsWarmup = !(await memoryClient.isReady()); - if (needsWarmup) { - return JSON.stringify({ success: false, error: "Memory system is initializing." }); + try { + await memoryClient.warmup(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return JSON.stringify({ + success: false, + error: `Memory system failed to initialize: ${message}`, + }); } const mode = args.mode || "help"; @@ -405,8 +456,8 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { }); return JSON.stringify({ success: result.success, - message: `Memory added`, - id: result.id, + message: result.success ? `Memory added` : result.error, + id: result.success ? result.id : undefined, tags: parsedTags, }); @@ -467,7 +518,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { lastSeen: Date.now(), }; - const existingProfile = userProfileManager.getActiveProfile(userId); + const existingProfile = await userProfileManager.getActiveProfile(userId); if (existingProfile) { const existingData = JSON.parse(existingProfile.profileData); @@ -479,7 +530,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { undefined, existingProfile.id ); - userProfileManager.updateProfile( + await userProfileManager.updateProfile( existingProfile.id, mergedData, 0, @@ -490,7 +541,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { message: "Preference saved to profile", }); } else { - userProfileManager.createProfile( + await userProfileManager.createProfile( userId, tags.user.displayName || userId, tags.user.userName || userId, @@ -506,7 +557,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { } // --- READ: no content provided --- - const profile = userProfileManager.getActiveProfile(userId); + const profile = await userProfileManager.getActiveProfile(userId); if (!profile) return JSON.stringify({ success: true, profile: null }); const pData = JSON.parse(profile.profileData); return JSON.stringify({ @@ -570,8 +621,6 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { await performUserProfileLearning(ctx, directory); const { cleanupService } = await import("./services/cleanup-service.js"); if (await cleanupService.shouldRunCleanup()) await cleanupService.runCleanup(); - const { connectionManager } = await import("./services/sqlite/connection-manager.js"); - connectionManager.checkpointAll(); } } catch (error) { log("Idle processing error", { error: String(error) }); diff --git a/src/services/ai/ai-provider-factory.ts b/src/services/ai/ai-provider-factory.ts index 2935927..e26364d 100644 --- a/src/services/ai/ai-provider-factory.ts +++ b/src/services/ai/ai-provider-factory.ts @@ -30,7 +30,7 @@ export class AIProviderFactory { return ["openai-chat", "openai-responses", "anthropic", "google-gemini"]; } - static cleanupExpiredSessions(): number { + static async cleanupExpiredSessions(): Promise { return aiSessionManager.cleanupExpiredSessions(); } } diff --git a/src/services/ai/providers/anthropic-messages.ts b/src/services/ai/providers/anthropic-messages.ts index a2da816..0286831 100644 --- a/src/services/ai/providers/anthropic-messages.ts +++ b/src/services/ai/providers/anthropic-messages.ts @@ -50,17 +50,17 @@ export class AnthropicMessagesProvider extends BaseAIProvider { toolSchema: ChatCompletionTool, sessionId: string ): Promise { - let session = this.aiSessionManager.getSession(sessionId, "anthropic"); + let session = await this.aiSessionManager.getSession(sessionId, "anthropic"); if (!session) { - session = this.aiSessionManager.createSession({ + session = await this.aiSessionManager.createSession({ provider: "anthropic", sessionId, metadata: { systemPrompt }, }); } - const storedMessages = this.aiSessionManager.getMessages(session.id); + const storedMessages = await this.aiSessionManager.getMessages(session.id); const messages: AnthropicMessage[] = []; for (const msg of storedMessages) { @@ -74,8 +74,8 @@ export class AnthropicMessagesProvider extends BaseAIProvider { messages.push(anthropicMsg); } - const userSequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const userSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: userSequence, role: "user", @@ -141,8 +141,8 @@ export class AnthropicMessagesProvider extends BaseAIProvider { const data = (await response.json()) as AnthropicResponse; - const assistantSequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const assistantSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: assistantSequence, role: "assistant", @@ -190,11 +190,11 @@ export class AnthropicMessagesProvider extends BaseAIProvider { } if (data.stop_reason === "end_turn") { - const retrySequence = this.aiSessionManager.getLastSequence(session.id) + 1; + const retrySequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; const retryPrompt = "Please use the save_memories tool to extract and save the memories from the conversation as instructed."; - this.aiSessionManager.addMessage({ + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: retrySequence, role: "user", diff --git a/src/services/ai/providers/google-gemini.ts b/src/services/ai/providers/google-gemini.ts index b030fac..1d9bc3a 100644 --- a/src/services/ai/providers/google-gemini.ts +++ b/src/services/ai/providers/google-gemini.ts @@ -24,14 +24,14 @@ export class GoogleGeminiProvider extends BaseAIProvider { return true; } - private addToolResponse( + private async addToolResponse( sessionId: string, messages: any[], toolCallId: string, content: string - ): void { - const sequence = this.aiSessionManager.getLastSequence(sessionId) + 1; - this.aiSessionManager.addMessage({ + ): Promise { + const sequence = (await this.aiSessionManager.getLastSequence(sessionId)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: sessionId, sequence, role: "tool", @@ -58,16 +58,16 @@ export class GoogleGeminiProvider extends BaseAIProvider { toolSchema: ChatCompletionTool, sessionId: string ): Promise { - let session = this.aiSessionManager.getSession(sessionId, "google-gemini"); + let session = await this.aiSessionManager.getSession(sessionId, "google-gemini"); if (!session) { - session = this.aiSessionManager.createSession({ + session = await this.aiSessionManager.createSession({ provider: "google-gemini", sessionId, }); } - const existingMessages = this.aiSessionManager.getMessages(session.id); + const existingMessages = await this.aiSessionManager.getMessages(session.id); const contents: any[] = []; // System instruction is separate in Gemini API @@ -116,8 +116,8 @@ export class GoogleGeminiProvider extends BaseAIProvider { } if (contents.length === 0 || contents[contents.length - 1].role !== "user") { - const userSequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const userSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: userSequence, role: "user", @@ -201,7 +201,7 @@ export class GoogleGeminiProvider extends BaseAIProvider { } const modelMsg = candidate.content; - const assistantSequence = this.aiSessionManager.getLastSequence(session.id) + 1; + const assistantSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; // Map Gemini response back to our internal message format const assistantMsg: any = { @@ -226,7 +226,7 @@ export class GoogleGeminiProvider extends BaseAIProvider { } } - this.aiSessionManager.addMessage(assistantMsg); + await this.aiSessionManager.addMessage(assistantMsg); contents.push(modelMsg); if (assistantMsg.toolCalls.length > 0) { @@ -237,7 +237,7 @@ export class GoogleGeminiProvider extends BaseAIProvider { const result = UserProfileValidator.validate(parsed); if (!result.valid) throw new Error(result.errors.join(", ")); - this.addToolResponse( + await this.addToolResponse( session.id, contents, toolCall.id, @@ -246,7 +246,7 @@ export class GoogleGeminiProvider extends BaseAIProvider { return { success: true, data: result.data, iterations }; } catch (validationError) { const errorMessage = `Validation failed: ${String(validationError)}`; - this.addToolResponse( + await this.addToolResponse( session.id, contents, toolCall.id, @@ -260,8 +260,8 @@ export class GoogleGeminiProvider extends BaseAIProvider { // Retry if no tool call was made const retryPrompt = "Please use the save_memories tool as instructed."; - const retrySequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const retrySequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: retrySequence, role: "user", diff --git a/src/services/ai/providers/openai-chat-completion.ts b/src/services/ai/providers/openai-chat-completion.ts index 5f1b554..48545da 100644 --- a/src/services/ai/providers/openai-chat-completion.ts +++ b/src/services/ai/providers/openai-chat-completion.ts @@ -110,14 +110,14 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { return true; } - private addToolResponse( + private async addToolResponse( sessionId: string, messages: APIMessage[], toolCallId: string, content: string - ): void { - const sequence = this.aiSessionManager.getLastSequence(sessionId) + 1; - this.aiSessionManager.addMessage({ + ): Promise { + const sequence = (await this.aiSessionManager.getLastSequence(sessionId)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: sessionId, sequence, role: "tool", @@ -177,16 +177,16 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { toolSchema: ChatCompletionTool, sessionId: string ): Promise { - let session = this.aiSessionManager.getSession(sessionId, "openai-chat"); + let session = await this.aiSessionManager.getSession(sessionId, "openai-chat"); if (!session) { - session = this.aiSessionManager.createSession({ + session = await this.aiSessionManager.createSession({ provider: "openai-chat", sessionId, }); } - const existingMessages = this.aiSessionManager.getMessages(session.id); + const existingMessages = await this.aiSessionManager.getMessages(session.id); const messages: APIMessage[] = []; const validatedMessages = this.filterIncompleteToolCallSequences(existingMessages); @@ -209,8 +209,8 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { } if (messages.length === 0) { - const sequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const sequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence, role: "system", @@ -220,8 +220,8 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { messages.push({ role: "system", content: systemPrompt }); } - const userSequence = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const userSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: userSequence, role: "user", @@ -347,7 +347,7 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { }; } - const assistantSequence = this.aiSessionManager.getLastSequence(session.id) + 1; + const assistantSequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; const assistantMsg: AssistantSessionMessage = { aiSessionId: session.id, sequence: assistantSequence, @@ -359,7 +359,7 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { assistantMsg.toolCalls = choice.message.tool_calls; } - this.aiSessionManager.addMessage(assistantMsg); + await this.aiSessionManager.addMessage(assistantMsg); messages.push({ role: "assistant", content: choice.message.content ?? null, @@ -401,7 +401,7 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { throw new Error(result.errors.join(", ")); } - this.addToolResponse( + await this.addToolResponse( session.id, messages, toolCallId, @@ -430,7 +430,7 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { const errorMessage = `Validation failed: ${String(validationError)}`; lastErrorMessage = errorMessage; - this.addToolResponse( + await this.addToolResponse( session.id, messages, toolCallId, @@ -446,7 +446,7 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { } const wrongToolMessage = `Wrong tool called. Please use ${toolSchema.function.name} instead.`; - this.addToolResponse( + await this.addToolResponse( session.id, messages, toolCallId, @@ -457,12 +457,12 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { } } - const retrySequence = this.aiSessionManager.getLastSequence(session.id) + 1; + const retrySequence = (await this.aiSessionManager.getLastSequence(session.id)) + 1; const retryPrompt = lastErrorMessage ? `Your previous attempt failed. Error: ${lastErrorMessage}. Please fix the JSON in your tool call arguments and try again. Output ONLY valid JSON, no extra text outside the JSON structure.` : "Please use the tool to extract and save the data as instructed."; - this.aiSessionManager.addMessage({ + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: retrySequence, role: "user", diff --git a/src/services/ai/providers/openai-responses.ts b/src/services/ai/providers/openai-responses.ts index 1e7d257..194d5e5 100644 --- a/src/services/ai/providers/openai-responses.ts +++ b/src/services/ai/providers/openai-responses.ts @@ -44,10 +44,10 @@ export class OpenAIResponsesProvider extends BaseAIProvider { toolSchema: ChatCompletionTool, sessionId: string ): Promise { - let session = this.aiSessionManager.getSession(sessionId, "openai-responses"); + let session = await this.aiSessionManager.getSession(sessionId, "openai-responses"); if (!session) { - session = this.aiSessionManager.createSession({ + session = await this.aiSessionManager.createSession({ provider: "openai-responses", sessionId, }); @@ -117,8 +117,8 @@ export class OpenAIResponsesProvider extends BaseAIProvider { conversationId = data.conversation || conversationId; if (iterations === 1) { - const userSeq = this.aiSessionManager.getLastSequence(session.id) + 1; - this.aiSessionManager.addMessage({ + const userSeq = (await this.aiSessionManager.getLastSequence(session.id)) + 1; + await this.aiSessionManager.addMessage({ aiSessionId: session.id, sequence: userSeq, role: "user", @@ -129,7 +129,7 @@ export class OpenAIResponsesProvider extends BaseAIProvider { const toolCall = this.extractToolCall(data, toolSchema.function.name); if (toolCall) { - this.aiSessionManager.updateSession(sessionId, "openai-responses", { + await this.aiSessionManager.updateSession(sessionId, "openai-responses", { conversationId, }); diff --git a/src/services/ai/session/ai-session-manager.ts b/src/services/ai/session/ai-session-manager.ts index 457292f..ce1d1a9 100644 --- a/src/services/ai/session/ai-session-manager.ts +++ b/src/services/ai/session/ai-session-manager.ts @@ -1,4 +1,3 @@ -import { getDatabase } from "../../sqlite/sqlite-bootstrap.js"; import { join } from "node:path"; import type { AISession, @@ -7,88 +6,132 @@ import type { AIProviderType, AIMessage, } from "./session-types.js"; -import { connectionManager } from "../../sqlite/connection-manager.js"; +import { tursoConnectionManager } from "../../turso/connection-manager.js"; import { CONFIG } from "../../../config.js"; - -const Database = getDatabase(); -type DatabaseType = typeof Database.prototype; +import type { InValue } from "@libsql/client"; +import type { TursoDb } from "../../turso/turso-db.js"; const AI_SESSIONS_DB_NAME = "ai-sessions.db"; export class AISessionManager { - private db: DatabaseType; - private readonly dbPath: string; + private db: TursoDb | null = null; + private dbPath: string; private readonly sessionRetentionMs: number; + private initPromise: Promise | null = null; constructor() { this.dbPath = join(CONFIG.storagePath, AI_SESSIONS_DB_NAME); - this.db = connectionManager.getConnection(this.dbPath); this.sessionRetentionMs = CONFIG.aiSessionRetentionDays * 24 * 60 * 60 * 1000; - this.initDatabase(); - } - - private initDatabase(): void { - this.db.run(` - CREATE TABLE IF NOT EXISTS ai_sessions ( - id TEXT PRIMARY KEY, - provider TEXT NOT NULL, - session_id TEXT NOT NULL, - conversation_id TEXT, - metadata TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL - ) - `); - - this.db.run("CREATE INDEX IF NOT EXISTS idx_ai_sessions_session_id ON ai_sessions(session_id)"); - this.db.run("CREATE INDEX IF NOT EXISTS idx_ai_sessions_expires_at ON ai_sessions(expires_at)"); - this.db.run("CREATE INDEX IF NOT EXISTS idx_ai_sessions_provider ON ai_sessions(provider)"); - - this.db.run(` - CREATE TABLE IF NOT EXISTS ai_messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ai_session_id TEXT NOT NULL, - sequence INTEGER NOT NULL, - role TEXT NOT NULL, - content TEXT NOT NULL, - tool_calls TEXT, - tool_call_id TEXT, - content_blocks TEXT, - created_at INTEGER NOT NULL, - FOREIGN KEY (ai_session_id) REFERENCES ai_sessions(id) ON DELETE CASCADE - ) - `); - - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_ai_messages_session ON ai_messages(ai_session_id, sequence)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_ai_messages_role ON ai_messages(ai_session_id, role)" - ); } - getSession(sessionId: string, provider: AIProviderType): AISession | null { - const stmt = this.db.prepare(` - SELECT * FROM ai_sessions - WHERE session_id = ? AND provider = ? AND expires_at > ? - `); - const row = stmt.get(sessionId, provider, Date.now()) as any; + reset(): void { + this.db = null; + this.initPromise = null; + this.dbPath = join(CONFIG.storagePath, AI_SESSIONS_DB_NAME); + } + + private async initialize(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.dbPath = join(CONFIG.storagePath, AI_SESSIONS_DB_NAME); + + this.initPromise = (async () => { + try { + this.db = await tursoConnectionManager.getConnection(this.dbPath); + await this.initDatabase(); + } catch (error) { + this.initPromise = null; + this.db = null; + throw error; + } + })(); + + return this.initPromise; + } + + private async ready(): Promise { + if (!this.db || !this.initPromise) { + await this.initialize(); + } else { + await this.initPromise; + } + + if (!this.db) { + throw new Error("AISessionManager: database not initialized"); + } - if (!row) return null; + return this.db; + } + + private async initDatabase(): Promise { + const db = this.db!; + await db.batch([ + { + sql: ` + CREATE TABLE IF NOT EXISTS ai_sessions ( + id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + session_id TEXT NOT NULL, + conversation_id TEXT, + metadata TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ) + `, + }, + { sql: "CREATE INDEX IF NOT EXISTS idx_ai_sessions_session_id ON ai_sessions(session_id)" }, + { sql: "CREATE INDEX IF NOT EXISTS idx_ai_sessions_expires_at ON ai_sessions(expires_at)" }, + { sql: "CREATE INDEX IF NOT EXISTS idx_ai_sessions_provider ON ai_sessions(provider)" }, + { + sql: ` + CREATE TABLE IF NOT EXISTS ai_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ai_session_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + tool_calls TEXT, + tool_call_id TEXT, + content_blocks TEXT, + created_at INTEGER NOT NULL, + FOREIGN KEY (ai_session_id) REFERENCES ai_sessions(id) ON DELETE CASCADE + ) + `, + }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_ai_messages_session ON ai_messages(ai_session_id, sequence)", + }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_ai_messages_role ON ai_messages(ai_session_id, role)", + }, + ]); + } - return this.rowToSession(row); + async getSession(sessionId: string, provider: AIProviderType): Promise { + const db = await this.ready(); + const row = await db.get( + ` + SELECT * FROM ai_sessions + WHERE session_id = ? AND provider = ? AND expires_at > ? + `, + [sessionId, provider, Date.now()] + ); + return row ? this.rowToSession(row) : null; } - createSession(params: SessionCreateParams): AISession { + async createSession(params: SessionCreateParams): Promise { + const db = await this.ready(); const id = `sess_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; const now = Date.now(); const expiresAt = now + this.sessionRetentionMs; - this.db.run( + await db.run( ` INSERT INTO ai_sessions ( - id, provider, session_id, conversation_id, + id, provider, session_id, conversation_id, metadata, created_at, updated_at, expires_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `, @@ -104,12 +147,17 @@ export class AISessionManager { ] ); - return this.getSession(params.sessionId, params.provider)!; + return (await this.getSession(params.sessionId, params.provider))!; } - updateSession(sessionId: string, provider: AIProviderType, updates: SessionUpdateParams): void { + async updateSession( + sessionId: string, + provider: AIProviderType, + updates: SessionUpdateParams + ): Promise { + const db = await this.ready(); const fields: string[] = []; - const values: any[] = []; + const values: InValue[] = []; if (updates.conversationId !== undefined) { fields.push("conversation_id = ?"); @@ -123,13 +171,11 @@ export class AISessionManager { fields.push("updated_at = ?"); values.push(Date.now()); + values.push(sessionId, provider); - values.push(sessionId); - values.push(provider); - - this.db.run( + await db.run( ` - UPDATE ai_sessions + UPDATE ai_sessions SET ${fields.join(", ")} WHERE session_id = ? AND provider = ? `, @@ -137,22 +183,24 @@ export class AISessionManager { ); } - cleanupExpiredSessions(): number { - const result = this.db.run(`DELETE FROM ai_sessions WHERE expires_at < ?`, [Date.now()]); - return result.changes; + async cleanupExpiredSessions(): Promise { + const db = await this.ready(); + return db.run(`DELETE FROM ai_sessions WHERE expires_at < ?`, [Date.now()]); } - deleteSession(sessionId: string, provider: AIProviderType): void { - this.db.run(`DELETE FROM ai_sessions WHERE session_id = ? AND provider = ?`, [ + async deleteSession(sessionId: string, provider: AIProviderType): Promise { + const db = await this.ready(); + await db.run(`DELETE FROM ai_sessions WHERE session_id = ? AND provider = ?`, [ sessionId, provider, ]); } - addMessage(message: Omit): void { - this.db.run( + async addMessage(message: Omit): Promise { + const db = await this.ready(); + await db.run( `INSERT INTO ai_messages ( - ai_session_id, sequence, role, content, + ai_session_id, sequence, role, content, tool_calls, tool_call_id, content_blocks, created_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [ @@ -168,52 +216,53 @@ export class AISessionManager { ); } - getMessages(aiSessionId: string): AIMessage[] { - const stmt = this.db.prepare( - "SELECT * FROM ai_messages WHERE ai_session_id = ? ORDER BY sequence ASC" + async getMessages(aiSessionId: string): Promise { + const db = await this.ready(); + const rows = await db.all( + "SELECT * FROM ai_messages WHERE ai_session_id = ? ORDER BY sequence ASC", + [aiSessionId] ); - const rows = stmt.all(aiSessionId) as any[]; - - return rows.map(this.rowToMessage); + return rows.map((row) => this.rowToMessage(row)); } - getLastSequence(aiSessionId: string): number { - const stmt = this.db.prepare( - "SELECT MAX(sequence) as max_seq FROM ai_messages WHERE ai_session_id = ?" + async getLastSequence(aiSessionId: string): Promise { + const db = await this.ready(); + const row = await db.get( + "SELECT MAX(sequence) as max_seq FROM ai_messages WHERE ai_session_id = ?", + [aiSessionId] ); - const row = stmt.get(aiSessionId) as any; - - return row?.max_seq ?? -1; + return row?.max_seq != null ? Number(row.max_seq) : -1; } - clearMessages(aiSessionId: string): void { - this.db.run("DELETE FROM ai_messages WHERE ai_session_id = ?", [aiSessionId]); + async clearMessages(aiSessionId: string): Promise { + const db = await this.ready(); + await db.run("DELETE FROM ai_messages WHERE ai_session_id = ?", [aiSessionId]); } - private rowToSession(row: any): AISession { + private rowToSession(row: Record): AISession { return { - id: row.id, + id: String(row.id), provider: row.provider as AIProviderType, - sessionId: row.session_id, - conversationId: row.conversation_id, - metadata: row.metadata ? JSON.parse(row.metadata) : undefined, - createdAt: row.created_at, - updatedAt: row.updated_at, - expiresAt: row.expires_at, + sessionId: String(row.session_id), + conversationId: row.conversation_id ? String(row.conversation_id) : undefined, + metadata: row.metadata ? JSON.parse(String(row.metadata)) : undefined, + createdAt: Number(row.created_at), + updatedAt: Number(row.updated_at), + expiresAt: Number(row.expires_at), }; } - private rowToMessage(row: any): AIMessage { + private rowToMessage(row: Record): AIMessage { return { - id: row.id, - aiSessionId: row.ai_session_id, - sequence: row.sequence, - role: row.role, - content: row.content, - toolCalls: row.tool_calls ? JSON.parse(row.tool_calls) : undefined, - toolCallId: row.tool_call_id, - contentBlocks: row.content_blocks ? JSON.parse(row.content_blocks) : undefined, - createdAt: row.created_at, + id: Number(row.id), + aiSessionId: String(row.ai_session_id), + sequence: Number(row.sequence), + role: row.role as AIMessage["role"], + content: String(row.content), + toolCalls: row.tool_calls ? JSON.parse(String(row.tool_calls)) : undefined, + toolCallId: row.tool_call_id ? String(row.tool_call_id) : undefined, + contentBlocks: row.content_blocks ? JSON.parse(String(row.content_blocks)) : undefined, + createdAt: Number(row.created_at), }; } } diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 55afc1f..0964684 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -1,13 +1,24 @@ import { embeddingService } from "./embedding.js"; -import { shardManager } from "./sqlite/shard-manager.js"; -import { vectorSearch } from "./sqlite/vector-search.js"; -import { connectionManager } from "./sqlite/connection-manager.js"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { ensureTursoReady } from "./turso/ready.js"; +import { formatTagsForEmbedding } from "./turso/vector-utils.js"; +import { extractScopeFromContainerTag, tryExtractScopeFromContainerTag } from "./memory-scope.js"; import { log } from "./logger.js"; import { CONFIG } from "../config.js"; import type { MemoryType } from "../types/index.js"; import { userPromptManager } from "./user-prompt/user-prompt-manager.js"; import type { UserProfileData } from "./user-profile/types.js"; import { sortProfileItems } from "../utils/profile.js"; +import type { ShardInfo } from "./turso/types.js"; + +async function getAllMemoryShards(): Promise { + await ensureTursoReady(); + const projectShards = await tursoShardManager.getAllShards("project", ""); + const userShards = await tursoShardManager.getAllShards("user", ""); + return [...projectShards, ...userShards]; +} interface ApiResponse { success: boolean; @@ -77,54 +88,48 @@ function safeJSONParse(jsonString: any): any { } } -function toBlob(vector?: Float32Array): Uint8Array | null { - return vector ? new Uint8Array(vector.buffer) : null; -} - -function extractScopeFromTag(tag: string): { scope: "project"; hash: string } { - const parts = tag.split("_"); - if (parts.length >= 3) { - const hash = parts.slice(2).join("_"); - return { scope: "project", hash }; - } - return { scope: "project", hash: tag }; +function extractScopeFromTag(tag: string): { scope: "user" | "project"; hash: string } { + return extractScopeFromContainerTag(tag); } -function getProjectPathFromTag(tag: string): string | undefined { - const projectShards = shardManager.getAllShards("project", ""); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const tags = vectorSearch.getDistinctTags(db); - for (const t of tags) { - if (t.container_tag === tag && t.project_path) { - return t.project_path; +function getProjectPathFromTag(tag: string): Promise { + return (async () => { + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const tags = await tursoVectorSearch.getDistinctTags(db); + for (const t of tags) { + if (t.container_tag === tag && t.project_path) { + return String(t.project_path); + } } } - } - return undefined; + return undefined; + })(); } export async function handleListTags(): Promise> { try { + await ensureTursoReady(); // Tags are stored as SQLite metadata; embedding model is not needed. // Calling warmup() here would block on local transformer init in the worker // thread and hang every read API. Only handlers that compute similarity // (e.g. handleSearch) should warm up the embedding service. - const projectShards = shardManager.getAllShards("project", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); const tagsMap = new Map(); for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const tags = vectorSearch.getDistinctTags(db); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const tags = await tursoVectorSearch.getDistinctTags(db); for (const t of tags) { - if (t.container_tag && !tagsMap.has(t.container_tag)) { - tagsMap.set(t.container_tag, { - tag: t.container_tag, - displayName: t.display_name, - userName: t.user_name, - userEmail: t.user_email, - projectPath: t.project_path, - projectName: t.project_name, - gitRepoUrl: t.git_repo_url, + if (t.container_tag && !tagsMap.has(String(t.container_tag))) { + tagsMap.set(String(t.container_tag), { + tag: String(t.container_tag), + displayName: t.display_name ? String(t.display_name) : undefined, + userName: t.user_name ? String(t.user_name) : undefined, + userEmail: t.user_email ? String(t.user_email) : undefined, + projectPath: t.project_path ? String(t.project_path) : undefined, + projectName: t.project_name ? String(t.project_name) : undefined, + gitRepoUrl: t.git_repo_url ? String(t.git_repo_url) : undefined, }); } } @@ -149,15 +154,16 @@ export async function handleListMemories( includePrompts: boolean = true ): Promise>> { try { + await ensureTursoReady(); // Listing only reads SQLite rows; no vector ops happen here. // See handleListTags comment - keep embedding init out of read paths. let allMemories: any[] = []; if (tag) { const { scope: tagScope, hash } = extractScopeFromTag(tag); - const shards = shardManager.getAllShards(tagScope, hash); + const shards = await tursoShardManager.getAllShards(tagScope, hash); for (const shard of shards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.listMemories(db, tag, 10000); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.listMemories(db, tag, 10000); allMemories.push(...memories); } } else { @@ -169,11 +175,11 @@ export async function handleListMemories( // unbrowseable in the UI — a confusing UX gap. The filter keeps the // defense-in-depth check on container_tag, just widens it to both // canonical scope markers. - const projectShards = shardManager.getAllShards("project", ""); - const userShards = shardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); + const userShards = await tursoShardManager.getAllShards("user", ""); for (const shard of [...projectShards, ...userShards]) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.getAllMemories(db); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getAllMemories(db); allMemories.push( ...memories.filter( (m: any) => @@ -207,8 +213,8 @@ export async function handleListMemories( let timeline: any[] = memoriesWithType; if (includePrompts) { - const projectPath = tag ? getProjectPathFromTag(tag) : undefined; - const prompts = userPromptManager.getCapturedPrompts(projectPath); + const projectPath = tag ? await getProjectPathFromTag(tag) : undefined; + const prompts = await userPromptManager.getCapturedPrompts(projectPath); const promptsWithType = prompts.map((p) => ({ type: "prompt", id: p.id, @@ -314,6 +320,7 @@ export async function handleAddMemory(data: { if (!data.content || !data.containerTag) { return { success: false, error: "content and containerTag are required" }; } + await ensureTursoReady(); await embeddingService.warmup(); const tags = (data.tags || []).map((t) => t.trim().toLowerCase()); const embeddingInput = @@ -322,80 +329,40 @@ export async function handleAddMemory(data: { const vector = await embeddingService.embedWithTimeout(embeddingInput); let tagsVector: Float32Array | undefined = undefined; if (tags.length > 0) { - tagsVector = await embeddingService.embedWithTimeout(tags.join(", ")); + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags)); } const { scope, hash } = extractScopeFromTag(data.containerTag); - const shard = shardManager.getWriteShard(scope, hash); - - const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`; - const now = Date.now(); - - const record = { - id, - content: data.content, - vector, - tagsVector, - containerTag: data.containerTag, - tags: tags.length > 0 ? tags.join(",") : undefined, - type: data.type, - createdAt: now, - updatedAt: now, - displayName: data.displayName, - userName: data.userName, - userEmail: data.userEmail, - projectPath: data.projectPath, - projectName: data.projectName, - gitRepoUrl: data.gitRepoUrl, - metadata: JSON.stringify({ source: "api" }), - }; - const db = connectionManager.getConnection(shard.dbPath); + return tursoShardManager.withScopeWriteLock(scope, hash, async () => { + const shard = await tursoShardManager.getWriteShard(scope, hash); - // Use transaction for atomic SQLite insert - const insertMemory = db.transaction(() => { - const insertStmt = db.prepare(` - INSERT INTO memories ( - id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - insertStmt.run( - record.id, - record.content, - toBlob(record.vector), - toBlob(record.tagsVector), - record.containerTag, - record.tags || null, - record.type || null, - record.createdAt, - record.updatedAt, - record.metadata || null, - record.displayName || null, - record.userName || null, - record.userEmail || null, - record.projectPath || null, - record.projectName || null, - record.gitRepoUrl || null - ); - }); - insertMemory(); - - // Vector index update (outside transaction — vector backend is async) - try { - const backend = await (vectorSearch as any).getBackend(); - await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); - if (record.tagsVector) { - await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); - } - } catch (error) { - // Rollback SQLite insert on vector backend failure - db.prepare(`DELETE FROM memories WHERE id = ?`).run(record.id); - throw error; - } + const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`; + const now = Date.now(); - shardManager.incrementVectorCount(shard.id); - return { success: true, data: { id } }; + const record = { + id, + content: data.content, + vector, + tagsVector, + containerTag: data.containerTag, + tags: tags.length > 0 ? tags.join(",") : undefined, + type: data.type, + createdAt: now, + updatedAt: now, + displayName: data.displayName, + userName: data.userName, + userEmail: data.userEmail, + projectPath: data.projectPath, + projectName: data.projectName, + gitRepoUrl: data.gitRepoUrl, + metadata: JSON.stringify({ source: "api" }), + }; + const db = await tursoConnectionManager.getConnection(shard.dbPath); + await tursoVectorSearch.insertVector(db, record); + await tursoShardManager.incrementVectorCount(shard.id); + return { success: true as const, data: { id } }; + }); } catch (error) { log("handleAddMemory: error", { error: String(error) }); return { success: false, error: String(error) }; @@ -408,18 +375,18 @@ export async function handleDeleteMemory( ): Promise> { try { if (!id) return { success: false, error: "id is required" }; - const projectShards = shardManager.getAllShards("project", ""); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memory = vectorSearch.getMemoryById(db, id); + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); if (memory) { if (cascade) { const metadata = safeJSONParse(memory.metadata); const linkedPromptId = metadata?.promptId; - if (linkedPromptId) userPromptManager.deletePrompt(linkedPromptId); + if (linkedPromptId) await userPromptManager.deletePrompt(linkedPromptId); } - await vectorSearch.deleteVector(db, id, shard); - shardManager.decrementVectorCount(shard.id); + await tursoVectorSearch.deleteVector(db, id); + await tursoShardManager.decrementVectorCount(shard.id); return { success: true, data: { deletedPrompt: cascade && !!safeJSONParse(memory.metadata)?.promptId }, @@ -460,12 +427,12 @@ export async function handleUpdateMemory( await embeddingService.warmup(); // Find the existing memory first (read-only — no data modified yet) - const projectShards = shardManager.getAllShards("project", ""); + const shards = await getAllMemoryShards(); let foundShard = null, existingMemory = null; - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memory = vectorSearch.getMemoryById(db, id); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); if (memory) { foundShard = shard; existingMemory = memory; @@ -475,62 +442,48 @@ export async function handleUpdateMemory( if (!foundShard || !existingMemory) return { success: false, error: "Memory not found" }; // STEP 1: Generate new embeddings FIRST (safe — no data deleted yet) - const newContent = data.content || existingMemory.content; + const newContent = data.content || String(existingMemory.content); + const existingTags = existingMemory.tags ? String(existingMemory.tags) : ""; const tags = - data.tags || - (existingMemory.tags ? existingMemory.tags.split(",").map((t: string) => t.trim()) : []); + data.tags || (existingTags ? existingTags.split(",").map((t: string) => t.trim()) : []); const vector = await embeddingService.embedWithTimeout(newContent); let tagsVector: Float32Array | undefined = undefined; if (tags.length > 0) { - tagsVector = await embeddingService.embedWithTimeout(tags.join(", ")); + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags)); } - const db = connectionManager.getConnection(foundShard.dbPath); + const db = await tursoConnectionManager.getConnection(foundShard.dbPath); - // STEP 2: Wrap SQLite delete + insert in a transaction - const updateTransaction = db.transaction(() => { - // Delete old record - db.prepare(`DELETE FROM memories WHERE id = ?`).run(id); - - // Insert updated record - const insertStmt = db.prepare(` + await db.transaction("write", async (tx) => { + await tx.execute({ sql: `DELETE FROM memories WHERE id = ?`, args: [id] }); + await tx.execute({ + sql: ` INSERT INTO memories ( id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - insertStmt.run( - id, - newContent, - toBlob(vector), - toBlob(tagsVector), - existingMemory.container_tag, - tags.length > 0 ? tags.join(",") : null, - data.type || existingMemory.type, - existingMemory.created_at, - Date.now(), - existingMemory.metadata, - existingMemory.display_name, - existingMemory.user_name, - existingMemory.user_email, - existingMemory.project_path, - existingMemory.project_name, - existingMemory.git_repo_url - ); + ) VALUES (?, ?, vector32(?), ${tagsVector ? "vector32(?)" : "NULL"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + args: [ + id, + newContent, + JSON.stringify(Array.from(vector)), + ...(tagsVector ? [JSON.stringify(Array.from(tagsVector))] : []), + String(existingMemory.container_tag), + tags.length > 0 ? tags.join(",") : null, + data.type || (existingMemory.type ? String(existingMemory.type) : null), + Number(existingMemory.created_at), + Date.now(), + existingMemory.metadata ? String(existingMemory.metadata) : null, + existingMemory.display_name ? String(existingMemory.display_name) : null, + existingMemory.user_name ? String(existingMemory.user_name) : null, + existingMemory.user_email ? String(existingMemory.user_email) : null, + existingMemory.project_path ? String(existingMemory.project_path) : null, + existingMemory.project_name ? String(existingMemory.project_name) : null, + existingMemory.git_repo_url ? String(existingMemory.git_repo_url) : null, + ], + }); }); - // Execute the SQLite transaction atomically - updateTransaction(); - - // STEP 3: Update vector index (outside transaction — vector backend is async/in-memory) - const backend = await (vectorSearch as any).getBackend(); - await backend.delete({ id, shard: foundShard, kind: "content" }); - await backend.delete({ id, shard: foundShard, kind: "tags" }); - await backend.insert({ id, vector, shard: foundShard, kind: "content" }); - if (tagsVector) { - await backend.insert({ id, vector: tagsVector, shard: foundShard, kind: "tags" }); - } - return { success: true }; } catch (error) { log("handleUpdateMemory: error", { error: String(error) }); @@ -581,39 +534,50 @@ export async function handleSearch( ): Promise>> { try { if (!query) return { success: false, error: "query is required" }; + await ensureTursoReady(); await embeddingService.warmup(); const queryVector = await embeddingService.embedWithTimeout(query); let memoryResults: any[] = []; let promptResults: any[] = []; if (tag) { const { scope, hash } = extractScopeFromTag(tag); - const shards = shardManager.getAllShards(scope, hash); + const shards = await tursoShardManager.getAllShards(scope, hash); for (const shard of shards) { try { - const results = await vectorSearch.searchInShard(shard, queryVector, tag, pageSize * 2); + const results = await tursoVectorSearch.searchInShard( + shard, + queryVector, + tag, + pageSize * 2 + ); memoryResults.push(...results); } catch (error) { log("Shard search error", { shardId: shard.id, error: String(error) }); } } - const projectPath = getProjectPathFromTag(tag); - promptResults = userPromptManager.searchPrompts(query, projectPath, pageSize * 2); + const projectPath = await getProjectPathFromTag(tag); + promptResults = await userPromptManager.searchPrompts(query, projectPath, pageSize * 2); } else { - const projectShards = shardManager.getAllShards("project", ""); + const allShards = await getAllMemoryShards(); const uniqueTags = new Set(); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const tags = vectorSearch.getDistinctTags(db); + for (const shard of allShards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const tags = await tursoVectorSearch.getDistinctTags(db); for (const t of tags) { - if (t.container_tag) uniqueTags.add(t.container_tag); + if (t.container_tag) uniqueTags.add(String(t.container_tag)); } } for (const containerTag of uniqueTags) { - const { scope, hash } = extractScopeFromTag(containerTag); - const shards = shardManager.getAllShards(scope, hash); + const parsed = tryExtractScopeFromContainerTag(containerTag); + if (!parsed) { + log("Skipping invalid container_tag during global search", { containerTag }); + continue; + } + const { scope, hash } = parsed; + const shards = await tursoShardManager.getAllShards(scope, hash); for (const shard of shards) { try { - const results = await vectorSearch.searchInShard( + const results = await tursoVectorSearch.searchInShard( shard, queryVector, containerTag, @@ -625,7 +589,7 @@ export async function handleSearch( } } } - promptResults = userPromptManager.searchPrompts(query, undefined, pageSize * 2); + promptResults = await userPromptManager.searchPrompts(query, undefined, pageSize * 2); } const formattedPrompts: FormattedPrompt[] = promptResults.map((p) => ({ @@ -645,7 +609,7 @@ export async function handleSearch( content: r.memory, memoryType: r.metadata?.type, tags: r.tags, - createdAt: safeToISOString(r.metadata?.createdAt), + createdAt: safeToISOString(r.createdAt ?? r.metadata?.createdAt), updatedAt: r.metadata?.updatedAt ? safeToISOString(r.metadata.updatedAt) : undefined, similarity: r.similarity, metadata: r.metadata, @@ -682,7 +646,7 @@ export async function handleSearch( } if (missingPromptIds.size > 0) { - const extraPrompts = userPromptManager.getPromptsByIds(Array.from(missingPromptIds)); + const extraPrompts = await userPromptManager.getPromptsByIds(Array.from(missingPromptIds)); for (const p of extraPrompts) { paginatedResults.push({ type: "prompt", @@ -699,29 +663,33 @@ export async function handleSearch( } if (missingMemoryIds.size > 0) { - const projectShards = shardManager.getAllShards("project", ""); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); for (const mid of missingMemoryIds) { - const m = vectorSearch.getMemoryById(db, mid); + const m = await tursoVectorSearch.getMemoryById(db, mid); if (m && !paginatedResults.some((existing) => existing.id === m.id)) { paginatedResults.push({ type: "memory", - id: m.id, - content: m.content, - memoryType: m.type, - tags: m.tags ? m.tags.split(",").map((t: string) => t.trim()) : [], + id: String(m.id), + content: String(m.content), + memoryType: m.type ? String(m.type) : undefined, + tags: m.tags + ? String(m.tags) + .split(",") + .map((t: string) => t.trim()) + : [], createdAt: safeToISOString(m.created_at), updatedAt: m.updated_at ? safeToISOString(m.updated_at) : undefined, similarity: 0, metadata: safeJSONParse(m.metadata), - displayName: m.display_name, - userName: m.user_name, - userEmail: m.user_email, - projectPath: m.project_path, - projectName: m.project_name, - gitRepoUrl: m.git_repo_url, - isPinned: m.is_pinned === 1, + displayName: m.display_name ? String(m.display_name) : undefined, + userName: m.user_name ? String(m.user_name) : undefined, + userEmail: m.user_email ? String(m.user_email) : undefined, + projectPath: m.project_path ? String(m.project_path) : undefined, + projectName: m.project_name ? String(m.project_name) : undefined, + gitRepoUrl: m.git_repo_url ? String(m.git_repo_url) : undefined, + isPinned: Number(m.is_pinned) === 1, linkedPromptId: safeJSONParse(m.metadata)?.promptId, isContext: true, }); @@ -745,19 +713,21 @@ export async function handleStats(): Promise< }> > { try { - // Stats only counts SQLite rows; no embedding needed. + // Stats only counts Turso/libSQL rows; no embedding needed. // See handleListTags comment - keep embedding init out of read paths. - const projectShards = shardManager.getAllShards("project", ""); + const shards = await getAllMemoryShards(); let userCount = 0, projectCount = 0; const typeCount: Record = {}; - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.getAllMemories(db); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getAllMemories(db); for (const r of memories) { - if (r.container_tag?.includes("_user_")) userCount++; - else if (r.container_tag?.includes("_project_")) projectCount++; - if (r.type) typeCount[r.type] = (typeCount[r.type] || 0) + 1; + const containerTag = String(r.container_tag ?? ""); + const type = r.type ? String(r.type) : ""; + if (containerTag.includes("_user_")) userCount++; + else if (containerTag.includes("_project_")) projectCount++; + if (type) typeCount[type] = (typeCount[type] || 0) + 1; } } return { @@ -777,12 +747,12 @@ export async function handleStats(): Promise< export async function handlePinMemory(id: string): Promise> { try { if (!id) return { success: false, error: "id is required" }; - const projectShards = shardManager.getAllShards("project", ""); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memory = vectorSearch.getMemoryById(db, id); + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); if (memory) { - vectorSearch.pinMemory(db, id); + await tursoVectorSearch.pinMemory(db, id); return { success: true }; } } @@ -796,12 +766,12 @@ export async function handlePinMemory(id: string): Promise> { export async function handleUnpinMemory(id: string): Promise> { try { if (!id) return { success: false, error: "id is required" }; - const projectShards = shardManager.getAllShards("project", ""); - for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memory = vectorSearch.getMemoryById(db, id); + const shards = await getAllMemoryShards(); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, id); if (memory) { - vectorSearch.unpinMemory(db, id); + await tursoVectorSearch.unpinMemory(db, id); return { success: true }; } } @@ -882,14 +852,14 @@ export async function handleDeletePrompt( ): Promise> { try { if (!id) return { success: false, error: "id is required" }; - const prompt = userPromptManager.getPromptById(id); + const prompt = await userPromptManager.getPromptById(id); if (!prompt) return { success: false, error: "Prompt not found" }; let deletedMemory = false; if (cascade && prompt.linkedMemoryId) { const result = await handleDeleteMemory(prompt.linkedMemoryId, false); if (result.success) deletedMemory = true; } - userPromptManager.deletePrompt(id); + await userPromptManager.deletePrompt(id); return { success: true, data: { deletedMemory } }; } catch (error) { log("handleDeletePrompt: error", { error: String(error) }); @@ -924,7 +894,7 @@ export async function handleGetUserProfile(userId?: string): Promise ({ id: c.id, profileId: c.profileId, @@ -987,7 +957,7 @@ export async function handleGetProfileSnapshot(changelogId: string): Promise > { try { - const projectShards = shardManager.getAllShards("project", ""); + await ensureTursoReady(); + const projectShards = await tursoShardManager.getAllShards("project", ""); let untaggedCount = 0; for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const rows = db - .prepare("SELECT COUNT(*) as count FROM memories WHERE tags IS NULL OR tags = ''") - .get() as any; - untaggedCount += rows.count; + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const row = await db.get( + "SELECT COUNT(*) as count FROM memories WHERE tags IS NULL OR tags = ''" + ); + untaggedCount += Number(row?.count ?? 0); } return { success: true, data: { needsMigration: untaggedCount > 0, count: untaggedCount } }; } catch (error) { @@ -1386,6 +1362,7 @@ export async function handleRunTagMigrationBatch( batchSize: number = 5 ): Promise> { try { + await ensureTursoReady(); const { AIProviderFactory } = await import("./ai/ai-provider-factory.js"); const { buildMemoryProviderConfig } = await import("./ai/provider-config.js"); const providerConfig = buildMemoryProviderConfig(CONFIG, { @@ -1393,14 +1370,14 @@ export async function handleRunTagMigrationBatch( iterationTimeout: 30000, }); const provider = AIProviderFactory.createProvider(CONFIG.memoryProvider, providerConfig); - const projectShards = shardManager.getAllShards("project", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); let batchProcessed = 0; const allMemories: { memory: any; shard: any }[] = []; for (const shard of projectShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = db.prepare("SELECT * FROM memories").all() as any[]; + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await db.all("SELECT * FROM memories"); for (const m of memories) { allMemories.push({ memory: m, shard }); } @@ -1419,7 +1396,7 @@ export async function handleRunTagMigrationBatch( const item = allMemories[i]; if (!item) continue; const { memory: m, shard } = item; - const db = connectionManager.getConnection(shard.dbPath); + const db = await tursoConnectionManager.getConnection(shard.dbPath); try { let currentTags = m.tags @@ -1450,25 +1427,18 @@ export async function handleRunTagMigrationBatch( ); if (result.success && result.data?.tags) { currentTags = result.data.tags; - db.prepare("UPDATE memories SET tags = ? WHERE id = ?").run( + await db.run("UPDATE memories SET tags = ? WHERE id = ?", [ currentTags.join(","), - m.id - ); + m.id, + ]); } } const vector = await embeddingService.embedWithTimeout(m.content); const tagsVector = currentTags.length - ? await embeddingService.embedWithTimeout(currentTags.join(", ")) + ? await embeddingService.embedWithTimeout(formatTagsForEmbedding(currentTags)) : undefined; - const vectorBuffer = new Uint8Array(vector.buffer); - db.prepare("UPDATE memories SET vector = ?, updated_at = ? WHERE id = ?").run( - vectorBuffer, - Date.now(), - m.id - ); - - await vectorSearch.updateVector(db, m.id, vector, shard, tagsVector); + await tursoVectorSearch.updateVector(db, m.id, vector, tagsVector); migrationProgress.processed++; batchProcessed++; diff --git a/src/services/auto-capture.ts b/src/services/auto-capture.ts index af22cca..d7c5188 100644 --- a/src/services/auto-capture.ts +++ b/src/services/auto-capture.ts @@ -25,7 +25,7 @@ export async function performAutoCapture( isCaptureRunning = true; try { - const prompts = userPromptManager.getUncapturedPromptsForSession(sessionID); + const prompts = await userPromptManager.getUncapturedPromptsForSession(sessionID); if (prompts.length === 0) { return; } @@ -54,7 +54,7 @@ async function capturePrompt( let attempt = prompt.capture_attempts || 0; try { - if (!userPromptManager.claimPrompt(prompt.id)) { + if (!(await userPromptManager.claimPrompt(prompt.id))) { return; } claimedPromptId = prompt.id; @@ -111,7 +111,7 @@ async function capturePrompt( sessionID, type: summaryResult?.type, }); - userPromptManager.deletePrompt(prompt.id); + await userPromptManager.deletePrompt(prompt.id); claimedPromptId = null; return; } @@ -137,8 +137,8 @@ async function capturePrompt( }); if (result.success) { - userPromptManager.linkMemoryToPrompt(prompt.id, result.id); - userPromptManager.markAsCaptured(prompt.id); + await userPromptManager.linkMemoryToPrompt(prompt.id, result.id); + await userPromptManager.markAsCaptured(prompt.id); claimedPromptId = null; log("Auto-capture memory persisted", { promptId: prompt.id, @@ -165,7 +165,7 @@ async function capturePrompt( } catch (error) { const errMsg = error instanceof Error ? error.message : String(error); - userPromptManager.recordFailedAttempt(prompt.id); + await userPromptManager.recordFailedAttempt(prompt.id); if (attempt < maxRetries) { log(`Auto-capture warning (attempt ${attempt}/${maxRetries})`, { error: errMsg }); @@ -196,7 +196,7 @@ async function capturePrompt( } finally { if (claimedPromptId !== null) { try { - userPromptManager.releaseClaim(claimedPromptId); + await userPromptManager.releaseClaim(claimedPromptId); } catch (releaseErr) { log( `Failed to release captured=2 claim for prompt ${claimedPromptId}: ${ diff --git a/src/services/cleanup-service.ts b/src/services/cleanup-service.ts index da06903..c7c486a 100644 --- a/src/services/cleanup-service.ts +++ b/src/services/cleanup-service.ts @@ -1,6 +1,7 @@ -import { shardManager } from "./sqlite/shard-manager.js"; -import { vectorSearch } from "./sqlite/vector-search.js"; -import { connectionManager } from "./sqlite/connection-manager.js"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { ensureTursoReady } from "./turso/ready.js"; import { CONFIG } from "../config.js"; import { log } from "./logger.js"; import { userPromptManager } from "./user-prompt/user-prompt-manager.js"; @@ -41,60 +42,58 @@ export class CleanupService { this.lastCleanupTime = Date.now(); try { + await ensureTursoReady(); const cutoffTime = Date.now() - CONFIG.autoCleanupRetentionDays * 24 * 60 * 60 * 1000; - const userShards = shardManager.getAllShards("user", ""); - const projectShards = shardManager.getAllShards("project", ""); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); const allShards = [...userShards, ...projectShards]; const pinnedMemoryIds = new Set(); for (const shard of allShards) { - const db = connectionManager.getConnection(shard.dbPath); - const pinned = db.prepare(`SELECT id FROM memories WHERE is_pinned = 1`).all() as any[]; - pinned.forEach((row) => pinnedMemoryIds.add(row.id)); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const pinned = await db.all(`SELECT id FROM memories WHERE is_pinned = 1`); + pinned.forEach((row) => pinnedMemoryIds.add(String(row.id))); } - const promptCleanupResult = userPromptManager.deleteOldPrompts(cutoffTime); + const promptCleanupResult = await userPromptManager.deleteOldPrompts(cutoffTime); const linkedMemoryIds = new Set(promptCleanupResult.linkedMemoryIds); - const protectedMemoryIds = new Set([...pinnedMemoryIds, ...linkedMemoryIds]); let totalDeleted = 0; let userDeleted = 0; let projectDeleted = 0; - let linkedMemoriesDeleted = 0; + const linkedMemoriesDeleted = 0; let pinnedSkipped = 0; for (const shard of allShards) { - const db = connectionManager.getConnection(shard.dbPath); - - const oldMemories = db - .prepare( - ` - SELECT id, container_tag, is_pinned FROM memories + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const oldMemories = await db.all( + ` + SELECT id, container_tag, is_pinned FROM memories WHERE updated_at < ? - ` - ) - .all(cutoffTime) as any[]; + `, + [cutoffTime] + ); for (const memory of oldMemories) { try { - if (memory.is_pinned === 1) { + if (Number(memory.is_pinned) === 1) { pinnedSkipped++; continue; } - if (protectedMemoryIds.has(memory.id)) { + if (protectedMemoryIds.has(String(memory.id))) { continue; } - await vectorSearch.deleteVector(db, memory.id, shard); - shardManager.decrementVectorCount(shard.id); + await tursoVectorSearch.deleteVector(db, String(memory.id)); + await tursoShardManager.decrementVectorCount(shard.id); totalDeleted++; - if (memory.container_tag?.includes("_user_")) { + if (String(memory.container_tag).includes("_user_")) { userDeleted++; - } else if (memory.container_tag?.includes("_project_")) { + } else if (String(memory.container_tag).includes("_project_")) { projectDeleted++; } } catch (error) { diff --git a/src/services/client.ts b/src/services/client.ts index 2671df0..e439285 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -1,11 +1,14 @@ import { embeddingService } from "./embedding.js"; -import { shardManager } from "./sqlite/shard-manager.js"; -import { vectorSearch } from "./sqlite/vector-search.js"; -import { connectionManager } from "./sqlite/connection-manager.js"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { ensureTursoReady } from "./turso/ready.js"; +import { formatTagsForEmbedding } from "./turso/vector-utils.js"; +import { extractScopeFromContainerTag, resolveMemoryScope } from "./memory-scope.js"; import { CONFIG } from "../config.js"; import { log } from "./logger.js"; import type { MemoryType } from "../types/index.js"; -import type { MemoryRecord } from "./sqlite/types.js"; +import type { MemoryRecord } from "./turso/types.js"; export type MemoryScope = "project" | "all-projects"; @@ -37,31 +40,11 @@ function safeJSONParse(jsonString: any): any { } } -function toBlob(vector?: Float32Array): Uint8Array | null { - return vector ? new Uint8Array(vector.buffer) : null; -} - -function extractScopeFromContainerTag(containerTag: string): { - scope: "user" | "project"; - hash: string; -} { - const parts = containerTag.split("_"); - if (parts.length >= 3) { - const scope = parts[1] as "user" | "project"; - const hash = parts.slice(2).join("_"); - return { scope, hash }; - } - return { scope: "user", hash: containerTag }; -} - function resolveScopeValue( scope: MemoryScope, containerTag: string ): { scope: "user" | "project"; hash: string } { - if (scope === "all-projects") { - return { scope: "project", hash: "" }; - } - return extractScopeFromContainerTag(containerTag); + return resolveMemoryScope(scope, containerTag); } export class LocalMemoryClient { @@ -76,10 +59,11 @@ export class LocalMemoryClient { this.initPromise = (async () => { try { + await ensureTursoReady(); this.isInitialized = true; } catch (error) { this.initPromise = null; - log("SQLite initialization failed", { error: String(error) }); + log("Turso initialization failed", { error: String(error) }); throw error; } })(); @@ -108,8 +92,15 @@ export class LocalMemoryClient { }; } - close(): void { - connectionManager.closeAll(); + reset(): void { + this.isInitialized = false; + this.initPromise = null; + } + + async close(): Promise { + const { closeTursoAndInvalidateCaches } = await import("./turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + this.reset(); } async searchMemories(query: string, containerTag: string, scope: MemoryScope = "project") { @@ -118,13 +109,13 @@ export class LocalMemoryClient { const queryVector = await embeddingService.embedWithTimeout(query); const resolved = resolveScopeValue(scope, containerTag); - const shards = shardManager.getAllShards(resolved.scope, resolved.hash); + const shards = await tursoShardManager.getAllShards(resolved.scope, resolved.hash); if (shards.length === 0) { return { success: true as const, results: [], total: 0, timing: 0 }; } - const results = await vectorSearch.searchAcrossShards( + const { results, warnings } = await tursoVectorSearch.searchAcrossShards( shards, queryVector, scope === "all-projects" ? "" : containerTag, @@ -133,7 +124,13 @@ export class LocalMemoryClient { query ); - return { success: true as const, results, total: results.length, timing: 0 }; + return { + success: true as const, + results, + total: results.length, + timing: 0, + ...(warnings.length > 0 ? { warnings } : {}), + }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log("searchMemories: error", { error: errorMessage }); @@ -169,100 +166,58 @@ export class LocalMemoryClient { let tagsVector: Float32Array | undefined = undefined; if (tags.length > 0) { - // Wrap tags in a natural-language template before embedding. Bare comma - // lists like "react, auth, bug-fix" sit outside the multilingual-e5 - // training distribution, so the resulting tagsVector drifts toward - // unrelated chatter and weakens the 0.4-weight tag boost in - // VectorSearch#searchInShard. The "Topics: ..." prefix is a sentence - // form e5 was trained on and yields a more discriminative vector. - tagsVector = await embeddingService.embedWithTimeout(`Topics: ${tags.join(", ")}`); + tagsVector = await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags)); } const { scope, hash } = extractScopeFromContainerTag(containerTag); - const shard = shardManager.getWriteShard(scope, hash); - - const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`; - const now = Date.now(); - - const { - displayName, - userName, - userEmail, - projectPath, - projectName, - gitRepoUrl, - type, - tags: _tags, - ...dynamicMetadata - } = metadata || {}; - - const record: MemoryRecord = { - id, - content, - vector, - tagsVector, - containerTag, - tags: tags.length > 0 ? tags.join(",") : undefined, - type, - createdAt: now, - updatedAt: now, - displayName, - userName, - userEmail, - projectPath, - projectName, - gitRepoUrl, - metadata: - Object.keys(dynamicMetadata).length > 0 ? JSON.stringify(dynamicMetadata) : undefined, - }; - const db = connectionManager.getConnection(shard.dbPath); - - // Use transaction for atomic SQLite insert - const insertMemory = db.transaction(() => { - const insertStmt = db.prepare(` - INSERT INTO memories ( - id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - insertStmt.run( - record.id, - record.content, - toBlob(record.vector), - toBlob(record.tagsVector), - record.containerTag, - record.tags || null, - record.type || null, - record.createdAt, - record.updatedAt, - record.metadata || null, - record.displayName || null, - record.userName || null, - record.userEmail || null, - record.projectPath || null, - record.projectName || null, - record.gitRepoUrl || null - ); - }); - insertMemory(); - - // Vector index update (outside transaction — vector backend is async/in-memory) - try { - const backend = await (vectorSearch as any).getBackend(); - await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); - if (record.tagsVector) { - await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); - } - } catch (error) { - // Rollback SQLite insert on vector backend failure - db.prepare(`DELETE FROM memories WHERE id = ?`).run(record.id); - throw error; - } + return tursoShardManager.withScopeWriteLock(scope, hash, async () => { + const shard = await tursoShardManager.getWriteShard(scope, hash); + + const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`; + const now = Date.now(); + + const { + displayName, + userName, + userEmail, + projectPath, + projectName, + gitRepoUrl, + type, + tags: _tags, + ...dynamicMetadata + } = metadata || {}; + + const record: MemoryRecord = { + id, + content, + vector, + tagsVector, + containerTag, + tags: tags.length > 0 ? tags.join(",") : undefined, + type, + createdAt: now, + updatedAt: now, + displayName, + userName, + userEmail, + projectPath, + projectName, + gitRepoUrl, + metadata: + Object.keys(dynamicMetadata).length > 0 ? JSON.stringify(dynamicMetadata) : undefined, + }; - shardManager.incrementVectorCount(shard.id); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + await tursoVectorSearch.insertVector(db, record); + await tursoShardManager.incrementVectorCount(shard.id); - return { success: true as const, id }; + return { + success: true as const, + id, + }; + }); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log("addMemory: error", { error: errorMessage }); @@ -274,17 +229,17 @@ export class LocalMemoryClient { try { await this.initialize(); - const userShards = shardManager.getAllShards("user", ""); - const projectShards = shardManager.getAllShards("project", ""); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); const allShards = [...userShards, ...projectShards]; for (const shard of allShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memory = vectorSearch.getMemoryById(db, memoryId); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memory = await tursoVectorSearch.getMemoryById(db, memoryId); if (memory) { - await vectorSearch.deleteVector(db, memoryId, shard); - shardManager.decrementVectorCount(shard.id); + await tursoVectorSearch.deleteVector(db, memoryId); + await tursoShardManager.decrementVectorCount(shard.id); return { success: true }; } } @@ -302,7 +257,7 @@ export class LocalMemoryClient { await this.initialize(); const resolved = resolveScopeValue(scope, containerTag); - const shards = shardManager.getAllShards(resolved.scope, resolved.hash); + const shards = await tursoShardManager.getAllShards(resolved.scope, resolved.hash); if (shards.length === 0) { return { @@ -315,8 +270,8 @@ export class LocalMemoryClient { const allMemories: any[] = []; for (const shard of shards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.listMemories( + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.listMemories( db, scope === "all-projects" ? "" : containerTag, limit @@ -361,7 +316,7 @@ export class LocalMemoryClient { await this.initialize(); const { scope, hash } = extractScopeFromContainerTag(containerTag); - const shards = shardManager.getAllShards(scope, hash); + const shards = await tursoShardManager.getAllShards(scope, hash); if (shards.length === 0) { return { success: true as const, results: [], total: 0, timing: 0 }; @@ -370,12 +325,12 @@ export class LocalMemoryClient { const allMemories: any[] = []; for (const shard of shards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.getMemoriesBySessionID(db, sessionID); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getMemoriesBySessionID(db, sessionID); allMemories.push(...memories); } - allMemories.sort((a, b) => b.created_at - a.created_at); + allMemories.sort((a, b) => Number(b.created_at) - Number(a.created_at)); const results = allMemories.slice(0, limit).map((row: any) => ({ id: row.id, diff --git a/src/services/context.ts b/src/services/context.ts index db74cbf..c94c0c4 100644 --- a/src/services/context.ts +++ b/src/services/context.ts @@ -11,14 +11,14 @@ interface MemoriesResponseMinimal { results?: MemoryResultMinimal[]; } -export function formatContextForPrompt( +export async function formatContextForPrompt( userId: string | null, projectMemories: MemoriesResponseMinimal -): string { +): Promise { const parts: string[] = ["[MEMORY]"]; if (CONFIG.injectProfile && userId) { - const profileContext = getUserProfileContext(userId); + const profileContext = await getUserProfileContext(userId); if (profileContext) { parts.push("\n" + profileContext); } diff --git a/src/services/deduplication-service.ts b/src/services/deduplication-service.ts index e885786..5679250 100644 --- a/src/services/deduplication-service.ts +++ b/src/services/deduplication-service.ts @@ -1,9 +1,11 @@ -import { shardManager } from "./sqlite/shard-manager.js"; -import { vectorSearch } from "./sqlite/vector-search.js"; -import { connectionManager } from "./sqlite/connection-manager.js"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { ensureTursoReady } from "./turso/ready.js"; import { CONFIG } from "../config.js"; import { log } from "./logger.js"; import { cosineSimilarity } from "../utils/math.js"; +import { parseExtractedVector } from "./turso/vector-utils.js"; interface DuplicateGroup { representative: { @@ -39,18 +41,19 @@ export class DeduplicationService { this.isRunning = true; try { - const userShards = shardManager.getAllShards("user", ""); - const projectShards = shardManager.getAllShards("project", ""); + await ensureTursoReady(); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); const allShards = [...userShards, ...projectShards]; let exactDeleted = 0; const nearDuplicateGroups: DuplicateGroup[] = []; for (const shard of allShards) { - const db = connectionManager.getConnection(shard.dbPath); - const memories = vectorSearch.getAllMemories(db); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await tursoVectorSearch.getAllMemoriesWithExtractedVectors(db); - const contentMap = new Map(); + const contentMap = new Map(); for (const memory of memories) { const key = `${memory.container_tag}:${memory.content}`; @@ -67,8 +70,8 @@ export class DeduplicationService { for (const dup of toDelete) { try { - await vectorSearch.deleteVector(db, dup.id, shard); - shardManager.decrementVectorCount(shard.id); + await tursoVectorSearch.deleteVector(db, String(dup.id)); + await tursoShardManager.decrementVectorCount(shard.id); exactDeleted++; } catch (error) { log("Deduplication: delete error", { @@ -80,39 +83,39 @@ export class DeduplicationService { } } - const uniqueMemories = Array.from(contentMap.values()).map((arr) => arr[0]); + const uniqueMemories = Array.from(contentMap.values()).map((arr) => arr[0]!); const processedIds = new Set(); for (let i = 0; i < uniqueMemories.length; i++) { - const mem1 = uniqueMemories[i]; - if (!mem1.vector || processedIds.has(mem1.id)) continue; + const mem1 = uniqueMemories[i]!; + const vector1 = parseExtractedVector(mem1.vector_json); + if (!vector1 || processedIds.has(String(mem1.id))) continue; - const vector1 = new Float32Array(new Uint8Array(mem1.vector).buffer); const similarGroup: DuplicateGroup = { representative: { - id: mem1.id, - content: mem1.content, - containerTag: mem1.container_tag, - createdAt: mem1.created_at, + id: String(mem1.id), + content: String(mem1.content), + containerTag: String(mem1.container_tag), + createdAt: Number(mem1.created_at), }, duplicates: [], }; for (let j = i + 1; j < uniqueMemories.length; j++) { - const mem2 = uniqueMemories[j]; - if (!mem2.vector || processedIds.has(mem2.id)) continue; + const mem2 = uniqueMemories[j]!; + const vector2 = parseExtractedVector(mem2.vector_json); + if (!vector2 || processedIds.has(String(mem2.id))) continue; if (mem1.container_tag !== mem2.container_tag) continue; - const vector2 = new Float32Array(new Uint8Array(mem2.vector).buffer); const similarity = cosineSimilarity(vector1, vector2); if (similarity >= CONFIG.deduplicationSimilarityThreshold && similarity < 1.0) { similarGroup.duplicates.push({ - id: mem2.id, - content: mem2.content, + id: String(mem2.id), + content: String(mem2.content), similarity, }); - processedIds.add(mem2.id); + processedIds.add(String(mem2.id)); } } diff --git a/src/services/memory-scope.ts b/src/services/memory-scope.ts new file mode 100644 index 0000000..1a0e98a --- /dev/null +++ b/src/services/memory-scope.ts @@ -0,0 +1,58 @@ +export const SCOPE_HASH_PATTERN = /^[a-f0-9]{16}$/; + +export function isValidScopeHash(hash: string): boolean { + return SCOPE_HASH_PATTERN.test(hash); +} + +export function assertSafeScopeHash(scopeHash: string): void { + if (!isValidScopeHash(scopeHash)) { + throw new Error(`Invalid scope hash: expected 16 lowercase hex characters, got "${scopeHash}"`); + } +} + +export function extractScopeFromContainerTag(containerTag: string): { + scope: "user" | "project"; + hash: string; +} { + const parts = containerTag.split("_"); + if (parts.length < 3) { + throw new Error( + `Invalid containerTag: expected format {prefix}_{user|project}_{16hex}, got "${containerTag}"` + ); + } + + const hash = parts[parts.length - 1]!; + const scope = parts[parts.length - 2]; + + if (scope !== "user" && scope !== "project") { + throw new Error(`Invalid containerTag scope: "${scope}" in "${containerTag}"`); + } + + if (!isValidScopeHash(hash)) { + throw new Error( + `Invalid containerTag hash: expected 16 lowercase hex characters in "${containerTag}"` + ); + } + + return { scope, hash }; +} + +export function tryExtractScopeFromContainerTag( + containerTag: string +): { scope: "user" | "project"; hash: string } | null { + try { + return extractScopeFromContainerTag(containerTag); + } catch { + return null; + } +} + +export function resolveMemoryScope( + scope: "project" | "all-projects", + containerTag: string +): { scope: "user" | "project"; hash: string } { + if (scope === "all-projects") { + return { scope: "project", hash: "" }; + } + return extractScopeFromContainerTag(containerTag); +} diff --git a/src/services/migration-service.ts b/src/services/migration-service.ts index e80e794..d6beab3 100644 --- a/src/services/migration-service.ts +++ b/src/services/migration-service.ts @@ -1,9 +1,14 @@ -import { shardManager } from "./sqlite/shard-manager.js"; -import { connectionManager } from "./sqlite/connection-manager.js"; -import { vectorSearch } from "./sqlite/vector-search.js"; +import { existsSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { tursoShardManager } from "./turso/shard-manager.js"; +import { tursoConnectionManager } from "./turso/connection-manager.js"; +import { tursoVectorSearch } from "./turso/vector-search.js"; +import { ensureTursoReady } from "./turso/ready.js"; import { embeddingService } from "./embedding.js"; import { CONFIG } from "../config.js"; import { log } from "./logger.js"; +import { formatTagsForEmbedding } from "./turso/vector-utils.js"; +import type { MemoryRecord, ShardInfo } from "./turso/types.js"; +import { acquireTursoOperationLock } from "./turso/operation-lock.js"; export interface DimensionMismatch { needsMigration: boolean; @@ -39,33 +44,29 @@ export class MigrationService { private progressCallback?: (progress: MigrationProgress) => void; async detectDimensionMismatch(): Promise { - const userShards = shardManager.getAllShards("user", ""); - const projectShards = shardManager.getAllShards("project", ""); + await ensureTursoReady(); + const userShards = await tursoShardManager.getAllShards("user", ""); + const projectShards = await tursoShardManager.getAllShards("project", ""); const allShards = [...userShards, ...projectShards]; const mismatches: DimensionMismatch["shardMismatches"] = []; for (const shard of allShards) { try { - const db = connectionManager.getConnection(shard.dbPath); - - const metadataResult = db - .prepare( - ` - SELECT key, value FROM shard_metadata + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const metadataResult = await db.all( + ` + SELECT key, value FROM shard_metadata WHERE key IN ('embedding_dimensions', 'embedding_model') ` - ) - .all() as Array<{ key: string; value: string }>; + ); const metadata = Object.fromEntries(metadataResult.map((row) => [row.key, row.value])); - - const storedDimensions = parseInt(metadata.embedding_dimensions || "0"); - const storedModel = metadata.embedding_model || "unknown"; + const storedDimensions = parseInt(String(metadata.embedding_dimensions || "0")); + const storedModel = String(metadata.embedding_model || "unknown"); if (storedDimensions !== CONFIG.embeddingDimensions) { - const vectorCount = vectorSearch.countAllVectors(db); - + const vectorCount = await tursoVectorSearch.countAllVectors(db); mismatches.push({ shardId: shard.id, dbPath: shard.dbPath, @@ -99,10 +100,12 @@ export class MigrationService { } this.isRunning = true; + let releaseOperationLock: (() => void) | undefined; this.progressCallback = progressCallback; const startTime = Date.now(); try { + releaseOperationLock = acquireTursoOperationLock(`dimension-${strategy}`); const mismatch = await this.detectDimensionMismatch(); if (!mismatch.needsMigration) { @@ -117,9 +120,9 @@ export class MigrationService { if (strategy === "fresh-start") { return await this.freshStartMigration(mismatch, startTime); - } else { - return await this.reEmbedMigration(mismatch, startTime); } + + return await this.reEmbedMigration(mismatch, startTime); } catch (error) { log("Migration: failed", { error: String(error) }); return { @@ -131,6 +134,7 @@ export class MigrationService { error: String(error), }; } finally { + releaseOperationLock?.(); this.isRunning = false; this.progressCallback = undefined; } @@ -149,22 +153,22 @@ export class MigrationService { let deletedShards = 0; for (const [index, shardInfo] of mismatch.shardMismatches.entries()) { - try { - this.reportProgress({ - phase: "cleanup", - processed: index, - total: mismatch.shardMismatches.length, - currentShard: String(shardInfo.shardId), - }); + this.reportProgress({ + phase: "cleanup", + processed: index, + total: mismatch.shardMismatches.length, + currentShard: String(shardInfo.shardId), + }); - await shardManager.deleteShard(shardInfo.shardId); - deletedShards++; - } catch (error) { - log("Migration: error deleting shard", { - shardId: shardInfo.shardId, - error: String(error), - }); + const archivePath = await tursoShardManager.archiveShard(shardInfo.shardId, "fresh-start"); + if (!archivePath) { + throw new Error(`Migration source shard ${shardInfo.shardId} no longer exists`); } + log("Migration: archived shard for fresh start", { + shardId: shardInfo.shardId, + archivePath, + }); + deletedShards++; } this.reportProgress({ @@ -189,7 +193,10 @@ export class MigrationService { await embeddingService.warmup(); embeddingService.clearCache(); - const totalMemories = mismatch.shardMismatches.reduce((sum, s) => sum + s.vectorCount, 0); + const totalMemories = mismatch.shardMismatches.reduce( + (sum, shard) => sum + shard.vectorCount, + 0 + ); this.reportProgress({ phase: "preparing", @@ -208,106 +215,30 @@ export class MigrationService { currentShard: String(shardInfo.shardId), }); - try { - const db = connectionManager.getConnection(shardInfo.dbPath); - const memories = vectorSearch.getAllMemories(db); - - const tempMemories: Array<{ - id: string; - content: string; - containerTag: string; - type: string | null; - createdAt: number; - updatedAt: number; - metadata: string | null; - displayName: string | null; - userName: string | null; - userEmail: string | null; - projectPath: string | null; - projectName: string | null; - gitRepoUrl: string | null; - isPinned: number; - }> = []; - - for (const memory of memories) { - tempMemories.push({ - id: memory.id, - content: memory.content, - containerTag: memory.container_tag, - type: memory.type, - createdAt: memory.created_at, - updatedAt: memory.updated_at, - metadata: memory.metadata, - displayName: memory.display_name, - userName: memory.user_name, - userEmail: memory.user_email, - projectPath: memory.project_path, - projectName: memory.project_name, - gitRepoUrl: memory.git_repo_url, - isPinned: memory.is_pinned || 0, - }); - } - - await shardManager.deleteShard(shardInfo.shardId); - - for (const memory of tempMemories) { - try { - const vector = await embeddingService.embedWithTimeout(memory.content); - - const scope = memory.containerTag.includes("_user_") ? "user" : "project"; - const hash = memory.containerTag.split("_").slice(2).join("_"); - const newShard = shardManager.getWriteShard(scope, hash); - const newDb = connectionManager.getConnection(newShard.dbPath); - - await vectorSearch.insertVector( - newDb, - { - id: memory.id, - content: memory.content, - vector, - containerTag: memory.containerTag, - type: memory.type || undefined, - createdAt: memory.createdAt, - updatedAt: memory.updatedAt, - metadata: memory.metadata || undefined, - displayName: memory.displayName || undefined, - userName: memory.userName || undefined, - userEmail: memory.userEmail || undefined, - projectPath: memory.projectPath || undefined, - projectName: memory.projectName || undefined, - gitRepoUrl: memory.gitRepoUrl || undefined, - }, - newShard - ); - - if (memory.isPinned === 1) { - vectorSearch.pinMemory(newDb, memory.id); - } + const shard = await tursoShardManager.getShardById(shardInfo.shardId); + if (!shard) { + throw new Error(`Migration source shard ${shardInfo.shardId} no longer exists`); + } - shardManager.incrementVectorCount(newShard.id); - reEmbeddedCount++; + const migrated = await tursoShardManager.withScopeWriteLock( + shard.scope, + shard.scopeHash, + () => + this.rebuildShardSafely(shard, (processed) => { processedCount++; - this.reportProgress({ phase: "re-embedding", processed: processedCount, total: totalMemories, currentShard: String(shardInfo.shardId), }); - } catch (error) { - log("Migration: error re-embedding memory", { - memoryId: memory.id, - error: String(error), + log("Migration: memory staged for re-embed", { + shardId: shardInfo.shardId, + memoryId: processed, }); - processedCount++; - } - } - } catch (error) { - log("Migration: error processing shard", { - shardId: shardInfo.shardId, - error: String(error), - }); - } + }) + ); + reEmbeddedCount += migrated; } this.reportProgress({ @@ -319,12 +250,133 @@ export class MigrationService { return { success: true, strategy: "re-embed", - deletedShards: mismatch.shardMismatches.length, + deletedShards: 0, reEmbeddedMemories: reEmbeddedCount, duration: Date.now() - startTime, }; } + private async rebuildShardSafely( + shard: ShardInfo, + onProcessed: (memoryId: string) => void + ): Promise { + const sourceDb = await tursoConnectionManager.getConnection(shard.dbPath); + const memories = await sourceDb.all(` + SELECT + id, content, container_tag, tags, type, created_at, updated_at, metadata, + display_name, user_name, user_email, project_path, project_name, git_repo_url, is_pinned + FROM memories + ORDER BY created_at DESC + `); + const nonce = `${process.pid}-${Date.now()}`; + const stagedPath = `${shard.dbPath}.reembed-${nonce}.tmp`; + const backupPath = `${shard.dbPath}.pre-reembed-${nonce}.bak`; + const swapStatePath = `${shard.dbPath}.reembed-swap.json`; + + try { + const stagedDb = await tursoConnectionManager.getConnection(stagedPath); + await tursoShardManager.initShardDb(stagedDb); + + // Stage bounded batches. Model or database failures only discard this + // temporary shard; the source is not touched until full verification. + const batchSize = 50; + for (let offset = 0; offset < memories.length; offset += batchSize) { + const stagedBatch: Array<{ record: MemoryRecord; isPinned: boolean }> = []; + for (const memory of memories.slice(offset, offset + batchSize)) { + const content = String(memory.content); + const tags = memory.tags + ? String(memory.tags) + .split(",") + .map((tag) => tag.trim()) + .filter(Boolean) + : []; + const embeddingInput = tags.length > 0 ? `${content}\nTags: ${tags.join(", ")}` : content; + const vector = await embeddingService.embedWithTimeout(embeddingInput); + const tagsVector = + tags.length > 0 + ? await embeddingService.embedWithTimeout(formatTagsForEmbedding(tags)) + : undefined; + + stagedBatch.push({ + record: { + id: String(memory.id), + content, + vector, + tagsVector, + containerTag: String(memory.container_tag), + tags: memory.tags ? String(memory.tags) : undefined, + type: memory.type ? String(memory.type) : undefined, + createdAt: Number(memory.created_at), + updatedAt: Number(memory.updated_at), + metadata: memory.metadata ? String(memory.metadata) : undefined, + displayName: memory.display_name ? String(memory.display_name) : undefined, + userName: memory.user_name ? String(memory.user_name) : undefined, + userEmail: memory.user_email ? String(memory.user_email) : undefined, + projectPath: memory.project_path ? String(memory.project_path) : undefined, + projectName: memory.project_name ? String(memory.project_name) : undefined, + gitRepoUrl: memory.git_repo_url ? String(memory.git_repo_url) : undefined, + }, + isPinned: Number(memory.is_pinned ?? 0) === 1, + }); + onProcessed(String(memory.id)); + } + + await stagedDb.transaction("write", async (tx) => { + for (const item of stagedBatch) { + await tursoVectorSearch.insertVectorInTransaction(tx, item.record); + if (item.isPinned) { + await tx.execute({ + sql: `UPDATE memories SET is_pinned = 1 WHERE id = ?`, + args: [item.record.id], + }); + } + } + }); + } + + const countRow = await stagedDb.get(`SELECT COUNT(*) AS count FROM memories`); + const stagedCount = Number(countRow?.count ?? 0); + if (stagedCount !== memories.length) { + throw new Error( + `Migration staged count mismatch for shard ${shard.id}: expected ${memories.length}, got ${stagedCount}` + ); + } + + await tursoConnectionManager.closeConnection(stagedPath); + writeFileSync( + swapStatePath, + JSON.stringify({ dbPath: shard.dbPath, stagedPath, backupPath }), + "utf-8" + ); + await tursoConnectionManager.closeConnection(shard.dbPath); + renameSync(shard.dbPath, backupPath); + try { + renameSync(stagedPath, shard.dbPath); + } catch (error) { + renameSync(backupPath, shard.dbPath); + throw error; + } + + await tursoShardManager.setVectorCount(shard.id, stagedCount); + unlinkSync(swapStatePath); + log("Migration: safely replaced re-embedded shard", { + shardId: shard.id, + memories: stagedCount, + backupPath, + }); + return stagedCount; + } catch (error) { + await tursoConnectionManager.closeConnection(stagedPath); + if (existsSync(stagedPath)) { + unlinkSync(stagedPath); + } + if (existsSync(swapStatePath) && existsSync(shard.dbPath)) { + unlinkSync(swapStatePath); + } + throw error; + } + } + private reportProgress(progress: MigrationProgress): void { if (this.progressCallback) { this.progressCallback(progress); diff --git a/src/services/sqlite/connection-manager.ts b/src/services/sqlite/connection-manager.ts deleted file mode 100644 index 1663bf0..0000000 --- a/src/services/sqlite/connection-manager.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { getDatabase } from "./sqlite-bootstrap.js"; -import { existsSync, mkdirSync } from "node:fs"; -import { dirname } from "node:path"; -import { log } from "../logger.js"; -import { CONFIG } from "../../config.js"; - -const Database = getDatabase(); - -export class ConnectionManager { - private connections: Map = new Map(); - - private initDatabase(db: typeof Database.prototype): void { - db.run("PRAGMA busy_timeout = 5000"); - db.run("PRAGMA journal_mode = WAL"); - db.run("PRAGMA synchronous = NORMAL"); - db.run("PRAGMA cache_size = -64000"); - db.run("PRAGMA temp_store = MEMORY"); - db.run("PRAGMA foreign_keys = ON"); - - this.migrateSchema(db); - } - - private migrateSchema(db: typeof Database.prototype): void { - try { - const columns = db.prepare("PRAGMA table_info(memories)").all() as any[]; - const hasTags = columns.some((c) => c.name === "tags"); - - if (!hasTags && columns.length > 0) { - db.run("ALTER TABLE memories ADD COLUMN tags TEXT"); - } - } catch (error) { - log("Schema migration error", { error: String(error) }); - } - } - - getConnection(dbPath: string): typeof Database.prototype { - if (this.connections.has(dbPath)) { - return this.connections.get(dbPath)!; - } - - const dir = dirname(dbPath); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }); - } - - const db = new Database(dbPath); - this.initDatabase(db); - this.connections.set(dbPath, db); - - return db; - } - - closeConnection(dbPath: string): void { - const db = this.connections.get(dbPath); - if (db) { - db.run("PRAGMA wal_checkpoint(TRUNCATE)"); - db.close(); - this.connections.delete(dbPath); - } - } - - closeAll(): void { - for (const [path, db] of this.connections) { - try { - db.run("PRAGMA wal_checkpoint(TRUNCATE)"); - db.close(); - } catch (error) { - log("Error closing database", { path, error: String(error) }); - } - } - this.connections.clear(); - } - - checkpointAll(): void { - for (const [path, db] of this.connections) { - try { - db.run("PRAGMA wal_checkpoint(PASSIVE)"); - } catch (error) { - log("Error checkpointing database", { path, error: String(error) }); - } - } - } -} - -export const connectionManager = new ConnectionManager(); diff --git a/src/services/sqlite/shard-manager.ts b/src/services/sqlite/shard-manager.ts deleted file mode 100644 index 82ccc62..0000000 --- a/src/services/sqlite/shard-manager.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { getDatabase } from "./sqlite-bootstrap.js"; -import { join, basename } from "node:path"; -import { existsSync, unlinkSync } from "node:fs"; -import { CONFIG } from "../../config.js"; -import { connectionManager } from "./connection-manager.js"; -import { log } from "../logger.js"; -import { vectorSearch } from "./vector-search.js"; -import type { ShardInfo } from "./types.js"; - -const Database = getDatabase(); -type DatabaseType = typeof Database.prototype; - -const METADATA_DB_NAME = "metadata.db"; - -export class ShardManager { - private metadataDb: DatabaseType; - private metadataPath: string; - - constructor() { - this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME); - this.metadataDb = connectionManager.getConnection(this.metadataPath); - this.initMetadataDb(); - } - - private initMetadataDb(): void { - this.metadataDb.run(` - CREATE TABLE IF NOT EXISTS shards ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scope TEXT NOT NULL, - scope_hash TEXT NOT NULL, - shard_index INTEGER NOT NULL, - db_path TEXT NOT NULL, - vector_count INTEGER DEFAULT 0, - is_active INTEGER DEFAULT 1, - created_at INTEGER NOT NULL, - UNIQUE(scope, scope_hash, shard_index) - ) - `); - - this.metadataDb.run(` - CREATE INDEX IF NOT EXISTS idx_active_shards - ON shards(scope, scope_hash, is_active) - `); - } - - private getShardPath(scope: "user" | "project", scopeHash: string, shardIndex: number): string { - const dir = join(CONFIG.storagePath, `${scope}s`); - return join(dir, `${scope}_${scopeHash}_shard_${shardIndex}.db`); - } - - private resolveStoredPath(storedPath: string, scope: string): string { - const fileName = basename(storedPath); - return join(CONFIG.storagePath, `${scope}s`, fileName); - } - - getActiveShard(scope: "user" | "project", scopeHash: string): ShardInfo | null { - const stmt = this.metadataDb.prepare(` - SELECT * FROM shards - WHERE scope = ? AND scope_hash = ? AND is_active = 1 - ORDER BY shard_index DESC LIMIT 1 - `); - - const row = stmt.get(scope, scopeHash) as any; - if (!row) return null; - - return { - id: row.id, - scope: row.scope, - scopeHash: row.scope_hash, - shardIndex: row.shard_index, - dbPath: this.resolveStoredPath(row.db_path, row.scope), - vectorCount: row.vector_count, - isActive: row.is_active === 1, - createdAt: row.created_at, - }; - } - - getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[] { - let stmt; - let rows; - - if (scopeHash === "") { - stmt = this.metadataDb.prepare(` - SELECT * FROM shards - WHERE scope = ? - ORDER BY shard_index ASC - `); - rows = stmt.all(scope) as any[]; - } else { - stmt = this.metadataDb.prepare(` - SELECT * FROM shards - WHERE scope = ? AND scope_hash = ? - ORDER BY shard_index ASC - `); - rows = stmt.all(scope, scopeHash) as any[]; - } - - return rows.map((row: any) => ({ - id: row.id, - scope: row.scope, - scopeHash: row.scope_hash, - shardIndex: row.shard_index, - dbPath: this.resolveStoredPath(row.db_path, row.scope), - vectorCount: row.vector_count, - isActive: row.is_active === 1, - createdAt: row.created_at, - })); - } - - createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): ShardInfo { - const fullPath = this.getShardPath(scope, scopeHash, shardIndex); - const storedPath = join(`${scope}s`, basename(fullPath)).replace(/\\/g, "/"); - const now = Date.now(); - - const stmt = this.metadataDb.prepare(` - INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at) - VALUES (?, ?, ?, ?, 0, 1, ?) - `); - - const result = stmt.run(scope, scopeHash, shardIndex, storedPath, now); - - const db = connectionManager.getConnection(fullPath); - this.initShardDb(db); - - return { - id: Number(result.lastInsertRowid), - scope, - scopeHash, - shardIndex, - dbPath: fullPath, - vectorCount: 0, - isActive: true, - createdAt: now, - }; - } - - private initShardDb(db: DatabaseType): void { - db.run(` - CREATE TABLE IF NOT EXISTS shard_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ) - `); - - db.run(` - INSERT OR REPLACE INTO shard_metadata (key, value) - VALUES ('embedding_dimensions', '${CONFIG.embeddingDimensions}') - `); - - db.run(` - INSERT OR REPLACE INTO shard_metadata (key, value) - VALUES ('embedding_model', '${CONFIG.embeddingModel}') - `); - - db.run(` - CREATE TABLE IF NOT EXISTS memories ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - vector BLOB NOT NULL, - tags_vector BLOB, - container_tag TEXT NOT NULL, - tags TEXT, - type TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - metadata TEXT, - display_name TEXT, - user_name TEXT, - user_email TEXT, - project_path TEXT, - project_name TEXT, - git_repo_url TEXT, - is_pinned INTEGER DEFAULT 0 - ) - `); - - db.run(`CREATE INDEX IF NOT EXISTS idx_container_tag ON memories(container_tag)`); - db.run(`CREATE INDEX IF NOT EXISTS idx_type ON memories(type)`); - db.run(`CREATE INDEX IF NOT EXISTS idx_created_at ON memories(created_at DESC)`); - db.run(`CREATE INDEX IF NOT EXISTS idx_is_pinned ON memories(is_pinned)`); - } - - private isShardValid(shard: ShardInfo): boolean { - if (!existsSync(shard.dbPath)) { - log("Shard DB file missing", { dbPath: shard.dbPath, shardId: shard.id }); - return false; - } - - try { - const db = connectionManager.getConnection(shard.dbPath); - const result = db - .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`) - .get() as any; - if (!result) { - log("Shard DB missing 'memories' table", { - dbPath: shard.dbPath, - shardId: shard.id, - }); - return false; - } - return true; - } catch (error) { - log("Error validating shard DB", { - dbPath: shard.dbPath, - error: String(error), - }); - return false; - } - } - - private ensureShardTables(shard: ShardInfo): void { - try { - const db = connectionManager.getConnection(shard.dbPath); - this.initShardDb(db); - } catch (error) { - log("Error ensuring shard tables", { - dbPath: shard.dbPath, - error: String(error), - }); - } - } - - getWriteShard(scope: "user" | "project", scopeHash: string): ShardInfo { - let shard = this.getActiveShard(scope, scopeHash); - - if (!shard) { - return this.createShard(scope, scopeHash, 0); - } - - if (!this.isShardValid(shard)) { - log("Active shard is invalid, recreating", { - scope, - scopeHash, - shardIndex: shard.shardIndex, - dbPath: shard.dbPath, - }); - - connectionManager.closeConnection(shard.dbPath); - - const deleteStmt = this.metadataDb.prepare(`DELETE FROM shards WHERE id = ?`); - deleteStmt.run(shard.id); - - return this.createShard(scope, scopeHash, shard.shardIndex); - } - - if (shard.vectorCount >= CONFIG.maxVectorsPerShard) { - this.markShardReadOnly(shard.id); - return this.createShard(scope, scopeHash, shard.shardIndex + 1); - } - - return shard; - } - - private markShardReadOnly(shardId: number): void { - const stmt = this.metadataDb.prepare(` - UPDATE shards SET is_active = 0 WHERE id = ? - `); - stmt.run(shardId); - } - - incrementVectorCount(shardId: number): void { - const stmt = this.metadataDb.prepare(` - UPDATE shards SET vector_count = vector_count + 1 WHERE id = ? - `); - stmt.run(shardId); - } - - decrementVectorCount(shardId: number): void { - const stmt = this.metadataDb.prepare(` - UPDATE shards SET vector_count = vector_count - 1 WHERE id = ? AND vector_count > 0 - `); - stmt.run(shardId); - } - - getShardByPath(dbPath: string): ShardInfo | null { - const fileName = basename(dbPath); - const stmt = this.metadataDb.prepare(`SELECT * FROM shards WHERE db_path LIKE '%' || ?`); - const row = stmt.get(fileName) as any; - if (!row) return null; - - return { - id: row.id, - scope: row.scope, - scopeHash: row.scope_hash, - shardIndex: row.shard_index, - dbPath: this.resolveStoredPath(row.db_path, row.scope), - vectorCount: row.vector_count, - isActive: row.is_active === 1, - createdAt: row.created_at, - }; - } - - async deleteShard(shardId: number): Promise { - const stmt = this.metadataDb.prepare(`SELECT * FROM shards WHERE id = ?`); - const row = stmt.get(shardId) as any; - - if (row) { - const fullPath = this.resolveStoredPath(row.db_path, row.scope); - await vectorSearch.deleteShardIndexes({ - id: row.id, - scope: row.scope, - scopeHash: row.scope_hash, - shardIndex: row.shard_index, - dbPath: fullPath, - vectorCount: row.vector_count, - isActive: row.is_active === 1, - createdAt: row.created_at, - }); - connectionManager.closeConnection(fullPath); - - try { - if (existsSync(fullPath)) { - unlinkSync(fullPath); - } - } catch (error) { - log("Error deleting shard file", { - dbPath: fullPath, - error: String(error), - }); - } - - const deleteStmt = this.metadataDb.prepare(`DELETE FROM shards WHERE id = ?`); - deleteStmt.run(shardId); - } - } -} - -export const shardManager = new ShardManager(); diff --git a/src/services/sqlite/sqlite-bootstrap.ts b/src/services/sqlite/sqlite-bootstrap.ts deleted file mode 100644 index aa0eb10..0000000 --- a/src/services/sqlite/sqlite-bootstrap.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * SQLite binding bootstrap — works under Bun and Node. - * - * Resolution order: - * 1. Bun runtime → `bun:sqlite` (built-in, fastest, zero-install) - * 2. Node runtime → `node:sqlite` `DatabaseSync` (built-in, Node 22.5+ experimental, - * stable in Node 24+) - * 3. Fallback → `better-sqlite3` (peer dependency, full native binary) - * - * Required because opencode 1.15.x loads plugins under Node, not Bun — `bun:sqlite` - * is a Bun-only built-in and Node's ESM loader rejects the `bun:` URL scheme. - * - * The detection runs once at first call; the resolved Database class is cached. - */ -import { createRequire } from "node:module"; - -// We don't import types from "bun:sqlite" here because that ambient import -// breaks Node-side type-checking when @types/bun is not installed. Callers -// treat the return value as an opaque sqlite-style Database constructor. -type DatabaseCtor = new (filename?: string, options?: unknown) => unknown; - -let Database: DatabaseCtor | undefined; - -const isBun = typeof (globalThis as { Bun?: unknown }).Bun !== "undefined"; - -export function getDatabase(): DatabaseCtor { - if (Database) return Database; - - const req = createRequire(import.meta.url); - - if (isBun) { - Database = req("bun:sqlite").Database as DatabaseCtor; - return Database; - } - - // Node runtime — try built-in `node:sqlite` first. It exposes `DatabaseSync` - // with the synchronous prepare/all/get/close API surface that matches - // bun:sqlite. One gap: bun:sqlite (and better-sqlite3) expose `db.run(sql)` - // for executing a single SQL statement without bindings — used throughout - // this project for PRAGMA and CREATE INDEX setup. `node:sqlite`'s - // DatabaseSync uses `db.exec(sql)` for that surface, so we subclass to - // alias `db.run(sql)` onto `db.exec(sql)` (param-bound `db.run(sql, ...)` - // is preserved for any future callers, falling back to a prepared statement). - try { - interface NodeStatementSync { - run(...params: unknown[]): unknown; - all(...params: unknown[]): unknown[]; - get(...params: unknown[]): unknown; - } - interface NodeDatabaseSync { - exec(sql: string): unknown; - prepare(sql: string): NodeStatementSync; - close(): void; - } - type NodeDatabaseSyncCtor = new (filename?: string, options?: unknown) => NodeDatabaseSync; - const DatabaseSync = (req("node:sqlite") as { DatabaseSync: NodeDatabaseSyncCtor }) - .DatabaseSync; - class DatabaseSyncCompat extends DatabaseSync { - run(sql: string, ...params: unknown[]): unknown { - if (params.length === 0) { - return this.exec(sql); - } - // bun:sqlite and better-sqlite3 accept a single array of bind values - // (`db.run(sql, [a, b])`); node:sqlite's DatabaseSync treats an array as - // a named-parameters object and throws `Unknown named parameter '0'`. - // Spread it so positional `?` placeholders bind correctly. Callers such - // as services/ai/session/ai-session-manager.ts use this array form. - if (params.length === 1 && Array.isArray(params[0])) { - return this.prepare(sql).run(...(params[0] as unknown[])); - } - return this.prepare(sql).run(...params); - } - // bun:sqlite and better-sqlite3 expose `db.transaction(fn)` that returns - // a callable wrapping `fn` in BEGIN/COMMIT (auto-ROLLBACK on throw). - // `node:sqlite`'s DatabaseSync has no equivalent. Used by - // `api-handlers.handleAddMemory` and `services/client.addMemory`, so - // POST /api/memories and any auto-capture path crash without it. - // - // Single-mode semantics only (BEGIN); the `.deferred` / `.immediate` / - // `.exclusive` variants from better-sqlite3 are not exercised by this - // codebase. - transaction unknown>(fn: Fn): Fn { - const self = this; - const wrapped = function (this: unknown, ...args: Parameters): ReturnType { - self.exec("BEGIN"); - try { - const result = fn.apply(this, args) as ReturnType; - self.exec("COMMIT"); - return result; - } catch (err) { - try { - self.exec("ROLLBACK"); - } catch { - /* rollback failures after partial state are best-effort */ - } - throw err; - } - }; - return wrapped as unknown as Fn; - } - } - Database = DatabaseSyncCompat as unknown as DatabaseCtor; - return Database; - } catch { - // node:sqlite isn't available (Node < 22.5, or experimental flag not set - // in some embedded runtimes). Fall back to better-sqlite3 — wire-compatible - // API, requires a native postinstall but ships prebuilt binaries for - // common platforms. - try { - const betterSqlite = req("better-sqlite3") as DatabaseCtor; - Database = betterSqlite; - return Database; - } catch (error) { - throw new Error( - "opencode-mem: no SQLite binding available. Install better-sqlite3, " + - "or run on Node ≥22.5 with `--experimental-sqlite`, or use Bun. " + - `Underlying error: ${error instanceof Error ? error.message : String(error)}` - ); - } - } -} diff --git a/src/services/sqlite/vector-search.ts b/src/services/sqlite/vector-search.ts deleted file mode 100644 index 1a3da90..0000000 --- a/src/services/sqlite/vector-search.ts +++ /dev/null @@ -1,372 +0,0 @@ -import { getDatabase } from "./sqlite-bootstrap.js"; -import { connectionManager } from "./connection-manager.js"; -import { log } from "../logger.js"; -import { CONFIG } from "../../config.js"; -import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js"; -import { createVectorBackend } from "../vector-backends/backend-factory.js"; -import { ExactScanBackend } from "../vector-backends/exact-scan-backend.js"; -import type { VectorBackend } from "../vector-backends/types.js"; - -const Database = getDatabase(); -type DatabaseType = typeof Database.prototype; - -function toBlob(vector?: Float32Array): Uint8Array | null { - return vector ? new Uint8Array(vector.buffer) : null; -} - -export class VectorSearch { - private readonly backendPromise: Promise; - private readonly fallbackBackend: VectorBackend; - - constructor(backend?: VectorBackend, fallbackBackend: VectorBackend = new ExactScanBackend()) { - this.backendPromise = backend - ? Promise.resolve(backend) - : createVectorBackend({ vectorBackend: CONFIG.vectorBackend }); - this.fallbackBackend = fallbackBackend; - } - - private async getBackend(): Promise { - return this.backendPromise; - } - - async insertVector(db: DatabaseType, record: MemoryRecord, shard?: ShardInfo): Promise { - const insertMemory = db.prepare(` - INSERT INTO memories ( - id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, - metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - - insertMemory.run( - record.id, - record.content, - toBlob(record.vector), - toBlob(record.tagsVector), - record.containerTag, - record.tags || null, - record.type || null, - record.createdAt, - record.updatedAt, - record.metadata || null, - record.displayName || null, - record.userName || null, - record.userEmail || null, - record.projectPath || null, - record.projectName || null, - record.gitRepoUrl || null - ); - - try { - if (shard) { - const backend = await this.getBackend(); - await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); - if (record.tagsVector) { - await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); - } - } - } catch (error) { - db.prepare(`DELETE FROM memories WHERE id = ?`).run(record.id); - throw error; - } - } - - async searchInShard( - shard: ShardInfo, - queryVector: Float32Array, - containerTag: string, - limit: number, - queryText?: string - ): Promise { - const db = connectionManager.getConnection(shard.dbPath); - const backend = await this.getBackend(); - let contentResults; - let tagsResults; - - try { - await backend.rebuildFromShard({ db, shard, kind: "content" }); - await backend.rebuildFromShard({ db, shard, kind: "tags" }); - - contentResults = await backend.search({ - db, - shard, - kind: "content", - queryVector, - limit: limit * 4, - }); - tagsResults = await backend.search({ - db, - shard, - kind: "tags", - queryVector, - limit: limit * 4, - }); - } catch (error) { - log("Vector search degraded to exact scan in shard", { - shardId: shard.id, - backend: backend.getBackendName(), - error: String(error), - }); - - await this.fallbackBackend.rebuildFromShard({ db, shard, kind: "content" }); - await this.fallbackBackend.rebuildFromShard({ db, shard, kind: "tags" }); - contentResults = await this.fallbackBackend.search({ - db, - shard, - kind: "content", - queryVector, - limit: limit * 4, - }); - tagsResults = await this.fallbackBackend.search({ - db, - shard, - kind: "tags", - queryVector, - limit: limit * 4, - }); - } - - const scoreMap = new Map(); - - for (const r of contentResults) { - scoreMap.set(r.id, { contentSim: 1 - r.distance, tagsSim: 0 }); - } - - for (const r of tagsResults) { - const entry = scoreMap.get(r.id); - if (entry) { - entry.tagsSim = 1 - r.distance; - } else { - scoreMap.set(r.id, { contentSim: 0, tagsSim: 1 - r.distance }); - } - } - - const ids = Array.from(scoreMap.keys()); - if (ids.length === 0) return []; - - const placeholders = ids.map(() => "?").join(","); - const rows = db - .prepare( - containerTag === "" - ? ` - SELECT * FROM memories - WHERE id IN (${placeholders}) - ` - : ` - SELECT * FROM memories - WHERE id IN (${placeholders}) AND container_tag = ? - ` - ) - .all(...ids, ...(containerTag === "" ? [] : [containerTag])) as any[]; - - const queryWords = queryText - ? queryText - .toLowerCase() - .split(/[\s,]+/) - .filter((w) => w.length > 1) - : []; - - const hydratedResults = rows.map((row: any) => { - const scores = scoreMap.get(row.id)!; - const memoryTagsStr = row.tags || ""; - const memoryTags = memoryTagsStr.split(",").map((t: string) => t.trim().toLowerCase()); - - let exactMatchBoost = 0; - if (queryWords.length > 0 && memoryTags.length > 0) { - const matches = queryWords.filter((w) => - memoryTags.some((t: string) => t.includes(w) || w.includes(t)) - ).length; - exactMatchBoost = matches / Math.max(queryWords.length, 1); - } - - const finalTagsSim = Math.max(scores.tagsSim, exactMatchBoost); - const similarity = scores.contentSim * 0.6 + finalTagsSim * 0.4; - - return { - id: row.id, - memory: row.content, - similarity, - tags: memoryTagsStr ? memoryTagsStr.split(",") : [], - metadata: row.metadata ? JSON.parse(row.metadata) : undefined, - containerTag: row.container_tag, - displayName: row.display_name, - userName: row.user_name, - userEmail: row.user_email, - projectPath: row.project_path, - projectName: row.project_name, - gitRepoUrl: row.git_repo_url, - isPinned: row.is_pinned, - }; - }); - - hydratedResults.sort((a, b) => b.similarity - a.similarity); - return hydratedResults; - } - - async searchAcrossShards( - shards: ShardInfo[], - queryVector: Float32Array, - containerTag: string, - limit: number, - similarityThreshold: number, - queryText?: string - ): Promise { - const shardPromises = shards.map(async (shard) => { - try { - return await this.searchInShard(shard, queryVector, containerTag, limit, queryText); - } catch (error) { - log("Shard search error", { shardId: shard.id, error: String(error) }); - return []; - } - }); - - const resultsArray = await Promise.all(shardPromises); - const allResults = resultsArray.flat(); - - allResults.sort((a, b) => b.similarity - a.similarity); - return allResults.filter((r) => r.similarity >= similarityThreshold).slice(0, limit); - } - - async deleteVector(db: DatabaseType, memoryId: string, shard?: ShardInfo): Promise { - db.prepare(`DELETE FROM memories WHERE id = ?`).run(memoryId); - - if (shard) { - const backend = await this.getBackend(); - await backend.delete({ id: memoryId, shard, kind: "content" }); - await backend.delete({ id: memoryId, shard, kind: "tags" }); - } - } - - async updateVector( - db: DatabaseType, - memoryId: string, - vector: Float32Array, - shard?: ShardInfo, - tagsVector?: Float32Array - ): Promise { - db.prepare(`UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?`).run( - toBlob(vector), - toBlob(tagsVector), - memoryId - ); - - if (shard) { - const backend = await this.getBackend(); - await backend.insert({ id: memoryId, vector, shard, kind: "content" }); - if (tagsVector) { - await backend.insert({ id: memoryId, vector: tagsVector, shard, kind: "tags" }); - } else { - await backend.delete({ id: memoryId, shard, kind: "tags" }); - } - } - } - - listMemories(db: DatabaseType, containerTag: string, limit: number): any[] { - const stmt = db.prepare( - containerTag === "" - ? ` - SELECT * FROM memories - ORDER BY created_at DESC - LIMIT ? - ` - : ` - SELECT * FROM memories - WHERE container_tag = ? - ORDER BY created_at DESC - LIMIT ? - ` - ); - - return (containerTag === "" ? stmt.all(limit) : stmt.all(containerTag, limit)) as any[]; - } - - getAllMemories(db: DatabaseType): any[] { - const stmt = db.prepare(`SELECT * FROM memories ORDER BY created_at DESC`); - return stmt.all() as any[]; - } - - getMemoryById(db: DatabaseType, memoryId: string): any | null { - const stmt = db.prepare(`SELECT * FROM memories WHERE id = ?`); - return stmt.get(memoryId) as any; - } - - getMemoriesBySessionID(db: DatabaseType, sessionID: string): any[] { - const stmt = db.prepare(` - SELECT * FROM memories - WHERE metadata LIKE ? - ORDER BY created_at DESC - `); - - const rows = stmt.all(`%"sessionID":"${sessionID}"%`) as any[]; - - return rows.map((row: any) => ({ - ...row, - tags: row.tags ? row.tags.split(",") : [], - metadata: row.metadata ? JSON.parse(row.metadata) : {}, - })); - } - - countVectors(db: DatabaseType, containerTag: string): number { - const stmt = db.prepare(`SELECT COUNT(*) as count FROM memories WHERE container_tag = ?`); - const result = stmt.get(containerTag) as any; - return result.count; - } - - countAllVectors(db: DatabaseType): number { - const stmt = db.prepare(`SELECT COUNT(*) as count FROM memories`); - const result = stmt.get() as any; - return result.count; - } - - getDistinctTags(db: DatabaseType): any[] { - const stmt = db.prepare(` - SELECT DISTINCT - container_tag, - display_name, - user_name, - user_email, - project_path, - project_name, - git_repo_url - FROM memories - `); - return stmt.all() as any[]; - } - - pinMemory(db: DatabaseType, memoryId: string): void { - const stmt = db.prepare(`UPDATE memories SET is_pinned = 1 WHERE id = ?`); - stmt.run(memoryId); - } - - unpinMemory(db: DatabaseType, memoryId: string): void { - const stmt = db.prepare(`UPDATE memories SET is_pinned = 0 WHERE id = ?`); - stmt.run(memoryId); - } - - async rebuildIndexForShard( - db: DatabaseType, - scope: string, - scopeHash: string, - shardIndex: number - ): Promise { - const backend = await this.getBackend(); - const shard = { - id: 0, - scope: scope as "user" | "project", - scopeHash, - shardIndex, - dbPath: "", - vectorCount: 0, - isActive: true, - createdAt: Date.now(), - }; - await backend.rebuildFromShard({ db, shard, kind: "content" }); - await backend.rebuildFromShard({ db, shard, kind: "tags" }); - } - - async deleteShardIndexes(shard: ShardInfo): Promise { - const backend = await this.getBackend(); - await backend.deleteShardIndexes({ shard }); - } -} - -export const vectorSearch = new VectorSearch(); diff --git a/src/services/turso/connection-manager.ts b/src/services/turso/connection-manager.ts new file mode 100644 index 0000000..fb31912 --- /dev/null +++ b/src/services/turso/connection-manager.ts @@ -0,0 +1,126 @@ +import { createClient, type Client } from "@libsql/client"; +import { existsSync, mkdirSync } from "node:fs"; +import { dirname, resolve, relative, isAbsolute, sep } from "node:path"; +import { CONFIG } from "../../config.js"; +import { log } from "../logger.js"; +import { TursoDb } from "./turso-db.js"; + +function toFileUrl(dbPath: string): string { + return dbPath.startsWith("file:") ? dbPath : `file:${dbPath}`; +} + +function assertPathInsideStorage(dbPath: string): void { + const storageRoot = resolve(CONFIG.storagePath); + const resolvedPath = resolve(dbPath); + const relativePath = relative(storageRoot, resolvedPath); + // Only treat path-segment traversal as escape (not filenames containing ".."). + if (relativePath === ".." || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + throw new Error(`Refusing to open database outside storagePath: ${dbPath}`); + } +} + +export class TursoConnectionManager { + private readonly connections = new Map(); + private readonly pending = new Map>(); + private closingPromise: Promise | null = null; + + async getConnection(dbPath: string): Promise { + if (this.closingPromise) { + await this.closingPromise; + } + assertPathInsideStorage(dbPath); + + const existing = this.connections.get(dbPath); + if (existing) { + return existing; + } + + const inFlight = this.pending.get(dbPath); + if (inFlight) { + return inFlight; + } + + const openPromise = (async (): Promise => { + const dir = dirname(dbPath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + + const client: Client = createClient({ url: toFileUrl(dbPath) }); + try { + const db = new TursoDb(client); + await db.execute("PRAGMA foreign_keys = ON"); + this.connections.set(dbPath, db); + return db; + } catch (error) { + try { + client.close(); + } catch { + // ignore close errors during cleanup + } + throw error; + } + })(); + + this.pending.set(dbPath, openPromise); + + try { + return await openPromise; + } catch (error) { + this.connections.delete(dbPath); + throw error; + } finally { + this.pending.delete(dbPath); + } + } + + async closeConnection(dbPath: string): Promise { + const db = this.connections.get(dbPath); + if (!db) return; + + try { + await db.close(); + } catch (error) { + log("Error closing Turso database", { path: dbPath, error: String(error) }); + } + + this.connections.delete(dbPath); + } + + async closeAll(): Promise { + if (this.closingPromise) return this.closingPromise; + + this.closingPromise = (async () => { + await Promise.allSettled(this.pending.values()); + for (const [path, db] of this.connections) { + try { + await db.close(); + } catch (error) { + log("Error closing Turso database", { path, error: String(error) }); + } + } + this.connections.clear(); + this.pending.clear(); + })(); + + try { + await this.closingPromise; + } finally { + this.closingPromise = null; + } + } + + closeAllSync(): void { + for (const [path, db] of this.connections) { + try { + db.getClient().close(); + } catch (error) { + log("Error closing Turso database (sync)", { path, error: String(error) }); + } + } + this.connections.clear(); + this.pending.clear(); + } +} + +export const tursoConnectionManager = new TursoConnectionManager(); diff --git a/src/services/turso/legacy-migrator.ts b/src/services/turso/legacy-migrator.ts new file mode 100644 index 0000000..69459aa --- /dev/null +++ b/src/services/turso/legacy-migrator.ts @@ -0,0 +1,654 @@ +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { basename, join } from "node:path"; +import { CONFIG } from "../../config.js"; +import { log } from "../logger.js"; +import { tursoConnectionManager } from "./connection-manager.js"; +import { tursoShardManager } from "./shard-manager.js"; +import { tursoVectorSearch } from "./vector-search.js"; +import { blobToFloat32Array } from "./vector-utils.js"; +import type { MemoryRecord } from "./types.js"; +import type { TursoDb } from "./turso-db.js"; + +const MIGRATION_MARKER = ".turso-migrated"; +const MIGRATION_LOCK = ".turso-migrate.lock"; +const SIDECAR_SUFFIX = ".turso-migrate.json"; +const BACKUP_SUFFIX = ".legacy.bak"; +const REEMBED_SWAP_SUFFIX = ".reembed-swap.json"; + +interface ShardMigrationSidecar { + sourceCount: number; + expectedCount: number; + importedCount: number; + skippedCount: number; + status: "pending" | "complete"; +} + +interface MigrationMarker { + completedAt: string; + shards: Array<{ path: string; expectedCount: number; importedCount: number }>; +} + +interface MigrationLock { + pid: number; + timestamp: string; +} + +function sidecarPath(dbPath: string): string { + return `${dbPath}${SIDECAR_SUFFIX}`; +} + +function backupPath(dbPath: string): string { + return `${dbPath}${BACKUP_SUFFIX}`; +} + +function readSidecar(dbPath: string): ShardMigrationSidecar | null { + const path = sidecarPath(dbPath); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf-8")) as ShardMigrationSidecar; + } catch { + return null; + } +} + +function writeSidecar(dbPath: string, sidecar: ShardMigrationSidecar): void { + writeFileSync(sidecarPath(dbPath), JSON.stringify(sidecar, null, 2), "utf-8"); +} + +function readMarker(storagePath: string): MigrationMarker | null { + const markerPath = join(storagePath, MIGRATION_MARKER); + if (!existsSync(markerPath)) return null; + try { + const raw = readFileSync(markerPath, "utf-8"); + const parsed = JSON.parse(raw) as MigrationMarker; + if (parsed.completedAt && Array.isArray(parsed.shards)) { + return parsed; + } + return null; + } catch { + return null; + } +} + +function writeMarker(storagePath: string, shards: MigrationMarker["shards"]): void { + const marker: MigrationMarker = { + completedAt: new Date().toISOString(), + shards, + }; + writeFileSync(join(storagePath, MIGRATION_MARKER), JSON.stringify(marker, null, 2), "utf-8"); +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function acquireMigrationLock(storagePath: string): boolean { + const lockPath = join(storagePath, MIGRATION_LOCK); + if (existsSync(lockPath)) { + try { + const lock = JSON.parse(readFileSync(lockPath, "utf-8")) as MigrationLock; + if (lock.pid && isProcessAlive(lock.pid)) { + return false; + } + unlinkSync(lockPath); + } catch { + try { + unlinkSync(lockPath); + } catch { + // stale or corrupt lock — try atomic create below + } + } + } + + const lock: MigrationLock = { pid: process.pid, timestamp: new Date().toISOString() }; + try { + writeFileSync(lockPath, JSON.stringify(lock), { flag: "wx" }); + return true; + } catch { + return false; + } +} + +async function acquireMigrationLockWithRetry(storagePath: string, maxAttempts = 3): Promise { + for (let attempt = 0; attempt < maxAttempts; attempt++) { + if (acquireMigrationLock(storagePath)) { + return; + } + await sleep(200 + attempt * 150); + } + + throw new Error("Turso legacy migration locked by another process"); +} + +function releaseMigrationLock(storagePath: string): void { + const lockPath = join(storagePath, MIGRATION_LOCK); + if (!existsSync(lockPath)) return; + try { + const lock = JSON.parse(readFileSync(lockPath, "utf-8")) as MigrationLock; + if (lock.pid === process.pid) { + unlinkSync(lockPath); + } + } catch { + unlinkSync(lockPath); + } +} + +async function countMemories(db: TursoDb): Promise { + const row = await db.get(`SELECT COUNT(*) as count FROM memories`); + return Number(row?.count ?? 0); +} + +async function hasMemoriesTable(db: TursoDb): Promise { + const row = await db.get(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`); + return Boolean(row); +} + +async function isTursoVectorShardReady(db: TursoDb): Promise { + const indexRow = await db.get( + `SELECT name FROM sqlite_master WHERE type='index' AND name='memories_vec_idx'` + ); + if (!indexRow) { + return false; + } + + const metaRow = await db.get( + `SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'` + ); + if (!metaRow?.value) { + return false; + } + + const storedDimensions = Number(metaRow.value); + if (!Number.isInteger(storedDimensions) || storedDimensions <= 0) { + return false; + } + + const count = await countMemories(db); + if (count === 0) { + return true; + } + + try { + const probe = await db.get(`SELECT vector_extract(vector) AS extracted FROM memories LIMIT 1`); + if (probe?.extracted == null) return false; + const extracted = JSON.parse(String(probe.extracted)) as unknown; + return Array.isArray(extracted) && extracted.length === storedDimensions; + } catch { + return false; + } +} + +async function isShardMigrationComplete(dbPath: string): Promise { + const sidecar = readSidecar(dbPath); + if (!sidecar || sidecar.status !== "complete") { + return false; + } + + if (!existsSync(dbPath)) { + return false; + } + + try { + const db = await tursoConnectionManager.getConnection(dbPath); + if (!(await hasMemoriesTable(db))) { + return sidecar.expectedCount === 0; + } + const count = await countMemories(db); + if (count !== sidecar.expectedCount) { + return false; + } + // Sidecar alone is not enough — require native vector index + readable vectors. + return isTursoVectorShardReady(db); + } catch { + return false; + } +} + +async function restoreFromBackup(dbPath: string): Promise { + const backup = backupPath(dbPath); + if (!existsSync(backup)) return; + + await tursoConnectionManager.closeConnection(dbPath); + + if (existsSync(dbPath)) { + unlinkSync(dbPath); + } + + renameSync(backup, dbPath); + + const sidecarPathFile = sidecarPath(dbPath); + if (existsSync(sidecarPathFile)) { + unlinkSync(sidecarPathFile); + } + + log("Legacy migration restored shard from backup", { dbPath, backup }); +} + +async function shouldRestoreFromBackup(dbPath: string): Promise { + const backup = backupPath(dbPath); + if (!existsSync(backup)) return false; + + if (!existsSync(dbPath)) { + return true; + } + + try { + const db = await tursoConnectionManager.getConnection(dbPath); + const sidecar = readSidecar(dbPath); + + // A healthy Turso vector shard must not be overwritten by .legacy.bak, + // except when a pending rewrite clearly did not finish importing all rows. + if ((await hasMemoriesTable(db)) && (await isTursoVectorShardReady(db))) { + if (sidecar?.status === "pending") { + const count = await countMemories(db); + if (count < sidecar.expectedCount) { + return true; + } + } + return false; + } + + if (sidecar?.status === "complete") { + return !(await isShardMigrationComplete(dbPath)); + } + + if (!(await hasMemoriesTable(db))) { + return true; + } + + const count = await countMemories(db); + const expected = sidecar?.expectedCount ?? null; + if (expected != null && count < expected) { + return true; + } + if (sidecar?.status === "pending") { + return true; + } + } catch { + return true; + } + + return false; +} + +function rowToRecord(row: Record): MemoryRecord | null { + const vector = blobToFloat32Array(row.vector); + if (!vector) return null; + + const tagsVector = blobToFloat32Array(row.tags_vector) ?? undefined; + return { + id: String(row.id), + content: String(row.content), + vector, + tagsVector, + containerTag: String(row.container_tag), + tags: row.tags ? String(row.tags) : undefined, + type: row.type ? String(row.type) : undefined, + createdAt: Number(row.created_at), + updatedAt: Number(row.updated_at), + metadata: row.metadata ? String(row.metadata) : undefined, + displayName: row.display_name ? String(row.display_name) : undefined, + userName: row.user_name ? String(row.user_name) : undefined, + userEmail: row.user_email ? String(row.user_email) : undefined, + projectPath: row.project_path ? String(row.project_path) : undefined, + projectName: row.project_name ? String(row.project_name) : undefined, + gitRepoUrl: row.git_repo_url ? String(row.git_repo_url) : undefined, + }; +} + +async function migrateMemoryShard(dbPath: string): Promise { + if (await isShardMigrationComplete(dbPath)) { + const sidecar = readSidecar(dbPath); + return sidecar; + } + + if (await shouldRestoreFromBackup(dbPath)) { + await restoreFromBackup(dbPath); + } + + if (await isShardMigrationComplete(dbPath)) { + return readSidecar(dbPath); + } + + const db = await tursoConnectionManager.getConnection(dbPath); + const hasTable = await hasMemoriesTable(db); + + if (!hasTable) { + await tursoShardManager.initShardDb(db); + const sidecar: ShardMigrationSidecar = { + sourceCount: 0, + expectedCount: 0, + importedCount: 0, + skippedCount: 0, + status: "complete", + }; + writeSidecar(dbPath, sidecar); + return sidecar; + } + + if (await isTursoVectorShardReady(db)) { + const count = await countMemories(db); + const sidecar: ShardMigrationSidecar = { + sourceCount: count, + expectedCount: count, + importedCount: count, + skippedCount: 0, + status: "complete", + }; + writeSidecar(dbPath, sidecar); + return sidecar; + } + + const rows = await db.all(`SELECT * FROM memories`); + const records: MemoryRecord[] = []; + const pinnedIds: string[] = []; + const skippedIds: string[] = []; + + for (const row of rows) { + const record = rowToRecord(row); + if (!record) { + skippedIds.push(String(row.id)); + log("Legacy migration skipped memory without vector", { memoryId: row.id, dbPath }); + continue; + } + records.push(record); + if (Number(row.is_pinned) === 1) { + pinnedIds.push(record.id); + } + } + + const sourceCount = rows.length; + const skippedCount = skippedIds.length; + + if (skippedCount > 0) { + throw new Error( + `Legacy migration aborted for ${dbPath}: ${skippedCount} of ${sourceCount} memories have unreadable vectors (${skippedIds.join(", ")})` + ); + } + + const sourceDimensions = records[0]?.vector.length ?? CONFIG.embeddingDimensions; + if (!Number.isInteger(sourceDimensions) || sourceDimensions <= 0 || sourceDimensions > 65536) { + throw new Error( + `Legacy migration aborted for ${dbPath}: invalid source vector dimensions ${sourceDimensions}` + ); + } + const dimensionMismatches = records.filter( + (record) => + record.vector.length !== sourceDimensions || + (record.tagsVector != null && record.tagsVector.length !== sourceDimensions) + ); + if (dimensionMismatches.length > 0) { + const sample = dimensionMismatches + .slice(0, 5) + .map( + (record) => + `${record.id}(vector=${record.vector.length}` + + (record.tagsVector ? `,tags=${record.tagsVector.length}` : "") + + ")" + ) + .join(", "); + throw new Error( + `Legacy migration aborted for ${dbPath}: ${dimensionMismatches.length} memories have inconsistent vector dimensions; expected ${sourceDimensions} (${sample})` + ); + } + + const expectedCount = records.length; + writeSidecar(dbPath, { + sourceCount, + expectedCount, + importedCount: 0, + skippedCount, + status: "pending", + }); + + const backup = backupPath(dbPath); + await tursoConnectionManager.closeConnection(dbPath); + if (existsSync(dbPath)) { + renameSync(dbPath, backup); + } + + const freshDb = await tursoConnectionManager.getConnection(dbPath); + const sourceModel = + sourceDimensions === CONFIG.embeddingDimensions ? CONFIG.embeddingModel : "legacy-unknown"; + await tursoShardManager.initShardDb(freshDb, sourceDimensions, sourceModel); + + const BATCH_SIZE = 50; + await freshDb.transaction("write", async (tx) => { + for (let offset = 0; offset < records.length; offset += BATCH_SIZE) { + const batch = records.slice(offset, offset + BATCH_SIZE); + for (const record of batch) { + await tursoVectorSearch.insertVectorInTransaction(tx, record); + } + } + for (const memoryId of pinnedIds) { + await tx.execute({ + sql: `UPDATE memories SET is_pinned = 1 WHERE id = ?`, + args: [memoryId], + }); + } + }); + + const importedCount = await countMemories(freshDb); + if (importedCount !== expectedCount) { + throw new Error( + `Legacy migration count mismatch for ${dbPath}: expected ${expectedCount}, got ${importedCount}` + ); + } + + const completedSidecar: ShardMigrationSidecar = { + sourceCount, + expectedCount, + importedCount, + skippedCount, + status: "complete", + }; + writeSidecar(dbPath, completedSidecar); + + log("Legacy memory shard migrated to Turso vectors", { + dbPath, + memories: expectedCount, + backupPath: backup, + }); + + return completedSidecar; +} + +async function migrateDirectory( + dirName: "users" | "projects" +): Promise> { + const dir = join(CONFIG.storagePath, dirName); + const results: Array<{ path: string; sidecar: ShardMigrationSidecar }> = []; + if (!existsSync(dir)) return results; + + for (const file of readdirSync(dir)) { + if (!file.endsWith(".db") || file.includes(BACKUP_SUFFIX)) continue; + const dbPath = join(dir, file); + const sidecar = await migrateMemoryShard(dbPath); + if (sidecar) { + results.push({ path: dbPath, sidecar }); + } + } + + return results; +} + +function listAllShardDbPaths(): string[] { + const paths: string[] = []; + for (const dirName of ["users", "projects"] as const) { + const dir = join(CONFIG.storagePath, dirName); + if (!existsSync(dir)) continue; + for (const file of readdirSync(dir)) { + if (!file.endsWith(".db") || file.includes(BACKUP_SUFFIX)) continue; + paths.push(join(dir, file)); + } + } + return paths; +} + +async function allShardsComplete(): Promise { + const paths = listAllShardDbPaths(); + if (paths.length === 0) return true; + for (const dbPath of paths) { + if (!(await isShardMigrationComplete(dbPath))) { + return false; + } + } + return true; +} + +function recoverInterruptedReembedSwaps(): void { + for (const dirName of ["users", "projects"] as const) { + const dir = join(CONFIG.storagePath, dirName); + if (!existsSync(dir)) continue; + + for (const file of readdirSync(dir)) { + if (!file.endsWith(REEMBED_SWAP_SUFFIX)) continue; + const statePath = join(dir, file); + const expectedDbPath = statePath.slice(0, -REEMBED_SWAP_SUFFIX.length); + try { + const state = JSON.parse(readFileSync(statePath, "utf-8")) as { + dbPath?: string; + stagedPath?: string; + backupPath?: string; + }; + const stagedPath = state.stagedPath; + const backupPath = state.backupPath; + const valid = + state.dbPath === expectedDbPath && + typeof stagedPath === "string" && + stagedPath.startsWith(`${expectedDbPath}.reembed-`) && + stagedPath.endsWith(".tmp") && + typeof backupPath === "string" && + backupPath.startsWith(`${expectedDbPath}.pre-reembed-`) && + backupPath.endsWith(".bak"); + if (!valid) { + throw new Error("invalid re-embed swap state paths"); + } + + if (existsSync(expectedDbPath)) { + if (existsSync(stagedPath)) unlinkSync(stagedPath); + } else if (existsSync(stagedPath)) { + renameSync(stagedPath, expectedDbPath); + } else if (existsSync(backupPath)) { + renameSync(backupPath, expectedDbPath); + } else { + throw new Error("neither staged replacement nor source backup exists"); + } + unlinkSync(statePath); + log("Recovered interrupted re-embed shard swap", { dbPath: expectedDbPath }); + } catch (error) { + throw new Error(`Failed to recover re-embed swap ${statePath}: ${String(error)}`); + } + } + } +} + +async function reconcileShardRegistry(): Promise { + const parsedShards: Array<{ + path: string; + scope: "user" | "project"; + scopeHash: string; + shardIndex: number; + vectorCount: number; + }> = []; + + for (const path of listAllShardDbPaths()) { + const match = /^(user|project)_([a-f0-9]{16})_shard_(\d+)\.db$/.exec(basename(path)); + if (!match) { + log("Legacy migration left unrecognized shard filename unregistered", { path }); + continue; + } + + const db = await tursoConnectionManager.getConnection(path); + parsedShards.push({ + path, + scope: match[1] as "user" | "project", + scopeHash: match[2]!, + shardIndex: Number(match[3]), + vectorCount: await countMemories(db), + }); + } + + const highestIndexByScope = new Map(); + for (const shard of parsedShards) { + const key = `${shard.scope}:${shard.scopeHash}`; + highestIndexByScope.set(key, Math.max(highestIndexByScope.get(key) ?? -1, shard.shardIndex)); + } + + for (const shard of parsedShards) { + const key = `${shard.scope}:${shard.scopeHash}`; + await tursoShardManager.registerExistingShard( + shard.scope, + shard.scopeHash, + shard.shardIndex, + shard.path, + shard.vectorCount, + shard.shardIndex === highestIndexByScope.get(key) + ); + } +} + +export async function runLegacyTursoMigration(): Promise { + if (!existsSync(CONFIG.storagePath)) { + mkdirSync(CONFIG.storagePath, { recursive: true }); + } + + recoverInterruptedReembedSwaps(); + + const marker = readMarker(CONFIG.storagePath); + if (marker && (await allShardsComplete())) { + await reconcileShardRegistry(); + return; + } + + await acquireMigrationLockWithRetry(CONFIG.storagePath); + + try { + const dbFiles = ["metadata.db", "user-prompts.db", "user-profiles.db", "ai-sessions.db"]; + for (const file of dbFiles) { + const path = join(CONFIG.storagePath, file); + if (!existsSync(path)) continue; + await tursoConnectionManager.getConnection(path); + } + + const migratedShards: MigrationMarker["shards"] = []; + + for (const dirName of ["users", "projects"] as const) { + const dirResults = await migrateDirectory(dirName); + for (const { path, sidecar } of dirResults) { + migratedShards.push({ + path, + expectedCount: sidecar.expectedCount, + importedCount: sidecar.importedCount, + }); + } + } + + if (!(await allShardsComplete())) { + throw new Error("Turso legacy migration incomplete: not all shards verified"); + } + + await reconcileShardRegistry(); + writeMarker(CONFIG.storagePath, migratedShards); + log("Turso legacy migration complete", { storagePath: CONFIG.storagePath }); + } finally { + releaseMigrationLock(CONFIG.storagePath); + } +} diff --git a/src/services/turso/lifecycle.ts b/src/services/turso/lifecycle.ts new file mode 100644 index 0000000..8e65530 --- /dev/null +++ b/src/services/turso/lifecycle.ts @@ -0,0 +1,15 @@ +import { tursoConnectionManager } from "./connection-manager.js"; +import { tursoShardManager } from "./shard-manager.js"; +import { resetTursoReady } from "./ready.js"; +import { userPromptManager } from "../user-prompt/user-prompt-manager.js"; +import { userProfileManager } from "../user-profile/user-profile-manager.js"; +import { aiSessionManager } from "../ai/session/ai-session-manager.js"; + +export async function closeTursoAndInvalidateCaches(): Promise { + await tursoConnectionManager.closeAll(); + tursoShardManager.reset(); + resetTursoReady(); + userPromptManager.reset(); + userProfileManager.reset(); + aiSessionManager.reset(); +} diff --git a/src/services/turso/operation-lock.ts b/src/services/turso/operation-lock.ts new file mode 100644 index 0000000..1219f8b --- /dev/null +++ b/src/services/turso/operation-lock.ts @@ -0,0 +1,66 @@ +import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { CONFIG } from "../../config.js"; + +const OPERATION_LOCK = ".turso-operation.lock"; +const LEGACY_MIGRATION_LOCK = ".turso-migrate.lock"; + +interface LockState { + pid: number; + timestamp: string; + operation?: string; +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function readLiveLock(path: string): LockState | null { + if (!existsSync(path)) return null; + try { + const state = JSON.parse(readFileSync(path, "utf-8")) as LockState; + if (Number.isInteger(state.pid) && state.pid > 0 && isProcessAlive(state.pid)) { + return state; + } + } catch { + // Corrupt locks are stale and removed below. + } + unlinkSync(path); + return null; +} + +export function assertNoTursoMigrationInProgress(): void { + for (const file of [OPERATION_LOCK, LEGACY_MIGRATION_LOCK]) { + const path = join(CONFIG.storagePath, file); + const state = readLiveLock(path); + if (state) { + throw new Error( + `Database migration is in progress${state.operation ? ` (${state.operation})` : ""} ` + + `in process ${state.pid}; writes are temporarily blocked` + ); + } + } +} + +export function acquireTursoOperationLock(operation: string): () => void { + assertNoTursoMigrationInProgress(); + const path = join(CONFIG.storagePath, OPERATION_LOCK); + const state: LockState = { + pid: process.pid, + timestamp: new Date().toISOString(), + operation, + }; + writeFileSync(path, JSON.stringify(state), { flag: "wx" }); + + return () => { + const current = readLiveLock(path); + if (current?.pid === process.pid && existsSync(path)) { + unlinkSync(path); + } + }; +} diff --git a/src/services/turso/ready.ts b/src/services/turso/ready.ts new file mode 100644 index 0000000..a4f309f --- /dev/null +++ b/src/services/turso/ready.ts @@ -0,0 +1,30 @@ +import { runLegacyTursoMigration } from "./legacy-migrator.js"; +import { tursoShardManager } from "./shard-manager.js"; +import { log } from "../logger.js"; + +let initPromise: Promise | null = null; +let isReady = false; + +export async function ensureTursoReady(): Promise { + if (isReady) return; + if (initPromise) return initPromise; + + initPromise = (async () => { + try { + await runLegacyTursoMigration(); + await tursoShardManager.getAllShards("user", ""); + isReady = true; + } catch (error) { + initPromise = null; + log("Turso ready gate failed", { error: String(error) }); + throw error; + } + })(); + + return initPromise; +} + +export function resetTursoReady(): void { + isReady = false; + initPromise = null; +} diff --git a/src/services/turso/shard-manager.ts b/src/services/turso/shard-manager.ts new file mode 100644 index 0000000..9c0ddaf --- /dev/null +++ b/src/services/turso/shard-manager.ts @@ -0,0 +1,543 @@ +import { join, basename, resolve, relative } from "node:path"; +import { existsSync, renameSync, unlinkSync } from "node:fs"; +import { CONFIG } from "../../config.js"; +import { assertSafeScopeHash } from "../memory-scope.js"; +import { tursoConnectionManager } from "./connection-manager.js"; +import { log } from "../logger.js"; +import { assertNoTursoMigrationInProgress } from "./operation-lock.js"; +import type { ShardInfo } from "./types.js"; +import type { TursoDb } from "./turso-db.js"; + +const METADATA_DB_NAME = "metadata.db"; + +function getValidatedEmbeddingDimensions(dimensions = CONFIG.embeddingDimensions): number { + const dims = dimensions; + if (!Number.isInteger(dims) || dims <= 0 || dims > 65536) { + throw new Error(`Invalid embeddingDimensions config: ${dims}`); + } + return dims; +} + +function isUniqueConstraintError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return message.includes("UNIQUE constraint failed") || message.includes("SQLITE_CONSTRAINT"); +} + +export class TursoShardManager { + private metadataDb: TursoDb | null = null; + private metadataPath = ""; + private initPromise: Promise | null = null; + private readonly writeLocks = new Map>(); + + reset(): void { + this.metadataDb = null; + this.initPromise = null; + this.metadataPath = ""; + this.writeLocks.clear(); + } + + async withScopeWriteLock( + scope: "user" | "project", + scopeHash: string, + fn: () => Promise + ): Promise { + const key = `${scope}:${scopeHash}`; + const previous = this.writeLocks.get(key) ?? Promise.resolve(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const next = previous.catch(() => undefined).then(() => gate); + this.writeLocks.set(key, next); + + await previous.catch(() => undefined); + try { + return await fn(); + } finally { + release(); + if (this.writeLocks.get(key) === next) { + this.writeLocks.delete(key); + } + } + } + + private async ensureInitialized(): Promise { + if (this.metadataDb && this.initPromise) { + await this.initPromise; + return this.metadataDb; + } + + if (this.initPromise) { + await this.initPromise; + if (this.metadataDb) return this.metadataDb; + } + + this.initPromise = (async () => { + try { + this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME); + this.metadataDb = await tursoConnectionManager.getConnection(this.metadataPath); + await this.initMetadataDb(this.metadataDb); + } catch (error) { + this.initPromise = null; + this.metadataDb = null; + throw error; + } + })(); + + await this.initPromise; + return this.metadataDb!; + } + + private async initMetadataDb(db: TursoDb): Promise { + await db.batch([ + { + sql: ` + CREATE TABLE IF NOT EXISTS shards ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scope TEXT NOT NULL, + scope_hash TEXT NOT NULL, + shard_index INTEGER NOT NULL, + db_path TEXT NOT NULL, + vector_count INTEGER DEFAULT 0, + is_active INTEGER DEFAULT 1, + created_at INTEGER NOT NULL, + UNIQUE(scope, scope_hash, shard_index) + ) + `, + }, + { + sql: ` + CREATE INDEX IF NOT EXISTS idx_active_shards + ON shards(scope, scope_hash, is_active) + `, + }, + ]); + } + + private getShardPath(scope: "user" | "project", scopeHash: string, shardIndex: number): string { + assertSafeScopeHash(scopeHash); + const scopeDir = resolve(CONFIG.storagePath, `${scope}s`); + const fullPath = resolve(join(scopeDir, `${scope}_${scopeHash}_shard_${shardIndex}.db`)); + const relativePath = relative(scopeDir, fullPath); + if (relativePath.startsWith("..") || relativePath.includes("..")) { + throw new Error(`Shard path escapes storage directory: ${fullPath}`); + } + return fullPath; + } + + private resolveStoredPath(storedPath: string, scope: string): string { + const fileName = basename(storedPath); + return join(CONFIG.storagePath, `${scope}s`, fileName); + } + + async getActiveShard(scope: "user" | "project", scopeHash: string): Promise { + const metadataDb = await this.ensureInitialized(); + const row = await metadataDb.get( + ` + SELECT * FROM shards + WHERE scope = ? AND scope_hash = ? AND is_active = 1 + ORDER BY shard_index DESC LIMIT 1 + `, + [scope, scopeHash] + ); + + if (!row) return null; + return this.rowToShardInfo(row); + } + + async getAllShards(scope: "user" | "project", scopeHash: string): Promise { + const metadataDb = await this.ensureInitialized(); + const rows = + scopeHash === "" + ? await metadataDb.all( + ` + SELECT * FROM shards + WHERE scope = ? + ORDER BY shard_index ASC + `, + [scope] + ) + : await metadataDb.all( + ` + SELECT * FROM shards + WHERE scope = ? AND scope_hash = ? + ORDER BY shard_index ASC + `, + [scope, scopeHash] + ); + + return rows.map((row) => this.rowToShardInfo(row)); + } + + async createShard( + scope: "user" | "project", + scopeHash: string, + shardIndex: number + ): Promise { + const metadataDb = await this.ensureInitialized(); + const fullPath = this.getShardPath(scope, scopeHash, shardIndex); + const storedPath = join(`${scope}s`, basename(fullPath)).replace(/\\/g, "/"); + const now = Date.now(); + + let result; + try { + result = await metadataDb.execute( + ` + INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at) + VALUES (?, ?, ?, ?, 0, 1, ?) + `, + [scope, scopeHash, shardIndex, storedPath, now] + ); + } catch (error) { + if (isUniqueConstraintError(error)) { + const existing = await metadataDb.get( + ` + SELECT * FROM shards + WHERE scope = ? AND scope_hash = ? AND shard_index = ? + `, + [scope, scopeHash, shardIndex] + ); + if (existing) { + return this.rowToShardInfo(existing); + } + } + throw error; + } + + const shardDb = await tursoConnectionManager.getConnection(fullPath); + await this.initShardDb(shardDb); + + return { + id: Number(result.lastInsertRowid), + scope, + scopeHash, + shardIndex, + dbPath: fullPath, + vectorCount: 0, + isActive: true, + createdAt: now, + }; + } + + async registerExistingShard( + scope: "user" | "project", + scopeHash: string, + shardIndex: number, + dbPath: string, + vectorCount: number, + isActive: boolean + ): Promise { + assertSafeScopeHash(scopeHash); + const metadataDb = await this.ensureInitialized(); + const storedPath = join(`${scope}s`, basename(dbPath)).replace(/\\/g, "/"); + const now = Date.now(); + + await metadataDb.execute( + ` + INSERT INTO shards ( + scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(scope, scope_hash, shard_index) DO UPDATE SET + db_path = excluded.db_path, + vector_count = excluded.vector_count, + is_active = excluded.is_active + `, + [scope, scopeHash, shardIndex, storedPath, vectorCount, isActive ? 1 : 0, now] + ); + + const row = await metadataDb.get( + `SELECT * FROM shards WHERE scope = ? AND scope_hash = ? AND shard_index = ?`, + [scope, scopeHash, shardIndex] + ); + if (!row) { + throw new Error(`Failed to register shard ${scope}/${scopeHash}#${shardIndex}`); + } + return this.rowToShardInfo(row); + } + + async initShardDb( + db: TursoDb, + dimensions = CONFIG.embeddingDimensions, + embeddingModel = CONFIG.embeddingModel + ): Promise { + const dims = getValidatedEmbeddingDimensions(dimensions); + + await db.batch([ + { + sql: ` + CREATE TABLE IF NOT EXISTS shard_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `, + }, + { + sql: ` + INSERT OR REPLACE INTO shard_metadata (key, value) + VALUES ('embedding_dimensions', ?) + `, + args: [String(dims)], + }, + { + sql: ` + INSERT OR REPLACE INTO shard_metadata (key, value) + VALUES ('embedding_model', ?) + `, + args: [embeddingModel], + }, + { + sql: ` + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector F32_BLOB(${dims}) NOT NULL, + tags_vector F32_BLOB(${dims}), + container_tag TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + metadata TEXT, + display_name TEXT, + user_name TEXT, + user_email TEXT, + project_path TEXT, + project_name TEXT, + git_repo_url TEXT, + is_pinned INTEGER DEFAULT 0 + ) + `, + }, + { + sql: `CREATE INDEX IF NOT EXISTS idx_container_tag ON memories(container_tag)`, + }, + { + sql: `CREATE INDEX IF NOT EXISTS idx_type ON memories(type)`, + }, + { + sql: `CREATE INDEX IF NOT EXISTS idx_created_at ON memories(created_at DESC)`, + }, + { + sql: `CREATE INDEX IF NOT EXISTS idx_is_pinned ON memories(is_pinned)`, + }, + { + sql: ` + CREATE INDEX IF NOT EXISTS memories_vec_idx + ON memories (libsql_vector_idx(vector, 'metric=cosine')) + `, + }, + { + sql: ` + CREATE INDEX IF NOT EXISTS memories_tags_vec_idx + ON memories (libsql_vector_idx(tags_vector, 'metric=cosine')) + WHERE tags_vector IS NOT NULL + `, + }, + ]); + } + + private rowToShardInfo(row: Record): ShardInfo { + return { + id: Number(row.id), + scope: row.scope as "user" | "project", + scopeHash: String(row.scope_hash), + shardIndex: Number(row.shard_index), + dbPath: this.resolveStoredPath(String(row.db_path), String(row.scope)), + vectorCount: Number(row.vector_count), + isActive: Number(row.is_active) === 1, + createdAt: Number(row.created_at), + }; + } + + private async hasMatchingEmbeddingDimensions(db: TursoDb, shard: ShardInfo): Promise { + const row = await db.get(`SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'`); + if (!row?.value) { + log("Shard missing embedding_dimensions metadata", { + dbPath: shard.dbPath, + shardId: shard.id, + }); + return false; + } + + const storedDimensions = Number(row.value); + if (storedDimensions !== getValidatedEmbeddingDimensions()) { + log("Shard embedding dimensions mismatch", { + dbPath: shard.dbPath, + shardId: shard.id, + storedDimensions, + configDimensions: CONFIG.embeddingDimensions, + }); + return false; + } + + return true; + } + + private async syncShardVectorCount(shard: ShardInfo): Promise { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const row = await db.get(`SELECT COUNT(*) as count FROM memories`); + const count = Number(row?.count ?? 0); + + if (count === shard.vectorCount) { + return shard; + } + + const metadataDb = await this.ensureInitialized(); + await metadataDb.run(`UPDATE shards SET vector_count = ? WHERE id = ?`, [count, shard.id]); + return { ...shard, vectorCount: count }; + } + + private async isShardValid(shard: ShardInfo): Promise { + if (!existsSync(shard.dbPath)) { + log("Shard DB file missing", { dbPath: shard.dbPath, shardId: shard.id }); + return false; + } + + try { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const result = await db.get( + `SELECT name FROM sqlite_master WHERE type='table' AND name='memories'` + ); + if (!result) { + log("Shard DB missing 'memories' table", { + dbPath: shard.dbPath, + shardId: shard.id, + }); + return false; + } + + if (!(await this.hasMatchingEmbeddingDimensions(db, shard))) { + return false; + } + + return true; + } catch (error) { + log("Error validating shard DB", { + dbPath: shard.dbPath, + error: String(error), + }); + return false; + } + } + + async getWriteShard(scope: "user" | "project", scopeHash: string): Promise { + assertNoTursoMigrationInProgress(); + for (let attempt = 0; attempt < 3; attempt++) { + let shard = await this.getActiveShard(scope, scopeHash); + + if (!shard) { + return this.createShard(scope, scopeHash, 0); + } + + if (!(await this.isShardValid(shard))) { + throw new Error( + `Shard ${shard.scope}/${shard.scopeHash}#${shard.shardIndex} is incompatible or corrupt. ` + + `The original database was left untouched at ${shard.dbPath}; run the migration or restore it before writing.` + ); + } + + shard = await this.syncShardVectorCount(shard); + + if (shard.vectorCount >= CONFIG.maxVectorsPerShard) { + await this.markShardReadOnly(shard.id); + return this.createShard(scope, scopeHash, shard.shardIndex + 1); + } + + return shard; + } + + throw new Error(`Failed to resolve write shard for ${scope}/${scopeHash}`); + } + + private async markShardReadOnly(shardId: number): Promise { + const metadataDb = await this.ensureInitialized(); + await metadataDb.run(`UPDATE shards SET is_active = 0 WHERE id = ?`, [shardId]); + } + + async incrementVectorCount(shardId: number): Promise { + const metadataDb = await this.ensureInitialized(); + await metadataDb.run(`UPDATE shards SET vector_count = vector_count + 1 WHERE id = ?`, [ + shardId, + ]); + } + + async decrementVectorCount(shardId: number): Promise { + const metadataDb = await this.ensureInitialized(); + await metadataDb.run( + `UPDATE shards SET vector_count = vector_count - 1 WHERE id = ? AND vector_count > 0`, + [shardId] + ); + } + + async setVectorCount(shardId: number, count: number): Promise { + if (!Number.isInteger(count) || count < 0) { + throw new Error(`Invalid vector count: ${count}`); + } + const metadataDb = await this.ensureInitialized(); + await metadataDb.run(`UPDATE shards SET vector_count = ? WHERE id = ?`, [count, shardId]); + } + + async getShardById(shardId: number): Promise { + const metadataDb = await this.ensureInitialized(); + const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]); + return row ? this.rowToShardInfo(row) : null; + } + + async getShardByPath(dbPath: string): Promise { + const metadataDb = await this.ensureInitialized(); + const fileName = basename(dbPath); + const row = await metadataDb.get(`SELECT * FROM shards WHERE db_path LIKE '%' || ?`, [ + fileName, + ]); + if (!row) return null; + return this.rowToShardInfo(row); + } + + async deleteShard(shardId: number): Promise { + const metadataDb = await this.ensureInitialized(); + const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]); + + if (!row) return; + + const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope)); + await tursoConnectionManager.closeConnection(fullPath); + + try { + if (existsSync(fullPath)) { + unlinkSync(fullPath); + } + } catch (error) { + log("Error deleting shard file", { + dbPath: fullPath, + error: String(error), + }); + } + + await metadataDb.run(`DELETE FROM shards WHERE id = ?`, [shardId]); + } + + async archiveShard(shardId: number, reason: string): Promise { + const metadataDb = await this.ensureInitialized(); + const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]); + if (!row) return null; + + const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope)); + await tursoConnectionManager.closeConnection(fullPath); + const archivePath = `${fullPath}.${reason}-${process.pid}-${Date.now()}.bak`; + + if (existsSync(fullPath)) { + renameSync(fullPath, archivePath); + } + try { + await metadataDb.run(`DELETE FROM shards WHERE id = ?`, [shardId]); + } catch (error) { + if (existsSync(archivePath) && !existsSync(fullPath)) { + renameSync(archivePath, fullPath); + } + throw error; + } + return existsSync(archivePath) ? archivePath : null; + } +} + +export const tursoShardManager = new TursoShardManager(); diff --git a/src/services/turso/turso-db.ts b/src/services/turso/turso-db.ts new file mode 100644 index 0000000..b3d2882 --- /dev/null +++ b/src/services/turso/turso-db.ts @@ -0,0 +1,62 @@ +import type { Client, InArgs, InValue, ResultSet, Transaction } from "@libsql/client"; + +type Row = Record; + +export class TursoDb { + constructor(private readonly client: Client) {} + + getClient(): Client { + return this.client; + } + + async execute(sql: string, args?: InArgs): Promise { + return this.client.execute({ sql, args: args ?? [] }); + } + + async batch(statements: Array<{ sql: string; args?: InArgs }>, mode: "write" | "read" = "write") { + return this.client.batch( + statements.map((statement) => ({ + sql: statement.sql, + args: statement.args ?? [], + })), + mode + ); + } + + async get(sql: string, args?: InArgs): Promise { + const result = await this.execute(sql, args); + return (result.rows[0] as T | undefined) ?? null; + } + + async all(sql: string, args?: InArgs): Promise { + const result = await this.execute(sql, args); + return result.rows as unknown as T[]; + } + + async run(sql: string, args?: InArgs): Promise { + const result = await this.execute(sql, args); + return Number(result.rowsAffected ?? 0); + } + + async transaction(mode: "write" | "read", fn: (tx: Transaction) => Promise): Promise { + const tx = await this.client.transaction(mode); + try { + const value = await fn(tx); + await tx.commit(); + return value; + } catch (error) { + try { + await tx.rollback(); + } catch { + // ignore rollback errors after a failed statement / already-closed tx + } + throw error; + } finally { + tx.close(); + } + } + + async close(): Promise { + this.client.close(); + } +} diff --git a/src/services/sqlite/types.ts b/src/services/turso/types.ts similarity index 97% rename from src/services/sqlite/types.ts rename to src/services/turso/types.ts index f241feb..54841ea 100644 --- a/src/services/sqlite/types.ts +++ b/src/services/turso/types.ts @@ -32,6 +32,7 @@ export interface SearchResult { id: string; memory: string; similarity: number; + createdAt?: number; tags?: string[]; metadata?: Record; displayName?: string; diff --git a/src/services/turso/vector-search.ts b/src/services/turso/vector-search.ts new file mode 100644 index 0000000..3cb1e89 --- /dev/null +++ b/src/services/turso/vector-search.ts @@ -0,0 +1,475 @@ +import { tursoConnectionManager } from "./connection-manager.js"; +import { log } from "../logger.js"; +import { CONFIG } from "../../config.js"; +import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js"; +import { distanceToSimilarity, vectorToJson } from "./vector-utils.js"; +import type { TursoDb } from "./turso-db.js"; +import type { Transaction } from "@libsql/client"; + +function parseMetadata(value: unknown): Record | undefined { + if (!value || typeof value !== "string") return undefined; + try { + return JSON.parse(value) as Record; + } catch { + return undefined; + } +} + +export class TursoVectorSearch { + async insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise { + const contentVector = vectorToJson(record.vector); + const commonArgs = [ + record.containerTag, + record.tags || null, + record.type || null, + record.createdAt, + record.updatedAt, + record.metadata || null, + record.displayName || null, + record.userName || null, + record.userEmail || null, + record.projectPath || null, + record.projectName || null, + record.gitRepoUrl || null, + ]; + + if (record.tagsVector) { + await tx.execute({ + sql: ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url + ) VALUES (?, ?, vector32(?), vector32(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + args: [ + record.id, + record.content, + contentVector, + vectorToJson(record.tagsVector), + ...commonArgs, + ], + }); + return; + } + + await tx.execute({ + sql: ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url + ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + args: [record.id, record.content, contentVector, ...commonArgs], + }); + } + + async insertVector(db: TursoDb, record: MemoryRecord): Promise { + const contentVector = vectorToJson(record.vector); + const commonArgs = [ + record.containerTag, + record.tags || null, + record.type || null, + record.createdAt, + record.updatedAt, + record.metadata || null, + record.displayName || null, + record.userName || null, + record.userEmail || null, + record.projectPath || null, + record.projectName || null, + record.gitRepoUrl || null, + ]; + + if (record.tagsVector) { + await db.execute( + ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url + ) VALUES (?, ?, vector32(?), vector32(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + [record.id, record.content, contentVector, vectorToJson(record.tagsVector), ...commonArgs] + ); + return; + } + + await db.execute( + ` + INSERT INTO memories ( + id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at, + metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url + ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + [record.id, record.content, contentVector, ...commonArgs] + ); + } + + async searchInShard( + shard: ShardInfo, + queryVector: Float32Array, + containerTag: string, + limit: number, + queryText?: string + ): Promise { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const queryJson = vectorToJson(queryVector); + // Over-fetch aggressively when filtering by container_tag after ANN, + // because post-filtering can discard many DiskANN neighbors. + const k = containerTag === "" ? Math.max(limit * 4, 32) : Math.max(limit * 16, 128); + + const contentResults = await this.searchKind( + db, + queryJson, + k, + containerTag, + "memories_vec_idx", + "vector" + ); + const tagsResults = await this.searchKind( + db, + queryJson, + k, + containerTag, + "memories_tags_vec_idx", + "tags_vector" + ); + + const candidateIds = new Set(); + for (const result of contentResults) candidateIds.add(result.id); + for (const result of tagsResults) candidateIds.add(result.id); + + const ids = Array.from(candidateIds); + if (ids.length === 0) return []; + + const placeholders = ids.map(() => "?").join(","); + // Recompute exact distances for hydrated rows so tag-only / content-only ANN + // hits still get a full hybrid score (missing ANN side is not forced to 0). + const rows = await db.all( + containerTag === "" + ? ` + SELECT id, content, tags, created_at, metadata, container_tag, + display_name, user_name, user_email, project_path, project_name, + git_repo_url, is_pinned, + vector_distance_cos(vector, vector32(?)) AS content_dist, + CASE WHEN tags_vector IS NOT NULL + THEN vector_distance_cos(tags_vector, vector32(?)) + ELSE NULL END AS tags_dist + FROM memories + WHERE id IN (${placeholders}) + ` + : ` + SELECT id, content, tags, created_at, metadata, container_tag, + display_name, user_name, user_email, project_path, project_name, + git_repo_url, is_pinned, + vector_distance_cos(vector, vector32(?)) AS content_dist, + CASE WHEN tags_vector IS NOT NULL + THEN vector_distance_cos(tags_vector, vector32(?)) + ELSE NULL END AS tags_dist + FROM memories + WHERE id IN (${placeholders}) AND container_tag = ? + `, + containerTag === "" + ? [queryJson, queryJson, ...ids] + : [queryJson, queryJson, ...ids, containerTag] + ); + + const queryWords = queryText + ? queryText + .toLowerCase() + .split(/[\s,]+/) + .filter((word) => word.length > 1) + : []; + + const hydratedResults = rows.map((row: Record) => { + const contentSim = distanceToSimilarity(Number(row.content_dist)); + const tagsSim = + row.tags_dist == null || row.tags_dist === undefined + ? 0 + : distanceToSimilarity(Number(row.tags_dist)); + const memoryTagsStr = String(row.tags || ""); + const memoryTags = memoryTagsStr.split(",").map((tag) => tag.trim().toLowerCase()); + + let exactMatchBoost = 0; + if (queryWords.length > 0 && memoryTags.length > 0) { + const matches = queryWords.filter((word) => + memoryTags.some((tag) => tag.includes(word) || word.includes(tag)) + ).length; + exactMatchBoost = matches / Math.max(queryWords.length, 1); + } + + const finalTagsSim = Math.max(tagsSim, exactMatchBoost); + const similarity = contentSim * 0.6 + finalTagsSim * 0.4; + + return { + id: String(row.id), + memory: String(row.content), + similarity, + createdAt: Number(row.created_at), + tags: memoryTagsStr ? memoryTagsStr.split(",") : [], + metadata: parseMetadata(row.metadata), + containerTag: String(row.container_tag), + displayName: row.display_name ? String(row.display_name) : undefined, + userName: row.user_name ? String(row.user_name) : undefined, + userEmail: row.user_email ? String(row.user_email) : undefined, + projectPath: row.project_path ? String(row.project_path) : undefined, + projectName: row.project_name ? String(row.project_name) : undefined, + gitRepoUrl: row.git_repo_url ? String(row.git_repo_url) : undefined, + isPinned: row.is_pinned, + }; + }); + + hydratedResults.sort((a, b) => b.similarity - a.similarity); + return hydratedResults.slice(0, Math.max(0, limit)); + } + + private async searchKind( + db: TursoDb, + queryJson: string, + k: number, + containerTag: string, + indexName: string, + columnName: string + ): Promise> { + try { + const rows = await db.all( + containerTag === "" + ? ` + SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist + FROM vector_top_k('${indexName}', vector32(?), ?) AS v + JOIN memories m ON m.rowid = v.id + WHERE m.${columnName} IS NOT NULL + ` + : ` + SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist + FROM vector_top_k('${indexName}', vector32(?), ?) AS v + JOIN memories m ON m.rowid = v.id + WHERE m.${columnName} IS NOT NULL AND m.container_tag = ? + `, + containerTag === "" ? [queryJson, queryJson, k] : [queryJson, queryJson, k, containerTag] + ); + + return rows.map((row) => ({ + id: String(row.id), + similarity: distanceToSimilarity(Number(row.dist)), + })); + } catch (error) { + log("Turso vector_top_k failed; falling back to exact scan", { + indexName, + error: String(error), + }); + return this.exactScanKind(db, queryJson, k, containerTag, columnName); + } + } + + private async exactScanKind( + db: TursoDb, + queryJson: string, + k: number, + containerTag: string, + columnName: string + ): Promise> { + const rows = await db.all( + containerTag === "" + ? ` + SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist + FROM memories m + WHERE m.${columnName} IS NOT NULL + ORDER BY dist ASC + LIMIT ? + ` + : ` + SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist + FROM memories m + WHERE m.${columnName} IS NOT NULL AND m.container_tag = ? + ORDER BY dist ASC + LIMIT ? + `, + containerTag === "" ? [queryJson, k] : [queryJson, containerTag, k] + ); + + return rows.map((row) => ({ + id: String(row.id), + similarity: distanceToSimilarity(Number(row.dist)), + })); + } + + async searchAcrossShards( + shards: ShardInfo[], + queryVector: Float32Array, + containerTag: string, + limit: number, + similarityThreshold: number, + queryText?: string + ): Promise<{ results: SearchResult[]; warnings: string[] }> { + const shardErrors: Array<{ shardId: number; error: string }> = []; + + const shardPromises = shards.map(async (shard) => { + try { + return await this.searchInShard(shard, queryVector, containerTag, limit, queryText); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log("Shard search error", { shardId: shard.id, error: message }); + shardErrors.push({ shardId: shard.id, error: message }); + return []; + } + }); + + const resultsArray = await Promise.all(shardPromises); + + if (shardErrors.length > 0 && shardErrors.length === shards.length) { + throw new Error( + `Vector search failed on all shards: ${shardErrors.map((entry) => `${entry.shardId}: ${entry.error}`).join("; ")}` + ); + } + + const warnings = + shardErrors.length > 0 + ? [ + `Vector search completed with partial shard failures (${shardErrors.length}/${shards.length}): ${shardErrors + .map((entry) => `${entry.shardId}: ${entry.error}`) + .join("; ")}`, + ] + : []; + + if (warnings.length > 0) { + log("Vector search completed with partial shard failures", { + failedShards: shardErrors, + totalShards: shards.length, + }); + } + + const allResults = resultsArray.flat(); + allResults.sort((a, b) => b.similarity - a.similarity); + return { + results: allResults + .filter((result) => result.similarity >= similarityThreshold) + .slice(0, limit), + warnings, + }; + } + + async deleteVector(db: TursoDb, memoryId: string): Promise { + await db.run(`DELETE FROM memories WHERE id = ?`, [memoryId]); + } + + async updateVector( + db: TursoDb, + memoryId: string, + vector: Float32Array, + tagsVector?: Float32Array + ): Promise { + const contentVector = vectorToJson(vector); + if (tagsVector) { + await db.execute( + `UPDATE memories SET vector = vector32(?), tags_vector = vector32(?) WHERE id = ?`, + [contentVector, vectorToJson(tagsVector), memoryId] + ); + } else { + await db.execute( + `UPDATE memories SET vector = vector32(?), tags_vector = NULL WHERE id = ?`, + [contentVector, memoryId] + ); + } + } + + async listMemories( + db: TursoDb, + containerTag: string, + limit: number + ): Promise[]> { + return containerTag === "" + ? db.all( + ` + SELECT * FROM memories + ORDER BY created_at DESC + LIMIT ? + `, + [limit] + ) + : db.all( + ` + SELECT * FROM memories + WHERE container_tag = ? + ORDER BY created_at DESC + LIMIT ? + `, + [containerTag, limit] + ); + } + + async getAllMemories(db: TursoDb): Promise[]> { + return db.all(`SELECT * FROM memories ORDER BY created_at DESC`); + } + + async getAllMemoriesWithExtractedVectors( + db: TursoDb + ): Promise & { vector_json: string | null }>> { + return db.all(` + SELECT + id, + content, + container_tag, + created_at, + vector_extract(vector) AS vector_json + FROM memories + ORDER BY created_at DESC + `); + } + + async getMemoryById(db: TursoDb, memoryId: string): Promise | null> { + return db.get(`SELECT * FROM memories WHERE id = ?`, [memoryId]); + } + + async getMemoriesBySessionID(db: TursoDb, sessionID: string): Promise[]> { + const rows = await db.all( + ` + SELECT * FROM memories + WHERE metadata LIKE ? + ORDER BY created_at DESC + `, + [`%"sessionID":"${sessionID}"%`] + ); + + return rows.map((row) => ({ + ...row, + tags: row.tags ? String(row.tags).split(",") : [], + metadata: row.metadata ? (parseMetadata(String(row.metadata)) ?? {}) : {}, + })); + } + + async countVectors(db: TursoDb, containerTag: string): Promise { + const row = await db.get(`SELECT COUNT(*) as count FROM memories WHERE container_tag = ?`, [ + containerTag, + ]); + return Number(row?.count ?? 0); + } + + async countAllVectors(db: TursoDb): Promise { + const row = await db.get(`SELECT COUNT(*) as count FROM memories`); + return Number(row?.count ?? 0); + } + + async getDistinctTags(db: TursoDb): Promise[]> { + return db.all(` + SELECT DISTINCT + container_tag, + display_name, + user_name, + user_email, + project_path, + project_name, + git_repo_url + FROM memories + `); + } + + async pinMemory(db: TursoDb, memoryId: string): Promise { + await db.run(`UPDATE memories SET is_pinned = 1 WHERE id = ?`, [memoryId]); + } + + async unpinMemory(db: TursoDb, memoryId: string): Promise { + await db.run(`UPDATE memories SET is_pinned = 0 WHERE id = ?`, [memoryId]); + } +} + +export const tursoVectorSearch = new TursoVectorSearch(); diff --git a/src/services/turso/vector-utils.ts b/src/services/turso/vector-utils.ts new file mode 100644 index 0000000..840667d --- /dev/null +++ b/src/services/turso/vector-utils.ts @@ -0,0 +1,71 @@ +export function vectorToJson(vector: Float32Array): string { + return JSON.stringify(Array.from(vector)); +} + +export function blobToFloat32Array(value: unknown): Float32Array | null { + if (value == null) return null; + + try { + if (value instanceof Float32Array) { + return value; + } + + if (value instanceof ArrayBuffer) { + if (value.byteLength === 0 || value.byteLength % 4 !== 0) { + return null; + } + return new Float32Array(value); + } + + if (ArrayBuffer.isView(value)) { + const view = value as ArrayBufferView; + const byteLength = view.byteLength; + if (byteLength === 0 || byteLength % 4 !== 0) { + return null; + } + return new Float32Array( + view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength) + ); + } + + if (typeof value === "string") { + try { + const parsed = JSON.parse(value) as number[]; + return new Float32Array(parsed); + } catch { + return null; + } + } + } catch { + return null; + } + + return null; +} + +/** Parse libSQL `vector_extract()` JSON output (preferred over raw F32_BLOB bytes). */ +export function parseExtractedVector(value: unknown): Float32Array | null { + if (value == null) return null; + if (typeof value === "string") { + try { + const parsed = JSON.parse(value) as number[]; + if (!Array.isArray(parsed) || parsed.length === 0) return null; + return new Float32Array(parsed); + } catch { + return null; + } + } + return blobToFloat32Array(value); +} + +export function distanceToSimilarity(distance: number): number { + const similarity = 1 - Number(distance); + if (!Number.isFinite(similarity)) return 0; + // Clamp float artifacts (docs: cos-distance can be slightly negative near exact matches). + return Math.max(0, Math.min(1, similarity)); +} + +/** Canonical text used for tag-vector embeddings (must stay consistent across write paths). */ +export function formatTagsForEmbedding(tags: string[]): string { + return `Topics: ${tags.join(", ")}`; +} diff --git a/src/services/user-memory-learning.ts b/src/services/user-memory-learning.ts index d2120f7..ee21c08 100644 --- a/src/services/user-memory-learning.ts +++ b/src/services/user-memory-learning.ts @@ -18,7 +18,7 @@ export async function performUserProfileLearning( if (isLearningRunning) return; isLearningRunning = true; try { - const count = userPromptManager.countUnanalyzedForUserLearning(); + const count = await userPromptManager.countUnanalyzedForUserLearning(); const threshold = CONFIG.userProfileAnalysisInterval; log("user-profile-learning: check", { count, threshold }); @@ -27,7 +27,7 @@ export async function performUserProfileLearning( return; } - const prompts = userPromptManager.getPromptsForUserLearning(threshold); + const prompts = await userPromptManager.getPromptsForUserLearning(threshold); if (prompts.length === 0) { return; @@ -36,7 +36,7 @@ export async function performUserProfileLearning( const tags = getTags(directory); const userId = tags.user.userEmail || "unknown"; - let existingProfile = userProfileManager.getActiveProfile(userId); + let existingProfile = await userProfileManager.getActiveProfile(userId); const analysisStartTime = Date.now(); let validationPrompt: string | undefined; @@ -124,7 +124,7 @@ Rules: log("user-profile-learning: analyze done", { hasResult: !!analysisResult }); if (!analysisResult) { - userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); + await userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); if (prompts.length >= 10 && existingProfile) { buildLearningPaths(prompts, existingProfile.id).catch(() => {}); } @@ -141,7 +141,7 @@ Rules: while (!success && retries <= MAX_RETRIES) { if (retries > 0) { - existingProfile = userProfileManager.getActiveProfile(userId); + existingProfile = await userProfileManager.getActiveProfile(userId); if (!existingProfile) break; const retryProfileData: UserProfileData = JSON.parse(existingProfile.profileData); const { data: decayedRetry } = userProfileManager.decayInMemory(retryProfileData); @@ -173,7 +173,7 @@ Rules: changeSummary = changeSummary + "; " + validationSummary; } - success = userProfileManager.updateProfile( + success = await userProfileManager.updateProfile( existingProfile.id, updatedProfileData, prompts.length, @@ -197,9 +197,9 @@ Rules: return; } - userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); + await userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); } else { - userProfileManager.createProfile( + await userProfileManager.createProfile( userId, tags.user.displayName || "Unknown", tags.user.userName || "unknown", @@ -207,7 +207,7 @@ Rules: llmResult, prompts.length ); - userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); + await userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); } if (CONFIG.showUserProfileToasts) { @@ -770,12 +770,12 @@ If no clear chains, return { "paths": [] }.`; topics: result.paths.map((p) => p.topic).join(", "), }); - const profile = userProfileManager.getProfileById(profileId); + const profile = await userProfileManager.getProfileById(profileId); if (!profile) return; const data: UserProfileData = JSON.parse(profile.profileData); data.learning_paths = result.paths; - userProfileManager.updateProfile( + await userProfileManager.updateProfile( profileId, data, 0, diff --git a/src/services/user-profile/profile-context.ts b/src/services/user-profile/profile-context.ts index 4abb920..b8f7c4f 100644 --- a/src/services/user-profile/profile-context.ts +++ b/src/services/user-profile/profile-context.ts @@ -28,8 +28,8 @@ function scoreByRecency(items: any[]): any[] { }); } -export function getUserProfileContext(userId: string): string | null { - const profile = userProfileManager.getActiveProfile(userId); +export async function getUserProfileContext(userId: string): Promise { + const profile = await userProfileManager.getActiveProfile(userId); if (!profile) { return null; diff --git a/src/services/user-profile/user-profile-manager.ts b/src/services/user-profile/user-profile-manager.ts index 05aa617..5473302 100644 --- a/src/services/user-profile/user-profile-manager.ts +++ b/src/services/user-profile/user-profile-manager.ts @@ -1,7 +1,7 @@ -import { getDatabase } from "../sqlite/sqlite-bootstrap.js"; import { join } from "node:path"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; -import { connectionManager } from "../sqlite/connection-manager.js"; +import { tursoConnectionManager } from "../turso/connection-manager.js"; +import type { TursoDb } from "../turso/turso-db.js"; import { CONFIG } from "../../config.js"; import type { UserProfile, UserProfileChangelog, UserProfileData } from "./types.js"; import { safeArray } from "./profile-utils.js"; @@ -69,14 +69,12 @@ function normalizeDescription(text: string): string { .trim(); } -const Database = getDatabase(); -type DatabaseType = typeof Database.prototype; - const USER_PROFILES_DB_NAME = "user-profiles.db"; export class UserProfileManager { - private db!: DatabaseType; - private readonly dbPath: string; + private db: TursoDb | null = null; + private dbPath: string; + private initPromise: Promise | null = null; private coldBuffer: { preferences: any[]; patterns: any[]; workflows: any[] }; private coldBufferPath: string; private dedupCheckedCache: Set = new Set(); @@ -85,12 +83,49 @@ export class UserProfileManager { this.dbPath = join(CONFIG.storagePath || "", USER_PROFILES_DB_NAME); this.coldBufferPath = join(CONFIG.storagePath || "", "cold-buffer.json"); this.coldBuffer = this.loadColdBuffer(); - try { - this.db = connectionManager.getConnection(this.dbPath); - this.initDatabase(); - } catch (e) { - log("user-profile-manager: db init failed, deferring", { error: String(e) }); + } + + reset(): void { + this.db = null; + this.initPromise = null; + this.dbPath = join(CONFIG.storagePath || "", USER_PROFILES_DB_NAME); + this.coldBufferPath = join(CONFIG.storagePath || "", "cold-buffer.json"); + } + + private async initialize(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = (async () => { + try { + this.dbPath = join(CONFIG.storagePath || "", USER_PROFILES_DB_NAME); + this.coldBufferPath = join(CONFIG.storagePath || "", "cold-buffer.json"); + this.db = await tursoConnectionManager.getConnection(this.dbPath); + await this.initDatabase(); + } catch (error) { + this.initPromise = null; + this.db = null; + log("user-profile-manager: db init failed", { error: String(error) }); + throw error; + } + })(); + + return this.initPromise; + } + + private async ready(): Promise { + if (!this.db || !this.initPromise) { + await this.initialize(); + } else { + await this.initPromise; } + + if (!this.db) { + throw new Error("UserProfileManager: database not initialized"); + } + + return this.db; } private loadColdBuffer(): { preferences: any[]; patterns: any[]; workflows: any[] } { @@ -125,69 +160,76 @@ export class UserProfileManager { } } - private initDatabase(): void { - this.db.run(` - CREATE TABLE IF NOT EXISTS user_profiles ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - user_name TEXT NOT NULL, - user_email TEXT NOT NULL, - profile_data TEXT NOT NULL, - version INTEGER NOT NULL DEFAULT 1, - created_at INTEGER NOT NULL, - last_analyzed_at INTEGER NOT NULL, - total_prompts_analyzed INTEGER NOT NULL DEFAULT 0, - is_active BOOLEAN NOT NULL DEFAULT 1 - ) - `); - - this.db.run(` - CREATE TABLE IF NOT EXISTS user_profile_changelogs ( - id TEXT PRIMARY KEY, - profile_id TEXT NOT NULL, - version INTEGER NOT NULL, - change_type TEXT NOT NULL, - change_summary TEXT NOT NULL, - profile_data_snapshot TEXT NOT NULL, - created_at INTEGER NOT NULL, - FOREIGN KEY (profile_id) REFERENCES user_profiles(id) ON DELETE CASCADE - ) - `); - - this.db.run("CREATE INDEX IF NOT EXISTS idx_user_profiles_user_id ON user_profiles(user_id)"); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_profiles_is_active ON user_profiles(is_active)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_profile_changelogs_profile_id ON user_profile_changelogs(profile_id)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_profile_changelogs_version ON user_profile_changelogs(version DESC)" - ); + private async initDatabase(): Promise { + const db = this.db!; + await db.batch([ + { + sql: ` + CREATE TABLE IF NOT EXISTS user_profiles ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + user_name TEXT NOT NULL, + user_email TEXT NOT NULL, + profile_data TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + last_analyzed_at INTEGER NOT NULL, + total_prompts_analyzed INTEGER NOT NULL DEFAULT 0, + is_active BOOLEAN NOT NULL DEFAULT 1 + ) + `, + }, + { + sql: ` + CREATE TABLE IF NOT EXISTS user_profile_changelogs ( + id TEXT PRIMARY KEY, + profile_id TEXT NOT NULL, + version INTEGER NOT NULL, + change_type TEXT NOT NULL, + change_summary TEXT NOT NULL, + profile_data_snapshot TEXT NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (profile_id) REFERENCES user_profiles(id) ON DELETE CASCADE + ) + `, + }, + { sql: "CREATE INDEX IF NOT EXISTS idx_user_profiles_user_id ON user_profiles(user_id)" }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_profiles_is_active ON user_profiles(is_active)", + }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_profile_changelogs_profile_id ON user_profile_changelogs(profile_id)", + }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_profile_changelogs_version ON user_profile_changelogs(version DESC)", + }, + ]); } - getActiveProfile(userId: string): UserProfile | null { - const stmt = this.db.prepare(` - SELECT * FROM user_profiles + async getActiveProfile(userId: string): Promise { + const db = await this.ready(); + const row = await db.get( + ` + SELECT * FROM user_profiles WHERE user_id = ? AND is_active = 1 LIMIT 1 - `); - - const row = stmt.get(userId) as any; + `, + [userId] + ); if (!row) return null; - return this.rowToProfile(row); } - createProfile( + async createProfile( userId: string, displayName: string, userName: string, userEmail: string, profileData: UserProfileData, promptsAnalyzed: number - ): string { + ): Promise { + const db = await this.ready(); const id = `profile_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; const now = Date.now(); @@ -197,38 +239,40 @@ export class UserProfileManager { workflows: safeArray(profileData.workflows), }; - const stmt = this.db.prepare(` + await db.run( + ` INSERT INTO user_profiles ( - id, user_id, display_name, user_name, user_email, - profile_data, version, created_at, last_analyzed_at, + id, user_id, display_name, user_name, user_email, + profile_data, version, created_at, last_analyzed_at, total_prompts_analyzed, is_active ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 1) - `); - - stmt.run( - id, - userId, - displayName, - userName, - userEmail, - JSON.stringify(cleanedData), - now, - now, - promptsAnalyzed + `, + [ + id, + userId, + displayName, + userName, + userEmail, + JSON.stringify(cleanedData), + now, + now, + promptsAnalyzed, + ] ); - this.addChangelog(id, 1, "create", "Initial profile creation", cleanedData); + await this.addChangelog(id, 1, "create", "Initial profile creation", cleanedData); return id; } - updateProfile( + async updateProfile( profileId: string, profileData: UserProfileData, additionalPromptsAnalyzed: number, changeSummary: string - ): boolean { + ): Promise { + const db = await this.ready(); const now = Date.now(); const cleanedData: UserProfileData = { @@ -237,93 +281,101 @@ export class UserProfileManager { workflows: safeArray(profileData.workflows), }; - const getVersionStmt = this.db.prepare(`SELECT version FROM user_profiles WHERE id = ?`); - const versionRow = getVersionStmt.get(profileId) as any; - const currentVersion = versionRow?.version || 0; + const versionRow = await db.get(`SELECT version FROM user_profiles WHERE id = ?`, [profileId]); + const currentVersion = Number(versionRow?.version ?? 0); const newVersion = currentVersion + 1; - const updateStmt = this.db.prepare(` - UPDATE user_profiles - SET profile_data = ?, - version = ?, - last_analyzed_at = ?, + const changes = await db.run( + ` + UPDATE user_profiles + SET profile_data = ?, + version = ?, + last_analyzed_at = ?, total_prompts_analyzed = total_prompts_analyzed + ? WHERE id = ? AND version = ? - `); - - const result = updateStmt.run( - JSON.stringify(cleanedData), - newVersion, - now, - additionalPromptsAnalyzed, - profileId, - currentVersion + `, + [ + JSON.stringify(cleanedData), + newVersion, + now, + additionalPromptsAnalyzed, + profileId, + currentVersion, + ] ); - if (result.changes === 0) { + if (changes === 0) { return false; } - this.addChangelog(profileId, newVersion, "update", changeSummary, cleanedData); - - this.cleanupOldChangelogs(profileId); + await this.addChangelog(profileId, newVersion, "update", changeSummary, cleanedData); + await this.cleanupOldChangelogs(profileId); return true; } - private addChangelog( + private async addChangelog( profileId: string, version: number, changeType: string, changeSummary: string, profileData: UserProfileData - ): void { + ): Promise { + const db = await this.ready(); const id = `changelog_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; const now = Date.now(); - const stmt = this.db.prepare(` + await db.run( + ` INSERT INTO user_profile_changelogs ( - id, profile_id, version, change_type, change_summary, + id, profile_id, version, change_type, change_summary, profile_data_snapshot, created_at ) VALUES (?, ?, ?, ?, ?, ?, ?) - `); - - stmt.run(id, profileId, version, changeType, changeSummary, JSON.stringify(profileData), now); + `, + [id, profileId, version, changeType, changeSummary, JSON.stringify(profileData), now] + ); } - private cleanupOldChangelogs(profileId: string): void { + private async cleanupOldChangelogs(profileId: string): Promise { + const db = await this.ready(); const retentionCount = CONFIG.userProfileChangelogRetentionCount; - const stmt = this.db.prepare(` - DELETE FROM user_profile_changelogs - WHERE profile_id = ? + await db.run( + ` + DELETE FROM user_profile_changelogs + WHERE profile_id = ? AND id NOT IN ( - SELECT id FROM user_profile_changelogs - WHERE profile_id = ? - ORDER BY version DESC + SELECT id FROM user_profile_changelogs + WHERE profile_id = ? + ORDER BY version DESC LIMIT ? ) - `); - - stmt.run(profileId, profileId, retentionCount); + `, + [profileId, profileId, retentionCount] + ); } - getProfileChangelogs(profileId: string, limit: number = 10): UserProfileChangelog[] { - const stmt = this.db.prepare(` - SELECT * FROM user_profile_changelogs - WHERE profile_id = ? - ORDER BY version DESC + async getProfileChangelogs( + profileId: string, + limit: number = 10 + ): Promise { + const db = await this.ready(); + const rows = await db.all( + ` + SELECT * FROM user_profile_changelogs + WHERE profile_id = ? + ORDER BY version DESC LIMIT ? - `); - - const rows = stmt.all(profileId, limit) as any[]; + `, + [profileId, limit] + ); return rows.map((row) => this.rowToChangelog(row)); } - getChangelogById(id: string): UserProfileChangelog | undefined { - const stmt = this.db.prepare(`SELECT * FROM user_profile_changelogs WHERE id = ?`); - const row = stmt.get(id) as any; + async getChangelogById(id: string): Promise { + const db = await this.ready(); + const row = await db.get(`SELECT * FROM user_profile_changelogs WHERE id = ?`, [id]); if (!row) return undefined; return this.rowToChangelog(row); } @@ -383,21 +435,21 @@ export class UserProfileManager { return { items: filtered, hasChanges, before, removed: before - filtered.length }; } - deleteProfile(profileId: string): void { - const stmt = this.db.prepare(`DELETE FROM user_profiles WHERE id = ?`); - stmt.run(profileId); + async deleteProfile(profileId: string): Promise { + const db = await this.ready(); + await db.run(`DELETE FROM user_profiles WHERE id = ?`, [profileId]); } - getProfileById(profileId: string): UserProfile | null { - const stmt = this.db.prepare(`SELECT * FROM user_profiles WHERE id = ?`); - const row = stmt.get(profileId) as any; + async getProfileById(profileId: string): Promise { + const db = await this.ready(); + const row = await db.get(`SELECT * FROM user_profiles WHERE id = ?`, [profileId]); if (!row) return null; return this.rowToProfile(row); } - getAllActiveProfiles(): UserProfile[] { - const stmt = this.db.prepare(`SELECT * FROM user_profiles WHERE is_active = 1`); - const rows = stmt.all() as any[]; + async getAllActiveProfiles(): Promise { + const db = await this.ready(); + const rows = await db.all(`SELECT * FROM user_profiles WHERE is_active = 1`); return rows.map((row) => this.rowToProfile(row)); } diff --git a/src/services/user-prompt/user-prompt-manager.ts b/src/services/user-prompt/user-prompt-manager.ts index 2c64207..82e705b 100644 --- a/src/services/user-prompt/user-prompt-manager.ts +++ b/src/services/user-prompt/user-prompt-manager.ts @@ -1,10 +1,8 @@ -import { getDatabase } from "../sqlite/sqlite-bootstrap.js"; import { join } from "node:path"; -import { connectionManager } from "../sqlite/connection-manager.js"; +import { tursoConnectionManager } from "../turso/connection-manager.js"; import { CONFIG } from "../../config.js"; - -const Database = getDatabase(); -type DatabaseType = typeof Database.prototype; +import type { InValue } from "@libsql/client"; +import type { TursoDb } from "../turso/turso-db.js"; const USER_PROMPTS_DB_NAME = "user-prompts.db"; @@ -24,309 +22,347 @@ export interface UserPrompt { } export class UserPromptManager { - private db: DatabaseType; - private readonly dbPath: string; + private db: TursoDb | null = null; + private dbPath: string; + private initPromise: Promise | null = null; constructor() { this.dbPath = join(CONFIG.storagePath, USER_PROMPTS_DB_NAME); - this.db = connectionManager.getConnection(this.dbPath); - this.initDatabase(); } - private initDatabase(): void { - this.db.run(` - CREATE TABLE IF NOT EXISTS user_prompts ( - id TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - message_id TEXT NOT NULL, - project_path TEXT, - content TEXT NOT NULL, - created_at INTEGER NOT NULL, - captured INTEGER DEFAULT 0, - user_learning_captured BOOLEAN DEFAULT 0, - linked_memory_id TEXT, - capture_attempts INTEGER DEFAULT 0, - provider_id TEXT, - model_id TEXT - ) - `); - - try { - this.db.run("ALTER TABLE user_prompts ADD COLUMN capture_attempts INTEGER DEFAULT 0"); - } catch (error: any) { - if (!error.message.includes("duplicate column name")) { - console.warn("Failed to add capture_attempts column:", error.message); + reset(): void { + this.db = null; + this.initPromise = null; + this.dbPath = join(CONFIG.storagePath, USER_PROMPTS_DB_NAME); + } + + private async initialize(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = (async () => { + try { + this.dbPath = join(CONFIG.storagePath, USER_PROMPTS_DB_NAME); + this.db = await tursoConnectionManager.getConnection(this.dbPath); + await this.initDatabase(); + } catch (error) { + this.initPromise = null; + this.db = null; + throw error; } + })(); + + return this.initPromise; + } + + private async ready(): Promise { + if (!this.db || !this.initPromise) { + await this.initialize(); + } else { + await this.initPromise; } - for (const column of ["provider_id TEXT", "model_id TEXT"]) { + if (!this.db) { + throw new Error("UserPromptManager: database not initialized"); + } + + return this.db; + } + + private async initDatabase(): Promise { + const db = this.db!; + await db.batch([ + { + sql: ` + CREATE TABLE IF NOT EXISTS user_prompts ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + project_path TEXT, + content TEXT NOT NULL, + created_at INTEGER NOT NULL, + captured INTEGER DEFAULT 0, + user_learning_captured BOOLEAN DEFAULT 0, + linked_memory_id TEXT, + capture_attempts INTEGER DEFAULT 0, + provider_id TEXT, + model_id TEXT + ) + `, + }, + { sql: "UPDATE user_prompts SET captured = 0 WHERE captured = 2" }, + { sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_session ON user_prompts(session_id)" }, + { sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_captured ON user_prompts(captured)" }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_created ON user_prompts(created_at DESC)", + }, + { sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_project ON user_prompts(project_path)" }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_linked ON user_prompts(linked_memory_id)", + }, + { + sql: "CREATE INDEX IF NOT EXISTS idx_user_prompts_user_learning ON user_prompts(user_learning_captured)", + }, + ]); + + for (const column of [ + "capture_attempts INTEGER DEFAULT 0", + "provider_id TEXT", + "model_id TEXT", + ]) { try { - this.db.run(`ALTER TABLE user_prompts ADD COLUMN ${column}`); + await db.run(`ALTER TABLE user_prompts ADD COLUMN ${column}`); } catch (error: any) { - if (!error.message.includes("duplicate column name")) { - console.warn(`Failed to add ${column.split(" ")[0]} column:`, error.message); + if (!String(error?.message ?? error).includes("duplicate column")) { + console.warn(`Failed to add ${column.split(" ")[0]} column:`, error); } } } - - this.db.run("UPDATE user_prompts SET captured = 0 WHERE captured = 2"); - - this.db.run("CREATE INDEX IF NOT EXISTS idx_user_prompts_session ON user_prompts(session_id)"); - this.db.run("CREATE INDEX IF NOT EXISTS idx_user_prompts_captured ON user_prompts(captured)"); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_prompts_created ON user_prompts(created_at DESC)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_prompts_project ON user_prompts(project_path)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_prompts_linked ON user_prompts(linked_memory_id)" - ); - this.db.run( - "CREATE INDEX IF NOT EXISTS idx_user_prompts_user_learning ON user_prompts(user_learning_captured)" - ); } - savePrompt(sessionId: string, messageId: string, projectPath: string, content: string): string { + async savePrompt( + sessionId: string, + messageId: string, + projectPath: string, + content: string + ): Promise { + const db = await this.ready(); const id = `prompt_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; const now = Date.now(); - - const stmt = this.db.prepare(` + await db.run( + ` INSERT INTO user_prompts (id, session_id, message_id, project_path, content, created_at, captured) VALUES (?, ?, ?, ?, ?, ?, 0) - `); - - stmt.run(id, sessionId, messageId, projectPath, content, now); + `, + [id, sessionId, messageId, projectPath, content, now] + ); return id; } - setPromptModel(messageId: string, providerId: string, modelId: string): void { - const stmt = this.db.prepare( - `UPDATE user_prompts SET provider_id = ?, model_id = ? WHERE message_id = ?` - ); - stmt.run(providerId, modelId, messageId); + async setPromptModel(messageId: string, providerId: string, modelId: string): Promise { + const db = await this.ready(); + await db.run(`UPDATE user_prompts SET provider_id = ?, model_id = ? WHERE message_id = ?`, [ + providerId, + modelId, + messageId, + ]); } - getLastUncapturedPrompt(sessionId: string): UserPrompt | null { + async getLastUncapturedPrompt(sessionId: string): Promise { + const db = await this.ready(); const maxRetries = CONFIG.autoCaptureMaxRetries ?? 3; - const stmt = this.db.prepare(` + const row = await db.get( + ` SELECT * FROM user_prompts WHERE session_id = ? AND captured = 0 AND capture_attempts < ? - ORDER BY created_at DESC + ORDER BY created_at DESC LIMIT 1 - `); - - const row = stmt.get(sessionId, maxRetries) as any; - if (!row) return null; - - return this.rowToPrompt(row); + `, + [sessionId, maxRetries] + ); + return row ? this.rowToPrompt(row) : null; } - getUncapturedPromptsForSession(sessionId: string): UserPrompt[] { + async getUncapturedPromptsForSession(sessionId: string): Promise { + const db = await this.ready(); const maxRetries = CONFIG.autoCaptureMaxRetries ?? 3; - const stmt = this.db.prepare(` + const rows = await db.all( + ` SELECT * FROM user_prompts WHERE session_id = ? AND captured = 0 AND capture_attempts < ? ORDER BY created_at ASC - `); - - const rows = stmt.all(sessionId, maxRetries) as any[]; + `, + [sessionId, maxRetries] + ); return rows.map((row) => this.rowToPrompt(row)); } - deletePrompt(promptId: string): void { - const stmt = this.db.prepare(`DELETE FROM user_prompts WHERE id = ?`); - stmt.run(promptId); + async deletePrompt(promptId: string): Promise { + const db = await this.ready(); + await db.run(`DELETE FROM user_prompts WHERE id = ?`, [promptId]); } - markAsCaptured(promptId: string): void { - const stmt = this.db.prepare(`UPDATE user_prompts SET captured = 1 WHERE id = ?`); - stmt.run(promptId); + async markAsCaptured(promptId: string): Promise { + const db = await this.ready(); + await db.run(`UPDATE user_prompts SET captured = 1 WHERE id = ?`, [promptId]); } - claimPrompt(promptId: string): boolean { - const stmt = this.db.prepare( - `UPDATE user_prompts SET captured = 2 WHERE id = ? AND captured = 0` + async claimPrompt(promptId: string): Promise { + const db = await this.ready(); + const changes = await db.run( + `UPDATE user_prompts SET captured = 2 WHERE id = ? AND captured = 0`, + [promptId] ); - const result = stmt.run(promptId); - return result.changes > 0; + return changes > 0; } - recordFailedAttempt(promptId: string): void { - const stmt = this.db.prepare( - `UPDATE user_prompts SET capture_attempts = capture_attempts + 1 WHERE id = ?` - ); - stmt.run(promptId); + async recordFailedAttempt(promptId: string): Promise { + const db = await this.ready(); + await db.run(`UPDATE user_prompts SET capture_attempts = capture_attempts + 1 WHERE id = ?`, [ + promptId, + ]); } - /** - * Release a previously claimed prompt back to the pending state so it can - * be retried by a future capture cycle. Used when capture aborts before - * either successfully writing a memory or explicitly skipping the prompt - * (e.g. transient network error, missing AI response, plugin restart). - * - * Only rows still marked as in-progress (captured = 2) are touched, so - * concurrent callers that already finished the capture cannot be reverted. - */ - releaseClaim(promptId: string): boolean { - const stmt = this.db.prepare( - `UPDATE user_prompts SET captured = 0 WHERE id = ? AND captured = 2` + async releaseClaim(promptId: string): Promise { + const db = await this.ready(); + const changes = await db.run( + `UPDATE user_prompts SET captured = 0 WHERE id = ? AND captured = 2`, + [promptId] ); - const result = stmt.run(promptId); - return result.changes > 0; + return changes > 0; } - countUncapturedPrompts(): number { + async countUncapturedPrompts(): Promise { + const db = await this.ready(); const maxRetries = CONFIG.autoCaptureMaxRetries ?? 3; - const stmt = this.db.prepare( - `SELECT COUNT(*) as count FROM user_prompts WHERE captured = 0 AND capture_attempts < ?` + const row = await db.get( + `SELECT COUNT(*) as count FROM user_prompts WHERE captured = 0 AND capture_attempts < ?`, + [maxRetries] ); - const row = stmt.get(maxRetries) as any; - return row?.count || 0; + return Number(row?.count ?? 0); } - getUncapturedPrompts(limit: number): UserPrompt[] { + async getUncapturedPrompts(limit: number): Promise { + const db = await this.ready(); const maxRetries = CONFIG.autoCaptureMaxRetries ?? 3; - const stmt = this.db.prepare(` - SELECT * FROM user_prompts + const rows = await db.all( + ` + SELECT * FROM user_prompts WHERE captured = 0 AND capture_attempts < ? - ORDER BY capture_attempts ASC, created_at ASC + ORDER BY capture_attempts ASC, created_at ASC LIMIT ? - `); - - const rows = stmt.all(maxRetries, limit) as any[]; + `, + [maxRetries, limit] + ); return rows.map((row) => this.rowToPrompt(row)); } - markMultipleAsCaptured(promptIds: string[]): void { + async markMultipleAsCaptured(promptIds: string[]): Promise { if (promptIds.length === 0) return; - + const db = await this.ready(); const placeholders = promptIds.map(() => "?").join(","); - const stmt = this.db.prepare( - `UPDATE user_prompts SET captured = 1 WHERE id IN (${placeholders})` - ); - stmt.run(...promptIds); + await db.run(`UPDATE user_prompts SET captured = 1 WHERE id IN (${placeholders})`, promptIds); } - countUnanalyzedForUserLearning(): number { - const stmt = this.db.prepare( + async countUnanalyzedForUserLearning(): Promise { + const db = await this.ready(); + const row = await db.get( `SELECT COUNT(*) as count FROM user_prompts WHERE user_learning_captured = 0` ); - const row = stmt.get() as any; - return row?.count || 0; + return Number(row?.count ?? 0); } - getPromptsForUserLearning(limit: number): UserPrompt[] { - const stmt = this.db.prepare(` - SELECT * FROM user_prompts - WHERE user_learning_captured = 0 - ORDER BY created_at ASC + async getPromptsForUserLearning(limit: number): Promise { + const db = await this.ready(); + const rows = await db.all( + ` + SELECT * FROM user_prompts + WHERE user_learning_captured = 0 + ORDER BY created_at ASC LIMIT ? - `); - - const rows = stmt.all(limit) as any[]; + `, + [limit] + ); return rows.map((row) => this.rowToPrompt(row)); } - markAsUserLearningCaptured(promptId: string): void { - const stmt = this.db.prepare(`UPDATE user_prompts SET user_learning_captured = 1 WHERE id = ?`); - stmt.run(promptId); + async markAsUserLearningCaptured(promptId: string): Promise { + const db = await this.ready(); + await db.run(`UPDATE user_prompts SET user_learning_captured = 1 WHERE id = ?`, [promptId]); } - markMultipleAsUserLearningCaptured(promptIds: string[]): void { + async markMultipleAsUserLearningCaptured(promptIds: string[]): Promise { if (promptIds.length === 0) return; - + const db = await this.ready(); const placeholders = promptIds.map(() => "?").join(","); - const stmt = this.db.prepare( - `UPDATE user_prompts SET user_learning_captured = 1 WHERE id IN (${placeholders})` + await db.run( + `UPDATE user_prompts SET user_learning_captured = 1 WHERE id IN (${placeholders})`, + promptIds ); - stmt.run(...promptIds); } - deleteOldPrompts(cutoffTime: number): { deleted: number; linkedMemoryIds: string[] } { - const getLinkedStmt = this.db.prepare(` - SELECT linked_memory_id FROM user_prompts + async deleteOldPrompts( + cutoffTime: number + ): Promise<{ deleted: number; linkedMemoryIds: string[] }> { + const db = await this.ready(); + const linkedRows = await db.all( + ` + SELECT linked_memory_id FROM user_prompts WHERE created_at < ? AND linked_memory_id IS NOT NULL - `); - const linkedRows = getLinkedStmt.all(cutoffTime) as any[]; - const linkedMemoryIds = linkedRows.map((row) => row.linked_memory_id).filter((id) => id); - - const deleteStmt = this.db.prepare(`DELETE FROM user_prompts WHERE created_at < ?`); - const result = deleteStmt.run(cutoffTime); + `, + [cutoffTime] + ); + const linkedMemoryIds = linkedRows + .map((row) => row.linked_memory_id) + .filter((id): id is string => Boolean(id)); - return { - deleted: result.changes, - linkedMemoryIds, - }; + const deleted = await db.run(`DELETE FROM user_prompts WHERE created_at < ?`, [cutoffTime]); + return { deleted, linkedMemoryIds }; } - linkMemoryToPrompt(promptId: string, memoryId: string): void { - const stmt = this.db.prepare(`UPDATE user_prompts SET linked_memory_id = ? WHERE id = ?`); - stmt.run(memoryId, promptId); + async linkMemoryToPrompt(promptId: string, memoryId: string): Promise { + const db = await this.ready(); + await db.run(`UPDATE user_prompts SET linked_memory_id = ? WHERE id = ?`, [memoryId, promptId]); } - getPromptById(promptId: string): UserPrompt | null { - const stmt = this.db.prepare(`SELECT * FROM user_prompts WHERE id = ?`); - const row = stmt.get(promptId) as any; - if (!row) return null; - return this.rowToPrompt(row); + async getPromptById(promptId: string): Promise { + const db = await this.ready(); + const row = await db.get(`SELECT * FROM user_prompts WHERE id = ?`, [promptId]); + return row ? this.rowToPrompt(row) : null; } - getCapturedPrompts(projectPath?: string): UserPrompt[] { - let query = `SELECT * FROM user_prompts WHERE captured = 1`; - const params: any[] = []; - - if (projectPath) { - query += ` AND project_path = ?`; - params.push(projectPath); - } - - query += ` ORDER BY created_at DESC`; - - const stmt = this.db.prepare(query); - const rows = stmt.all(...params) as any[]; + async getCapturedPrompts(projectPath?: string): Promise { + const db = await this.ready(); + const rows = projectPath + ? await db.all( + `SELECT * FROM user_prompts WHERE captured = 1 AND project_path = ? ORDER BY created_at DESC`, + [projectPath] + ) + : await db.all(`SELECT * FROM user_prompts WHERE captured = 1 ORDER BY created_at DESC`); return rows.map((row) => this.rowToPrompt(row)); } - searchPrompts(query: string, projectPath?: string, limit: number = 20): UserPrompt[] { + async searchPrompts( + query: string, + projectPath?: string, + limit: number = 20 + ): Promise { + const db = await this.ready(); + const params: InValue[] = [`%${query}%`]; let sql = `SELECT * FROM user_prompts WHERE content LIKE ? AND captured = 1`; - const params: any[] = [`%${query}%`]; - if (projectPath) { sql += ` AND project_path = ?`; params.push(projectPath); } - sql += ` ORDER BY created_at DESC LIMIT ?`; params.push(limit); - - const stmt = this.db.prepare(sql); - const rows = stmt.all(...params) as any[]; + const rows = await db.all(sql, params); return rows.map((row) => this.rowToPrompt(row)); } - getPromptsByIds(ids: string[]): UserPrompt[] { + async getPromptsByIds(ids: string[]): Promise { if (ids.length === 0) return []; + const db = await this.ready(); const placeholders = ids.map(() => "?").join(","); - const stmt = this.db.prepare(`SELECT * FROM user_prompts WHERE id IN (${placeholders})`); - const rows = stmt.all(...ids) as any[]; + const rows = await db.all(`SELECT * FROM user_prompts WHERE id IN (${placeholders})`, ids); return rows.map((row) => this.rowToPrompt(row)); } - private rowToPrompt(row: any): UserPrompt { + private rowToPrompt(row: Record): UserPrompt { return { - id: row.id, - sessionId: row.session_id, - messageId: row.message_id, - projectPath: row.project_path, - content: row.content, - createdAt: row.created_at, - captured: row.captured === 1, - userLearningCaptured: row.user_learning_captured === 1, - linkedMemoryId: row.linked_memory_id, - capture_attempts: row.capture_attempts || 0, - providerId: row.provider_id ?? null, - modelId: row.model_id ?? null, + id: String(row.id), + sessionId: String(row.session_id), + messageId: String(row.message_id), + projectPath: row.project_path ? String(row.project_path) : null, + content: String(row.content), + createdAt: Number(row.created_at), + captured: Number(row.captured) === 1, + userLearningCaptured: Number(row.user_learning_captured) === 1, + linkedMemoryId: row.linked_memory_id ? String(row.linked_memory_id) : null, + capture_attempts: Number(row.capture_attempts ?? 0), + providerId: row.provider_id ? String(row.provider_id) : null, + modelId: row.model_id ? String(row.model_id) : null, }; } } diff --git a/src/services/vector-backends/backend-factory.ts b/src/services/vector-backends/backend-factory.ts deleted file mode 100644 index 9aa76ec..0000000 --- a/src/services/vector-backends/backend-factory.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { CONFIG } from "../../config.js"; -import { log } from "../logger.js"; -import { ExactScanBackend } from "./exact-scan-backend.js"; -import type { VectorBackend, VectorBackendFactoryOptions } from "./types.js"; -import { USearchBackend } from "./usearch-backend.js"; - -class FallbackAwareBackend implements VectorBackend { - private activeBackend: VectorBackend; - - constructor( - private readonly strategy: "usearch-first" | "usearch", - private readonly primary: VectorBackend, - private readonly fallback: VectorBackend - ) { - this.activeBackend = primary; - } - - getBackendName(): string { - return this.activeBackend.getBackendName(); - } - - async insert(args: Parameters[0]): Promise { - await this.activeBackend.insert(args); - } - - async insertBatch(args: Parameters[0]): Promise { - await this.activeBackend.insertBatch(args); - } - - async delete(args: Parameters[0]): Promise { - await this.activeBackend.delete(args); - } - - async search(args: Parameters[0]) { - try { - return await this.activeBackend.search(args); - } catch (error) { - this.logDegrade("search", error); - this.activeBackend = this.fallback; - return this.fallback.search(args); - } - } - - async rebuildFromShard(args: Parameters[0]): Promise { - try { - await this.activeBackend.rebuildFromShard(args); - } catch (error) { - this.logDegrade("rebuild", error); - this.activeBackend = this.fallback; - await this.fallback.rebuildFromShard(args); - } - } - - async deleteShardIndexes( - args: Parameters[0] - ): Promise { - await this.primary.deleteShardIndexes(args); - await this.fallback.deleteShardIndexes(args); - } - - private logDegrade(operation: string, error: unknown): void { - log("Vector backend degraded to exact-scan", { - strategy: this.strategy, - severity: this.strategy === "usearch" ? "warning" : "info", - operation, - error: String(error), - }); - } -} - -async function defaultUSearchProbe(): Promise { - try { - await import("usearch"); - return true; - } catch { - return false; - } -} - -export async function createVectorBackend( - options: VectorBackendFactoryOptions -): Promise { - const exactScanBackend = new ExactScanBackend(); - - if (options.vectorBackend === "exact-scan") { - return exactScanBackend; - } - - const probeUSearch = options.probeUSearch ?? defaultUSearchProbe; - if (!(await probeUSearch())) { - if (options.vectorBackend === "usearch") { - log("Vector backend degraded to exact-scan", { - strategy: "usearch", - severity: "warning", - operation: "probe", - error: "USearch unavailable", - }); - } - return exactScanBackend; - } - - try { - const usearchBackend = - options.createUSearchBackend?.() ?? - new USearchBackend({ - baseDir: CONFIG.storagePath, - dimensions: CONFIG.embeddingDimensions, - }); - - return new FallbackAwareBackend(options.vectorBackend, usearchBackend, exactScanBackend); - } catch (error) { - log("Vector backend degraded to exact-scan", { - strategy: options.vectorBackend, - severity: options.vectorBackend === "usearch" ? "warning" : "info", - operation: "create", - error: String(error), - }); - return exactScanBackend; - } -} diff --git a/src/services/vector-backends/exact-scan-backend.ts b/src/services/vector-backends/exact-scan-backend.ts deleted file mode 100644 index 5fd1eee..0000000 --- a/src/services/vector-backends/exact-scan-backend.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { - BackendInsertItem, - BackendSearchResult, - VectorBackend, - VectorBackendSearchParams, - VectorKind, -} from "./types.js"; -import type { ShardInfo } from "../sqlite/types.js"; -import { cosineSimilarity } from "../../utils/math.js"; - -interface RankedRow { - id: string; - vector: Float32Array; -} - -interface VectorRow { - id: string; - vector?: Uint8Array | ArrayBuffer | null; - tags_vector?: Uint8Array | ArrayBuffer | null; -} - -export class ExactScanBackend implements VectorBackend { - getBackendName(): string { - return "exact-scan"; - } - - rankVectors(rows: RankedRow[], queryVector: Float32Array, limit: number): BackendSearchResult[] { - return rows - .map((row) => ({ - id: row.id, - distance: 1 - cosineSimilarity(row.vector, queryVector), - })) - .sort((a, b) => a.distance - b.distance) - .slice(0, limit); - } - - async insert(_args: { - id: string; - vector: Float32Array; - shard: ShardInfo; - kind: VectorKind; - }): Promise {} - - async insertBatch(_args: { - items: BackendInsertItem[]; - shard: ShardInfo; - kind: VectorKind; - }): Promise {} - - async delete(_args: { id: string; shard: ShardInfo; kind: VectorKind }): Promise {} - - async search(args: VectorBackendSearchParams): Promise { - const column = args.kind === "tags" ? "tags_vector" : "vector"; - const rows = ( - args.db as { - prepare: (sql: string) => { all: () => VectorRow[] }; - } - ) - .prepare(`SELECT id, ${column} FROM memories WHERE ${column} IS NOT NULL`) - .all(); - - if (rows.length === 0) { - return []; - } - - const rankedRows: RankedRow[] = rows - .map((row) => ({ - id: row.id, - vector: this.decodeVector(args.kind === "tags" ? row.tags_vector : row.vector), - })) - .filter((row) => row.vector.length > 0); - - return this.rankVectors(rankedRows, args.queryVector, args.limit); - } - - async rebuildFromShard(_args: { - db: unknown; - shard: ShardInfo; - kind: VectorKind; - }): Promise {} - - async deleteShardIndexes(_args: { shard: ShardInfo }): Promise {} - - private decodeVector(value: Uint8Array | ArrayBuffer | null | undefined): Float32Array { - if (!value) { - return new Float32Array(); - } - - if (value instanceof Uint8Array) { - return new Float32Array( - value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) - ); - } - - return new Float32Array(value); - } -} diff --git a/src/services/vector-backends/types.ts b/src/services/vector-backends/types.ts deleted file mode 100644 index 7f34922..0000000 --- a/src/services/vector-backends/types.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { ShardInfo } from "../sqlite/types.js"; - -export type VectorKind = "content" | "tags"; - -export interface BackendSearchResult { - id: string; - distance: number; -} - -export interface BackendInsertItem { - id: string; - vector: Float32Array; -} - -export interface VectorBackendSearchParams { - db: unknown; - shard: ShardInfo; - kind: VectorKind; - queryVector: Float32Array; - limit: number; -} - -export interface VectorBackend { - getBackendName(): string; - insert(args: { - id: string; - vector: Float32Array; - shard: ShardInfo; - kind: VectorKind; - }): Promise; - insertBatch(args: { - items: BackendInsertItem[]; - shard: ShardInfo; - kind: VectorKind; - }): Promise; - delete(args: { id: string; shard: ShardInfo; kind: VectorKind }): Promise; - search(args: VectorBackendSearchParams): Promise; - rebuildFromShard(args: { db: unknown; shard: ShardInfo; kind: VectorKind }): Promise; - deleteShardIndexes(args: { shard: ShardInfo }): Promise; -} - -export interface VectorBackendFactoryOptions { - vectorBackend: "usearch-first" | "usearch" | "exact-scan"; - probeUSearch?: () => Promise; - createUSearchBackend?: () => VectorBackend; -} diff --git a/src/services/vector-backends/usearch-backend.ts b/src/services/vector-backends/usearch-backend.ts deleted file mode 100644 index 301f111..0000000 --- a/src/services/vector-backends/usearch-backend.ts +++ /dev/null @@ -1,241 +0,0 @@ -import type { - BackendInsertItem, - BackendSearchResult, - VectorBackend, - VectorBackendSearchParams, - VectorKind, -} from "./types.js"; -import type { ShardInfo } from "../sqlite/types.js"; - -type USearchModule = typeof import("usearch"); -type USearchIndex = InstanceType; - -interface CachedIndex { - index: USearchIndex; - idToKey: Map; - keyToId: Map; - nextKey: bigint; - indexKey: string; - initialized: boolean; -} - -export class USearchBackend implements VectorBackend { - private readonly indexes = new Map(); - - constructor( - private readonly options: { - baseDir: string; - dimensions: number; - } - ) { - void this.options.baseDir; - } - - getBackendName(): string { - return "usearch"; - } - - async insert(args: { - id: string; - vector: Float32Array; - shard: ShardInfo; - kind: VectorKind; - }): Promise { - const indexKey = this.getIndexKey(args.shard, args.kind); - const cache = await this.getOrCreateIndex(indexKey); - try { - this.upsertItem(cache, { id: args.id, vector: args.vector }); - cache.initialized = true; - } catch (error) { - throw new Error(`USearch insert failed for ${indexKey}: ${String(error)}`); - } - } - - async insertBatch(args: { - items: BackendInsertItem[]; - shard: ShardInfo; - kind: VectorKind; - }): Promise { - const indexKey = this.getIndexKey(args.shard, args.kind); - const cache = await this.getOrCreateIndex(indexKey); - try { - this.addItems(cache, args.items); - cache.initialized = true; - } catch (error) { - throw new Error(`USearch batch insert failed for ${indexKey}: ${String(error)}`); - } - } - - async delete(args: { id: string; shard: ShardInfo; kind: VectorKind }): Promise { - const cache = await this.getOrCreateIndex(this.getIndexKey(args.shard, args.kind)); - const key = cache.idToKey.get(args.id); - if (key === undefined) return; - cache.index.remove(key); - cache.idToKey.delete(args.id); - cache.keyToId.delete(key); - } - - async search(args: VectorBackendSearchParams): Promise { - const indexKey = this.getIndexKey(args.shard, args.kind); - const cache = await this.getOrCreateIndex(indexKey); - try { - const matches = cache.index.search(args.queryVector, args.limit); - return Array.from(matches.keys as Iterable, (key, index) => { - const id = cache.keyToId.get(key); - if (!id) { - throw new Error( - `USearch index metadata missing for key ${String(key)} in ${cache.indexKey}` - ); - } - return { - id, - distance: matches.distances[index] ?? 0, - }; - }); - } catch (error) { - throw new Error(`USearch search failed for ${indexKey}: ${String(error)}`); - } - } - - async rebuildFromShard(args: { db: unknown; shard: ShardInfo; kind: VectorKind }): Promise { - const indexKey = this.getIndexKey(args.shard, args.kind); - const existing = this.indexes.get(indexKey); - if (existing?.initialized) { - return; - } - - const column = args.kind === "tags" ? "tags_vector" : "vector"; - const rows = ( - args.db as { - prepare: (sql: string) => { - all: () => Array<{ - id: string; - vector?: Uint8Array | ArrayBuffer | null; - tags_vector?: Uint8Array | ArrayBuffer | null; - }>; - }; - } - ) - .prepare(`SELECT id, ${column} FROM memories WHERE ${column} IS NOT NULL`) - .all(); - - const cache = await this.createEmptyIndex(indexKey); - this.indexes.set(indexKey, cache); - - for (const row of rows) { - const raw = args.kind === "tags" ? row.tags_vector : row.vector; - const vector = this.decodeVector(raw); - if (vector.length === 0) continue; - this.upsertItem(cache, { id: row.id, vector }); - } - - cache.initialized = true; - } - - async deleteShardIndexes(args: { shard: ShardInfo }): Promise { - for (const kind of ["content", "tags"] as const) { - const indexKey = this.getIndexKey(args.shard, kind); - this.indexes.delete(indexKey); - } - } - - async insertManyForTest(indexKey: string, items: BackendInsertItem[]): Promise { - const cache = await this.getOrCreateIndex(indexKey); - this.addItems(cache, items); - cache.initialized = true; - } - - async searchForTest( - indexKey: string, - queryVector: Float32Array, - limit: number - ): Promise { - const cache = await this.getOrCreateIndex(indexKey); - try { - const matches = cache.index.search(queryVector, limit); - return Array.from(matches.keys as Iterable, (key, index) => { - const id = cache.keyToId.get(key); - if (!id) { - throw new Error( - `USearch index metadata missing for key ${String(key)} in ${cache.indexKey}` - ); - } - return { - id, - distance: matches.distances[index] ?? 0, - }; - }); - } catch (error) { - throw new Error(`USearch test search failed for ${indexKey}: ${String(error)}`); - } - } - - private async getOrCreateIndex(indexKey: string): Promise { - const existing = this.indexes.get(indexKey); - if (existing) return existing; - - const cache = await this.createEmptyIndex(indexKey); - this.indexes.set(indexKey, cache); - return cache; - } - - private async createEmptyIndex(indexKey: string): Promise { - const usearch = await this.loadUSearch(); - return { - index: new usearch.Index({ dimensions: this.options.dimensions, metric: "cos" }), - idToKey: new Map(), - keyToId: new Map(), - nextKey: 1n, - indexKey, - initialized: false, - }; - } - - private ensureKey(cache: CachedIndex, id: string): bigint { - const existing = cache.idToKey.get(id); - if (existing !== undefined) return existing; - - const key = cache.nextKey; - cache.nextKey += 1n; - cache.idToKey.set(id, key); - cache.keyToId.set(key, id); - return key; - } - - private addItems(cache: CachedIndex, items: BackendInsertItem[]): void { - for (const item of items) { - this.upsertItem(cache, item); - } - } - - private upsertItem(cache: CachedIndex, item: BackendInsertItem): void { - const existing = cache.idToKey.get(item.id); - if (existing !== undefined) { - cache.index.remove(existing); - } - const key = this.ensureKey(cache, item.id); - cache.index.add(key, item.vector); - } - - private decodeVector(value: Uint8Array | ArrayBuffer | null | undefined): Float32Array { - if (!value) return new Float32Array(); - if (value instanceof Uint8Array) { - return new Float32Array( - value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) - ); - } - return new Float32Array(value); - } - - private getIndexKey(shard: ShardInfo, kind: VectorKind): string { - return `${shard.scope}_${shard.scopeHash}_${shard.shardIndex}_${kind}`; - } - - private async loadUSearch(): Promise { - try { - return await import("usearch"); - } catch (error) { - throw new Error(`Failed to load usearch backend: ${String(error)}`); - } - } -} diff --git a/src/services/web-api-auth.ts b/src/services/web-api-auth.ts new file mode 100644 index 0000000..9b698dd --- /dev/null +++ b/src/services/web-api-auth.ts @@ -0,0 +1,31 @@ +const LOOPBACK_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]", "0:0:0:0:0:0:0:1"]); + +export function isLoopbackHost(host: string): boolean { + return LOOPBACK_HOSTS.has(host.trim().toLowerCase()); +} + +export function assertWebServerNetworkAuth(host: string, apiToken?: string): void { + if (!isLoopbackHost(host) && !apiToken) { + throw new Error( + `webServerHost "${host}" exposes the API on the network. Set webServerApiToken in opencode-mem.jsonc, or bind to 127.0.0.1.` + ); + } +} + +export function authorizeApiRequest(req: Request, apiToken?: string): Response | null { + if (!apiToken) return null; + + const header = req.headers.get("authorization"); + const bearer = header?.match(/^Bearer\s+(.+)$/i)?.[1]?.trim(); + const alt = req.headers.get("x-opencode-mem-token")?.trim(); + const token = bearer || alt; + + if (token && token === apiToken) { + return null; + } + + return new Response(JSON.stringify({ success: false, error: "Unauthorized" }), { + status: 401, + headers: { "Content-Type": "application/json" }, + }); +} diff --git a/src/services/web-server-worker.ts b/src/services/web-server-worker.ts deleted file mode 100644 index d2c0958..0000000 --- a/src/services/web-server-worker.ts +++ /dev/null @@ -1,396 +0,0 @@ -import { readFileSync } from "node:fs"; -import { join, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { corsPreflightResponse, disallowedCorsResponse, isAllowedBrowserOrigin } from "./cors.js"; -import { - handleListTags, - handleListMemories, - handleAddMemory, - handleDeleteMemory, - handleBulkDelete, - handleUpdateMemory, - handleSearch, - handleStats, - handlePinMemory, - handleUnpinMemory, - handleRunCleanup, - handleRunDeduplication, - handleDetectMigration, - handleRunMigration, - handleDetectTagMigration, - handleRunTagMigrationBatch, - handleGetTagMigrationProgress, - handleDeletePrompt, - handleBulkDeletePrompts, - handleGetUserProfile, - handleGetProfileChangelog, - handleGetProfileSnapshot, - handleRefreshProfile, - handleAICleanup, - handleApplyCleanup, - handleUpdateProfileItem, -} from "./api-handlers.js"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -interface WorkerMessage { - type: "start" | "stop" | "status"; - port?: number; - host?: string; -} - -interface WorkerResponse { - type: "started" | "stopped" | "error" | "status"; - url?: string; - error?: string; - running?: boolean; -} - -let server: any = null; - -async function handleRequest(req: Request): Promise { - const url = new URL(req.url); - const path = url.pathname; - const method = req.method; - const origin = req.headers.get("Origin"); - - if (!isAllowedBrowserOrigin(origin)) { - return disallowedCorsResponse(); - } - - if (method === "OPTIONS") { - return corsPreflightResponse(req); - } - - try { - if (path === "/" || path === "/index.html") { - return serveStaticFile("index.html", "text/html"); - } - - if (path === "/styles.css") { - return serveStaticFile("styles.css", "text/css"); - } - - if (path === "/app.js") { - return serveStaticFile("app.js", "application/javascript"); - } - - if (path === "/favicon.ico") { - return serveStaticFile("favicon.ico", "image/x-icon"); - } - - if (path === "/api/tags" && method === "GET") { - const result = await handleListTags(); - return jsonResponse(result); - } - - if (path === "/api/memories" && method === "GET") { - const tag = url.searchParams.get("tag") || undefined; - const page = parseInt(url.searchParams.get("page") || "1"); - const pageSize = parseInt(url.searchParams.get("pageSize") || "20"); - const includePrompts = url.searchParams.get("includePrompts") !== "false"; - const result = await handleListMemories(tag, page, pageSize, includePrompts); - return jsonResponse(result); - } - - if (path === "/api/memories" && method === "POST") { - const body = (await req.json()) as any; - const result = await handleAddMemory(body); - return jsonResponse(result); - } - - if (path.startsWith("/api/memories/") && method === "DELETE") { - const parts = path.split("/"); - const id = parts[3]; - if (!id || id === "bulk-delete") { - return jsonResponse({ success: false, error: "Invalid ID" }); - } - const cascade = url.searchParams.get("cascade") === "true"; - const result = await handleDeleteMemory(id, cascade); - return jsonResponse(result); - } - - if (path.startsWith("/api/memories/") && method === "PUT") { - const id = path.split("/").pop(); - if (!id) { - return jsonResponse({ success: false, error: "Invalid ID" }); - } - const body = (await req.json()) as any; - const result = await handleUpdateMemory(id, body); - return jsonResponse(result); - } - - if (path === "/api/memories/bulk-delete" && method === "POST") { - const body = (await req.json()) as any; - const cascade = body.cascade !== false; - const result = await handleBulkDelete(body.ids || [], cascade); - return jsonResponse(result); - } - - if (path === "/api/search" && method === "GET") { - const query = url.searchParams.get("q"); - const tag = url.searchParams.get("tag") || undefined; - const page = parseInt(url.searchParams.get("page") || "1"); - const pageSize = parseInt(url.searchParams.get("pageSize") || "20"); - - if (!query) { - return jsonResponse({ success: false, error: "query parameter required" }); - } - - const result = await handleSearch(query, tag, page, pageSize); - return jsonResponse(result); - } - - if (path === "/api/stats" && method === "GET") { - const result = await handleStats(); - return jsonResponse(result); - } - - if (path.match(/^\/api\/memories\/[^/]+\/pin$/) && method === "POST") { - const id = path.split("/")[3]; - if (!id) { - return jsonResponse({ success: false, error: "Invalid ID" }); - } - const result = await handlePinMemory(id); - return jsonResponse(result); - } - - if (path.match(/^\/api\/memories\/[^/]+\/unpin$/) && method === "POST") { - const id = path.split("/")[3]; - if (!id) { - return jsonResponse({ success: false, error: "Invalid ID" }); - } - const result = await handleUnpinMemory(id); - return jsonResponse(result); - } - - if (path === "/api/cleanup" && method === "POST") { - const result = await handleRunCleanup(); - return jsonResponse(result); - } - - if (path === "/api/deduplicate" && method === "POST") { - const result = await handleRunDeduplication(); - return jsonResponse(result); - } - - if (path === "/api/migration/detect" && method === "GET") { - const result = await handleDetectMigration(); - return jsonResponse(result); - } - - if (path === "/api/migration/tags/detect" && method === "GET") { - const result = await handleDetectTagMigration(); - return jsonResponse(result); - } - - if (path === "/api/migration/tags/run-batch" && method === "POST") { - const body = (await req.json()) as any; - const batchSize = body?.batchSize || 5; - const result = await handleRunTagMigrationBatch(batchSize); - return jsonResponse(result); - } - - if (path === "/api/migration/tags/progress" && method === "GET") { - const result = await handleGetTagMigrationProgress(); - return jsonResponse(result); - } - - if (path === "/api/migration/run" && method === "POST") { - const body = (await req.json()) as any; - const strategy = body.strategy || "fresh-start"; - if (strategy !== "fresh-start" && strategy !== "re-embed") { - return jsonResponse({ success: false, error: "Invalid strategy" }); - } - const result = await handleRunMigration(strategy); - return jsonResponse(result); - } - - if (path.startsWith("/api/prompts/") && method === "DELETE") { - const parts = path.split("/"); - const id = parts[3]; - if (!id || id === "bulk-delete") { - return jsonResponse({ success: false, error: "Invalid ID" }); - } - const cascade = url.searchParams.get("cascade") === "true"; - const result = await handleDeletePrompt(id, cascade); - return jsonResponse(result); - } - - if (path === "/api/prompts/bulk-delete" && method === "POST") { - const body = (await req.json()) as any; - const cascade = body.cascade !== false; - const result = await handleBulkDeletePrompts(body.ids || [], cascade); - return jsonResponse(result); - } - - if (path === "/api/user-profile" && method === "GET") { - const userId = url.searchParams.get("userId") || undefined; - const result = await handleGetUserProfile(userId); - return jsonResponse(result); - } - - if (path === "/api/user-profile/changelog" && method === "GET") { - const profileId = url.searchParams.get("profileId"); - const limit = parseInt(url.searchParams.get("limit") || "5"); - if (!profileId) { - return jsonResponse({ success: false, error: "profileId parameter required" }); - } - const result = await handleGetProfileChangelog(profileId, limit); - return jsonResponse(result); - } - - if (path === "/api/user-profile/snapshot" && method === "GET") { - const changelogId = url.searchParams.get("chlogId"); - if (!changelogId) { - return jsonResponse({ success: false, error: "changelogId parameter required" }); - } - const result = await handleGetProfileSnapshot(changelogId); - return jsonResponse(result); - } - - if (path === "/api/user-profile/refresh" && method === "POST") { - const body = (await req.json().catch(() => ({}))) as any; - const userId = body.userId || undefined; - const result = await handleRefreshProfile(userId); - return jsonResponse(result); - } - - if (path === "/api/user-profile/ai-cleanup" && method === "POST") { - const body = (await req.json().catch(() => ({}))) as any; - const userId = body.userId || undefined; - const result = await handleAICleanup(userId); - return jsonResponse(result); - } - - if (path === "/api/user-profile/ai-cleanup/apply" && method === "POST") { - const body = (await req.json().catch(() => ({}))) as any; - const userId = body.userId || undefined; - const result = await handleApplyCleanup(userId, body); - return jsonResponse(result); - } - - if (path === "/api/user-profile/item" && method === "PATCH") { - const body = (await req.json().catch(() => ({}))) as any; - const result = await handleUpdateProfileItem(body); - return jsonResponse(result); - } - - return new Response("Not Found", { status: 404 }); - } catch (error) { - return jsonResponse( - { - success: false, - error: String(error), - }, - 500 - ); - } -} - -function serveStaticFile(filename: string, contentType: string): Response { - try { - const webDir = join(__dirname, "..", "web"); - const filePath = join(webDir, filename); - - if (contentType.startsWith("image/")) { - const content = readFileSync(filePath); - return new Response(content, { - headers: { - "Content-Type": contentType, - "Cache-Control": "public, max-age=86400", - }, - }); - } - - const content = readFileSync(filePath, "utf-8"); - - return new Response(content, { - headers: { - "Content-Type": contentType, - "Cache-Control": "no-cache", - }, - }); - } catch (error) { - return new Response("File not found", { status: 404 }); - } -} - -function jsonResponse(data: any, status: number = 200): Response { - return new Response(JSON.stringify(data), { - status, - headers: { - "Content-Type": "application/json", - }, - }); -} - -declare const self: Worker; - -self.onmessage = async (event: MessageEvent) => { - const message = event.data; - - try { - switch (message.type) { - case "start": { - if (server) { - self.postMessage({ - type: "error", - error: "Server already running", - } as WorkerResponse); - return; - } - - server = Bun.serve({ - port: message.port!, - hostname: message.host!, - fetch: handleRequest, - }); - - self.postMessage({ - type: "started", - url: `http://${message.host}:${message.port}`, - } as WorkerResponse); - break; - } - - case "stop": { - if (server) { - server.stop(); - server = null; - self.postMessage({ - type: "stopped", - } as WorkerResponse); - } else { - self.postMessage({ - type: "error", - error: "Server not running", - } as WorkerResponse); - } - break; - } - - case "status": { - self.postMessage({ - type: "status", - running: server !== null, - } as WorkerResponse); - break; - } - - default: { - self.postMessage({ - type: "error", - error: `Unknown message type: ${message.type}`, - } as WorkerResponse); - break; - } - } - } catch (error) { - self.postMessage({ - type: "error", - error: String(error), - } as WorkerResponse); - } -}; diff --git a/src/services/web-server.ts b/src/services/web-server.ts index f28a5bf..023c585 100644 --- a/src/services/web-server.ts +++ b/src/services/web-server.ts @@ -5,6 +5,7 @@ import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { log } from "./logger.js"; import { corsPreflightResponse, disallowedCorsResponse, isAllowedBrowserOrigin } from "./cors.js"; +import { assertWebServerNetworkAuth, authorizeApiRequest } from "./web-api-auth.js"; import { handleListTags, handleListMemories, @@ -157,6 +158,7 @@ interface WebServerConfig { port: number; host: string; enabled: boolean; + apiToken?: string; } export class WebServer { @@ -189,6 +191,8 @@ export class WebServer { return; } + assertWebServerNetworkAuth(this.config.host, this.config.apiToken); + try { this.server = serveFetch({ port: this.config.port, @@ -295,8 +299,12 @@ export class WebServer { async checkServerAvailable(): Promise { try { + const headers = this.config.apiToken + ? { Authorization: `Bearer ${this.config.apiToken}` } + : undefined; const response = await fetch(`${this.getUrl()}/api/stats`, { method: "GET", + headers, signal: AbortSignal.timeout(2000), }); return response.ok; @@ -305,7 +313,7 @@ export class WebServer { } } - // --- HTTP request handling (inlined from web-server-worker.ts) --- + // --- HTTP request handling --- private async handleRequest(req: Request): Promise { const url = new URL(req.url); @@ -321,6 +329,11 @@ export class WebServer { return corsPreflightResponse(req); } + if (path.startsWith("/api/")) { + const unauthorized = authorizeApiRequest(req, this.config.apiToken); + if (unauthorized) return unauthorized; + } + try { if (path === "/" || path === "/index.html") { return this.serveStaticFile("index.html", "text/html"); diff --git a/src/types/usearch.d.ts b/src/types/usearch.d.ts deleted file mode 100644 index d2b4dcd..0000000 --- a/src/types/usearch.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module "usearch"; diff --git a/src/web/app.js b/src/web/app.js index de6c887..43e3100 100644 --- a/src/web/app.js +++ b/src/web/app.js @@ -1,5 +1,21 @@ const API_BASE = ""; +function resolveApiToken() { + try { + const params = new URLSearchParams(window.location.search); + const fromQuery = params.get("apiToken") || params.get("token"); + if (fromQuery) { + sessionStorage.setItem("opencode-mem-api-token", fromQuery); + return fromQuery; + } + return sessionStorage.getItem("opencode-mem-api-token") || ""; + } catch { + return ""; + } +} + +const API_TOKEN = resolveApiToken(); + const state = { tags: { project: [] }, memories: [], @@ -37,8 +53,14 @@ async function fetchAPI(endpoint, options = {}) { (options.method === "POST" && endpoint.includes("/ai-cleanup") ? 180000 : 60000); const { timeout: _, ...fetchOptions } = options; const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + const headers = new Headers(fetchOptions.headers || {}); + if (API_TOKEN) { + headers.set("Authorization", `Bearer ${API_TOKEN}`); + headers.set("X-Opencode-Mem-Token", API_TOKEN); + } const response = await fetch(API_BASE + endpoint, { ...fetchOptions, + headers, signal: controller.signal, }); clearTimeout(timeoutId); @@ -840,7 +862,7 @@ async function runMigration(strategy) { if ( !confirm( - `Run ${strategyName} migration?\n\nThis operation is IRREVERSIBLE and will:\n${strategy === "fresh-start" ? "- DELETE all existing memories\n- Remove all shards" : "- Re-embed all memories with new model\n- This may take several minutes"}\n\nContinue?` + `Run ${strategyName} migration?\n\n${strategy === "fresh-start" ? "- Remove all existing memories from the active store\n- Archive old shards as .fresh-start-*.bak files" : "- Re-embed all memories with the new model\n- Stage and verify replacement shards before swapping\n- Keep previous shards as .pre-reembed-*.bak files\n- This may take several minutes"}\n\nContinue?` ) ) { return; diff --git a/tests/memory-scope-helper.test.ts b/tests/memory-scope-helper.test.ts new file mode 100644 index 0000000..8285d8c --- /dev/null +++ b/tests/memory-scope-helper.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "bun:test"; +import { + assertSafeScopeHash, + extractScopeFromContainerTag, + isValidScopeHash, + resolveMemoryScope, + tryExtractScopeFromContainerTag, +} from "../src/services/memory-scope.js"; + +const PROJECT_HASH = "a1b2c3d4e5f67890"; +const USER_HASH = "b1b2c3d4e5f67890"; + +describe("memory scope helper", () => { + it("parses project container tags", () => { + expect(extractScopeFromContainerTag(`opencode_project_${PROJECT_HASH}`)).toEqual({ + scope: "project", + hash: PROJECT_HASH, + }); + }); + + it("parses user container tags", () => { + expect(extractScopeFromContainerTag(`opencode_user_${USER_HASH}`)).toEqual({ + scope: "user", + hash: USER_HASH, + }); + }); + + it("resolves all-projects scope to empty project hash", () => { + expect(resolveMemoryScope("all-projects", `opencode_project_${PROJECT_HASH}`)).toEqual({ + scope: "project", + hash: "", + }); + }); + + it("validates scope hash format", () => { + expect(isValidScopeHash(PROJECT_HASH)).toBe(true); + expect(isValidScopeHash("abc123")).toBe(false); + expect(isValidScopeHash("gggggggggggggggg")).toBe(false); + }); + + it("rejects container tags with invalid hash", () => { + expect(() => extractScopeFromContainerTag("opencode_project_abc123")).toThrow( + /16 lowercase hex/ + ); + }); + + it("rejects container tags with path traversal segments", () => { + expect(() => + extractScopeFromContainerTag("opencode_project_abcd1234567890ef/../../evil") + ).toThrow(/16 lowercase hex/); + }); + + it("assertSafeScopeHash throws for invalid hashes", () => { + expect(() => assertSafeScopeHash("../outside")).toThrow(/Invalid scope hash/); + }); + + it("tryExtractScopeFromContainerTag returns null for legacy tags", () => { + expect(tryExtractScopeFromContainerTag("opencode_project_abc")).toBeNull(); + expect(tryExtractScopeFromContainerTag(`opencode_project_${PROJECT_HASH}`)).toEqual({ + scope: "project", + hash: PROJECT_HASH, + }); + }); +}); diff --git a/tests/memory-scope.test.ts b/tests/memory-scope.test.ts index 0c338f2..7b1eb92 100644 --- a/tests/memory-scope.test.ts +++ b/tests/memory-scope.test.ts @@ -5,13 +5,13 @@ import { join } from "node:path"; const tempDirs: string[] = []; const clientUrl = new URL("../src/services/client.js", import.meta.url).href; -const connectionManagerUrl = new URL( - "../src/services/sqlite/connection-manager.js", - import.meta.url -).href; +const connectionManagerUrl = new URL("../src/services/turso/connection-manager.js", import.meta.url) + .href; const embeddingUrl = new URL("../src/services/embedding.js", import.meta.url).href; -const shardManagerUrl = new URL("../src/services/sqlite/shard-manager.js", import.meta.url).href; -const vectorSearchUrl = new URL("../src/services/sqlite/vector-search.js", import.meta.url).href; +const shardManagerUrl = new URL("../src/services/turso/shard-manager.js", import.meta.url).href; +const vectorSearchUrl = new URL("../src/services/turso/vector-search.js", import.meta.url).href; +const legacyMigratorUrl = new URL("../src/services/turso/legacy-migrator.js", import.meta.url).href; +const readyUrl = new URL("../src/services/turso/ready.js", import.meta.url).href; function runScenario(scriptBody: string) { const dir = mkdtempSync(join(tmpdir(), "opencode-mem-memory-scope-")); @@ -40,7 +40,7 @@ function makeDb(path) { ? [{ id: "a", content: "A", created_at: 2, container_tag: "tag-a" }] : path.includes("shard-b") ? [{ id: "b", content: "B", created_at: 1, container_tag: "tag-b" }] - : [{ id: "c", content: "C", created_at: 3, container_tag: "current" }]; + : [{ id: "c", content: "C", created_at: 3, container_tag: "opencode_project_a1b2c3d4e5f67890" }]; return { prepare(sql) { @@ -74,14 +74,14 @@ function makeDb(path) { } mock.module(${JSON.stringify(connectionManagerUrl)}, () => ({ - connectionManager: { - getConnection(path) { + tursoConnectionManager: { + async getConnection(path) { if (!dbByPath.has(path)) { dbByPath.set(path, makeDb(path)); } return dbByPath.get(path); }, - closeAll() {}, + async closeAll() {}, }, })); @@ -93,25 +93,35 @@ mock.module(${JSON.stringify(embeddingUrl)}, () => ({ }, })); +mock.module(${JSON.stringify(legacyMigratorUrl)}, () => ({ + runLegacyTursoMigration: async () => {}, +})); + +mock.module(${JSON.stringify(readyUrl)}, () => ({ + ensureTursoReady: async () => {}, +})); + mock.module(${JSON.stringify(shardManagerUrl)}, () => ({ - shardManager: { - getAllShards(scope, hash) { + tursoShardManager: { + async getAllShards(scope, hash) { return scope === "project" && hash === "" ? [makeShard("shard-a"), makeShard("shard-b")] : [makeShard("shard-current")]; }, - getWriteShard() { + async getWriteShard() { return makeShard("shard-write"); }, - incrementVectorCount() {}, + async incrementVectorCount() {}, }, })); mock.module(${JSON.stringify(vectorSearchUrl)}, () => ({ - vectorSearch: { - searchAcrossShards: async (shards) => - shards.map((s) => ({ id: s.id, memory: s.id, similarity: 1 })), - listMemories: (db, containerTag) => db.listMemories(containerTag), + tursoVectorSearch: { + searchAcrossShards: async (shards) => ({ + results: shards.map((s) => ({ id: s.id, memory: s.id, similarity: 1 })), + warnings: [], + }), + listMemories: async (db, containerTag) => db.listMemories(containerTag), insertVector: async () => {}, }, })); @@ -150,7 +160,7 @@ afterEach(() => { describe("memory scope", () => { it("defaults to project scope", () => { const result = runScenario(` -const res = await memoryClient.listMemories("current", 10); +const res = await memoryClient.listMemories("opencode_project_a1b2c3d4e5f67890", 10); console.log(JSON.stringify(res)); `); diff --git a/tests/memory-tool-warmup.test.ts b/tests/memory-tool-warmup.test.ts new file mode 100644 index 0000000..3deb5eb --- /dev/null +++ b/tests/memory-tool-warmup.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, afterEach, mock } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; + +describe("memory tool warmup gate", () => { + let baseDir: string; + let scriptPath: string; + + afterEach(async () => { + mock.restore(); + if (scriptPath) rmSync(scriptPath, { force: true }); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("calls warmup instead of rejecting when client is not yet initialized", async () => { + baseDir = mkdtempSync(join(tmpdir(), "memory-tool-warmup-")); + scriptPath = join(baseDir, "run.mjs"); + + const clientUrl = pathToFileURL(join(import.meta.dirname, "../src/services/client.ts")).href; + + const script = ` +import { mock } from "bun:test"; + +let warmupCalls = 0; +mock.module(${JSON.stringify(clientUrl)}, () => ({ + memoryClient: { + isReady: async () => false, + warmup: async () => { + warmupCalls += 1; + }, + addMemory: async () => ({ success: true, id: "mem_1" }), + searchMemories: async () => ({ success: true, results: [], total: 0 }), + listMemories: async () => ({ success: true, memories: [], pagination: {} }), + deleteMemory: async () => ({ success: true }), + }, +})); + +// Minimal stand-in for the tool gate logic from index.ts +const memoryClient = (await import(${JSON.stringify(clientUrl)})).memoryClient; +await memoryClient.warmup(); +const readyAfter = await memoryClient.isReady(); +console.log(JSON.stringify({ warmupCalls, readyAfter: typeof readyAfter === "boolean" })); +`; + + writeFileSync(scriptPath, script, "utf-8"); + const result = Bun.spawnSync({ + cmd: [process.execPath, scriptPath], + stdout: "pipe", + stderr: "pipe", + }); + + const stdout = Buffer.from(result.stdout).toString("utf8").trim(); + const jsonLine = stdout + .split("\n") + .reverse() + .find((line) => line.trim().startsWith("{")); + const parsed = jsonLine ? JSON.parse(jsonLine) : null; + + expect(result.exitCode).toBe(0); + expect(parsed?.warmupCalls).toBe(1); + }); +}); diff --git a/tests/package-dependencies.test.ts b/tests/package-dependencies.test.ts index 773c37d..fc07272 100644 --- a/tests/package-dependencies.test.ts +++ b/tests/package-dependencies.test.ts @@ -2,12 +2,12 @@ import { describe, expect, it } from "bun:test"; import pkg from "../package.json"; describe("published dependency constraints", () => { + it("uses @libsql/client for Turso persistence and vector search", () => { + expect(pkg.dependencies["@libsql/client"]).toBeTruthy(); + expect(pkg.dependencies).not.toHaveProperty("usearch"); + }); + it("uses @huggingface/transformers (v4+) as the local embedding backend", () => { - // Migrated from @xenova/transformers@^2.17.2 (frozen, sharp@^0.32 postinstall) - // to @huggingface/transformers@^4 (active successor, sharp@^0.34 prebuilt @img, - // installs cleanly under script-skipping plugin installers). The earlier revert - // (8fb0836) was specific to the v4.0.1 native-ONNX runtime on Windows, not a - // permanent rejection — see PR for the platform verification matrix. expect(pkg.dependencies["@huggingface/transformers"]).toMatch(/^\^?4\./); expect(pkg.dependencies).not.toHaveProperty("@xenova/transformers"); }); diff --git a/tests/plugin-loader-contract.test.ts b/tests/plugin-loader-contract.test.ts index 1137bd2..b1678fc 100644 --- a/tests/plugin-loader-contract.test.ts +++ b/tests/plugin-loader-contract.test.ts @@ -29,7 +29,7 @@ describe("OpenCode 1.3.x plugin-loader contract", () => { it("dist shard manager avoids CommonJS fs require in ESM output", () => { const source = readFileSync( - new URL("../dist/services/sqlite/shard-manager.js", import.meta.url), + new URL("../dist/services/turso/shard-manager.js", import.meta.url), "utf-8" ); @@ -70,7 +70,7 @@ describe("OpenCode 1.3.x plugin-loader contract", () => { expect(typeof serverFn).toBe("function"); // Attempt to invoke server with a minimal mock PluginInput. - // The plugin may throw during warmup (missing sqlite/usearch in test env) — that is expected. + // The plugin may throw during warmup (missing Turso runtime in test env) — that is expected. // If it succeeds, assert the returned hooks have the expected shape. const mockInput = { client: {}, @@ -89,7 +89,7 @@ describe("OpenCode 1.3.x plugin-loader contract", () => { expect(typeof hooks["chat.message"]).toBe("function"); expect(typeof hooks["event"]).toBe("function"); } catch { - // Warmup/sqlite/usearch failure in test environment is acceptable. + // Warmup/Turso failure in test environment is acceptable. // The callable surface assertion above is sufficient for contract verification. } }); diff --git a/tests/profile-write.test.ts b/tests/profile-write.test.ts index a17487b..b900740 100644 --- a/tests/profile-write.test.ts +++ b/tests/profile-write.test.ts @@ -7,7 +7,7 @@ import { describe, it, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { tursoConnectionManager } from "../src/services/turso/connection-manager.js"; // We patch CONFIG.storagePath before importing the manager so the DB lands in tmp. let tmpDir: string; @@ -29,7 +29,7 @@ describe("UserProfileManager – explicit preference writes", () => { }); afterEach(async () => { - connectionManager.closeAll(); + await tursoConnectionManager.closeAll(); await new Promise((r) => setTimeout(r, 100)); try { rmSync(tmpDir, { recursive: true, force: true }); @@ -40,7 +40,7 @@ describe("UserProfileManager – explicit preference writes", () => { const mgr = await makeManager(); const userId = "test@example.com"; - mgr.createProfile( + await mgr.createProfile( userId, "Test User", "testuser", @@ -62,7 +62,7 @@ describe("UserProfileManager – explicit preference writes", () => { 0 ); - const profile = mgr.getActiveProfile(userId); + const profile = await mgr.getActiveProfile(userId); expect(profile).not.toBeNull(); const data = JSON.parse(profile!.profileData); expect(data.preferences).toHaveLength(1); @@ -76,7 +76,7 @@ describe("UserProfileManager – explicit preference writes", () => { const userId = "test@example.com"; // Seed with one AI-learned preference - mgr.createProfile( + await mgr.createProfile( userId, "Test User", "testuser", @@ -98,7 +98,7 @@ describe("UserProfileManager – explicit preference writes", () => { 3 ); - const existingProfile = mgr.getActiveProfile(userId)!; + const existingProfile = await mgr.getActiveProfile(userId)!; const existingData = JSON.parse(existingProfile.profileData); const newPref = { @@ -111,14 +111,14 @@ describe("UserProfileManager – explicit preference writes", () => { }; const merged = await mgr.mergeProfileData(existingData, { preferences: [newPref] }); - mgr.updateProfile( + await mgr.updateProfile( existingProfile.id, merged, 0, "Explicit preference added: Always use numbered lists" ); - const updated = mgr.getActiveProfile(userId)!; + const updated = await mgr.getActiveProfile(userId)!; const updatedData = JSON.parse(updated.profileData); expect(updatedData.preferences).toHaveLength(2); @@ -142,7 +142,7 @@ describe("UserProfileManager – explicit preference writes", () => { lastSeen: Date.now(), }; - mgr.createProfile( + await mgr.createProfile( userId, "Test User", "testuser", @@ -156,14 +156,14 @@ describe("UserProfileManager – explicit preference writes", () => { ); // Write the same preference again (simulates calling profile+content twice) - const p1 = mgr.getActiveProfile(userId)!; + const p1 = await mgr.getActiveProfile(userId)!; const d1 = JSON.parse(p1.profileData); const merged = await mgr.mergeProfileData(d1, { preferences: [{ ...pref, lastSeen: Date.now() }], }); - mgr.updateProfile(p1.id, merged, 0, "Explicit preference added: Prefer short answers"); + await mgr.updateProfile(p1.id, merged, 0, "Explicit preference added: Prefer short answers"); - const p2 = mgr.getActiveProfile(userId)!; + const p2 = await mgr.getActiveProfile(userId)!; const d2 = JSON.parse(p2.profileData); // Still only one entry (deduplicated by category+description) expect(d2.preferences.filter((p: any) => p.description === description)).toHaveLength(1); @@ -175,7 +175,7 @@ describe("UserProfileManager – explicit preference writes", () => { it("returns null profile for unknown user (no auto-create on read)", async () => { const mgr = await makeManager(); - const profile = mgr.getActiveProfile("nobody@example.com"); + const profile = await mgr.getActiveProfile("nobody@example.com"); expect(profile).toBeNull(); }); @@ -184,7 +184,7 @@ describe("UserProfileManager – explicit preference writes", () => { const userId = "test@example.com"; const summary = "Explicit preference added: Use snake_case"; - mgr.createProfile( + await mgr.createProfile( userId, "Test User", "testuser", @@ -197,7 +197,7 @@ describe("UserProfileManager – explicit preference writes", () => { 0 ); - const p = mgr.getActiveProfile(userId)!; + const p = await mgr.getActiveProfile(userId)!; const d = JSON.parse(p.profileData); const merged = await mgr.mergeProfileData(d, { preferences: [ @@ -211,9 +211,9 @@ describe("UserProfileManager – explicit preference writes", () => { }, ], }); - mgr.updateProfile(p.id, merged, 0, summary); + await mgr.updateProfile(p.id, merged, 0, summary); - const changelogs = mgr.getProfileChangelogs(p.id); + const changelogs = await mgr.getProfileChangelogs(p.id); const last = changelogs[0]; expect(last.changeSummary).toBe(summary); expect(last.changeType).toBe("update"); diff --git a/tests/sqlite-bootstrap-array-params.test.ts b/tests/sqlite-bootstrap-array-params.test.ts deleted file mode 100644 index 0936a54..0000000 --- a/tests/sqlite-bootstrap-array-params.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, it, expect } from "bun:test"; -import { getDatabase } from "../src/services/sqlite/sqlite-bootstrap.js"; - -/** - * Regression for `Unknown named parameter '0'`. - * - * `db.run(sql, [a, b])` — passing a single array of bind values — is valid - * better-sqlite3 / bun:sqlite usage and is used by - * services/ai/session/ai-session-manager.ts (e.g. `cleanupExpiredSessions`, - * `addMessage`, `deleteSession`, `clearMessages`). - * - * Under Node, getDatabase() resolves to the `DatabaseSyncCompat` wrapper around - * node:sqlite. Its `run(sql, ...params)` forwarded `this.prepare(sql).run(...params)`, - * so a single array argument reached the statement as one object value; - * node:sqlite then read its indices ("0", "1", …) as named parameters and threw - * `Unknown named parameter '0'`. opencode 1.15.x loads plugins under Node, so the - * auto-capture path crashed there while passing under Bun. - * - * Note on runtime: under `bun test` getDatabase() returns bun:sqlite, which accepts - * arrays natively — so this asserts runtime parity. Run under Node (node:sqlite) to - * reproduce/guard the original regression. - */ -describe("sqlite-bootstrap: db.run with a single array of params", () => { - function freshDb() { - const Database = getDatabase() as unknown as new (filename?: string) => { - exec(sql: string): unknown; - run(sql: string, ...params: unknown[]): unknown; - prepare(sql: string): { get(...params: unknown[]): any; all(...params: unknown[]): any[] }; - }; - const db = new Database(":memory:"); - db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER, label TEXT)"); - return db; - } - - it("binds a single-element array positionally", () => { - const db = freshDb(); - db.run("INSERT INTO t (v) VALUES (?)", [42]); - const row = db.prepare("SELECT v FROM t").get(); - expect(row.v).toBe(42); - }); - - it("binds a multi-element array positionally", () => { - const db = freshDb(); - db.run("INSERT INTO t (v, label) VALUES (?, ?)", [7, "seven"]); - const row = db.prepare("SELECT v, label FROM t").get(); - expect(row.v).toBe(7); - expect(row.label).toBe("seven"); - }); - - it("still supports spread positional params", () => { - const db = freshDb(); - db.run("INSERT INTO t (v, label) VALUES (?, ?)", 9, "nine"); - const row = db.prepare("SELECT v, label FROM t").get(); - expect(row.v).toBe(9); - expect(row.label).toBe("nine"); - }); - - it("still execs paramless statements", () => { - const db = freshDb(); - db.run("INSERT INTO t (v) VALUES (1)"); - const row = db.prepare("SELECT COUNT(*) AS n FROM t").get(); - expect(row.n).toBe(1); - }); -}); diff --git a/tests/turso-api-scope.test.ts b/tests/turso-api-scope.test.ts new file mode 100644 index 0000000..8f67be1 --- /dev/null +++ b/tests/turso-api-scope.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("api memory shard scope", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("includes user-scope memories in stats, pin, and global search handlers", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-api-scope-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + const { handleStats, handlePinMemory, handleSearch } = + await import("../src/services/api-handlers.js"); + const { embeddingService } = await import("../src/services/embedding.js"); + const { userPromptManager } = + await import("../src/services/user-prompt/user-prompt-manager.js"); + + const vector = new Float32Array(768); + vector[0] = 1; + + const scopeHash = "c1c2c3d4e5f67890"; + const containerTag = `opencode_user_${scopeHash}`; + + const shard = await tursoShardManager.createShard("user", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + await tursoVectorSearch.insertVector(db, { + id: "mem_user_scope_1", + content: "user scoped memory", + vector, + containerTag, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + + const stats = await handleStats(); + expect(stats.success).toBe(true); + expect(stats.data?.byScope.user).toBe(1); + expect(stats.data?.total).toBe(1); + + const pin = await handlePinMemory("mem_user_scope_1"); + expect(pin.success).toBe(true); + + const pinned = await tursoVectorSearch.getMemoryById(db, "mem_user_scope_1"); + expect(Number(pinned?.is_pinned)).toBe(1); + + const originalWarmup = embeddingService.warmup; + const originalEmbedWithTimeout = embeddingService.embedWithTimeout; + const originalSearchPrompts = userPromptManager.searchPrompts; + embeddingService.warmup = async () => {}; + embeddingService.embedWithTimeout = async () => vector; + userPromptManager.searchPrompts = async () => []; + try { + const search = await handleSearch("user scoped memory"); + expect(search.success).toBe(true); + expect(search.data?.items.some((item) => item.id === "mem_user_scope_1")).toBe(true); + } finally { + embeddingService.warmup = originalWarmup; + embeddingService.embedWithTimeout = originalEmbedWithTimeout; + userPromptManager.searchPrompts = originalSearchPrompts; + } + }); +}); diff --git a/tests/turso-auxiliary-db-upgrade.test.ts b/tests/turso-auxiliary-db-upgrade.test.ts new file mode 100644 index 0000000..2620aea --- /dev/null +++ b/tests/turso-auxiliary-db-upgrade.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createClient } from "@libsql/client"; + +describe("turso auxiliary database upgrades", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("preserves legacy prompts while adding newer columns", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-prompt-upgrade-")); + const dbPath = join(baseDir, "user-prompts.db"); + const legacy = createClient({ url: `file:${dbPath}` }); + await legacy.batch( + [ + `CREATE TABLE user_prompts ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + project_path TEXT, + content TEXT NOT NULL, + created_at INTEGER NOT NULL, + captured INTEGER DEFAULT 0, + user_learning_captured BOOLEAN DEFAULT 0, + linked_memory_id TEXT + )`, + { + sql: `INSERT INTO user_prompts ( + id, session_id, message_id, project_path, content, created_at, captured + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + args: [ + "prompt_legacy", + "session_legacy", + "message_legacy", + "/legacy/project", + "preserve this prompt", + 123, + 1, + ], + }, + ], + "write" + ); + legacy.close(); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + const { userPromptManager } = + await import("../src/services/user-prompt/user-prompt-manager.js"); + userPromptManager.reset(); + + const prompt = await userPromptManager.getPromptById("prompt_legacy"); + expect(prompt?.content).toBe("preserve this prompt"); + expect(prompt?.capture_attempts).toBe(0); + expect(prompt?.providerId).toBeNull(); + expect(prompt?.modelId).toBeNull(); + + const verify = createClient({ url: `file:${dbPath}` }); + const columns = await verify.execute(`PRAGMA table_info(user_prompts)`); + const names = columns.rows.map((row) => String(row.name)); + expect(names).toContain("capture_attempts"); + expect(names).toContain("provider_id"); + expect(names).toContain("model_id"); + verify.close(); + }); +}); diff --git a/tests/turso-connection-lifecycle.test.ts b/tests/turso-connection-lifecycle.test.ts new file mode 100644 index 0000000..8eaaf66 --- /dev/null +++ b/tests/turso-connection-lifecycle.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso connection lifecycle", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("reopens databases after closeTursoAndInvalidateCaches", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-lifecycle-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + const { userPromptManager } = + await import("../src/services/user-prompt/user-prompt-manager.js"); + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + + const id = await userPromptManager.savePrompt("sess-1", "msg-1", "/tmp/project", "hello"); + expect(id).toBeTruthy(); + + await closeTursoAndInvalidateCaches(); + + const prompt = await userPromptManager.getPromptById(id); + expect(prompt?.content).toBe("hello"); + }); + + it("does not leak an in-flight open across closeAll", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-lifecycle-race-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const dbPath = join(baseDir, "race.db"); + + const opening = tursoConnectionManager.getConnection(dbPath); + await Promise.all([opening, tursoConnectionManager.closeAll()]); + + const reopened = await tursoConnectionManager.getConnection(dbPath); + await reopened.run(`CREATE TABLE IF NOT EXISTS lifecycle_probe (id INTEGER PRIMARY KEY)`); + expect( + await reopened.get(`SELECT name FROM sqlite_master WHERE name = 'lifecycle_probe'`) + ).toBeTruthy(); + }); +}); diff --git a/tests/turso-connection-manager.test.ts b/tests/turso-connection-manager.test.ts new file mode 100644 index 0000000..4280725 --- /dev/null +++ b/tests/turso-connection-manager.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso connection manager", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("deduplicates concurrent getConnection calls for the same path", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-conn-race-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + const dbPath = join(baseDir, "single.db"); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + + const [a, b, c] = await Promise.all([ + tursoConnectionManager.getConnection(dbPath), + tursoConnectionManager.getConnection(dbPath), + tursoConnectionManager.getConnection(dbPath), + ]); + + expect(a).toBe(b); + expect(b).toBe(c); + }); + + it("enables foreign keys on new connections", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-conn-fk-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + const dbPath = join(baseDir, "fk.db"); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const db = await tursoConnectionManager.getConnection(dbPath); + const row = await db.get(`PRAGMA foreign_keys`); + expect(Number((row as { foreign_keys?: number } | null)?.foreign_keys)).toBe(1); + }); + + it("refuses paths outside storagePath", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-conn-outside-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + await expect( + tursoConnectionManager.getConnection(join(tmpdir(), "outside-opencode-mem.db")) + ).rejects.toThrow(/outside storagePath/); + }); +}); diff --git a/tests/turso-dedup-extract.test.ts b/tests/turso-dedup-extract.test.ts new file mode 100644 index 0000000..fb08f97 --- /dev/null +++ b/tests/turso-dedup-extract.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso dedup vector_extract", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("detects near-duplicates using vector_extract JSON", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-dedup-extract-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 8; + CONFIG.deduplicationEnabled = true; + CONFIG.deduplicationSimilarityThreshold = 0.9; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + const { deduplicationService } = await import("../src/services/deduplication-service.js"); + + const scopeHash = "f1f2f3f4f5f67890"; + const containerTag = `opencode_project_${scopeHash}`; + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + + const vectorA = new Float32Array(8); + vectorA[0] = 1; + const vectorB = new Float32Array(8); + vectorB[0] = 0.99; + vectorB[1] = 0.1; + + await tursoVectorSearch.insertVector(db, { + id: "mem_a", + content: "alpha", + vector: vectorA, + containerTag, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + await tursoVectorSearch.insertVector(db, { + id: "mem_b", + content: "beta", + vector: vectorB, + containerTag, + createdAt: Date.now() + 1, + updatedAt: Date.now() + 1, + }); + + const extracted = await tursoVectorSearch.getAllMemoriesWithExtractedVectors(db); + expect(extracted.length).toBe(2); + expect(typeof extracted[0]?.vector_json).toBe("string"); + expect(String(extracted[0]?.vector_json).length).toBeGreaterThan(2); + + const result = await deduplicationService.detectAndRemoveDuplicates(); + expect( + result.nearDuplicateGroups.length + result.exactDuplicatesDeleted + ).toBeGreaterThanOrEqual(0); + // Near-dup group should be detected when vectors are highly similar + expect(result.nearDuplicateGroups.length).toBeGreaterThanOrEqual(1); + }); +}); diff --git a/tests/turso-exact-fallback.test.ts b/tests/turso-exact-fallback.test.ts new file mode 100644 index 0000000..0982462 --- /dev/null +++ b/tests/turso-exact-fallback.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso exact scan fallback", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("falls back to exact scan when vector_top_k index is missing", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-exact-fallback-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 8; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const scopeHash = "e1e2e3e4e5f67890"; + const containerTag = `opencode_project_${scopeHash}`; + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + + await db.run(`DROP INDEX IF EXISTS memories_vec_idx`); + await db.run(`DROP INDEX IF EXISTS memories_tags_vec_idx`); + + const vector = new Float32Array(8); + vector[0] = 1; + await tursoVectorSearch.insertVector(db, { + id: "mem_exact_1", + content: "exact fallback memory", + vector, + containerTag, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + + const results = await tursoVectorSearch.searchInShard(shard, vector, containerTag, 5); + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.id).toBe("mem_exact_1"); + }); +}); diff --git a/tests/turso-legacy-migrator.test.ts b/tests/turso-legacy-migrator.test.ts new file mode 100644 index 0000000..4c84600 --- /dev/null +++ b/tests/turso-legacy-migrator.test.ts @@ -0,0 +1,358 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createClient } from "@libsql/client"; + +describe("turso legacy migrator", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + async function createLegacyShard( + dir: string, + fileName: string, + memories: Array<{ id: string; vector: Float32Array; content: string; containerTag: string }> + ): Promise { + const dbPath = join(dir, fileName); + const client = createClient({ url: `file:${dbPath}` }); + const dims = memories[0]?.vector.length ?? 768; + + await client.batch( + [ + `CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + tags_vector BLOB, + container_tag TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + metadata TEXT, + display_name TEXT, + user_name TEXT, + user_email TEXT, + project_path TEXT, + project_name TEXT, + git_repo_url TEXT, + is_pinned INTEGER DEFAULT 0 + )`, + ], + "write" + ); + + for (const memory of memories) { + const blob = new Uint8Array(memory.vector.buffer); + await client.execute({ + sql: `INSERT INTO memories (id, content, vector, container_tag, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`, + args: [memory.id, memory.content, blob, memory.containerTag, Date.now(), Date.now()], + }); + } + + await client.close(); + return dbPath; + } + + it("migrates legacy shard and writes sidecar plus global marker", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const vector = new Float32Array(768); + vector[0] = 1; + const scopeHash = "a1b2c3d4e5f67890"; + await createLegacyShard(projectsDir, `project_${scopeHash}_shard_0.db`, [ + { + id: "mem_legacy_1", + vector, + content: "legacy memory", + containerTag: `opencode_project_${scopeHash}`, + }, + ]); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + const sidecarPath = join(projectsDir, `project_${scopeHash}_shard_0.db.turso-migrate.json`); + expect(existsSync(sidecarPath)).toBe(true); + const sidecar = JSON.parse(readFileSync(sidecarPath, "utf-8")); + expect(sidecar.status).toBe("complete"); + expect(sidecar.expectedCount).toBe(1); + expect(sidecar.sourceCount).toBe(1); + expect(sidecar.skippedCount).toBe(0); + + const markerPath = join(baseDir, ".turso-migrated"); + expect(existsSync(markerPath)).toBe(true); + const marker = JSON.parse(readFileSync(markerPath, "utf-8")); + expect(marker.shards?.length).toBe(1); + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const registered = await tursoShardManager.getAllShards("project", ""); + expect(registered).toHaveLength(1); + expect(registered[0]?.scopeHash).toBe(scopeHash); + expect(registered[0]?.vectorCount).toBe(1); + }); + + it("resumes from backup after interrupted migration", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-resume-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const vector = new Float32Array(768); + vector[0] = 1; + const dbPath = await createLegacyShard(projectsDir, "project_resume_shard_0.db", [ + { + id: "mem_resume_1", + vector, + content: "resume me", + containerTag: "opencode_project_resume", + }, + ]); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + + const backupPath = `${dbPath}.legacy.bak`; + + await tursoConnectionManager.closeConnection(dbPath); + const { renameSync } = await import("node:fs"); + renameSync(dbPath, backupPath); + + const freshDb = await tursoConnectionManager.getConnection(dbPath); + await tursoShardManager.initShardDb(freshDb); + writeFileSync( + `${dbPath}.turso-migrate.json`, + JSON.stringify({ expectedCount: 1, importedCount: 0, status: "pending" }), + "utf-8" + ); + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + const migratedDb = await tursoConnectionManager.getConnection(dbPath); + const countRow = await migratedDb.get(`SELECT COUNT(*) as count FROM memories`); + expect(Number(countRow?.count)).toBe(1); + + const restored = await migratedDb.get(`SELECT content FROM memories WHERE id = ?`, [ + "mem_resume_1", + ]); + expect(String(restored?.content)).toBe("resume me"); + }); + + it("does not skip migration when marker exists but shard sidecar is incomplete", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-marker-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const vector = new Float32Array(768); + vector[0] = 1; + await createLegacyShard(projectsDir, "project_marker_shard_0.db", [ + { + id: "mem_marker_1", + vector, + content: "marker test", + containerTag: "opencode_project_marker", + }, + ]); + + writeFileSync( + join(baseDir, ".turso-migrated"), + JSON.stringify({ completedAt: new Date().toISOString(), shards: [] }), + "utf-8" + ); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + const sidecarPath = join(projectsDir, "project_marker_shard_0.db.turso-migrate.json"); + expect(existsSync(sidecarPath)).toBe(true); + const sidecar = JSON.parse(readFileSync(sidecarPath, "utf-8")); + expect(sidecar.status).toBe("complete"); + }); + + it("aborts migration when legacy vector blob is unreadable", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-corrupt-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const dbPath = join(projectsDir, "project_corrupt_shard_0.db"); + const client = createClient({ url: `file:${dbPath}` }); + await client.batch( + [ + `CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + container_tag TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`, + ], + "write" + ); + await client.execute({ + sql: `INSERT INTO memories (id, content, vector, container_tag, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`, + args: [ + "mem_corrupt_1", + "bad vector", + new Uint8Array([1, 2, 3]), + "opencode_project_corrupt", + Date.now(), + Date.now(), + ], + }); + await client.close(); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await expect(runLegacyTursoMigration()).rejects.toThrow(/unreadable vectors/); + + expect(existsSync(join(baseDir, ".turso-migrated"))).toBe(false); + }); + + it("does not skip migration when complete sidecar lacks vector index", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-fake-sidecar-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const vector = new Float32Array(768); + vector[0] = 1; + const dbPath = await createLegacyShard(projectsDir, "project_fake_sidecar_shard_0.db", [ + { + id: "mem_fake_1", + vector, + content: "needs real migration", + containerTag: "opencode_project_fakesidecar", + }, + ]); + + writeFileSync( + `${dbPath}.turso-migrate.json`, + JSON.stringify({ + sourceCount: 1, + expectedCount: 1, + importedCount: 1, + skippedCount: 0, + status: "complete", + }), + "utf-8" + ); + writeFileSync( + join(baseDir, ".turso-migrated"), + JSON.stringify({ completedAt: new Date().toISOString(), shards: [] }), + "utf-8" + ); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const db = await tursoConnectionManager.getConnection(dbPath); + const indexRow = await db.get( + `SELECT name FROM sqlite_master WHERE type='index' AND name='memories_vec_idx'` + ); + expect(indexRow).toBeTruthy(); + const probe = await db.get(`SELECT vector_extract(vector) AS extracted FROM memories LIMIT 1`); + expect(String(probe?.extracted || "").length).toBeGreaterThan(0); + }); + + it("does not skip migration when index exists but embedding dimensions mismatch", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-dims-mismatch-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const scopeHash = "d1d2d3d4e5f67890"; + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const vector = new Float32Array(768); + vector[0] = 1; + await tursoVectorSearch.insertVector(db, { + id: "mem_dims_mismatch_1", + content: "turso shard with wrong metadata dims", + vector, + containerTag: `opencode_project_${scopeHash}`, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + await db.run( + `INSERT OR REPLACE INTO shard_metadata (key, value) VALUES ('embedding_dimensions', ?)`, + ["384"] + ); + await tursoConnectionManager.closeConnection(shard.dbPath); + + const sidecarPath = `${shard.dbPath}.turso-migrate.json`; + if (existsSync(sidecarPath)) { + rmSync(sidecarPath); + } + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + expect(existsSync(`${shard.dbPath}.legacy.bak`)).toBe(true); + const sidecar = JSON.parse(readFileSync(sidecarPath, "utf-8")); + expect(sidecar.status).toBe("complete"); + + const migratedDb = await tursoConnectionManager.getConnection(shard.dbPath); + const meta = await migratedDb.get( + `SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'` + ); + expect(Number(meta?.value)).toBe(768); + }); + + it("repairs the active shard flag while reconciling an existing registry", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-registry-")); + const scopeHash = "1122334455667788"; + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 4; + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const metadataDb = await tursoConnectionManager.getConnection(join(baseDir, "metadata.db")); + await metadataDb.run(`UPDATE shards SET is_active = 0 WHERE id = ?`, [shard.id]); + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + const active = await tursoShardManager.getActiveShard("project", scopeHash); + expect(active?.id).toBe(shard.id); + expect(active?.isActive).toBe(true); + }); +}); diff --git a/tests/turso-migrate-dims-preflight.test.ts b/tests/turso-migrate-dims-preflight.test.ts new file mode 100644 index 0000000..8703941 --- /dev/null +++ b/tests/turso-migrate-dims-preflight.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createClient } from "@libsql/client"; + +describe("turso legacy migrator dimension preflight", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("preserves legacy dimensions so startup can expose the re-embed migration", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-migrate-dim-preflight-")); + const projectsDir = join(baseDir, "projects"); + mkdirSync(projectsDir, { recursive: true }); + + const scopeHash = "0011223344556677"; + const dbPath = join(projectsDir, `project_${scopeHash}_shard_0.db`); + const client = createClient({ url: `file:${dbPath}` }); + await client.batch( + [ + `CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + container_tag TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`, + ], + "write" + ); + + const wrongDims = new Float32Array(4); + wrongDims[0] = 1; + await client.execute({ + sql: `INSERT INTO memories (id, content, vector, container_tag, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`, + args: [ + "mem_wrong_dims", + "wrong dims", + new Uint8Array(wrongDims.buffer), + "opencode_project_dimpreflight", + Date.now(), + Date.now(), + ], + }); + await client.close(); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + expect(existsSync(dbPath)).toBe(true); + expect(existsSync(`${dbPath}.legacy.bak`)).toBe(true); + expect(existsSync(join(baseDir, ".turso-migrated"))).toBe(true); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const migratedDb = await tursoConnectionManager.getConnection(dbPath); + const dimensions = await migratedDb.get( + `SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'` + ); + expect(Number(dimensions?.value)).toBe(4); + + const count = await migratedDb.get(`SELECT COUNT(*) AS count FROM memories`); + expect(Number(count?.count)).toBe(1); + + const { migrationService } = await import("../src/services/migration-service.js"); + const mismatch = await migrationService.detectDimensionMismatch(); + expect(mismatch.needsMigration).toBe(true); + expect(mismatch.shardMismatches).toHaveLength(1); + expect(mismatch.shardMismatches[0]?.storedDimensions).toBe(4); + }); +}); diff --git a/tests/turso-ready-gate.test.ts b/tests/turso-ready-gate.test.ts new file mode 100644 index 0000000..a99822b --- /dev/null +++ b/tests/turso-ready-gate.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, afterEach } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso ready gate", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("runs legacy migration once and initializes metadata", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-ready-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + const { ensureTursoReady } = await import("../src/services/turso/ready.js"); + await ensureTursoReady(); + await ensureTursoReady(); + + expect(existsSync(join(baseDir, "metadata.db"))).toBe(true); + }); + + it("throws when migration lock is held by another live process", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-ready-lock-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + writeFileSync( + join(baseDir, ".turso-migrate.lock"), + JSON.stringify({ pid: process.pid, timestamp: new Date().toISOString() }), + "utf-8" + ); + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await expect(runLegacyTursoMigration()).rejects.toThrow(/locked by another process/); + }); +}); diff --git a/tests/turso-reembed-migration.test.ts b/tests/turso-reembed-migration.test.ts new file mode 100644 index 0000000..0e0a84d --- /dev/null +++ b/tests/turso-reembed-migration.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + copyFileSync, + existsSync, + mkdtempSync, + readdirSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +const SCOPE_HASH = "0123456789abcdef"; + +describe("turso re-embed migration safety", () => { + let baseDir: string; + let restoreEmbedding: (() => void) | undefined; + + afterEach(async () => { + restoreEmbedding?.(); + restoreEmbedding = undefined; + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + async function createSourceShard() { + baseDir = mkdtempSync(join(tmpdir(), "turso-reembed-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 2; + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const shard = await tursoShardManager.createShard("project", SCOPE_HASH, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + for (const [index, id] of ["mem_one", "mem_two"].entries()) { + await tursoVectorSearch.insertVector(db, { + id, + content: `content ${index}`, + vector: new Float32Array([1, index]), + tagsVector: new Float32Array([0, 1]), + containerTag: `opencode_project_${SCOPE_HASH}`, + tags: "alpha,beta", + type: "project", + createdAt: 100 + index, + updatedAt: 200 + index, + metadata: JSON.stringify({ source: "test", index }), + displayName: `Memory ${index}`, + projectPath: `/project/${index}`, + }); + await tursoShardManager.incrementVectorCount(shard.id); + } + await tursoVectorSearch.pinMemory(db, "mem_one"); + return shard; + } + + async function stubEmbedding(failOn?: string) { + const { embeddingService } = await import("../src/services/embedding.js"); + const service = embeddingService as any; + const original = { + warmup: service.warmup, + clearCache: service.clearCache, + embedWithTimeout: service.embedWithTimeout, + }; + service.warmup = async () => {}; + service.clearCache = () => {}; + service.embedWithTimeout = async (text: string) => { + if (failOn && text.includes(failOn)) throw new Error("synthetic embedding failure"); + return new Float32Array([1, 0, 0, 0]); + }; + restoreEmbedding = () => Object.assign(service, original); + } + + it("stages a complete replacement and preserves metadata, tags, and pins", async () => { + const shard = await createSourceShard(); + await stubEmbedding(); + const { CONFIG } = await import("../src/config.js"); + CONFIG.embeddingDimensions = 4; + + const { migrationService } = await import("../src/services/migration-service.js"); + const result = await migrationService.migrateToNewModel("re-embed"); + + expect(result.success).toBe(true); + expect(result.deletedShards).toBe(0); + expect(result.reEmbeddedMemories).toBe(2); + expect( + readdirSync(join(baseDir, "projects")).some((name) => name.includes(".pre-reembed-")) + ).toBe(true); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const rows = await db.all( + `SELECT *, vector_extract(vector) AS vector_json FROM memories ORDER BY id` + ); + expect(rows).toHaveLength(2); + expect(String(rows[0]?.tags)).toBe("alpha,beta"); + expect(String(rows[0]?.display_name)).toBe("Memory 0"); + expect(Number(rows[0]?.is_pinned)).toBe(1); + expect(JSON.parse(String(rows[0]?.vector_json))).toHaveLength(4); + const metadata = await db.get( + `SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'` + ); + expect(Number(metadata?.value)).toBe(4); + }); + + it("leaves the source shard untouched when any embedding fails", async () => { + const shard = await createSourceShard(); + await stubEmbedding("content 1"); + const { CONFIG } = await import("../src/config.js"); + CONFIG.embeddingDimensions = 4; + + const { migrationService } = await import("../src/services/migration-service.js"); + const result = await migrationService.migrateToNewModel("re-embed"); + + expect(result.success).toBe(false); + expect(result.error).toContain("synthetic embedding failure"); + expect( + readdirSync(join(baseDir, "projects")).some((name) => name.includes(".pre-reembed-")) + ).toBe(false); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const count = await db.get(`SELECT COUNT(*) AS count FROM memories`); + expect(Number(count?.count)).toBe(2); + const metadata = await db.get( + `SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'` + ); + expect(Number(metadata?.value)).toBe(2); + }); + + it("archives source shards instead of deleting them during fresh start", async () => { + const shard = await createSourceShard(); + const { CONFIG } = await import("../src/config.js"); + CONFIG.embeddingDimensions = 4; + + const { migrationService } = await import("../src/services/migration-service.js"); + const result = await migrationService.migrateToNewModel("fresh-start"); + + expect(result.success).toBe(true); + expect(result.deletedShards).toBe(1); + const files = readdirSync(join(baseDir, "projects")); + expect(files.some((name) => name.includes(".fresh-start-") && name.endsWith(".bak"))).toBe( + true + ); + expect(files).not.toContain(shard.dbPath.split("/").at(-1)); + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + expect(await tursoShardManager.getAllShards("project", SCOPE_HASH)).toHaveLength(0); + }); + + it("recovers a crash between source backup and replacement rename", async () => { + const shard = await createSourceShard(); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + await tursoConnectionManager.closeConnection(shard.dbPath); + + const stagedPath = `${shard.dbPath}.reembed-crash.tmp`; + const backupPath = `${shard.dbPath}.pre-reembed-crash.bak`; + const statePath = `${shard.dbPath}.reembed-swap.json`; + copyFileSync(shard.dbPath, stagedPath); + renameSync(shard.dbPath, backupPath); + writeFileSync( + statePath, + JSON.stringify({ dbPath: shard.dbPath, stagedPath, backupPath }), + "utf-8" + ); + + const { runLegacyTursoMigration } = await import("../src/services/turso/legacy-migrator.js"); + await runLegacyTursoMigration(); + + expect(existsSync(shard.dbPath)).toBe(true); + expect(existsSync(statePath)).toBe(false); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const count = await db.get(`SELECT COUNT(*) AS count FROM memories`); + expect(Number(count?.count)).toBe(2); + }); +}); diff --git a/tests/turso-shard-recreate.test.ts b/tests/turso-shard-recreate.test.ts new file mode 100644 index 0000000..9a76475 --- /dev/null +++ b/tests/turso-shard-recreate.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +const TEST_SCOPE_HASH = "a1b2c3d4e5f67890"; + +describe("turso invalid shard protection", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("rejects writes on dimension mismatch without moving or emptying the shard", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-shard-protect-dims-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const shard = await tursoShardManager.createShard("project", TEST_SCOPE_HASH, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const vector = new Float32Array(768); + vector[0] = 1; + await tursoVectorSearch.insertVector(db, { + id: "mem_protected_1", + content: "must remain available for migration", + vector, + containerTag: `opencode_project_${TEST_SCOPE_HASH}`, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + await db.run( + `INSERT OR REPLACE INTO shard_metadata (key, value) VALUES ('embedding_dimensions', ?)`, + ["384"] + ); + + await expect(tursoShardManager.getWriteShard("project", TEST_SCOPE_HASH)).rejects.toThrow( + /left untouched/ + ); + + expect(existsSync(shard.dbPath)).toBe(true); + expect(existsSync(`${shard.dbPath}.invalid.bak`)).toBe(false); + const count = await db.get(`SELECT COUNT(*) AS count FROM memories`); + expect(Number(count?.count)).toBe(1); + }); + + it("rejects writes when dimension metadata is missing without deleting data", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-shard-protect-meta-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 768; + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const shard = await tursoShardManager.createShard("project", TEST_SCOPE_HASH, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const vector = new Float32Array(768); + vector[0] = 1; + await tursoVectorSearch.insertVector(db, { + id: "mem_missing_meta_1", + content: "must not be quarantined", + vector, + containerTag: `opencode_project_${TEST_SCOPE_HASH}`, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + await db.run(`DELETE FROM shard_metadata WHERE key = 'embedding_dimensions'`); + + await expect(tursoShardManager.getWriteShard("project", TEST_SCOPE_HASH)).rejects.toThrow( + /left untouched/ + ); + + expect(existsSync(shard.dbPath)).toBe(true); + const count = await db.get(`SELECT COUNT(*) AS count FROM memories`); + expect(Number(count?.count)).toBe(1); + }); +}); diff --git a/tests/turso-shard-rotation.test.ts b/tests/turso-shard-rotation.test.ts new file mode 100644 index 0000000..998dc3a --- /dev/null +++ b/tests/turso-shard-rotation.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso shard rotation", () => { + let baseDir: string; + + afterEach(async () => { + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + if (baseDir) rmSync(baseDir, { recursive: true, force: true }); + }); + + it("marks a full shard read-only and creates the next shard", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-rotation-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 4; + CONFIG.maxVectorsPerShard = 1; + + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + const hash = "abcdef0123456789"; + + const first = await tursoShardManager.getWriteShard("project", hash); + const db = await tursoConnectionManager.getConnection(first.dbPath); + await tursoVectorSearch.insertVector(db, { + id: "mem_rotation_1", + content: "fills the first shard", + vector: new Float32Array([1, 0, 0, 0]), + containerTag: `opencode_project_${hash}`, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + await tursoShardManager.incrementVectorCount(first.id); + + const second = await tursoShardManager.getWriteShard("project", hash); + expect(second.shardIndex).toBe(1); + expect(second.id).not.toBe(first.id); + + const all = await tursoShardManager.getAllShards("project", hash); + expect(all).toHaveLength(2); + expect(all.find((shard) => shard.id === first.id)?.isActive).toBe(false); + expect(all.find((shard) => shard.id === second.id)?.isActive).toBe(true); + }); + + it("blocks writes while a migration operation owns the storage lock", async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-operation-lock-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.embeddingDimensions = 4; + const { acquireTursoOperationLock } = await import("../src/services/turso/operation-lock.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + + const release = acquireTursoOperationLock("test"); + await expect(tursoShardManager.getWriteShard("project", "abcdef0123456789")).rejects.toThrow( + /writes are temporarily blocked/ + ); + release(); + + expect((await tursoShardManager.getWriteShard("project", "abcdef0123456789")).shardIndex).toBe( + 0 + ); + }); +}); diff --git a/tests/turso-vector-search.test.ts b/tests/turso-vector-search.test.ts new file mode 100644 index 0000000..39b7ce1 --- /dev/null +++ b/tests/turso-vector-search.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +describe("turso vector search", () => { + it("inserts and searches memories with native vector index", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "turso-vector-test-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + + const dims = CONFIG.embeddingDimensions; + const vector = new Float32Array(dims); + vector[0] = 1; + const tagsVector = new Float32Array(dims); + tagsVector[1] = 1; + + const scopeHash = "a1b2c3d4e5f67890"; + const containerTag = `opencode_project_${scopeHash}`; + + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + + await tursoVectorSearch.insertVector(db, { + id: "mem_test_1", + content: "Turso native vector search", + vector, + tagsVector, + containerTag, + tags: "turso,vector", + createdAt: Date.now(), + updatedAt: Date.now(), + }); + + await tursoVectorSearch.insertVector(db, { + id: "mem_test_no_tags", + content: "Content only vector", + vector, + containerTag, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + + const results = await tursoVectorSearch.searchInShard(shard, vector, containerTag, 5, "turso"); + + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.id).toBe("mem_test_1"); + expect(results[0]?.similarity).toBeGreaterThan(0.5); + + const limitedResults = await tursoVectorSearch.searchInShard( + shard, + vector, + containerTag, + 1, + "turso" + ); + expect(limitedResults).toHaveLength(1); + + await tursoConnectionManager.closeAll(); + const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js"); + await closeTursoAndInvalidateCaches(); + rmSync(baseDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/turso-vector-utils.test.ts b/tests/turso-vector-utils.test.ts new file mode 100644 index 0000000..2e38fe3 --- /dev/null +++ b/tests/turso-vector-utils.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "bun:test"; +import { + distanceToSimilarity, + formatTagsForEmbedding, +} from "../src/services/turso/vector-utils.js"; + +describe("turso vector utils", () => { + it("clamps cosine distance float artifacts into [0, 1] similarity", () => { + expect(distanceToSimilarity(0)).toBe(1); + expect(distanceToSimilarity(1)).toBe(0); + expect(distanceToSimilarity(-1e-14)).toBe(1); + expect(distanceToSimilarity(2)).toBe(0); + expect(distanceToSimilarity(Number.NaN)).toBe(0); + }); + + it("formats tag embedding text consistently", () => { + expect(formatTagsForEmbedding(["auth", "jwt"])).toBe("Topics: auth, jwt"); + }); +}); diff --git a/tests/user-prompt-manager-claim.test.ts b/tests/user-prompt-manager-claim.test.ts index 77b9b6f..1a35009 100644 --- a/tests/user-prompt-manager-claim.test.ts +++ b/tests/user-prompt-manager-claim.test.ts @@ -2,9 +2,8 @@ import { afterAll, beforeEach, afterEach, describe, expect, it } from "bun:test" import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { TursoDb } from "../src/services/turso/turso-db.js"; -// Redirect home to a temporary directory BEFORE importing config so that -// CONFIG.storagePath resolves under the test sandbox. const sandbox = mkdtempSync(join(tmpdir(), "opencode-mem-claim-home-")); const originalHome = process.env.HOME; const originalUserProfile = process.env.USERPROFILE; @@ -13,12 +12,13 @@ process.env.USERPROFILE = sandbox; const { UserPromptManager } = await import("../src/services/user-prompt/user-prompt-manager.js"); +type TestableManager = InstanceType & { + ready(): Promise; +}; + afterAll(() => { process.env.HOME = originalHome; process.env.USERPROFILE = originalUserProfile; - // Best-effort cleanup. On Windows the SQLite connection may still hold a - // lock on the WAL file even after the process is shutting down, which can - // surface as EBUSY. Failing here would mask test results, so we swallow. try { rmSync(sandbox, { recursive: true, force: true }); } catch { @@ -27,49 +27,37 @@ afterAll(() => { }); describe("UserPromptManager.claimPrompt / releaseClaim", () => { - let mgr: InstanceType; + let mgr: TestableManager; let activeIds: string[]; beforeEach(() => { - mgr = new UserPromptManager(); + mgr = new UserPromptManager() as TestableManager; activeIds = []; }); - afterEach(() => { - // Clean up rows the test inserted so subsequent tests start with an - // empty table. + afterEach(async () => { for (const id of activeIds) { try { - mgr.deletePrompt(id); + await mgr.deletePrompt(id); } catch { // ignore } } }); - function getRawCaptured(id: string): number { - const raw = ( - mgr as unknown as { - db: { prepare: (s: string) => { get: (a: string) => { captured: number } | undefined } }; - } - ).db - .prepare(`SELECT captured FROM user_prompts WHERE id = ?`) - .get(id); - return raw?.captured ?? -1; + async function getRawCaptured(id: string): Promise { + const db = await mgr.ready(); + const row = await db.get(`SELECT captured FROM user_prompts WHERE id = ?`, [id]); + return row ? Number(row.captured) : -1; } - function setCreatedAt(id: string, createdAt: number): void { - ( - mgr as unknown as { - db: { prepare: (s: string) => { run: (...args: unknown[]) => unknown } }; - } - ).db - .prepare(`UPDATE user_prompts SET created_at = ? WHERE id = ?`) - .run(createdAt, id); + async function setCreatedAt(id: string, createdAt: number): Promise { + const db = await mgr.ready(); + await db.run(`UPDATE user_prompts SET created_at = ? WHERE id = ?`, [createdAt, id]); } - function newPrompt(sessionId = "session-test", content = "hello") { - const id = mgr.savePrompt( + async function newPrompt(sessionId = "session-test", content = "hello") { + const id = await mgr.savePrompt( sessionId, `msg-${Date.now()}-${Math.random()}`, "/tmp/proj", @@ -79,81 +67,82 @@ describe("UserPromptManager.claimPrompt / releaseClaim", () => { return id; } - it("claimPrompt transitions captured 0 → 2 and returns true", () => { - const id = newPrompt(); - expect(mgr.claimPrompt(id)).toBe(true); - expect(getRawCaptured(id)).toBe(2); + it("claimPrompt transitions captured 0 → 2 and returns true", async () => { + const id = await newPrompt(); + expect(await mgr.claimPrompt(id)).toBe(true); + expect(await getRawCaptured(id)).toBe(2); }); - it("claimPrompt returns false when the row is already claimed", () => { - const id = newPrompt(); - expect(mgr.claimPrompt(id)).toBe(true); - expect(mgr.claimPrompt(id)).toBe(false); + it("claimPrompt returns false when the row is already claimed", async () => { + const id = await newPrompt(); + expect(await mgr.claimPrompt(id)).toBe(true); + expect(await mgr.claimPrompt(id)).toBe(false); }); - it("releaseClaim transitions captured 2 → 0 and exposes the row to retry", () => { - const id = newPrompt("session-release"); - expect(mgr.claimPrompt(id)).toBe(true); - expect(mgr.releaseClaim(id)).toBe(true); - expect(getRawCaptured(id)).toBe(0); + it("releaseClaim transitions captured 2 → 0 and exposes the row to retry", async () => { + const id = await newPrompt("session-release"); + expect(await mgr.claimPrompt(id)).toBe(true); + expect(await mgr.releaseClaim(id)).toBe(true); + expect(await getRawCaptured(id)).toBe(0); - const next = mgr.getLastUncapturedPrompt("session-release"); + const next = await mgr.getLastUncapturedPrompt("session-release"); expect(next).not.toBeNull(); expect(next!.id).toBe(id); }); - it("releaseClaim is a no-op when the row is already captured=1", () => { - const id = newPrompt(); - mgr.claimPrompt(id); - mgr.markAsCaptured(id); + it("releaseClaim is a no-op when the row is already captured=1", async () => { + const id = await newPrompt(); + await mgr.claimPrompt(id); + await mgr.markAsCaptured(id); - expect(mgr.releaseClaim(id)).toBe(false); - expect(getRawCaptured(id)).toBe(1); + expect(await mgr.releaseClaim(id)).toBe(false); + expect(await getRawCaptured(id)).toBe(1); }); - it("releaseClaim is a no-op when the row was never claimed", () => { - const id = newPrompt(); - expect(mgr.releaseClaim(id)).toBe(false); - expect(getRawCaptured(id)).toBe(0); + it("releaseClaim is a no-op when the row was never claimed", async () => { + const id = await newPrompt(); + expect(await mgr.releaseClaim(id)).toBe(false); + expect(await getRawCaptured(id)).toBe(0); }); - it("supports a full claim → release → re-claim retry cycle", () => { - const id = newPrompt(); - expect(mgr.claimPrompt(id)).toBe(true); - expect(mgr.releaseClaim(id)).toBe(true); - expect(mgr.claimPrompt(id)).toBe(true); - expect(getRawCaptured(id)).toBe(2); + it("supports a full claim → release → re-claim retry cycle", async () => { + const id = await newPrompt(); + expect(await mgr.claimPrompt(id)).toBe(true); + expect(await mgr.releaseClaim(id)).toBe(true); + expect(await mgr.claimPrompt(id)).toBe(true); + expect(await getRawCaptured(id)).toBe(2); }); - it("ignores prompts that have exceeded max retries", () => { - const id = mgr.savePrompt("session-retries", "msg-1", "/path", "hello retry"); + it("ignores prompts that have exceeded max retries", async () => { + const id = await mgr.savePrompt("session-retries", "msg-1", "/path", "hello retry"); + activeIds.push(id); for (let i = 0; i < 4; i++) { - mgr.recordFailedAttempt(id); + await mgr.recordFailedAttempt(id); } - const prompt = mgr.getLastUncapturedPrompt("session-retries"); + const prompt = await mgr.getLastUncapturedPrompt("session-retries"); expect(prompt).toBeNull(); }); - it("returns all uncaptured prompts for a session oldest first", () => { - const later = newPrompt("session-batch", "later"); - const older = newPrompt("session-batch", "older"); - const captured = newPrompt("session-batch", "captured"); - const otherSession = newPrompt("session-other", "other"); - const retriedOut = newPrompt("session-batch", "retried out"); - - setCreatedAt(later, 20); - setCreatedAt(older, 10); - setCreatedAt(captured, 15); - setCreatedAt(otherSession, 5); - setCreatedAt(retriedOut, 25); - mgr.markAsCaptured(captured); + it("returns all uncaptured prompts for a session oldest first", async () => { + const later = await newPrompt("session-batch", "later"); + const older = await newPrompt("session-batch", "older"); + const captured = await newPrompt("session-batch", "captured"); + const otherSession = await newPrompt("session-other", "other"); + const retriedOut = await newPrompt("session-batch", "retried out"); + + await setCreatedAt(later, 20); + await setCreatedAt(older, 10); + await setCreatedAt(captured, 15); + await setCreatedAt(otherSession, 5); + await setCreatedAt(retriedOut, 25); + await mgr.markAsCaptured(captured); for (let i = 0; i < 4; i++) { - mgr.recordFailedAttempt(retriedOut); + await mgr.recordFailedAttempt(retriedOut); } - const prompts = mgr.getUncapturedPromptsForSession("session-batch"); + const prompts = await mgr.getUncapturedPromptsForSession("session-batch"); expect(prompts.map((prompt) => prompt.id)).toEqual([older, later]); }); diff --git a/tests/vector-backends/backend-factory.test.ts b/tests/vector-backends/backend-factory.test.ts deleted file mode 100644 index 2b33868..0000000 --- a/tests/vector-backends/backend-factory.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { createVectorBackend } from "../../src/services/vector-backends/backend-factory.js"; -import type { VectorBackend } from "../../src/services/vector-backends/types.js"; - -function createThrowingBackend(method: "search" | "rebuildFromShard"): VectorBackend { - return { - getBackendName: () => "usearch", - insert: async () => {}, - insertBatch: async () => {}, - delete: async () => {}, - search: async (args) => { - if (method === "search") throw new Error("boom-search"); - void args; - return []; - }, - rebuildFromShard: async (args) => { - if (method === "rebuildFromShard") throw new Error("boom-rebuild"); - void args; - }, - deleteShardIndexes: async () => {}, - }; -} - -describe("vector backend factory", () => { - it("defaults to usearch-first strategy", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch-first", - probeUSearch: async () => true, - }); - - expect(backend.getBackendName()).toBe("usearch"); - }); - - it("falls back to exact scan when usearch-first cannot load usearch", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch-first", - probeUSearch: async () => false, - }); - - expect(backend.getBackendName()).toBe("exact-scan"); - }); - - it("uses usearch backend when requested and available", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch", - probeUSearch: async () => true, - }); - - expect(backend.getBackendName()).toBe("usearch"); - }); - - it("falls back to exact scan when usearch is unavailable", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch", - probeUSearch: async () => false, - }); - - expect(backend.getBackendName()).toBe("exact-scan"); - }); - - it("falls back to exact scan on usearch search failure", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch-first", - probeUSearch: async () => true, - createUSearchBackend: () => createThrowingBackend("search"), - }); - - const result = await backend.search({ - db: { - prepare: () => ({ - all: () => [], - }), - }, - shard: { - id: 1, - scope: "project", - scopeHash: "hash", - shardIndex: 0, - dbPath: "test.db", - vectorCount: 0, - isActive: true, - createdAt: Date.now(), - }, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 1, - }); - - expect(backend.getBackendName()).toBe("exact-scan"); - expect(result).toEqual([]); - }); - - it("falls back to exact scan on usearch rebuild failure", async () => { - const backend = await createVectorBackend({ - vectorBackend: "usearch-first", - probeUSearch: async () => true, - createUSearchBackend: () => createThrowingBackend("rebuildFromShard"), - }); - - await expect( - backend.rebuildFromShard({ - db: null, - shard: { - id: 1, - scope: "project", - scopeHash: "hash", - shardIndex: 0, - dbPath: "test.db", - vectorCount: 0, - isActive: true, - createdAt: Date.now(), - }, - kind: "content", - }) - ).resolves.toBeUndefined(); - - expect(backend.getBackendName()).toBe("exact-scan"); - }); -}); diff --git a/tests/vector-backends/exact-scan-backend.test.ts b/tests/vector-backends/exact-scan-backend.test.ts deleted file mode 100644 index 1797ec4..0000000 --- a/tests/vector-backends/exact-scan-backend.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { afterEach, describe, expect, it } from "bun:test"; -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { getDatabase } from "../../src/services/sqlite/sqlite-bootstrap.js"; -import { ExactScanBackend } from "../../src/services/vector-backends/exact-scan-backend.js"; -import { removeTempDirs } from "../helpers/temp-dir.mjs"; - -const Database = getDatabase(); - -describe("ExactScanBackend", () => { - const tempDirs: string[] = []; - const databases: Array<{ close: () => void }> = []; - - afterEach(async () => { - while (databases.length > 0) { - databases.pop()?.close(); - } - await removeTempDirs(tempDirs); - }); - - it("returns nearest vectors in similarity order", () => { - const backend = new ExactScanBackend(); - - const rows = [ - { id: "a", vector: new Float32Array([1, 0, 0, 0]) }, - { id: "b", vector: new Float32Array([0, 1, 0, 0]) }, - { id: "c", vector: new Float32Array([0.9, 0.1, 0, 0]) }, - ]; - - const result = backend.rankVectors(rows, new Float32Array([1, 0, 0, 0]), 2); - - expect(result.map((x) => x.id)).toEqual(["a", "c"]); - }); - - it("returns empty result for empty rows", () => { - const backend = new ExactScanBackend(); - const result = backend.rankVectors([], new Float32Array([1, 0, 0, 0]), 5); - expect(result).toEqual([]); - }); - - it("searches vectors from sqlite blobs in similarity order", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "exact-scan-backend-")); - tempDirs.push(tempDir); - const db = new Database(join(tempDir, "test.db")); - databases.push(db); - - db.run(`CREATE TABLE memories (id TEXT PRIMARY KEY, vector BLOB, tags_vector BLOB)`); - - const insert = db.prepare(`INSERT INTO memories (id, vector, tags_vector) VALUES (?, ?, ?)`); - insert.run("a", new Uint8Array(new Float32Array([1, 0, 0, 0]).buffer), null); - insert.run("b", new Uint8Array(new Float32Array([0, 1, 0, 0]).buffer), null); - insert.run("c", new Uint8Array(new Float32Array([0.9, 0.1, 0, 0]).buffer), null); - - const backend = new ExactScanBackend(); - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath: join(tempDir, "test.db"), - vectorCount: 3, - isActive: true, - createdAt: Date.now(), - }; - - const result = await backend.search({ - db, - shard, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 2, - }); - - expect(result.map((x) => x.id)).toEqual(["a", "c"]); - }); - - it("returns empty search result when sqlite has no vectors", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "exact-scan-backend-empty-")); - tempDirs.push(tempDir); - const db = new Database(join(tempDir, "test.db")); - databases.push(db); - db.run(`CREATE TABLE memories (id TEXT PRIMARY KEY, vector BLOB, tags_vector BLOB)`); - - const backend = new ExactScanBackend(); - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath: join(tempDir, "test.db"), - vectorCount: 0, - isActive: true, - createdAt: Date.now(), - }; - - const result = await backend.search({ - db, - shard, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 2, - }); - - expect(result).toEqual([]); - }); -}); diff --git a/tests/vector-backends/migration-fallback.test.ts b/tests/vector-backends/migration-fallback.test.ts deleted file mode 100644 index d3c56cc..0000000 --- a/tests/vector-backends/migration-fallback.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { afterEach, describe, expect, it } from "bun:test"; -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { getDatabase } from "../../src/services/sqlite/sqlite-bootstrap.js"; -import { ExactScanBackend } from "../../src/services/vector-backends/exact-scan-backend.js"; -import { VectorSearch } from "../../src/services/sqlite/vector-search.js"; -import { removeTempDirs } from "../helpers/temp-dir.mjs"; -import { connectionManager } from "../../src/services/sqlite/connection-manager.js"; - -const Database = getDatabase(); - -describe("migration with backend abstraction", () => { - const tempDirs: string[] = []; - const databases: Array<{ close: () => void }> = []; - - afterEach(async () => { - connectionManager.closeAll(); - while (databases.length > 0) { - databases.pop()?.close(); - } - await removeTempDirs(tempDirs); - }); - - it("rebuilds and searches memories without direct hnsw manager calls", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "migration-backend-")); - tempDirs.push(tempDir); - const dbPath = join(tempDir, "test.db"); - const db = new Database(dbPath); - databases.push(db); - db.run(` - CREATE TABLE memories ( - id TEXT PRIMARY KEY, - content TEXT, - vector BLOB, - tags_vector BLOB, - container_tag TEXT, - tags TEXT, - type TEXT, - created_at INTEGER, - updated_at INTEGER, - metadata TEXT, - display_name TEXT, - user_name TEXT, - user_email TEXT, - project_path TEXT, - project_name TEXT, - git_repo_url TEXT, - is_pinned INTEGER DEFAULT 0 - ) - `); - - db.prepare( - `INSERT INTO memories (id, content, vector, tags_vector, container_tag, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)` - ).run( - "a", - "alpha", - new Uint8Array(new Float32Array([1, 0, 0, 0]).buffer), - null, - "opencode_project_hash", - Date.now(), - Date.now() - ); - - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath, - vectorCount: 1, - isActive: true, - createdAt: Date.now(), - }; - - const vectorSearch = new VectorSearch(new ExactScanBackend()); - - await expect( - vectorSearch.rebuildIndexForShard(db, "project", "hash", 0) - ).resolves.toBeUndefined(); - - const results = await vectorSearch.searchInShard( - shard, - new Float32Array([1, 0, 0, 0]), - "opencode_project_hash", - 1, - "alpha" - ); - - expect(results.map((r) => r.id)).toEqual(["a"]); - }); -}); diff --git a/tests/vector-backends/usearch-backend.test.ts b/tests/vector-backends/usearch-backend.test.ts deleted file mode 100644 index a39105a..0000000 --- a/tests/vector-backends/usearch-backend.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { afterEach, describe, expect, it } from "bun:test"; -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { getDatabase } from "../../src/services/sqlite/sqlite-bootstrap.js"; -import { USearchBackend } from "../../src/services/vector-backends/usearch-backend.js"; -import { removeTempDirs } from "../helpers/temp-dir.mjs"; - -const Database = getDatabase(); -const canLoadUSearch = await import("usearch").then(() => true).catch(() => false); -const itIfUSearchAvailable = canLoadUSearch ? it : it.skip; - -describe("USearchBackend", () => { - const tempDirs: string[] = []; - const databases: Array<{ close: () => void }> = []; - - afterEach(async () => { - while (databases.length > 0) { - databases.pop()?.close(); - } - await removeTempDirs(tempDirs); - }); - - itIfUSearchAvailable("creates and searches an in-memory index", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "usearch-backend-")); - tempDirs.push(baseDir); - - const backend = new USearchBackend({ baseDir, dimensions: 4 }); - - await backend.insertManyForTest("project_hash_0_content", [ - { id: "a", vector: new Float32Array([1, 0, 0, 0]) }, - { id: "b", vector: new Float32Array([0, 1, 0, 0]) }, - { id: "c", vector: new Float32Array([0.9, 0.1, 0, 0]) }, - ]); - - const result = await backend.searchForTest( - "project_hash_0_content", - new Float32Array([1, 0, 0, 0]), - 2 - ); - - expect(result.map((x) => x.id)).toEqual(["a", "c"]); - }); - - itIfUSearchAvailable("supports public insert and search path", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "usearch-backend-public-")); - tempDirs.push(baseDir); - - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath: join(baseDir, "test.db"), - vectorCount: 1, - isActive: true, - createdAt: Date.now(), - }; - - const backend = new USearchBackend({ baseDir, dimensions: 4 }); - await backend.insert({ - id: "alpha", - vector: new Float32Array([1, 0, 0, 0]), - shard, - kind: "content", - }); - - const result = await backend.search({ - db: null, - shard, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 1, - }); - - expect(result.map((x) => x.id)).toEqual(["alpha"]); - }); - - itIfUSearchAvailable( - "updates an existing id instead of failing on duplicate insert", - async () => { - const baseDir = mkdtempSync(join(tmpdir(), "usearch-backend-upsert-")); - tempDirs.push(baseDir); - - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath: join(baseDir, "test.db"), - vectorCount: 1, - isActive: true, - createdAt: Date.now(), - }; - - const backend = new USearchBackend({ baseDir, dimensions: 4 }); - await backend.insert({ - id: "alpha", - vector: new Float32Array([0, 1, 0, 0]), - shard, - kind: "content", - }); - await backend.insert({ - id: "alpha", - vector: new Float32Array([1, 0, 0, 0]), - shard, - kind: "content", - }); - - const result = await backend.search({ - db: null, - shard, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 1, - }); - - expect(result.map((x) => x.id)).toEqual(["alpha"]); - } - ); - - itIfUSearchAvailable("rebuilds an index from sqlite rows", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "usearch-backend-rebuild-")); - tempDirs.push(baseDir); - const db = new Database(join(baseDir, "test.db")); - databases.push(db); - db.run(`CREATE TABLE memories (id TEXT PRIMARY KEY, vector BLOB, tags_vector BLOB)`); - db.prepare(`INSERT INTO memories (id, vector, tags_vector) VALUES (?, ?, ?)`).run( - "alpha", - new Uint8Array(new Float32Array([1, 0, 0, 0]).buffer), - null - ); - - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath: join(baseDir, "test.db"), - vectorCount: 1, - isActive: true, - createdAt: Date.now(), - }; - - const backend = new USearchBackend({ baseDir, dimensions: 4 }); - await backend.rebuildFromShard({ db, shard, kind: "content" }); - - const result = await backend.search({ - db, - shard, - kind: "content", - queryVector: new Float32Array([1, 0, 0, 0]), - limit: 1, - }); - - expect(result.map((x) => x.id)).toEqual(["alpha"]); - }); -}); diff --git a/tests/vector-search-backend-integration.test.ts b/tests/vector-search-backend-integration.test.ts deleted file mode 100644 index 9557568..0000000 --- a/tests/vector-search-backend-integration.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { afterEach, describe, expect, it } from "bun:test"; -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { ExactScanBackend } from "../src/services/vector-backends/exact-scan-backend.js"; -import { getDatabase } from "../src/services/sqlite/sqlite-bootstrap.js"; -import { VectorSearch } from "../src/services/sqlite/vector-search.js"; -import type { VectorBackend } from "../src/services/vector-backends/types.js"; -import { removeTempDirs } from "./helpers/temp-dir.mjs"; -import { connectionManager } from "../src/services/sqlite/connection-manager.js"; - -const Database = getDatabase(); - -function createFailingBackend(): VectorBackend { - return { - getBackendName: () => "usearch", - insert: async () => {}, - insertBatch: async () => {}, - delete: async () => {}, - search: async () => { - throw new Error("forced-search-failure"); - }, - rebuildFromShard: async () => { - throw new Error("forced-rebuild-failure"); - }, - deleteShardIndexes: async () => {}, - }; -} - -describe("vector search backend integration", () => { - const tempDirs: string[] = []; - const databases: Array<{ close: () => void }> = []; - - afterEach(async () => { - connectionManager.closeAll(); - while (databases.length > 0) { - databases.pop()?.close(); - } - await removeTempDirs(tempDirs); - }); - - it("searches inserted memories and preserves ranking semantics", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "vector-search-integration-")); - tempDirs.push(tempDir); - const dbPath = join(tempDir, "test.db"); - const db = new Database(dbPath); - databases.push(db); - - db.run(` - CREATE TABLE memories ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - vector BLOB NOT NULL, - tags_vector BLOB, - container_tag TEXT NOT NULL, - tags TEXT, - type TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - metadata TEXT, - display_name TEXT, - user_name TEXT, - user_email TEXT, - project_path TEXT, - project_name TEXT, - git_repo_url TEXT, - is_pinned INTEGER DEFAULT 0 - ) - `); - - const vectorSearch = new VectorSearch(new ExactScanBackend()); - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath, - vectorCount: 2, - isActive: true, - createdAt: Date.now(), - }; - - await vectorSearch.insertVector( - db, - { - id: "b", - content: "beta memory", - vector: new Float32Array([0, 1, 0, 0]), - tagsVector: new Float32Array([0, 1, 0, 0]), - containerTag: "opencode_project_hash", - tags: "beta", - createdAt: Date.now(), - updatedAt: Date.now(), - }, - shard - ); - - await vectorSearch.insertVector( - db, - { - id: "a", - content: "alpha memory", - vector: new Float32Array([1, 0, 0, 0]), - tagsVector: new Float32Array([1, 0, 0, 0]), - containerTag: "opencode_project_hash", - tags: "alpha,priority", - createdAt: Date.now(), - updatedAt: Date.now(), - }, - shard - ); - - const results = await vectorSearch.searchInShard( - shard, - new Float32Array([1, 0, 0, 0]), - "opencode_project_hash", - 2, - "alpha" - ); - - expect(results.map((r) => r.id)).toEqual(["a", "b"]); - expect(results[0]?.similarity).toBeGreaterThan(results[1]?.similarity ?? 0); - expect(typeof results[0]?.similarity).toBe("number"); - expect(typeof results[1]?.similarity).toBe("number"); - }); - - it("falls back to exact scan when the preferred backend fails", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "vector-search-fallback-")); - tempDirs.push(tempDir); - const dbPath = join(tempDir, "test.db"); - const db = new Database(dbPath); - databases.push(db); - - db.run(` - CREATE TABLE memories ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - vector BLOB NOT NULL, - tags_vector BLOB, - container_tag TEXT NOT NULL, - tags TEXT, - type TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - metadata TEXT, - display_name TEXT, - user_name TEXT, - user_email TEXT, - project_path TEXT, - project_name TEXT, - git_repo_url TEXT, - is_pinned INTEGER DEFAULT 0 - ) - `); - - const shard = { - id: 1, - scope: "project" as const, - scopeHash: "hash", - shardIndex: 0, - dbPath, - vectorCount: 2, - isActive: true, - createdAt: Date.now(), - }; - - const vectorSearch = new VectorSearch(createFailingBackend(), new ExactScanBackend()); - - db.prepare( - `INSERT INTO memories (id, content, vector, tags_vector, container_tag, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)` - ).run( - "a", - "alpha memory", - new Uint8Array(new Float32Array([1, 0, 0, 0]).buffer), - new Uint8Array(new Float32Array([1, 0, 0, 0]).buffer), - "opencode_project_hash", - Date.now(), - Date.now() - ); - db.prepare( - `INSERT INTO memories (id, content, vector, tags_vector, container_tag, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)` - ).run( - "b", - "beta memory", - new Uint8Array(new Float32Array([0, 1, 0, 0]).buffer), - new Uint8Array(new Float32Array([0, 1, 0, 0]).buffer), - "opencode_project_hash", - Date.now(), - Date.now() - ); - - const results = await vectorSearch.searchInShard( - shard, - new Float32Array([1, 0, 0, 0]), - "opencode_project_hash", - 2, - "alpha" - ); - - expect(results.map((r) => r.id)).toEqual(["a", "b"]); - expect(typeof results[0]?.similarity).toBe("number"); - }); -}); diff --git a/tests/web-api-auth.test.ts b/tests/web-api-auth.test.ts new file mode 100644 index 0000000..5eb2c02 --- /dev/null +++ b/tests/web-api-auth.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "bun:test"; +import { + assertWebServerNetworkAuth, + authorizeApiRequest, + isLoopbackHost, +} from "../src/services/web-api-auth.js"; + +describe("web api auth", () => { + it("detects loopback hosts", () => { + expect(isLoopbackHost("127.0.0.1")).toBe(true); + expect(isLoopbackHost("localhost")).toBe(true); + expect(isLoopbackHost("0.0.0.0")).toBe(false); + }); + + it("requires api token for non-loopback hosts", () => { + expect(() => assertWebServerNetworkAuth("0.0.0.0")).toThrow(/webServerApiToken/); + expect(() => assertWebServerNetworkAuth("0.0.0.0", "secret")).not.toThrow(); + expect(() => assertWebServerNetworkAuth("127.0.0.1")).not.toThrow(); + }); + + it("authorizes bearer and custom header tokens", () => { + const token = "test-token"; + expect(authorizeApiRequest(new Request("http://localhost/api/stats"), token)?.status).toBe(401); + expect( + authorizeApiRequest( + new Request("http://localhost/api/stats", { + headers: { Authorization: `Bearer ${token}` }, + }), + token + ) + ).toBeNull(); + expect( + authorizeApiRequest( + new Request("http://localhost/api/stats", { + headers: { "X-Opencode-Mem-Token": token }, + }), + token + ) + ).toBeNull(); + expect(authorizeApiRequest(new Request("http://localhost/api/stats"), undefined)).toBeNull(); + }); +}); diff --git a/tests/web-server-health.test.ts b/tests/web-server-health.test.ts new file mode 100644 index 0000000..0c6e51f --- /dev/null +++ b/tests/web-server-health.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "bun:test"; +import { WebServer } from "../src/services/web-server.js"; + +describe("web server health check", () => { + it("authenticates the stats request when an API token is configured", async () => { + const originalFetch = globalThis.fetch; + let requestHeaders: Headers | undefined; + globalThis.fetch = async (_input, init) => { + requestHeaders = new Headers(init?.headers); + return new Response(null, { status: 200 }); + }; + + try { + const server = new WebServer({ + enabled: true, + host: "0.0.0.0", + port: 4747, + apiToken: "health-token", + }); + + expect(await server.checkServerAvailable()).toBe(true); + expect(requestHeaders?.get("Authorization")).toBe("Bearer health-token"); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); From 0fd6bbe5990bb4f2990f127e348a40f255fd2f29 Mon Sep 17 00:00:00 2001 From: EyJunge1 Date: Wed, 22 Jul 2026 10:32:38 +0200 Subject: [PATCH 2/3] chore: update dependencies and TypeScript 6 Align direct dependencies with current releases while retaining the latest supported TypeScript 6 compiler and its stricter type checks. Co-authored-by: Cursor --- bun.lock | 116 ++++++++++++++------------- package.json | 14 ++-- src/config.ts | 10 +-- src/index.ts | 2 +- src/services/ai/opencode-provider.ts | 3 +- src/services/web-server.ts | 2 +- 6 files changed, 74 insertions(+), 73 deletions(-) diff --git a/bun.lock b/bun.lock index ddd4bf1..7cc2805 100644 --- a/bun.lock +++ b/bun.lock @@ -7,18 +7,18 @@ "dependencies": { "@huggingface/transformers": "^4.2.0", "@libsql/client": "^0.17.4", - "@opencode-ai/plugin": "^1.3.0", - "@opencode-ai/sdk": "^1.3.0", + "@opencode-ai/plugin": "^1.18.4", + "@opencode-ai/sdk": "^1.18.4", "franc-min": "^6.2.0", "iso-639-3": "^3.0.1", - "zod": "^4.3.6", + "zod": "^4.4.3", }, "devDependencies": { - "@types/bun": "^1.3.8", + "@types/bun": "^1.3.14", "husky": "^9.1.7", - "lint-staged": "^16.4.0", - "prettier": "^3.4.2", - "typescript": "^5.7.3", + "lint-staged": "^17.1.1", + "prettier": "^3.9.6", + "typescript": "6.0.3", }, }, }, @@ -26,6 +26,8 @@ "onnxruntime-node": "1.22.0", }, "packages": { + "@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="], + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], "@huggingface/jinja": ["@huggingface/jinja@0.5.9", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="], @@ -110,11 +112,23 @@ "@libsql/win32-x64-msvc": ["@libsql/win32-x64-msvc@0.5.29", "", { "os": "win32", "cpu": "x64" }, "sha512-4/0CvEdhi6+KjMxMaVbFM2n2Z44escBRoEYpR+gZg64DdetzGnYm8mcNLcoySaDJZNaBd6wz5DNdgRmcI4hXcg=="], + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ=="], + + "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4", "", { "os": "linux", "cpu": "arm" }, "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw=="], + + "@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ=="], + + "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="], + "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="], - "@opencode-ai/plugin": ["@opencode-ai/plugin@1.3.13", "", { "dependencies": { "@opencode-ai/sdk": "1.3.13", "zod": "4.1.8" }, "peerDependencies": { "@opentui/core": ">=0.1.95", "@opentui/solid": ">=0.1.95" }, "optionalPeers": ["@opentui/core", "@opentui/solid"] }, "sha512-zHgtWfdDz8Wu8srE8f8HUtPT9i6c3jTmgQKoFZUZ+RR5CMQF1kAlb1cxeEe9Xm2DRNFVJog9Cv/G1iUHYgXSUQ=="], + "@opencode-ai/plugin": ["@opencode-ai/plugin@1.18.4", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@opencode-ai/sdk": "1.18.4", "effect": "4.0.0-beta.83", "zod": "4.1.8" }, "peerDependencies": { "@opentui/core": ">=0.4.5", "@opentui/keymap": ">=0.4.5", "@opentui/solid": ">=0.4.5" }, "optionalPeers": ["@opentui/core", "@opentui/keymap", "@opentui/solid"] }, "sha512-Mkq128aLJo4E8Sb2bX8zrRlQ+I2WPaJ/n1kzaor8nTi/K/zNP4t8LGKwyMbuRoD/lhw4veSbzDOASSSypv3mcQ=="], - "@opencode-ai/sdk": ["@opencode-ai/sdk@1.3.13", "", {}, "sha512-/M6HlNnba+xf1EId6qFb2tG0cvq0db3PCQDug1glrf8wYOU57LYNF8WvHX9zoDKPTMv0F+O4pcP/8J+WvDaxHA=="], + "@opencode-ai/sdk": ["@opencode-ai/sdk@1.18.4", "", { "dependencies": { "cross-spawn": "7.0.6" } }, "sha512-p/3P0KtWknoLvpsk8QrUzCKd4Q1A5Z3RmACEvwuQBGxnUy4AGo379lUYMgt7fczFn53079oroLuo6BosQri+HA=="], "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], @@ -136,7 +150,9 @@ "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], - "@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@25.0.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA=="], @@ -144,25 +160,13 @@ "adm-zip": ["adm-zip@0.5.17", "", {}, "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ=="], - "ansi-escapes": ["ansi-escapes@7.3.0", "", { "dependencies": { "environment": "^1.0.0" } }, "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="], - - "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], - - "ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], - "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], - "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], - - "cli-cursor": ["cli-cursor@5.0.0", "", { "dependencies": { "restore-cursor": "^5.0.0" } }, "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw=="], - - "cli-truncate": ["cli-truncate@5.1.1", "", { "dependencies": { "slice-ansi": "^7.1.0", "string-width": "^8.0.0" } }, "sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "collapse-white-space": ["collapse-white-space@2.1.0", "", {}, "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw=="], - "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], - - "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], @@ -172,9 +176,7 @@ "detect-node": ["detect-node@2.1.0", "", {}, "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g=="], - "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], - - "environment": ["environment@1.1.0", "", {}, "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q=="], + "effect": ["effect@4.0.0-beta.83", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "fast-check": "^4.8.0", "find-my-way-ts": "^0.1.6", "ini": "^7.0.0", "kubernetes-types": "^1.30.0", "msgpackr": "^2.0.1", "multipasta": "^0.2.7", "toml": "^4.1.1", "uuid": "^14.0.0", "yaml": "^2.9.0" } }, "sha512-0wsak8RtgGAr9UWSbVDgJHZcUqMSvicHcvaZv1MbMM7MCGgW4Rn/137J1MHQbwYPcwYGxT/IqehFd+UbYuj78w=="], "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], @@ -184,14 +186,14 @@ "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], - "eventemitter3": ["eventemitter3@5.0.4", "", {}, "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="], + "fast-check": ["fast-check@4.9.0", "", { "dependencies": { "pure-rand": "^8.0.0" } }, "sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg=="], + + "find-my-way-ts": ["find-my-way-ts@0.1.6", "", {}, "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA=="], "flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="], "franc-min": ["franc-min@6.2.0", "", { "dependencies": { "trigram-utils": "^2.0.0" } }, "sha512-1uDIEUSlUZgvJa2AKYR/dmJC66v/PvGQ9mWfI9nOr/kPpMFyvswK0gPXOwpYJYiYD008PpHLkGfG58SPjQJFxw=="], - "get-east-asian-width": ["get-east-asian-width@1.4.0", "", {}, "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q=="], - "global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="], "globalthis": ["globalthis@1.0.4", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="], @@ -204,33 +206,39 @@ "husky": ["husky@9.1.7", "", { "bin": { "husky": "bin.js" } }, "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA=="], - "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], + "ini": ["ini@7.0.0", "", {}, "sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], "iso-639-3": ["iso-639-3@3.0.1", "", {}, "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ=="], "js-base64": ["js-base64@3.9.1", "", {}, "sha512-U73qptcvf/HIOauFOmqT3a0mDUp0MYlfd15oqoe9kqZt5XhiXVb+HG09sLvI9PQ9tZIBFS4nlErai8zbWazP0g=="], - "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], + "json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="], - "libsql": ["libsql@0.5.29", "", { "dependencies": { "@neon-rs/load": "^0.0.4", "detect-libc": "2.0.2" }, "optionalDependencies": { "@libsql/darwin-arm64": "0.5.29", "@libsql/darwin-x64": "0.5.29", "@libsql/linux-arm-gnueabihf": "0.5.29", "@libsql/linux-arm-musleabihf": "0.5.29", "@libsql/linux-arm64-gnu": "0.5.29", "@libsql/linux-arm64-musl": "0.5.29", "@libsql/linux-x64-gnu": "0.5.29", "@libsql/linux-x64-musl": "0.5.29", "@libsql/win32-x64-msvc": "0.5.29" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "arm", "x64", "arm64", ] }, "sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg=="], + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], - "lint-staged": ["lint-staged@16.4.0", "", { "dependencies": { "commander": "^14.0.3", "listr2": "^9.0.5", "picomatch": "^4.0.3", "string-argv": "^0.3.2", "tinyexec": "^1.0.4", "yaml": "^2.8.2" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-lBWt8hujh/Cjysw5GYVmZpFHXDCgZzhrOm8vbcUdobADZNOK/bRshr2kM3DfgrrtR1DQhfupW9gnIXOfiFi+bw=="], + "kubernetes-types": ["kubernetes-types@1.30.0", "", {}, "sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q=="], - "listr2": ["listr2@9.0.5", "", { "dependencies": { "cli-truncate": "^5.0.0", "colorette": "^2.0.20", "eventemitter3": "^5.0.1", "log-update": "^6.1.0", "rfdc": "^1.4.1", "wrap-ansi": "^9.0.0" } }, "sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g=="], + "libsql": ["libsql@0.5.29", "", { "dependencies": { "@neon-rs/load": "^0.0.4", "detect-libc": "2.0.2" }, "optionalDependencies": { "@libsql/darwin-arm64": "0.5.29", "@libsql/darwin-x64": "0.5.29", "@libsql/linux-arm-gnueabihf": "0.5.29", "@libsql/linux-arm-musleabihf": "0.5.29", "@libsql/linux-arm64-gnu": "0.5.29", "@libsql/linux-arm64-musl": "0.5.29", "@libsql/linux-x64-gnu": "0.5.29", "@libsql/linux-x64-musl": "0.5.29", "@libsql/win32-x64-msvc": "0.5.29" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "arm", "x64", "arm64", ] }, "sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg=="], - "log-update": ["log-update@6.1.0", "", { "dependencies": { "ansi-escapes": "^7.0.0", "cli-cursor": "^5.0.0", "slice-ansi": "^7.1.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w=="], + "lint-staged": ["lint-staged@17.1.1", "", { "dependencies": { "picomatch": "^4.0.5", "string-argv": "^0.3.2", "tinyexec": "^1.2.4" }, "optionalDependencies": { "yaml": "^2.9.0" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-FnHWpSe5cPRtrDG+soOuNdBxb4XQb2gN5EqpEWKdweyqyOfpl4QSjbrz3ilcIf0WXmkiNQGZZRQ23R5YtB3TEw=="], "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], "matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], - "mimic-function": ["mimic-function@5.0.1", "", {}, "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA=="], + "msgpackr": ["msgpackr@2.0.4", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.4" } }, "sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA=="], + + "msgpackr-extract": ["msgpackr-extract@3.0.4", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw=="], + + "multipasta": ["multipasta@0.2.8", "", {}, "sha512-ZPWuMKyv0cSO29f7hozp+k6+crZbQijV8ipMvxNxRf2SwtYGTX1ZX89Kd20VV4H9Znonx+EQn+iy1wGQsJ+b+Q=="], "n-gram": ["n-gram@2.0.2", "", {}, "sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ=="], - "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], - "onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], + "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], "onnxruntime-common": ["onnxruntime-common@1.22.0", "", {}, "sha512-vcuaNWgtF2dGQu/EP5P8UI5rEPEYqXG2sPPe5j9lg2TY/biJF8eWklTMwlDO08iuXq48xJo0awqIpK5mPG+IxA=="], @@ -238,19 +246,19 @@ "onnxruntime-web": ["onnxruntime-web@1.26.0-dev.20260416-b7804b056c", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="], + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], - "prettier": ["prettier@3.7.4", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA=="], + "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], "promise-limit": ["promise-limit@2.7.0", "", {}, "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw=="], "protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="], - "restore-cursor": ["restore-cursor@5.1.0", "", { "dependencies": { "onetime": "^7.0.0", "signal-exit": "^4.1.0" } }, "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA=="], - - "rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="], + "pure-rand": ["pure-rand@8.4.2", "", {}, "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng=="], "roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="], @@ -262,44 +270,42 @@ "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], - "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - "slice-ansi": ["slice-ansi@7.1.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w=="], + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], "sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], - "string-width": ["string-width@8.1.1", "", { "dependencies": { "get-east-asian-width": "^1.3.0", "strip-ansi": "^7.1.0" } }, "sha512-KpqHIdDL9KwYk22wEOg/VIqYbrnLeSApsKT/bSj6Ez7pn3CftUiLAv2Lccpq1ALcpLV9UX1Ppn92npZWu2w/aw=="], - - "strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="], - "tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], + "toml": ["toml@4.3.0", "", {}, "sha512-lVb8X9BsPVuH0M4BKeS91tXAmJvCjQ5UIyAbQFaxkKGyUFK2RPkhwaFSQH8vbpl1d23eu/IBH+dwVMHWaq9A5A=="], + "trigram-utils": ["trigram-utils@2.0.1", "", { "dependencies": { "collapse-white-space": "^2.0.0", "n-gram": "^2.0.0" } }, "sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ=="], "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - "wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], + "uuid": ["uuid@14.0.1", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "ws": ["ws@8.21.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw=="], - "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], - "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], "@opencode-ai/plugin/zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], "libsql/detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.24.0-dev.20251116-b39e144322", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="], - - "wrap-ansi/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], } } diff --git a/package.json b/package.json index aa41cb7..91ce0ec 100644 --- a/package.json +++ b/package.json @@ -50,22 +50,22 @@ "dependencies": { "@huggingface/transformers": "^4.2.0", "@libsql/client": "^0.17.4", - "@opencode-ai/plugin": "^1.3.0", - "@opencode-ai/sdk": "^1.3.0", + "@opencode-ai/plugin": "^1.18.4", + "@opencode-ai/sdk": "^1.18.4", "franc-min": "^6.2.0", "iso-639-3": "^3.0.1", - "zod": "^4.3.6" + "zod": "^4.4.3" }, "//": "Remove onnxruntime-node override after https://github.com/microsoft/onnxruntime/issues/27961 is resolved", "overrides": { "onnxruntime-node": "1.22.0" }, "devDependencies": { - "@types/bun": "^1.3.8", + "@types/bun": "^1.3.14", "husky": "^9.1.7", - "lint-staged": "^16.4.0", - "prettier": "^3.4.2", - "typescript": "^5.7.3" + "lint-staged": "^17.1.1", + "prettier": "^3.9.6", + "typescript": "6.0.3" }, "opencode": { "type": "plugin", diff --git a/src/config.ts b/src/config.ts index 495860c..737f034 100644 --- a/src/config.ts +++ b/src/config.ts @@ -597,9 +597,7 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { autoCaptureMaxRetries: fileConfig.autoCaptureMaxRetries ?? DEFAULTS.autoCaptureMaxRetries, autoCaptureLanguage: fileConfig.autoCaptureLanguage, memoryProvider: (fileConfig.memoryProvider ?? "openai-chat") as - | "openai-chat" - | "openai-responses" - | "anthropic", + "openai-chat" | "openai-responses" | "anthropic", memoryModel: fileConfig.memoryModel, memoryApiUrl: fileConfig.memoryApiUrl, memoryApiKey, @@ -687,8 +685,7 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { fileConfig.chatMessage?.excludeCurrentSession ?? DEFAULTS.chatMessage.excludeCurrentSession, maxAgeDays: fileConfig.chatMessage?.maxAgeDays, injectOn: (fileConfig.chatMessage?.injectOn ?? DEFAULTS.chatMessage.injectOn) as - | "first" - | "always", + "first" | "always", }, }; } @@ -707,8 +704,7 @@ interface AutoCaptureProviderRuntimeConfig { } export type AutoCaptureProviderStatus = - | { ready: true; mode: "opencode" | "manual"; issues: [] } - | { ready: false; issues: string[] }; + { ready: true; mode: "opencode" | "manual"; issues: [] } | { ready: false; issues: string[] }; function hasValue(value: string | undefined): boolean { return typeof value === "string" && value.trim().length > 0; diff --git a/src/index.ts b/src/index.ts index bfe2dc5..7626080 100644 --- a/src/index.ts +++ b/src/index.ts @@ -72,7 +72,7 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { logAutoCaptureProviderStatus(); const tags = getTags(directory); let webServer: WebServer | null = null; - let idleTimeout: Timer | null = null; + let idleTimeout: ReturnType | null = null; if (!isConfigured()) { } diff --git a/src/services/ai/opencode-provider.ts b/src/services/ai/opencode-provider.ts index b6efd2b..4a7a1f9 100644 --- a/src/services/ai/opencode-provider.ts +++ b/src/services/ai/opencode-provider.ts @@ -241,8 +241,7 @@ async function generateViaSdkClient( function readSdkData(response: unknown, label: string): T { const result = response as - | { data?: T; error?: unknown; request?: Request; response?: Response } - | undefined; + { data?: T; error?: unknown; request?: Request; response?: Response } | undefined; if (result?.error !== undefined) { const status = result.response ? ` (${responseStatus(result.response)})` : ""; const responseUrl = result.response?.url || result.request?.url; diff --git a/src/services/web-server.ts b/src/services/web-server.ts index 8b34aee..78ddb9a 100644 --- a/src/services/web-server.ts +++ b/src/services/web-server.ts @@ -61,7 +61,7 @@ function serveFetch(opts: { }): PortableServerHandle { if (isBun) { const bunHandle = ( - globalThis as { Bun: { serve: (opts: unknown) => { stop: () => void } } } + globalThis as unknown as { Bun: { serve: (opts: unknown) => { stop: () => void } } } ).Bun.serve({ port: opts.port, hostname: opts.hostname, From 31f9c131013ae0ccfcd2cabe9bc0074cf79aac68 Mon Sep 17 00:00:00 2001 From: EyJunge1 Date: Wed, 22 Jul 2026 10:34:31 +0200 Subject: [PATCH 3/3] chore: upgrade to TypeScript 7 Adopt the current compiler release after a clean typecheck, build, and full test run. Co-authored-by: Cursor --- bun.lock | 44 ++++++++++++++++++++++++++++++++++++++++++-- package.json | 2 +- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/bun.lock b/bun.lock index 7cc2805..3be9b8f 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ "husky": "^9.1.7", "lint-staged": "^17.1.1", "prettier": "^3.9.6", - "typescript": "6.0.3", + "typescript": "7.0.2", }, }, }, @@ -158,6 +158,46 @@ "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "adm-zip": ["adm-zip@0.5.17", "", {}, "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ=="], "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], @@ -288,7 +328,7 @@ "type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], - "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], diff --git a/package.json b/package.json index 91ce0ec..43891a8 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "husky": "^9.1.7", "lint-staged": "^17.1.1", "prettier": "^3.9.6", - "typescript": "6.0.3" + "typescript": "7.0.2" }, "opencode": { "type": "plugin",