fix(sqlite): make startup migrations concurrency-safe

.tests/library/library-search-index.test.js

@@ -15,6 +15,36 @@ test.after(async () => {
   await cleanupIsolatedState(isolatedState);
 });
 
+test("startup restores a missing search index trigger and rebuilds documents", async () => {
+  const { initializeLibrarySearchIndex } = await importFromRepo(
+    "backend/config/library-search-index.js",
+  );
+  db.exec("DROP TRIGGER library_search_documents_ai");
+  db.prepare(
+    "INSERT INTO library_search_documents (entity_kind, entity_id, title) VALUES ('artist', 998, 'Unindexed Artist')",
+  ).run();
+
+  assert.equal(initializeLibrarySearchIndex(db), true);
+  assert.ok(db.prepare(
+    "SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = 'library_search_documents_ai'",
+  ).get());
+  assert.equal(db.prepare("SELECT 1 FROM library_search_documents WHERE entity_id = 998").get(), undefined);
+});
+
+test("startup rebuilds search documents when their version is outdated", async () => {
+  const { initializeLibrarySearchIndex } = await importFromRepo(
+    "backend/config/library-search-index.js",
+  );
+  db.prepare(
+    "INSERT INTO library_search_documents (entity_kind, entity_id, title) VALUES ('artist', 999, 'Stale Artist')",
+  ).run();
+  db.prepare("UPDATE settings SET value = '1' WHERE key = 'librarySearchIndexVersion'").run();
+
+  assert.equal(initializeLibrarySearchIndex(db), true);
+  assert.equal(db.prepare("SELECT 1 FROM library_search_documents WHERE entity_id = 999").get(), undefined);
+  assert.equal(db.prepare("SELECT value FROM settings WHERE key = 'librarySearchIndexVersion'").get().value, "2");
+});
+
 test("search index service remains usable when the FTS schema is unavailable", async () => {
   db.exec(`
     DROP TRIGGER IF EXISTS library_search_documents_ai;

.tests/migration/lidarr-artist-index.test.js

@@ -0,0 +1,155 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import Database from "better-sqlite3";
+import test from "node:test";
+import { ensureUniqueLidarrArtistIdIndex } from "../../backend/config/lidarr-artist-index.js";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+const indexName = "idx_lidarr_artist_id_map_foreign_id";
+
+function boot(dbPath, dataDir) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, ["--input-type=module", "-e",
+      'const { db } = await import("./backend/config/db-sqlite.js"); db.close();',
+    ], {
+      cwd: repoRoot,
+      env: { ...process.env, AURRAL_DB_PATH: dbPath, AURRAL_DATA_DIR: dataDir },
+      stdio: ["ignore", "ignore", "pipe"],
+    });
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    child.once("error", reject);
+    child.once("close", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`Database startup exited ${code}: ${stderr}`));
+    });
+  });
+}
+
+function withDatabase(name, run) {
+  const dataDir = mkdtempSync(path.join(tmpdir(), `aurral-${name}-`));
+  const dbPath = path.join(dataDir, "aurral.db");
+  return Promise.resolve()
+    .then(() => run(dbPath, dataDir))
+    .finally(() => rmSync(dataDir, { recursive: true, force: true }));
+}
+
+function indexState(db) {
+  return db.prepare("PRAGMA index_list(lidarr_artist_id_map)").all()
+    .find((index) => index.name === indexName);
+}
+
+test("concurrent startup creates a unique Lidarr artist index on a fresh database", () =>
+  withDatabase("artist-index-fresh", async (dbPath, dataDir) => {
+    await Promise.all(Array.from({ length: 4 }, () => boot(dbPath, dataDir)));
+    const db = new Database(dbPath);
+    try {
+      assert.equal(indexState(db)?.unique, 1);
+    } finally {
+      db.close();
+    }
+  }));
+
+test("concurrent startup upgrades a nonunique index and preserves the winning mappings", () =>
+  withDatabase("artist-index-upgrade", async (dbPath, dataDir) => {
+    await boot(dbPath, dataDir);
+    const seedDb = new Database(dbPath);
+    seedDb.exec(`
+      DROP INDEX ${indexName};
+      CREATE INDEX ${indexName} ON lidarr_artist_id_map (lidarr_foreign_artist_id);
+      INSERT INTO lidarr_artist_id_map VALUES ('older', 'provider-a', 1);
+      INSERT INTO lidarr_artist_id_map VALUES ('newer', 'provider-a', 2);
+      INSERT INTO lidarr_artist_id_map VALUES ('tie-z', 'provider-b', 3);
+      INSERT INTO lidarr_artist_id_map VALUES ('tie-a', 'provider-b', 3);
+      INSERT INTO lidarr_artist_id_map VALUES ('unrelated', 'provider-c', 4);
+    `);
+    seedDb.close();
+
+    await Promise.all(Array.from({ length: 4 }, () => boot(dbPath, dataDir)));
+    const db = new Database(dbPath);
+    try {
+      assert.equal(indexState(db)?.unique, 1);
+      assert.deepEqual(db.prepare(
+        "SELECT musicbrainz_id, lidarr_foreign_artist_id FROM lidarr_artist_id_map ORDER BY lidarr_foreign_artist_id",
+      ).all(), [
+        { musicbrainz_id: "newer", lidarr_foreign_artist_id: "provider-a" },
+        { musicbrainz_id: "tie-a", lidarr_foreign_artist_id: "provider-b" },
+        { musicbrainz_id: "unrelated", lidarr_foreign_artist_id: "provider-c" },
+      ]);
+    } finally {
+      db.close();
+    }
+  }));
+
+test("startup keeps an existing unique index without changing the schema", () =>
+  withDatabase("artist-index-repeat", async (dbPath, dataDir) => {
+    await boot(dbPath, dataDir);
+    const db = new Database(dbPath);
+    const schemaVersion = db.pragma("schema_version", { simple: true });
+    assert.equal(indexState(db)?.unique, 1);
+    db.close();
+
+    await boot(dbPath, dataDir);
+    const reopened = new Database(dbPath);
+    try {
+      assert.equal(indexState(reopened)?.unique, 1);
+      assert.equal(reopened.pragma("schema_version", { simple: true }), schemaVersion);
+    } finally {
+      reopened.close();
+    }
+  }));
+
+test("index migration needs no write lock when the index is already unique", () =>
+  withDatabase("artist-index-writer", async (dbPath, dataDir) => {
+    await boot(dbPath, dataDir);
+    const writer = new Database(dbPath);
+    const reader = new Database(dbPath);
+    reader.pragma("busy_timeout = 20");
+    writer.exec("BEGIN IMMEDIATE");
+    try {
+      ensureUniqueLidarrArtistIdIndex(reader);
+    } finally {
+      writer.exec("ROLLBACK");
+      reader.close();
+      writer.close();
+    }
+    const db = new Database(dbPath);
+    try {
+      assert.equal(indexState(db)?.unique, 1);
+    } finally {
+      db.close();
+    }
+  }));
+
+test("failed unique index creation restores the legacy index and mappings", () =>
+  withDatabase("artist-index-rollback", async (dbPath, dataDir) => {
+    await boot(dbPath, dataDir);
+    const seedDb = new Database(dbPath);
+    seedDb.exec(`
+      DROP INDEX ${indexName};
+      CREATE INDEX ${indexName} ON lidarr_artist_id_map (lidarr_foreign_artist_id);
+      INSERT INTO lidarr_artist_id_map VALUES ('first', 'provider-a', 1);
+      INSERT INTO lidarr_artist_id_map VALUES ('second', 'provider-a', 2);
+      CREATE TRIGGER prevent_mapping_delete BEFORE DELETE ON lidarr_artist_id_map
+      BEGIN SELECT RAISE(IGNORE); END;
+    `);
+    const schemaVersion = seedDb.pragma("schema_version", { simple: true });
+    seedDb.close();
+
+    await assert.rejects(boot(dbPath, dataDir), /UNIQUE constraint failed/);
+    const db = new Database(dbPath);
+    try {
+      assert.equal(indexState(db)?.unique, 0);
+      assert.equal(db.pragma("schema_version", { simple: true }), schemaVersion);
+      assert.deepEqual(db.prepare(
+        "SELECT musicbrainz_id FROM lidarr_artist_id_map ORDER BY musicbrainz_id",
+      ).all().map(({ musicbrainz_id }) => musicbrainz_id), ["first", "second"]);
+    } finally {
+      db.close();
+    }
+  }));

.tests/migration/v2-migration.test.js

@@ -249,3 +249,27 @@ test("startup applies the consolidated migration once", async () => {
 
   db.close();
 });
+
+test("current schema needs no write lock while another connection writes", async () => {
+  const { dbPath } = createPreMigrationDb();
+  const { initializeSchemaOnStartup, TARGET_SCHEMA_VERSION } = await import(
+    "../../backend/config/schema-migration-v2.js"
+  );
+  const writer = new Database(dbPath);
+  writer.pragma("journal_mode = WAL");
+  writer.prepare("INSERT INTO settings (key, value) VALUES ('schemaVersion', ?)")
+    .run(String(TARGET_SCHEMA_VERSION));
+  const reader = new Database(dbPath);
+  reader.pragma("busy_timeout = 20");
+  writer.exec("BEGIN IMMEDIATE");
+  try {
+    assert.deepEqual(initializeSchemaOnStartup(reader, dbHelpers), {
+      migrated: false,
+      schemaVersion: TARGET_SCHEMA_VERSION,
+    });
+  } finally {
+    writer.exec("ROLLBACK");
+    reader.close();
+    writer.close();
+  }
+});

backend/config/db-sqlite.js

@@ -3,6 +3,7 @@ import path from "path";
 import fs from "fs";
 import { initializeSchemaOnStartup } from "./schema-migration-v2.js";
 import { initializeLibrarySearchIndex } from "./library-search-index.js";
+import { ensureUniqueLidarrArtistIdIndex } from "./lidarr-artist-index.js";
 import { syncDownloadFolderPath } from "../services/downloadFolderConfig.js";
 import { ensureDataDir } from "./data-dir.js";
 
@@ -19,8 +20,18 @@ if (!fs.existsSync(path.dirname(DB_PATH))) {
 const db = new Database(DB_PATH);
 
 db.pragma("foreign_keys = ON");
-db.pragma("journal_mode = WAL");
 db.pragma("busy_timeout = 5000");
+for (let attempt = 0; attempt < 5; attempt++) {
+  try {
+    if (db.pragma("journal_mode", { simple: true }) !== "wal") {
+      db.pragma("journal_mode = WAL");
+    }
+    break;
+  } catch (error) {
+    if (!String(error?.code || "").startsWith("SQLITE_BUSY") || attempt === 4) throw error;
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+  }
+}
 db.pragma("synchronous = NORMAL");
 db.pragma("cache_size = -24000");
 db.pragma("mmap_size = 25165824");
@@ -544,42 +555,7 @@ db.exec(`
     ON library_media_files (track_id, album_id, source, available, created_at DESC);
 `);
 
-const duplicateLidarrArtistIds = db
-  .prepare(
-    `SELECT lidarr_foreign_artist_id
-     FROM lidarr_artist_id_map
-     GROUP BY lidarr_foreign_artist_id
-     HAVING COUNT(*) > 1`,
-  )
-  .all();
-
-if (duplicateLidarrArtistIds.length > 0) {
-  const deleteDuplicateLidarrArtistId = db.prepare(
-    `DELETE FROM lidarr_artist_id_map
-     WHERE lidarr_foreign_artist_id = ?
-       AND musicbrainz_id NOT IN (
-         SELECT musicbrainz_id
-         FROM lidarr_artist_id_map
-         WHERE lidarr_foreign_artist_id = ?
-         ORDER BY updated_at DESC, musicbrainz_id ASC
-         LIMIT 1
-       )`,
-  );
-  db.transaction((duplicates) => {
-    for (const duplicate of duplicates) {
-      deleteDuplicateLidarrArtistId.run(
-        duplicate.lidarr_foreign_artist_id,
-        duplicate.lidarr_foreign_artist_id,
-      );
-    }
-  })(duplicateLidarrArtistIds);
-}
-
-db.exec(`
-  DROP INDEX IF EXISTS idx_lidarr_artist_id_map_foreign_id;
-  CREATE UNIQUE INDEX idx_lidarr_artist_id_map_foreign_id
-    ON lidarr_artist_id_map (lidarr_foreign_artist_id);
-`);
+ensureUniqueLidarrArtistIdIndex(db);
 
 const tableColumns = db
   .prepare("PRAGMA table_info(playlist_download_jobs)")

backend/config/library-search-index.js

@@ -193,31 +193,45 @@ export const populateLibrarySearchDocuments = (db) => {
 };
 
 export function initializeLibrarySearchIndex(db) {
-  const hadSearchIndex = Boolean(
-    db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
-      .get("library_search_fts"),
-  );
-  const version = db
-    .prepare("SELECT value FROM settings WHERE key = ?")
-    .get("librarySearchIndexVersion")?.value;
-  if (version && version !== SEARCH_INDEX_VERSION) {
+  const fts5Enabled = db.prepare("SELECT sqlite_compileoption_used(?) AS enabled")
+    .get("ENABLE_FTS5")?.enabled;
+  const hasCurrentSearchIndex = () => {
+    const hasIndex = Boolean(
+      db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
+        .get("library_search_fts"),
+    );
+    const hasTriggers = db.prepare(`
+      SELECT COUNT(*) AS count FROM sqlite_master
+      WHERE type = 'trigger' AND name IN (
+        'library_search_documents_ai',
+        'library_search_documents_au',
+        'library_search_documents_ad'
+      )
+    `).get().count === 3;
+    const version = db.prepare("SELECT value FROM settings WHERE key = ?")
+      .get("librarySearchIndexVersion")?.value;
+    return { hasIndex, hasTriggers, version };
+  };
+  if (!fts5Enabled) return false;
+  const current = hasCurrentSearchIndex();
+  if (current.hasIndex && current.hasTriggers && current.version === SEARCH_INDEX_VERSION) return true;
+
+  return db.transaction(() => {
+    const { hasIndex, hasTriggers, version } = hasCurrentSearchIndex();
+    if (hasIndex && hasTriggers && version === SEARCH_INDEX_VERSION) return true;
     db.exec(`
       DROP TRIGGER IF EXISTS library_search_documents_ai;
       DROP TRIGGER IF EXISTS library_search_documents_au;
       DROP TRIGGER IF EXISTS library_search_documents_ad;
       DROP TABLE IF EXISTS library_search_fts;
+      DROP TABLE IF EXISTS library_search_documents;
     `);
-  }
-  if (!createSearchSchema(db)) return false;
-  if (version === SEARCH_INDEX_VERSION && hadSearchIndex) return true;
-
-  db.transaction(() => {
-    db.prepare("DELETE FROM library_search_documents").run();
+    if (!createSearchSchema(db)) return false;
     populateLibrarySearchDocuments(db);
     db.prepare("INSERT INTO library_search_fts(library_search_fts) VALUES ('rebuild')").run();
     db.prepare("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)")
       .run("librarySearchIndexVersion", SEARCH_INDEX_VERSION);
     rebuildStoredLibraryGenreStats(db);
-  })();
-  return true;
+    return true;
+  }).immediate();
 }