diff --git a/docs/AUTHENTICATION.md b/docs/AUTHENTICATION.md index 4f95f104..789af012 100644 --- a/docs/AUTHENTICATION.md +++ b/docs/AUTHENTICATION.md @@ -170,13 +170,13 @@ curl -X POST https://your-server.com/api/videos/download \ # List active API keys docker exec youtarr-db mysql -u root -p123qweasd youtarr -e " SELECT id, name, key_prefix, created_at, last_used_at -FROM ApiKeys +FROM apikeys WHERE is_active = 1; " # Revoke a key by ID docker exec youtarr-db mysql -u root -p123qweasd youtarr -e " -UPDATE ApiKeys SET is_active = 0 WHERE id = 1; +UPDATE apikeys SET is_active = 0 WHERE id = 1; " ``` @@ -186,7 +186,7 @@ For detailed API documentation and examples (bookmarklets, mobile shortcuts, Pyt ### Session Configuration - **Duration**: 7 days -- **Storage**: Database table `Sessions` +- **Storage**: Database table `sessions` - **Browser storage**: Token persisted as `authToken` in `localStorage` - **Client header**: Token forwarded on each API request via `x-access-token` @@ -202,7 +202,7 @@ For detailed API documentation and examples (bookmarklets, mobile shortcuts, Pyt ```bash docker exec youtarr-db mysql -u root -p123qweasd youtarr -e " SELECT id, session_token, username, expires_at, is_active -FROM Sessions +FROM sessions WHERE expires_at > NOW() AND is_active = 1; " @@ -211,7 +211,7 @@ WHERE expires_at > NOW() #### Clear All Sessions (Force Re-login) ```bash docker exec youtarr-db mysql -u root -p123qweasd youtarr -e " -DELETE FROM Sessions; +DELETE FROM sessions; " ``` diff --git a/docs/BACKUP_RESTORE.md b/docs/BACKUP_RESTORE.md index 8ce0e69a..f0a01488 100644 --- a/docs/BACKUP_RESTORE.md +++ b/docs/BACKUP_RESTORE.md @@ -12,6 +12,12 @@ On Debian/Ubuntu, install jq with: `sudo apt install jq` ## Quick Start +The `backup.sh` and database restore steps below operate on the bundled +`youtarr-db` container. They do not back up or restore an external database +configured through `DB_HOST`. External database users must use their database +provider's backup tools or a database dump connected to the actual external +server, and preserve their local configuration and metadata separately. + ### Create a Backup ```bash @@ -125,6 +131,49 @@ Skips the confirmation prompt. Use with caution in scripts. ## Migration Scenarios +### Before Updating or Returning to an Earlier Version + +Create a backup before updating Youtarr, especially when an update changes the +database schema. Stop Youtarr first so downloads, settings, and metadata do not +change while the backup is taken. For the bundled database: + +```bash +./stop.sh +./scripts/backup.sh +``` + +Wait for the backup to finish successfully before updating. Keep the archive +and record the Youtarr image version you were running. For an external database, +back up the actual external database using your provider's tools or a database +dump; `backup.sh` alone does not cover that database. + +The table and column rename migration changes the names expected by older +Youtarr versions. **Changing the Docker image back to an older version alone +does not undo the migration.** To return to the version you backed up: + +1. Stop Youtarr and keep it stopped throughout recovery. +2. Restore the backup taken before the update. For the bundled database, use + `./scripts/restore.sh /path/to/pre-update-backup.tar.gz`. For an external + database, restore the pre-update database backup and matching local settings + and metadata using your external backup procedure. +3. After restoring, select the previous image version before starting Youtarr. + For the standard Compose setup, set + `YOUTARR_IMAGE=dialmaster/youtarr:` in `.env`, replacing + the placeholder with the recorded version. Do this after restoration because + `restore.sh` also restores `.env`. Avoid `latest` or `dev` for this recovery. +4. Start Youtarr using the normal startup command for your database setup and + verify your settings and download history. + +The bundled database restore replaces the database, including its schema and +`SequelizeMeta` migration history. Starting the newer image against the restored +database would apply the new migrations again. + +Restoring returns database state, settings, and backed-up metadata to the backup +time; changes made since then are not retained in the restored database. Video +files are not included in these backups. Files downloaded or deleted after the +backup are not reverted, so the restored history may differ from the files on +disk. Back up the output directory separately if you need to restore those files. + ### Moving to a New Computer 1. **On the old computer:** diff --git a/docs/DATABASE.md b/docs/DATABASE.md index eeca9e18..e5e02d6e 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -23,17 +23,17 @@ Youtarr uses MariaDB/MySQL for storing: | Table | Model | Description | | :----------------- | :------------- | :-------------------------------- | | `channels` | `Channel` | YouTube channel information. `m3u_enabled` (boolean, default false): generate a `.m3u` playlist file in the channel folder. `m3u_sort_order` (string, default `oldest_first`): `.m3u` entry order, `oldest_first` or `newest_first`. `auto_removal_protected` (boolean, default false): exclude every video of this channel from auto-removal; only applies while the channel is subscribed (`enabled`), and enabling it clears `auto_removal_keep_recent_count`. `auto_removal_keep_recent_count` (int, nullable): auto-removal always keeps this many of the channel's most recently downloaded videos; mutually exclusive with `auto_removal_protected`, dormant while the channel is unsubscribed. | -| `Videos` | `Video` | Downloaded video metadata. `video_resolution` (VARCHAR(20), nullable): actual pixel dimensions of the downloaded file, e.g. `"1920x1080"`, measured by ffprobe at download time and backfilled by the filesystem rescan; the displayed tier label (e.g. "1080p") is derived client-side (`client/src/utils/videoResolution.ts`), so labeling rules can change without re-probing; NULL = not yet checked or audio-only, `"0x0"` = probe failed (never shown in the UI). | -| `channelvideos` | `ChannelVideo` | Channel <-> video associations. `published_at_source` tracks publishedAt provenance: `exact` (.info.json), `approximate` (yt-dlp flat-playlist date), `estimated` (ordering-only placeholder assigned when YouTube returns a listing with no dates; never displayed), NULL (legacy, treated as approximate) | -| `Jobs` | `Job` | Download job queue. `aux_data` (MEDIUMTEXT, nullable): JSON snapshot of job data other than videos (failed downloads, diagnoses, skip counts, terminated channels), written on job save and merged back on startup load; NULL for jobs recorded before the column existed. | -| `JobVideos` | `JobVideo` | Job <-> video associations | -| `JobVideoDownloads`| `JobVideoDownload`| Download progress tracking | -| `Sessions` | `Session` | User authentication sessions | -| `ApiKeys` | `ApiKey` | API key credentials for external integrations (bookmarklets, shortcuts, automation) | +| `videos` | `Video` | Downloaded video metadata. `video_resolution` (VARCHAR(20), nullable): actual pixel dimensions of the downloaded file, e.g. `"1920x1080"`, measured by ffprobe at download time and backfilled by the filesystem rescan; the displayed tier label (e.g. "1080p") is derived client-side (`client/src/utils/videoResolution.ts`), so labeling rules can change without re-probing; NULL = not yet checked or audio-only, `"0x0"` = probe failed (never shown in the UI). | +| `channelvideos` | `ChannelVideo` | Channel <-> video associations. `published_at_source` tracks `published_at` provenance: `exact` (.info.json), `approximate` (yt-dlp flat-playlist date), `estimated` (ordering-only placeholder assigned when YouTube returns a listing with no dates; never displayed), NULL (legacy, treated as approximate) | +| `jobs` | `Job` | Download job queue. `aux_data` (MEDIUMTEXT, nullable): JSON snapshot of job data other than videos (failed downloads, diagnoses, skip counts, terminated channels), written on job save and merged back on startup load; NULL for jobs recorded before the column existed. | +| `jobvideos` | `JobVideo` | Job <-> video associations | +| `jobvideodownloads`| `JobVideoDownload`| Download progress tracking | +| `sessions` | `Session` | User authentication sessions | +| `apikeys` | `ApiKey` | API key credentials for external integrations (bookmarklets, shortcuts, automation) | | `playlists` | `Playlist` | Subscribed YouTube playlists with per-playlist sync targets and seeded settings. `auto_download_baseline_at` (DATETIME, nullable): seed-then-track baseline for playlist auto-downloads; NULL until the first auto-download run. `sort_order` (STRING NOT NULL, default `'default'`): saved output order for the `.m3u` file and media server sync; `'reversed'` flips the YouTube playlist order. | | `playlistvideos` | `PlaylistVideo` | One row per (playlist, video) with the YouTube playlist position | | `playlist_sync_state` | `PlaylistSyncState` | Per-(playlist, server) sync state: server playlist id, last_synced_at, last_error | -| `subfolders` | `Subfolder` | Durable registry of known subfolder names (id, name unique, createdAt, updatedAt). Backfilled from channels, playlists, and video file paths by the `add-subfolders-table` migration; kept current by register-on-create and register-on-download-override. | +| `subfolders` | `Subfolder` | Durable registry of known subfolder names (id, name unique, created_at, updated_at). Backfilled from channels, playlists, and video file paths by the `add-subfolders-table` migration; kept current by register-on-create and register-on-download-override. | | `video_watch_status` | `VideoWatchStatus` | Per-video, per-media-server, per-user watch state pulled by the watch status sync. Absence of a row means never synced/unknown, not unwatched. Columns: `video_id`, `server_type` (`plex`/`jellyfin`/`emby`), `server_user_id` (Plex owner is `'1'`), `played`, `play_count`, `position_ms`, `percent_watched`, `last_watched_at`, `last_synced_at`. Unique index on `(video_id, server_type, server_user_id)`. | | `media_server_users` | `MediaServerUser` | Media-server account directory populated during watch status sync: `server_type`, `server_user_id`, `server_user_name`. Unique index on `(server_type, server_user_id)`. Used to display which users watched a video. | | `watch_status_sync_cursors` | `WatchStatusSyncCursor` | Durable per-server watch-status sync cursor (unique `server_type`, `cursor` DATETIME). Today only Plex uses it: the newest play-history event scanned, so incremental pulls never permanently skip events. Deleting a row forces a full history re-scan on the next sync. | diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 0f34f484..22d80c01 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -451,7 +451,7 @@ ERROR 1396 (HY000) at line 21: Operation CREATE USER failed for 'root'@'%' 2. If the error persists, check `docker compose logs youtarr` to see which migration is still failing. 3. Manually reconcile the schema for that migration: - Connect to MariaDB: `docker compose exec youtarr-db mysql -u root -p youtarr` - - Drop the duplicate column or table mentioned in the error (for example `ALTER TABLE Videos DROP COLUMN media_type;`), **or** restore a known-good backup. + - Drop the duplicate column or table mentioned in the error (for example `ALTER TABLE videos DROP COLUMN media_type;`), **or** restore a known-good backup. - Exit MySQL and restart the stack. 4. Once the stack is back online, confirm the schema is healthy: `curl http://localhost:3087/api/db-status` reports whether the database connection and schema checks passed, and the startup logs (`docker compose logs youtarr`) show the migration results. diff --git a/migrations/20260830201917-lowercased-table-column-names.js b/migrations/20260830201917-lowercased-table-column-names.js new file mode 100644 index 00000000..e54dacc8 --- /dev/null +++ b/migrations/20260830201917-lowercased-table-column-names.js @@ -0,0 +1,125 @@ +'use strict'; + +const { extractTableName, columnExists } = require('./helpers'); + +// The tableExists function in helpers is case-insensitive so we cannot use it for this migration. +async function tableExists(queryInterface, tableName) { + const tablesRaw = await queryInterface.showAllTables(); + return tablesRaw.some((table) => extractTableName(table) === tableName); +} + +async function tableAndColumnExist(queryInterface, tableName, columnName) { + return await tableExists(queryInterface, tableName) && await columnExists(queryInterface, tableName, columnName); +}; + +const TABLES = [ + ['ApiKeys', 'apikeys'], + ['JobVideoDownloads', 'jobvideodownloads'], + ['JobVideos', 'jobvideos'], + ['Jobs', 'jobs'], + ['Sessions', 'sessions'], + ['Videos', 'videos'], +]; + +async function renameTableResumably(queryInterface, original, lowercase, destination) { + // Recognize temporary names left by either an upgrade or a rollback. + // Compare stored names exactly so distinct tables on LCTN=0 are conflicts, + // while LCTN=1 naturally returns only the lowercase stored name. + const candidates = new Set([original, lowercase, `${original}-tmp`, `${lowercase}-tmp`]); + const tables = (await queryInterface.showAllTables()).map(extractTableName); + const matches = tables.filter((name) => candidates.has(name)); + if (matches.length !== 1) { + throw new Error(`Cannot rename ${original}: expected one table, found ${matches.length} (${matches.join(', ')})`); + } + + const source = matches[0]; + if (source === destination) return; + + if (source.endsWith('-tmp')) { + await queryInterface.renameTable(source, destination); + } else { + // A case-only rename fails on LCTN=2. Each step commits independently, + // so the temporary name must remain recoverable on the next invocation. + const temporary = `${source}-tmp`; + await queryInterface.renameTable(source, temporary); + await queryInterface.renameTable(temporary, destination); + } +} + +async function normalizeTableNames(queryInterface) { + for (const [original, lowercase] of TABLES) { + await renameTableResumably(queryInterface, original, lowercase, lowercase); + } +} + +const COLUMNS = [ + ['channels', 'lastFetchedByTab', 'last_fetched_by_tab'], + ['channelvideos', 'publishedAt', 'published_at'], + ['jobs', 'jobType', 'job_type'], + ['jobs', 'timeCreated', 'time_created'], + ['jobs', 'timeInitiated', 'time_initiated'], + ['media_server_users', 'createdAt', 'created_at'], + ['media_server_users', 'updatedAt', 'updated_at'], + ['playlist_sync_state', 'createdAt', 'created_at'], + ['playlist_sync_state', 'updatedAt', 'updated_at'], + ['playlists', 'createdAt', 'created_at'], + ['playlists', 'lastFetched', 'last_fetched'], + ['playlists', 'updatedAt', 'updated_at'], + ['playlistvideos', 'createdAt', 'created_at'], + ['playlistvideos', 'updatedAt', 'updated_at'], + ['subfolders', 'createdAt', 'created_at'], + ['subfolders', 'updatedAt', 'updated_at'], + ['video_watch_status', 'createdAt', 'created_at'], + ['video_watch_status', 'updatedAt', 'updated_at'], + ['videos', 'audioFilePath', 'audio_file_path'], + ['videos', 'audioFileSize', 'audio_file_size'], + ['videos', 'filePath', 'file_path'], + ['videos', 'fileSize', 'file_size'], + ['videos', 'originalDate', 'original_date'], + ['videos', 'youTubeChannelName', 'youtube_channel_name'], + ['videos', 'youTubeVideoName', 'youtube_video_name'], + ['videos', 'youtubeId', 'youtube_id'], + ['watch_status_sync_cursors', 'createdAt', 'created_at'], + ['watch_status_sync_cursors', 'updatedAt', 'updated_at'], +]; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up (queryInterface, Sequelize) { + await normalizeTableNames(queryInterface); + for (const [tableName, from, to] of COLUMNS) { + if (await tableAndColumnExist(queryInterface, tableName, from)) { + await queryInterface.renameColumn(tableName, from, to); + } + } + + // Sequelize generates invalid SQL for these columns since their default (in the database) is `CURRENT_TIMESTAMP`, see https://github.com/sequelize/sequelize/issues/8868 + if (await tableAndColumnExist(queryInterface, 'sessions', 'createdAt')) { + await queryInterface.sequelize.query('ALTER TABLE sessions CHANGE createdAt created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP'); + } + if (await tableAndColumnExist(queryInterface, 'sessions', 'updatedAt')) { + await queryInterface.sequelize.query('ALTER TABLE sessions CHANGE updatedAt updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'); + } + }, + + async down (queryInterface, Sequelize) { + // Recover tables before touching columns, including an interrupted upgrade + // or a rollback that already restored some original table names. + await normalizeTableNames(queryInterface); + if (await tableAndColumnExist(queryInterface, 'sessions', 'created_at')) { + await queryInterface.sequelize.query('ALTER TABLE sessions CHANGE created_at createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP'); + } + if (await tableAndColumnExist(queryInterface, 'sessions', 'updated_at')) { + await queryInterface.sequelize.query('ALTER TABLE sessions CHANGE updated_at updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'); + } + + for (const [tableName, to, from] of COLUMNS) { + if (await tableAndColumnExist(queryInterface, tableName, from)) { + await queryInterface.renameColumn(tableName, from, to); + } + } + for (const [original, lowercase] of TABLES) { + await renameTableResumably(queryInterface, original, lowercase, original); + } + } +}; diff --git a/migrations/__tests__/lowercasedTableColumnNames.test.js b/migrations/__tests__/lowercasedTableColumnNames.test.js new file mode 100644 index 00000000..ccbcb9a4 --- /dev/null +++ b/migrations/__tests__/lowercasedTableColumnNames.test.js @@ -0,0 +1,282 @@ +'use strict'; + +const migration = require('../20260830201917-lowercased-table-column-names'); + +// Transformations to simulate create/compare behaviour for different values of lower_case_table_names. +const LCTNS = { + 0: { + store: (name) => name, + compare: (a, b) => a === b, + }, + 1: { + store: (name) => name.toLowerCase(), + compare: (a, b) => a.toLowerCase() === b.toLowerCase(), + }, + 2: { + store: (name) => name, + compare: (a, b) => a.toLowerCase() === b.toLowerCase(), + }, +}; + +// The whole schema before this migration. +const PRE_RENAME = { + ApiKeys: ['id', 'name', 'key_hash', 'key_prefix', 'created_at', 'last_used_at', 'is_active', 'usage_count'], + channels: ['id', 'channel_id', 'title', 'url', 'description', 'uploader', 'enabled', 'available_tabs', 'auto_download_enabled_tabs', 'lastFetchedByTab', 'sub_folder', 'video_quality', 'min_duration', 'max_duration', 'title_filter_regex', 'folder_name', 'default_rating', 'audio_format', 'skip_video_folder', 'hidden_tabs', 'terminated_at', 'm3u_enabled', 'm3u_sort_order', 'auto_removal_protected', 'auto_removal_keep_recent_count'], + channelvideos: ['id', 'youtube_id', 'channel_id', 'title', 'thumbnail', 'duration', 'publishedAt', 'availability', 'media_type', 'youtube_removed', 'youtube_removed_checked_at', 'live_status', 'ignored', 'ignored_at', 'published_at_source'], + Jobs: ['id', 'jobType', 'status', 'output', 'timeInitiated', 'timeCreated', 'aux_data'], + JobVideoDownloads: ['id', 'job_id', 'youtube_id', 'file_path', 'status', 'created_at'], + JobVideos: ['id', 'job_id', 'video_id'], + media_server_users: ['id', 'server_type', 'server_user_id', 'server_user_name', 'createdAt', 'updatedAt'], + playlist_sync_state: ['id', 'playlist_id', 'server_type', 'server_playlist_id', 'last_synced_at', 'last_error', 'createdAt', 'updatedAt'], + playlists: ['id', 'playlist_id', 'title', 'url', 'description', 'uploader', 'thumbnail', 'video_count', 'enabled', 'auto_download', 'sync_to_plex', 'sync_to_jellyfin', 'sync_to_emby', 'public_on_servers', 'default_sub_folder', 'video_quality', 'min_duration', 'max_duration', 'title_filter_regex', 'audio_format', 'default_rating', 'lastFetched', 'createdAt', 'updatedAt', 'auto_download_baseline_at', 'sort_order'], + playlistvideos: ['id', 'playlist_id', 'youtube_id', 'position', 'added_at', 'channel_id', 'ignored', 'ignored_at', 'createdAt', 'updatedAt', 'title', 'thumbnail', 'duration', 'channel_name', 'published_at'], + Sessions: ['id', 'session_token', 'username', 'user_agent', 'ip_address', 'expires_at', 'last_used_at', 'is_active', 'createdAt', 'updatedAt'], + subfolders: ['id', 'name', 'createdAt', 'updatedAt'], + video_watch_status: ['id', 'video_id', 'server_type', 'server_user_id', 'played', 'play_count', 'position_ms', 'percent_watched', 'last_watched_at', 'last_synced_at', 'createdAt', 'updatedAt'], + Videos: ['id', 'youtubeId', 'youTubeChannelName', 'youTubeVideoName', 'duration', 'originalDate', 'description', 'channel_id', 'filePath', 'fileSize', 'removed', 'media_type', 'youtube_removed', 'youtube_removed_checked_at', 'last_downloaded_at', 'content_rating', 'age_limit', 'normalized_rating', 'rating_source', 'audioFilePath', 'audioFileSize', 'protected', 'video_resolution'], + watch_status_sync_cursors: ['id', 'server_type', 'cursor', 'createdAt', 'updatedAt'], +}; + +// The whole schema after this migration. +const POST_RENAME = { + apikeys: ['id', 'name', 'key_hash', 'key_prefix', 'created_at', 'last_used_at', 'is_active', 'usage_count'], + channels: ['id', 'channel_id', 'title', 'url', 'description', 'uploader', 'enabled', 'available_tabs', 'auto_download_enabled_tabs', 'last_fetched_by_tab', 'sub_folder', 'video_quality', 'min_duration', 'max_duration', 'title_filter_regex', 'folder_name', 'default_rating', 'audio_format', 'skip_video_folder', 'hidden_tabs', 'terminated_at', 'm3u_enabled', 'm3u_sort_order', 'auto_removal_protected', 'auto_removal_keep_recent_count'], + channelvideos: ['id', 'youtube_id', 'channel_id', 'title', 'thumbnail', 'duration', 'published_at', 'availability', 'media_type', 'youtube_removed', 'youtube_removed_checked_at', 'live_status', 'ignored', 'ignored_at', 'published_at_source'], + jobs: ['id', 'job_type', 'status', 'output', 'time_initiated', 'time_created', 'aux_data'], + jobvideodownloads: ['id', 'job_id', 'youtube_id', 'file_path', 'status', 'created_at'], + jobvideos: ['id', 'job_id', 'video_id'], + media_server_users: ['id', 'server_type', 'server_user_id', 'server_user_name', 'created_at', 'updated_at'], + playlist_sync_state: ['id', 'playlist_id', 'server_type', 'server_playlist_id', 'last_synced_at', 'last_error', 'created_at', 'updated_at'], + playlists: ['id', 'playlist_id', 'title', 'url', 'description', 'uploader', 'thumbnail', 'video_count', 'enabled', 'auto_download', 'sync_to_plex', 'sync_to_jellyfin', 'sync_to_emby', 'public_on_servers', 'default_sub_folder', 'video_quality', 'min_duration', 'max_duration', 'title_filter_regex', 'audio_format', 'default_rating', 'last_fetched', 'created_at', 'updated_at', 'auto_download_baseline_at', 'sort_order'], + playlistvideos: ['id', 'playlist_id', 'youtube_id', 'position', 'added_at', 'channel_id', 'ignored', 'ignored_at', 'created_at', 'updated_at', 'title', 'thumbnail', 'duration', 'channel_name', 'published_at'], + sessions: ['id', 'session_token', 'username', 'user_agent', 'ip_address', 'expires_at', 'last_used_at', 'is_active', 'created_at', 'updated_at'], + subfolders: ['id', 'name', 'created_at', 'updated_at'], + video_watch_status: ['id', 'video_id', 'server_type', 'server_user_id', 'played', 'play_count', 'position_ms', 'percent_watched', 'last_watched_at', 'last_synced_at', 'created_at', 'updated_at'], + videos: ['id', 'youtube_id', 'youtube_channel_name', 'youtube_video_name', 'duration', 'original_date', 'description', 'channel_id', 'file_path', 'file_size', 'removed', 'media_type', 'youtube_removed', 'youtube_removed_checked_at', 'last_downloaded_at', 'content_rating', 'age_limit', 'normalized_rating', 'rating_source', 'audio_file_path', 'audio_file_size', 'protected', 'video_resolution'], + watch_status_sync_cursors: ['id', 'server_type', 'cursor', 'created_at', 'updated_at'], +}; + +// A schema that's somewhere inbetween the pre and post schema, simulating the result of a partial run of the migration/rollback. +const MIXED = { + ...PRE_RENAME, + videos: PRE_RENAME.Videos, + // Because of the order of operations in the migration we cannot ever have a case where the columns of a table are renamed but the table itself is not, the column renames _always_ occur on the lowercased table names. +}; +delete MIXED.Videos; + +// Helper to transform a schema with an LCTN. +const applyLCTN = (template, lctn) => { + const transforms = LCTNS[lctn]; + return Object.fromEntries( + Object.entries(template).map(([table, columns]) => [transforms.store(table), [...columns]]) + ); +}; + +const createMockInterface = ({ template, lctn, interruptTable }) => { + const transforms = LCTNS[lctn]; + const schema = applyLCTN(template, lctn); + + const findTable = (table) => Object.keys(schema).find((name) => transforms.compare(table, name)); + + const mockInterface = { + getSchema: () => schema, + + // Used in tableExists helper. + showAllTables: async () => Object.keys(schema), + + // Used in columnExists helper. + describeTable: async (table) => { + const realTable = findTable(table); + if (realTable === undefined) { + throw new Error(`Table ${table} doesn't exist.`); + } + + return Object.fromEntries(schema[realTable].map((name) => [name, true])); + }, + + renameTable: async (from, to) => { + // Fail after the first rename has committed, before the second begins. + if (from.endsWith('-tmp') && from.toLowerCase() === `${interruptTable}-tmp`) { + interruptTable = undefined; + throw new Error('Simulated interruption between table renames'); + } + const realFrom = findTable(from); + if (realFrom === undefined) { + throw new Error(`Table ${from} doesn't exist.`); + } + + const realTo = transforms.store(to); + if (realFrom === realTo) { + throw new Error(`From (${from}) and to (${to}) resolve to the same name (${realTo}).`); + } + + const existingTo = findTable(to); + if (existingTo !== undefined) { + throw new Error(`Table ${to} already exists (${existingTo}).`); + } + + schema[realTo] = schema[realFrom]; + delete schema[realFrom]; + }, + + renameColumn: async (table, from, to) => { + const realTable = findTable(table); + if (realTable === undefined) { + throw new Error(`Table ${table} doesn't exist.`); + } + + const columns = schema[realTable]; + if (!(columns.includes(from))) { + throw new Error(`Column ${from} doesn't exist.`); + } + if (columns.includes(to)) { + throw new Error(`Column ${to} already exists.`); + } + + columns[columns.indexOf(from)] = to; + }, + + sequelize: { + query: jest.fn().mockImplementation(async (query) => { + const match = /^ALTER TABLE (\w+) CHANGE (\w+) (\w+)/.exec(query); + if (match) { + await mockInterface.renameColumn(match[1], match[2], match[3]); + } + }), + }, + }; + + return mockInterface; +}; + +describe('lowercasedTableColumnNames', () => { + for (const lctn of [0, 1, 2]) { + describe(`with lower_case_table_names=${lctn}`, () => { + for (const direction of ['up', 'down']) { + for (const recovery of ['up', 'down']) { + for (const table of ['apikeys', 'jobvideodownloads', 'jobvideos', 'jobs', 'sessions', 'videos']) { + // LCTN=1 upgrades already have lowercase names and do not rename. + if (direction === 'up' && lctn === 1) continue; + test(`${recovery} recovers ${direction} interrupted at ${table}`, async () => { + const mockQueryInterface = createMockInterface({ + template: direction === 'up' ? PRE_RENAME : POST_RENAME, + lctn, + interruptTable: table, + }); + + await expect(migration[direction](mockQueryInterface)).rejects.toThrow('Simulated interruption'); + expect(Object.keys(mockQueryInterface.getSchema()).some((name) => name.endsWith('-tmp'))).toBe(true); + + await migration[recovery](mockQueryInterface); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(recovery === 'up' ? POST_RENAME : PRE_RENAME, lctn)); + }); + } + } + } + + for (const direction of ['up', 'down']) { + for (const temporary of ['Videos-tmp', 'videos-tmp']) { + test(`${direction} recovers stored ${temporary}`, async () => { + const template = { ...PRE_RENAME, [temporary]: PRE_RENAME.Videos }; + delete template.Videos; + const mockQueryInterface = createMockInterface({ template, lctn }); + + await migration[direction](mockQueryInterface); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(direction === 'up' ? POST_RENAME : PRE_RENAME, lctn)); + }); + } + + test(`${direction} rejects a missing required table`, async () => { + const template = { ...PRE_RENAME }; + delete template.ApiKeys; + const mockQueryInterface = createMockInterface({ template, lctn }); + + await expect(migration[direction](mockQueryInterface)).rejects.toThrow('expected one table, found 0'); + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(template, lctn)); + }); + + test(`${direction} rejects a temporary table alongside the original`, async () => { + const template = { ...PRE_RENAME, 'ApiKeys-tmp': PRE_RENAME.ApiKeys }; + const mockQueryInterface = createMockInterface({ template, lctn }); + + await expect(migration[direction](mockQueryInterface)).rejects.toThrow('expected one table, found 2'); + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(template, lctn)); + }); + + if (lctn === 0) { + test(`${direction} rejects distinct original and lowercase tables`, async () => { + const template = { ...PRE_RENAME, apikeys: PRE_RENAME.ApiKeys }; + const mockQueryInterface = createMockInterface({ template, lctn }); + + await expect(migration[direction](mockQueryInterface)).rejects.toThrow('expected one table, found 2'); + expect(mockQueryInterface.getSchema()).toEqual(template); + }); + } + } + + test('up from clean state', async () => { + const mockQueryInterface = createMockInterface({ + template: PRE_RENAME, + lctn, + }); + + await migration.up(mockQueryInterface); + + expect(mockQueryInterface.sequelize.query).toHaveBeenCalledTimes(2); + expect(mockQueryInterface.sequelize.query).toHaveBeenNthCalledWith( + 1, + 'ALTER TABLE sessions CHANGE createdAt created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP', + ); + expect(mockQueryInterface.sequelize.query).toHaveBeenNthCalledWith( + 2, + 'ALTER TABLE sessions CHANGE updatedAt updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', + ); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(POST_RENAME, lctn)); + }); + + test('up from mixed state', async () => { + const mockQueryInterface = createMockInterface({ + template: MIXED, + lctn, + }); + + await migration.up(mockQueryInterface); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(POST_RENAME, lctn)); + }); + + test('down from clean state', async () => { + const mockQueryInterface = createMockInterface({ + template: POST_RENAME, + lctn, + }); + + await migration.down(mockQueryInterface); + + expect(mockQueryInterface.sequelize.query).toHaveBeenCalledTimes(2); + expect(mockQueryInterface.sequelize.query).toHaveBeenNthCalledWith( + 1, + 'ALTER TABLE sessions CHANGE created_at createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP', + ); + expect(mockQueryInterface.sequelize.query).toHaveBeenNthCalledWith( + 2, + 'ALTER TABLE sessions CHANGE updated_at updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', + ); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(PRE_RENAME, lctn)); + }); + + test('down from mixed state', async () => { const mockQueryInterface = createMockInterface({ + template: MIXED, lctn, + }); + + await migration.down(mockQueryInterface); + + expect(mockQueryInterface.getSchema()).toEqual(applyLCTN(PRE_RENAME, lctn)); + }); + }); + } +}); diff --git a/migrations/helpers.js b/migrations/helpers.js index a6885d9f..f6eb3b8e 100644 --- a/migrations/helpers.js +++ b/migrations/helpers.js @@ -169,6 +169,7 @@ async function removeIndexIfExists(queryInterface, tableName, identifier) { } module.exports = { + extractTableName, tableExists, columnExists, createTableIfNotExists, diff --git a/migrations/lib/jobsUuidCollation.js b/migrations/lib/jobsUuidCollation.js index 24909d68..9ce423b7 100644 --- a/migrations/lib/jobsUuidCollation.js +++ b/migrations/lib/jobsUuidCollation.js @@ -7,6 +7,13 @@ // column created afterwards (errno 150). These helpers detect and // restore the canonical utf8mb4_bin collation so referencing and referenced // columns always match. +// +// LEGACY: do not call from new migrations. This helper targets the +// pre-rename table names (Jobs, JobVideos, JobVideoDownloads) and exists +// only to serve the three historical migrations that already use it, +// all of which sort before 20260830201917-lowercased-table-column-names. +// Any new collation repair must target jobs, jobvideos, and +// jobvideodownloads and should live in a new helper. const REQUIRED_COLLATION = 'utf8mb4_bin'; diff --git a/server/models/apikey.js b/server/models/apikey.js index e893a305..05e860f6 100644 --- a/server/models/apikey.js +++ b/server/models/apikey.js @@ -45,7 +45,7 @@ ApiKey.init( { sequelize, modelName: 'ApiKey', - tableName: 'ApiKeys', + tableName: 'apikeys', timestamps: false, } ); diff --git a/server/models/channel.js b/server/models/channel.js index 460be27d..cf7439fe 100644 --- a/server/models/channel.js +++ b/server/models/channel.js @@ -136,6 +136,7 @@ Channel.init( modelName: 'Channel', timestamps: false, tableName: 'channels', + underscored: true, } ); diff --git a/server/models/channelvideo.js b/server/models/channelvideo.js index 7824abbb..96f99dc8 100644 --- a/server/models/channelvideo.js +++ b/server/models/channelvideo.js @@ -84,6 +84,7 @@ ChannelVideo.init( modelName: 'ChannelVideo', tableName: 'channelvideos', timestamps: false, + underscored: true, } ); diff --git a/server/models/job.js b/server/models/job.js index 6cd8a030..2aa98403 100644 --- a/server/models/job.js +++ b/server/models/job.js @@ -40,8 +40,9 @@ Job.init( { sequelize, modelName: 'Job', - tableName: 'Jobs', + tableName: 'jobs', timestamps: false, + underscored: true, } ); diff --git a/server/models/jobvideo.js b/server/models/jobvideo.js index 59921ed6..b76fa981 100644 --- a/server/models/jobvideo.js +++ b/server/models/jobvideo.js @@ -15,7 +15,7 @@ JobVideo.init( type: DataTypes.UUID, allowNull: false, references: { - model: 'Jobs', + model: 'jobs', key: 'id', }, onUpdate: 'CASCADE', @@ -24,7 +24,7 @@ JobVideo.init( type: DataTypes.INTEGER, allowNull: false, references: { - model: 'Videos', + model: 'videos', key: 'id', }, onUpdate: 'CASCADE', @@ -34,7 +34,7 @@ JobVideo.init( sequelize, modelName: 'JobVideo', timestamps: false, - tableName: 'JobVideos', + tableName: 'jobvideos', } ); diff --git a/server/models/jobvideodownload.js b/server/models/jobvideodownload.js index 2770f83b..b4cc2ac5 100644 --- a/server/models/jobvideodownload.js +++ b/server/models/jobvideodownload.js @@ -15,7 +15,7 @@ JobVideoDownload.init( type: DataTypes.UUID, allowNull: false, references: { - model: 'Jobs', + model: 'jobs', key: 'id', }, onUpdate: 'CASCADE', @@ -44,7 +44,7 @@ JobVideoDownload.init( sequelize, modelName: 'JobVideoDownload', timestamps: false, - tableName: 'JobVideoDownloads', + tableName: 'jobvideodownloads', indexes: [ { fields: ['job_id'] diff --git a/server/models/mediaserveruser.js b/server/models/mediaserveruser.js index e1e3d4ad..92c6cb6d 100644 --- a/server/models/mediaserveruser.js +++ b/server/models/mediaserveruser.js @@ -10,7 +10,13 @@ MediaServerUser.init( server_user_id: { type: DataTypes.STRING, allowNull: false }, server_user_name: { type: DataTypes.STRING, allowNull: true }, }, - { sequelize, modelName: 'MediaServerUser', tableName: 'media_server_users', timestamps: true } + { + sequelize, + modelName: 'MediaServerUser', + tableName: 'media_server_users', + timestamps: true, + underscored: true, + } ); module.exports = MediaServerUser; diff --git a/server/models/playlist.js b/server/models/playlist.js index 32c50c19..2a456ce7 100644 --- a/server/models/playlist.js +++ b/server/models/playlist.js @@ -38,7 +38,13 @@ Playlist.init( // baseline for "download videos added after this timestamp". auto_download_baseline_at: { type: DataTypes.DATE, allowNull: true }, }, - { sequelize, modelName: 'Playlist', tableName: 'playlists', timestamps: true } + { + sequelize, + modelName: 'Playlist', + tableName: 'playlists', + timestamps: true, + underscored: true, + } ); module.exports = Playlist; diff --git a/server/models/playlistsyncstate.js b/server/models/playlistsyncstate.js index 4da93c0a..cde4c03e 100644 --- a/server/models/playlistsyncstate.js +++ b/server/models/playlistsyncstate.js @@ -12,7 +12,13 @@ PlaylistSyncState.init( last_synced_at: { type: DataTypes.DATE, allowNull: true }, last_error: { type: DataTypes.TEXT, allowNull: true }, }, - { sequelize, modelName: 'PlaylistSyncState', tableName: 'playlist_sync_state', timestamps: true } + { + sequelize, + modelName: 'PlaylistSyncState', + tableName: 'playlist_sync_state', + timestamps: true, + underscored: true, + } ); module.exports = PlaylistSyncState; diff --git a/server/models/playlistvideo.js b/server/models/playlistvideo.js index 44ae6d08..dbaeb434 100644 --- a/server/models/playlistvideo.js +++ b/server/models/playlistvideo.js @@ -19,7 +19,13 @@ PlaylistVideo.init( ignored: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, ignored_at: { type: DataTypes.DATE, allowNull: true }, }, - { sequelize, modelName: 'PlaylistVideo', tableName: 'playlistvideos', timestamps: true } + { + sequelize, + modelName: 'PlaylistVideo', + tableName: 'playlistvideos', + timestamps: true, + underscored: true, + } ); module.exports = PlaylistVideo; diff --git a/server/models/session.js b/server/models/session.js index de46f4e6..4c853eef 100644 --- a/server/models/session.js +++ b/server/models/session.js @@ -56,8 +56,9 @@ Session.init( { sequelize, modelName: 'Session', - tableName: 'Sessions', + tableName: 'sessions', timestamps: true, + underscored: true, indexes: [ { fields: ['session_token'] }, { fields: ['expires_at'] }, diff --git a/server/models/subfolder.js b/server/models/subfolder.js index 4a6c43f0..370b28c8 100644 --- a/server/models/subfolder.js +++ b/server/models/subfolder.js @@ -9,7 +9,13 @@ Subfolder.init( // Stored clean (no __ prefix). Unique under utf8mb4_unicode_ci (case-insensitive). name: { type: DataTypes.STRING(100), allowNull: false, unique: true }, }, - { sequelize, modelName: 'Subfolder', tableName: 'subfolders', timestamps: true } + { + sequelize, + modelName: 'Subfolder', + tableName: 'subfolders', + timestamps: true, + underscored: true, + } ); module.exports = Subfolder; diff --git a/server/models/video.js b/server/models/video.js index 669f7f24..1eb7dcfc 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -19,10 +19,12 @@ Video.init( youTubeChannelName: { type: DataTypes.STRING, allowNull: false, + field: 'youtube_channel_name', }, youTubeVideoName: { type: DataTypes.STRING, allowNull: false, + field: 'youtube_video_name', }, duration: { type: DataTypes.INTEGER, @@ -116,8 +118,9 @@ Video.init( { sequelize, modelName: 'Video', - tableName: 'Videos', + tableName: 'videos', timestamps: false, + underscored: true, } ); diff --git a/server/models/videowatchstatus.js b/server/models/videowatchstatus.js index 330bd686..72c5e305 100644 --- a/server/models/videowatchstatus.js +++ b/server/models/videowatchstatus.js @@ -16,7 +16,13 @@ VideoWatchStatus.init( last_watched_at: { type: DataTypes.DATE, allowNull: true }, last_synced_at: { type: DataTypes.DATE, allowNull: false }, }, - { sequelize, modelName: 'VideoWatchStatus', tableName: 'video_watch_status', timestamps: true } + { + sequelize, + modelName: 'VideoWatchStatus', + tableName: 'video_watch_status', + timestamps: true, + underscored: true, + } ); module.exports = VideoWatchStatus; diff --git a/server/models/watchstatussynccursor.js b/server/models/watchstatussynccursor.js index bda5edbe..4fd008b4 100644 --- a/server/models/watchstatussynccursor.js +++ b/server/models/watchstatussynccursor.js @@ -9,7 +9,13 @@ WatchStatusSyncCursor.init( server_type: { type: DataTypes.ENUM('plex', 'jellyfin', 'emby'), allowNull: false, unique: true }, cursor: { type: DataTypes.DATE, allowNull: true }, }, - { sequelize, modelName: 'WatchStatusSyncCursor', tableName: 'watch_status_sync_cursors', timestamps: true } + { + sequelize, + modelName: 'WatchStatusSyncCursor', + tableName: 'watch_status_sync_cursors', + timestamps: true, + underscored: true, + } ); module.exports = WatchStatusSyncCursor; diff --git a/server/modules/__tests__/autoRemovalQueries.test.js b/server/modules/__tests__/autoRemovalQueries.test.js index 488ad7ab..44c096ef 100644 --- a/server/modules/__tests__/autoRemovalQueries.test.js +++ b/server/modules/__tests__/autoRemovalQueries.test.js @@ -48,7 +48,7 @@ describe('autoRemovalQueries', () => { expect(ids).toEqual([5, 3, 9]); const [sql, options] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('Videos.removed = 0'); + expect(sql).toContain('videos.removed = 0'); expect(sql).toContain('ORDER BY timeCreated DESC'); expect(sql).toContain('LIMIT :count'); expect(options.replacements).toEqual({ count: 3 }); @@ -60,7 +60,7 @@ describe('autoRemovalQueries', () => { await autoRemovalQueries.getRecentVideoIds(5); const [sql] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('Videos.protected = 0'); + expect(sql).toContain('videos.protected = 0'); }); test('rethrows when the query fails so callers can fail closed', async () => { @@ -76,8 +76,8 @@ describe('autoRemovalQueries', () => { await autoRemovalQueries.getRecentVideoIds(5); const [sql] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1'); - expect(sql).toContain('COALESCE(ProtChannel.auto_removal_protected, 0) = 0'); + expect(sql).toContain('LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1'); + expect(sql).toContain('COALESCE(protchannel.auto_removal_protected, 0) = 0'); }); }); @@ -102,8 +102,8 @@ describe('autoRemovalQueries', () => { minDaysSinceWatched: 0 }); const [sql, options] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('Videos.removed = 0'); - expect(sql).toContain('Videos.protected = 0'); + expect(sql).toContain('videos.removed = 0'); + expect(sql).toContain('videos.protected = 0'); expect(sql).toContain('EXISTS (WATCHED_PROBE)'); expect(sql).not.toContain(':minVideoAgeDays'); expect(sql).not.toContain(':excludeIds'); @@ -114,7 +114,7 @@ describe('autoRemovalQueries', () => { await autoRemovalQueries.getWatchedRemovalCandidates(); const [sql] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('GROUP BY Videos.id'); + expect(sql).toContain('GROUP BY videos.id'); expect(sql).toMatch(/MAX\(COALESCE\(/); expect(sql).not.toContain('DISTINCT'); }); @@ -147,7 +147,7 @@ describe('autoRemovalQueries', () => { await autoRemovalQueries.getWatchedRemovalCandidates({ excludeIds: [4, 8] }); const [sql, options] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('Videos.id NOT IN (:excludeIds)'); + expect(sql).toContain('videos.id NOT IN (:excludeIds)'); expect(options.replacements).toEqual({ excludeIds: [4, 8] }); }); @@ -162,8 +162,8 @@ describe('autoRemovalQueries', () => { await autoRemovalQueries.getWatchedRemovalCandidates(); const [sql] = mockSequelize.query.mock.calls[0]; - expect(sql).toContain('LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1'); - expect(sql).toContain('COALESCE(ProtChannel.auto_removal_protected, 0) = 0'); + expect(sql).toContain('LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1'); + expect(sql).toContain('COALESCE(protchannel.auto_removal_protected, 0) = 0'); }); }); @@ -196,8 +196,8 @@ describe('autoRemovalQueries', () => { expect(result).toEqual({ channelCount: 2, ids: [10, 11, 20] }); expect(mockSequelize.query).toHaveBeenCalledTimes(3); const [perChannelSql, perChannelOptions] = mockSequelize.query.mock.calls[1]; - expect(perChannelSql).toContain('Videos.channel_id = :channelId'); - expect(perChannelSql).toContain('Videos.protected = 0'); + expect(perChannelSql).toContain('videos.channel_id = :channelId'); + expect(perChannelSql).toContain('videos.protected = 0'); expect(perChannelSql).toContain('ORDER BY timeCreated DESC'); expect(perChannelSql).toContain('LIMIT :count'); expect(perChannelOptions.replacements).toEqual({ channelId: 'UC-aaa', count: 2 }); diff --git a/server/modules/__tests__/databaseHealthModule.test.js b/server/modules/__tests__/databaseHealthModule.test.js index d64104dd..e85b413f 100644 --- a/server/modules/__tests__/databaseHealthModule.test.js +++ b/server/modules/__tests__/databaseHealthModule.test.js @@ -41,9 +41,9 @@ describe('DatabaseHealthModule', () => { Channel: { getTableName: () => 'channels', rawAttributes: { - id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false }, - channel_id: { type: { constructor: { name: 'STRING' } }, allowNull: true }, - title: { type: { constructor: { name: 'STRING' } }, allowNull: true }, + id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false, field: 'id' }, + channelId: { type: { constructor: { name: 'STRING' } }, allowNull: true, field: 'channel_id' }, + title: { type: { constructor: { name: 'STRING' } }, allowNull: true, field: 'title' }, }, }, }; @@ -69,13 +69,13 @@ describe('DatabaseHealthModule', () => { Channel: { getTableName: () => 'channels', rawAttributes: { - id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false }, + id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false, field: 'id' }, }, }, Video: { getTableName: () => 'videos', rawAttributes: { - id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false }, + id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false, field: 'id' }, }, }, }; @@ -101,9 +101,9 @@ describe('DatabaseHealthModule', () => { Channel: { getTableName: () => 'channels', rawAttributes: { - id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false }, - channel_id: { type: { constructor: { name: 'STRING' } }, allowNull: true }, - new_field: { type: { constructor: { name: 'STRING' } }, allowNull: true }, + id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false, field: 'id' }, + channel_id: { type: { constructor: { name: 'STRING' } }, allowNull: true, field: 'channel_id' }, + new_field: { type: { constructor: { name: 'STRING' } }, allowNull: true, field: 'new_field' }, }, }, }; @@ -130,8 +130,8 @@ describe('DatabaseHealthModule', () => { Channel: { getTableName: () => 'channels', rawAttributes: { - id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false }, - channel_id: { type: { constructor: { name: 'STRING' } }, allowNull: false }, // Model says not nullable + id: { type: { constructor: { name: 'INTEGER' } }, allowNull: false, field: 'id' }, + channel_id: { type: { constructor: { name: 'STRING' } }, allowNull: false, field: 'channel_id' }, // Model says not nullable }, }, }; diff --git a/server/modules/__tests__/fileCheckModule.test.js b/server/modules/__tests__/fileCheckModule.test.js index 2db5b362..951c59be 100644 --- a/server/modules/__tests__/fileCheckModule.test.js +++ b/server/modules/__tests__/fileCheckModule.test.js @@ -528,7 +528,7 @@ describe('FileCheckModule', () => { expect(mockSequelize.query).toHaveBeenCalledTimes(1); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET fileSize = ? WHERE id = ?', + 'UPDATE videos SET file_size = ? WHERE id = ?', { replacements: [2000, 1], type: 'UPDATE' @@ -545,7 +545,7 @@ describe('FileCheckModule', () => { expect(mockSequelize.query).toHaveBeenCalledTimes(1); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET removed = ? WHERE id = ?', + 'UPDATE videos SET removed = ? WHERE id = ?', { replacements: [1, 1], type: 'UPDATE' @@ -562,7 +562,7 @@ describe('FileCheckModule', () => { expect(mockSequelize.query).toHaveBeenCalledTimes(1); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET fileSize = ?, removed = ? WHERE id = ?', + 'UPDATE videos SET file_size = ?, removed = ? WHERE id = ?', { replacements: [2000, 0, 1], type: 'UPDATE' @@ -582,7 +582,7 @@ describe('FileCheckModule', () => { expect(mockSequelize.query).toHaveBeenCalledTimes(3); expect(mockSequelize.query).toHaveBeenNthCalledWith( 1, - 'UPDATE Videos SET fileSize = ?, removed = ? WHERE id = ?', + 'UPDATE videos SET file_size = ?, removed = ? WHERE id = ?', { replacements: [2000, 0, 1], type: 'UPDATE' @@ -590,7 +590,7 @@ describe('FileCheckModule', () => { ); expect(mockSequelize.query).toHaveBeenNthCalledWith( 2, - 'UPDATE Videos SET removed = ? WHERE id = ?', + 'UPDATE videos SET removed = ? WHERE id = ?', { replacements: [1, 2], type: 'UPDATE' @@ -598,7 +598,7 @@ describe('FileCheckModule', () => { ); expect(mockSequelize.query).toHaveBeenNthCalledWith( 3, - 'UPDATE Videos SET fileSize = ? WHERE id = ?', + 'UPDATE videos SET file_size = ? WHERE id = ?', { replacements: [5000, 3], type: 'UPDATE' @@ -614,7 +614,7 @@ describe('FileCheckModule', () => { await fileCheckModule.applyVideoUpdates(mockSequelize, mockSequelizeLib, updates); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET removed = ? WHERE id = ?', + 'UPDATE videos SET removed = ? WHERE id = ?', { replacements: [0, 1], type: 'UPDATE' @@ -630,7 +630,7 @@ describe('FileCheckModule', () => { await fileCheckModule.applyVideoUpdates(mockSequelize, mockSequelizeLib, updates); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET removed = ? WHERE id = ?', + 'UPDATE videos SET removed = ? WHERE id = ?', { replacements: [1, 1], type: 'UPDATE' @@ -657,7 +657,7 @@ describe('FileCheckModule', () => { await fileCheckModule.applyVideoUpdates(mockSequelize, mockSequelizeLib, updates); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET fileSize = ? WHERE id = ?', + 'UPDATE videos SET file_size = ? WHERE id = ?', { replacements: [largeSize, 1], type: 'UPDATE' @@ -673,7 +673,7 @@ describe('FileCheckModule', () => { await fileCheckModule.applyVideoUpdates(mockSequelize, mockSequelizeLib, updates); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET fileSize = ? WHERE id = ?', + 'UPDATE videos SET file_size = ? WHERE id = ?', { replacements: [0, 1], type: 'UPDATE' @@ -702,7 +702,7 @@ describe('FileCheckModule', () => { await fileCheckModule.applyVideoUpdates(mockSequelize, mockSequelizeLib, updates); expect(mockSequelize.query).toHaveBeenCalledWith( - 'UPDATE Videos SET filePath = ?, fileSize = ?, removed = ? WHERE id = ?', + 'UPDATE videos SET file_path = ?, file_size = ?, removed = ? WHERE id = ?', expect.objectContaining({ replacements: ['/videos/channel/video [abc123].mkv', 5000, 0, 1] }) diff --git a/server/modules/__tests__/videoDeletionModule.test.js b/server/modules/__tests__/videoDeletionModule.test.js index 7ec037cd..e9fd78f1 100644 --- a/server/modules/__tests__/videoDeletionModule.test.js +++ b/server/modules/__tests__/videoDeletionModule.test.js @@ -927,7 +927,7 @@ describe('VideoDeletionModule', () => { await VideoDeletionModule.getVideosOlderThanThreshold(30); const queryString = mockSequelize.query.mock.calls[0][0]; - expect(queryString).toContain('Videos.protected = 0'); + expect(queryString).toContain('videos.protected = 0'); }); test('should exclude the provided video ids', async () => { @@ -936,7 +936,7 @@ describe('VideoDeletionModule', () => { await VideoDeletionModule.getVideosOlderThanThreshold(30, [7, 9]); const [queryString, options] = mockSequelize.query.mock.calls[0]; - expect(queryString).toContain('Videos.id NOT IN (:excludeIds)'); + expect(queryString).toContain('videos.id NOT IN (:excludeIds)'); expect(options.replacements).toEqual({ ageInDays: 30, excludeIds: [7, 9] }); }); @@ -1068,7 +1068,7 @@ describe('VideoDeletionModule', () => { await VideoDeletionModule.getOldestVideos(10); const queryString = mockSequelize.query.mock.calls[0][0]; - expect(queryString).toContain('Videos.protected = 0'); + expect(queryString).toContain('videos.protected = 0'); }); }); diff --git a/server/modules/__tests__/videosModule.test.js b/server/modules/__tests__/videosModule.test.js index 649403ad..aeca7c1e 100644 --- a/server/modules/__tests__/videosModule.test.js +++ b/server/modules/__tests__/videosModule.test.js @@ -34,7 +34,7 @@ describe('VideosModule', () => { mockWatchStatusQueries = { getWatchedByMap: jest.fn().mockResolvedValue(new Map()), buildWatchedExistsSql: jest.fn().mockReturnValue({ - sql: 'EXISTS (SELECT 1 FROM video_watch_status vws WHERE vws.video_id = Videos.id AND vws.played = 1)', + sql: 'EXISTS (SELECT 1 FROM video_watch_status vws WHERE vws.video_id = videos.id AND vws.played = 1)', replacements: {} }) }; @@ -232,33 +232,33 @@ describe('VideosModule', () => { const sqlQuery = mockSequelize.query.mock.calls[1][0]; // Verify the query contains all expected columns - expect(sqlQuery).toContain('Videos.id'); - expect(sqlQuery).toContain('Videos.youtubeId'); - expect(sqlQuery).toContain('Videos.youTubeChannelName'); - expect(sqlQuery).toContain('Videos.youTubeVideoName'); - expect(sqlQuery).toContain('Videos.duration'); - expect(sqlQuery).toContain('Videos.originalDate'); - expect(sqlQuery).toContain('Videos.description'); - expect(sqlQuery).toContain('Videos.channel_id'); - expect(sqlQuery).toContain('Videos.filePath'); - expect(sqlQuery).toContain('Videos.fileSize'); - expect(sqlQuery).toContain('Videos.removed'); - expect(sqlQuery).toContain('Videos.youtube_removed'); + expect(sqlQuery).toContain('videos.id'); + expect(sqlQuery).toContain('videos.youtube_id AS "youtubeId"'); + expect(sqlQuery).toContain('videos.youtube_channel_name AS "youTubeChannelName"'); + expect(sqlQuery).toContain('videos.youtube_video_name AS "youTubeVideoName"'); + expect(sqlQuery).toContain('videos.duration'); + expect(sqlQuery).toContain('videos.original_date AS "originalDate"'); + expect(sqlQuery).toContain('videos.description'); + expect(sqlQuery).toContain('videos.channel_id'); + expect(sqlQuery).toContain('videos.file_path AS "filePath"'); + expect(sqlQuery).toContain('videos.file_size AS "fileSize"'); + expect(sqlQuery).toContain('videos.removed'); + expect(sqlQuery).toContain('videos.youtube_removed'); // Feeds the 24h existence-check throttle; without it every page load // fires a YouTube oembed check per video. - expect(sqlQuery).toContain('Videos.youtube_removed_checked_at'); - expect(sqlQuery).toContain('Videos.media_type'); - expect(sqlQuery).toContain('Videos.video_resolution'); - expect(sqlQuery).toContain('COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, \'%Y%m%d\')) AS timeCreated'); + expect(sqlQuery).toContain('videos.youtube_removed_checked_at'); + expect(sqlQuery).toContain('videos.media_type'); + expect(sqlQuery).toContain('videos.video_resolution'); + expect(sqlQuery).toContain('COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, \'%Y%m%d\')) AS timeCreated'); // Verify the JOINs expect(sqlQuery).toContain('LEFT JOIN'); - expect(sqlQuery).toContain('JobVideos ON Videos.id = JobVideos.video_id'); - expect(sqlQuery).toContain('Jobs ON Jobs.id = JobVideos.job_id'); + expect(sqlQuery).toContain('jobvideos ON videos.id = jobvideos.video_id'); + expect(sqlQuery).toContain('jobs ON jobs.id = jobvideos.job_id'); // Verify the ORDER BY clause expect(sqlQuery).toContain('ORDER BY'); - expect(sqlQuery).toContain('COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, \'%Y%m%d\')) DESC'); + expect(sqlQuery).toContain('COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, \'%Y%m%d\')) DESC'); // Verify the LIMIT and OFFSET expect(sqlQuery).toContain('LIMIT :limit OFFSET :offset'); @@ -306,7 +306,7 @@ describe('VideosModule', () => { }); test('should handle sequelize-specific errors', async () => { - const sequelizeError = new Error('SequelizeDatabaseError: Table Videos does not exist'); + const sequelizeError = new Error('SequelizeDatabaseError: Table videos does not exist'); mockSequelize.query.mockRejectedValue(sequelizeError); await expect(VideosModule.getVideosPaginated()).rejects.toThrow('SequelizeDatabaseError'); @@ -326,8 +326,8 @@ describe('VideosModule', () => { const replacements = mockSequelize.query.mock.calls[1][1].replacements; expect(countQuery).toContain('WHERE'); - expect(countQuery).toContain('(Videos.youTubeVideoName LIKE :search OR Videos.youTubeChannelName LIKE :search)'); - expect(videosQuery).toContain('(Videos.youTubeVideoName LIKE :search OR Videos.youTubeChannelName LIKE :search)'); + expect(countQuery).toContain('(videos.youtube_video_name LIKE :search OR videos.youtube_channel_name LIKE :search)'); + expect(videosQuery).toContain('(videos.youtube_video_name LIKE :search OR videos.youtube_channel_name LIKE :search)'); expect(replacements.search).toBe('%test video%'); }); @@ -353,7 +353,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated({ sortBy: 'published', sortOrder: 'asc' }); const query1 = mockSequelize.query.mock.calls[1][0]; - expect(query1).toContain('ORDER BY Videos.originalDate ASC'); + expect(query1).toContain('ORDER BY videos.original_date ASC'); jest.clearAllMocks(); @@ -365,7 +365,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated(); const query2 = mockSequelize.query.mock.calls[1][0]; - expect(query2).toContain('ORDER BY COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, \'%Y%m%d\')) DESC'); + expect(query2).toContain('ORDER BY COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, \'%Y%m%d\')) DESC'); }); test('should handle date filters correctly', async () => { @@ -381,8 +381,8 @@ describe('VideosModule', () => { const query = mockSequelize.query.mock.calls[0][0]; const replacements = mockSequelize.query.mock.calls[0][1].replacements; - expect(query).toContain('Videos.originalDate >= :dateFrom'); - expect(query).toContain('Videos.originalDate <= :dateTo'); + expect(query).toContain('videos.original_date >= :dateFrom'); + expect(query).toContain('videos.original_date <= :dateTo'); expect(replacements.dateFrom).toBe('20240101'); // Dates formatted without dashes expect(replacements.dateTo).toBe('20241231'); }); @@ -397,7 +397,7 @@ describe('VideosModule', () => { const query = mockSequelize.query.mock.calls[0][0]; const replacements = mockSequelize.query.mock.calls[0][1].replacements; - expect(query).toContain('Videos.youTubeChannelName = :channelFilter'); + expect(query).toContain('videos.youtube_channel_name = :channelFilter'); expect(replacements.channelFilter).toBe('Test Channel'); }); @@ -409,7 +409,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated({ protectedFilter: 'only' }); const query = mockSequelize.query.mock.calls[0][0]; - expect(query).toContain('Videos.protected = 1'); + expect(query).toContain('videos.protected = 1'); }); test('should apply protectedFilter=exclude to the WHERE clause', async () => { @@ -420,7 +420,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated({ protectedFilter: 'exclude' }); const query = mockSequelize.query.mock.calls[0][0]; - expect(query).toContain('Videos.protected = 0'); + expect(query).toContain('videos.protected = 0'); }); test('should apply missingFilter=only to the WHERE clause', async () => { @@ -431,7 +431,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated({ missingFilter: 'only' }); const query = mockSequelize.query.mock.calls[0][0]; - expect(query).toContain('Videos.removed = 1'); + expect(query).toContain('videos.removed = 1'); }); test('should apply missingFilter=exclude to the WHERE clause', async () => { @@ -442,7 +442,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated({ missingFilter: 'exclude' }); const query = mockSequelize.query.mock.calls[0][0]; - expect(query).toContain('Videos.removed = 0'); + expect(query).toContain('videos.removed = 0'); }); test('should omit missingFilter from the WHERE clause by default', async () => { @@ -453,7 +453,7 @@ describe('VideosModule', () => { await VideosModule.getVideosPaginated(); const query = mockSequelize.query.mock.calls[0][0]; - expect(query).not.toContain('Videos.removed'); + expect(query).not.toContain('videos.removed'); }); test('should apply watchedFilter=only as an EXISTS clause in count and page queries', async () => { @@ -484,7 +484,7 @@ describe('VideosModule', () => { test('should merge watched-rule replacements into the query replacements', async () => { mockWatchStatusQueries.buildWatchedExistsSql.mockReturnValue({ - sql: 'EXISTS (SELECT 1 FROM video_watch_status vws WHERE vws.video_id = Videos.id AND vws.played = 1 AND vws.server_user_id = :watchedPlexOwnerId)', + sql: 'EXISTS (SELECT 1 FROM video_watch_status vws WHERE vws.video_id = videos.id AND vws.played = 1 AND vws.server_user_id = :watchedPlexOwnerId)', replacements: { watchedPlexOwnerId: '1' } }); mockSequelize.query.mockResolvedValueOnce([{ total: 0 }]); @@ -541,7 +541,7 @@ describe('VideosModule', () => { // Check update query was called expect(mockSequelize.query).toHaveBeenCalledTimes(4); const updateQuery = mockSequelize.query.mock.calls[2][0]; - expect(updateQuery).toContain('UPDATE Videos SET'); + expect(updateQuery).toContain('UPDATE videos SET'); }); test('should mark video as removed when file does not exist', async () => { @@ -575,7 +575,7 @@ describe('VideosModule', () => { // Check update query was called expect(mockSequelize.query).toHaveBeenCalledTimes(4); const updateQuery = mockSequelize.query.mock.calls[2][0]; - expect(updateQuery).toContain('UPDATE Videos SET'); + expect(updateQuery).toContain('UPDATE videos SET'); expect(updateQuery).toContain('removed = ?'); }); @@ -942,7 +942,7 @@ describe('VideosModule', () => { expect.any(Function) ); const updateCalls = mockSequelize.query.mock.calls.filter( - ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE Videos SET') + ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE videos SET') ); expect(updateCalls.length).toBe(1); const [, options] = updateCalls[0]; @@ -989,7 +989,7 @@ describe('VideosModule', () => { expect(result.timedOut).toBe(true); const updateCalls = mockSequelize.query.mock.calls.filter( - ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE Videos SET') + ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE videos SET') ); // The first flush of 100 completed probes was persisted before the abort. expect(updateCalls.length).toBe(100); @@ -1038,7 +1038,7 @@ describe('VideosModule', () => { expect(maxInFlight).toBeGreaterThan(1); expect(maxInFlight).toBeLessThanOrEqual(4); const updateCalls = mockSequelize.query.mock.calls.filter( - ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE Videos SET') + ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE videos SET') ); expect(updateCalls.length).toBe(VIDEO_COUNT); }); @@ -1067,7 +1067,7 @@ describe('VideosModule', () => { await VideosModule.backfillVideoMetadata(); const updateCalls = mockSequelize.query.mock.calls.filter( - ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE Videos SET') + ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE videos SET') ); expect(updateCalls.length).toBe(1); const [sql, options] = updateCalls[0]; @@ -1101,7 +1101,7 @@ describe('VideosModule', () => { expect(mockExecFile).not.toHaveBeenCalled(); const updateCalls = mockSequelize.query.mock.calls.filter( - ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE Videos SET') + ([sql]) => typeof sql === 'string' && sql.startsWith('UPDATE videos SET') ); expect(updateCalls.length).toBe(1); const [sql, options] = updateCalls[0]; diff --git a/server/modules/autoRemovalQueries.js b/server/modules/autoRemovalQueries.js index cdde2bdd..6b3c5ab2 100644 --- a/server/modules/autoRemovalQueries.js +++ b/server/modules/autoRemovalQueries.js @@ -4,7 +4,7 @@ const watchStatusQueries = require('./mediaServers/watchStatusQueries'); // Matches the timeCreated calculation used by videosModule.js and the other // auto-removal candidate queries in videoDeletionModule.js. const DOWNLOAD_TIME_SQL = - 'COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, \'%Y%m%d\'))'; + 'COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, \'%Y%m%d\'))'; // Read-only candidate queries for auto-removal (the watched strategy and // the keep-most-recent guard, including per-channel keep-recent); deletion itself stays in videoDeletionModule. @@ -26,15 +26,15 @@ class AutoRemovalQueries { try { const query = ` - SELECT Videos.id, MAX(${DOWNLOAD_TIME_SQL}) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1 - WHERE Videos.removed = 0 - AND Videos.protected = 0 - AND COALESCE(ProtChannel.auto_removal_protected, 0) = 0 - GROUP BY Videos.id + SELECT videos.id, MAX(${DOWNLOAD_TIME_SQL}) AS timeCreated + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1 + WHERE videos.removed = 0 + AND videos.protected = 0 + AND COALESCE(protchannel.auto_removal_protected, 0) = 0 + GROUP BY videos.id HAVING timeCreated IS NOT NULL ORDER BY timeCreated DESC LIMIT :count @@ -64,7 +64,7 @@ class AutoRemovalQueries { try { const channels = await sequelize.query( - `SELECT channel_id, auto_removal_keep_recent_count AS keepCount + `SELECT channel_id, auto_removal_keep_recent_count AS "keepCount" FROM channels WHERE auto_removal_keep_recent_count > 0 AND auto_removal_protected = 0 @@ -76,14 +76,14 @@ class AutoRemovalQueries { const ids = []; for (const channel of channels) { const query = ` - SELECT Videos.id, MAX(${DOWNLOAD_TIME_SQL}) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - WHERE Videos.removed = 0 - AND Videos.protected = 0 - AND Videos.channel_id = :channelId - GROUP BY Videos.id + SELECT videos.id, MAX(${DOWNLOAD_TIME_SQL}) AS timeCreated + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + WHERE videos.removed = 0 + AND videos.protected = 0 + AND videos.channel_id = :channelId + GROUP BY videos.id HAVING timeCreated IS NOT NULL ORDER BY timeCreated DESC LIMIT :count @@ -132,27 +132,27 @@ class AutoRemovalQueries { let excludeClause = ''; if (excludeIds.length > 0) { - excludeClause = ' AND Videos.id NOT IN (:excludeIds)\n'; + excludeClause = ' AND videos.id NOT IN (:excludeIds)\n'; replacements.excludeIds = excludeIds; } const query = ` SELECT - Videos.id, - Videos.youtubeId, - Videos.youTubeVideoName, - Videos.youTubeChannelName, - Videos.fileSize, + videos.id, + videos.youtube_id AS "youtubeId", + videos.youtube_video_name AS "youTubeVideoName", + videos.youtube_channel_name AS "youTubeChannelName", + videos.file_size AS "fileSize", MAX(${DOWNLOAD_TIME_SQL}) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1 - WHERE Videos.removed = 0 - AND Videos.protected = 0 - AND COALESCE(ProtChannel.auto_removal_protected, 0) = 0 + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1 + WHERE videos.removed = 0 + AND videos.protected = 0 + AND COALESCE(protchannel.auto_removal_protected, 0) = 0 AND ${watched.sql} -${excludeClause} GROUP BY Videos.id +${excludeClause} GROUP BY videos.id ${havingClause} ORDER BY timeCreated ASC `; diff --git a/server/modules/channel/__tests__/tabState.test.js b/server/modules/channel/__tests__/tabState.test.js index 2104a4c6..289fe6fd 100644 --- a/server/modules/channel/__tests__/tabState.test.js +++ b/server/modules/channel/__tests__/tabState.test.js @@ -155,7 +155,7 @@ describe('tabState', () => { // Verify it uses JSON_SET with COALESCE to handle NULL const sqlQuery = sequelize.query.mock.calls[0][0]; expect(sqlQuery).toContain('JSON_SET'); - expect(sqlQuery).toContain('COALESCE(lastFetchedByTab, \'{}\')'); + expect(sqlQuery).toContain('COALESCE(last_fetched_by_tab, \'{}\')'); expect(sequelize.query).toHaveBeenCalledWith( expect.any(String), { diff --git a/server/modules/channel/tabState.js b/server/modules/channel/tabState.js index 82f41309..1a894d34 100644 --- a/server/modules/channel/tabState.js +++ b/server/modules/channel/tabState.js @@ -66,8 +66,8 @@ class TabState { // COALESCE handles the case where lastFetchedByTab is NULL await sequelize.query(` UPDATE channels - SET lastFetchedByTab = JSON_SET( - COALESCE(lastFetchedByTab, '{}'), + SET last_fetched_by_tab = JSON_SET( + COALESCE(last_fetched_by_tab, '{}'), :jsonPath, :timestamp ) diff --git a/server/modules/databaseHealthModule.js b/server/modules/databaseHealthModule.js index 4231234c..d639abaa 100644 --- a/server/modules/databaseHealthModule.js +++ b/server/modules/databaseHealthModule.js @@ -46,7 +46,7 @@ async function validateDatabaseSchema(sequelize, models) { const dbColumnSet = new Set(dbColumns); // Get model attributes - const modelAttributes = model.rawAttributes; + const modelAttributes = Object.fromEntries(Object.values(model.rawAttributes).map((attr) => [attr.field, attr])); const modelColumns = Object.keys(modelAttributes); // Check for missing columns in database diff --git a/server/modules/fileCheckModule.js b/server/modules/fileCheckModule.js index e3698db8..6e166e81 100644 --- a/server/modules/fileCheckModule.js +++ b/server/modules/fileCheckModule.js @@ -153,19 +153,19 @@ class FileCheckModule { const values = []; if (update.filePath !== undefined) { - setClauses.push('filePath = ?'); + setClauses.push('file_path = ?'); values.push(update.filePath); } if (update.fileSize !== undefined) { - setClauses.push('fileSize = ?'); + setClauses.push('file_size = ?'); values.push(update.fileSize); } if (update.audioFilePath !== undefined) { - setClauses.push('audioFilePath = ?'); + setClauses.push('audio_file_path = ?'); values.push(update.audioFilePath); } if (update.audioFileSize !== undefined) { - setClauses.push('audioFileSize = ?'); + setClauses.push('audio_file_size = ?'); values.push(update.audioFileSize); } if (update.removed !== undefined) { @@ -176,7 +176,7 @@ class FileCheckModule { if (setClauses.length > 0) { values.push(update.id); await sequelize.query( - `UPDATE Videos SET ${setClauses.join(', ')} WHERE id = ?`, + `UPDATE videos SET ${setClauses.join(', ')} WHERE id = ?`, { replacements: values, type: Sequelize.QueryTypes.UPDATE diff --git a/server/modules/mediaServers/__tests__/watchStatusQueries.test.js b/server/modules/mediaServers/__tests__/watchStatusQueries.test.js index 66f8bea5..433d9ef6 100644 --- a/server/modules/mediaServers/__tests__/watchStatusQueries.test.js +++ b/server/modules/mediaServers/__tests__/watchStatusQueries.test.js @@ -67,7 +67,7 @@ describe('watchStatusQueries', () => { const { sql, replacements } = watchStatusQueries.buildWatchedExistsSql(); expect(sql).toMatch(/^EXISTS \(SELECT 1 FROM video_watch_status/); - expect(sql).toContain('video_id = Videos.id'); + expect(sql).toContain('video_id = videos.id'); expect(sql).toContain('played = 1'); expect(sql).not.toContain('server_type'); expect(replacements).toEqual({}); diff --git a/server/modules/mediaServers/watchStatusQueries.js b/server/modules/mediaServers/watchStatusQueries.js index a1e35a8c..3438bc43 100644 --- a/server/modules/mediaServers/watchStatusQueries.js +++ b/server/modules/mediaServers/watchStatusQueries.js @@ -42,7 +42,7 @@ class WatchStatusQueries { // 'primary' only the Plex owner + configured Jellyfin/Emby users). _watchedRuleConditions() { const config = configModule.getConfig(); - const conditions = ['vws.video_id = Videos.id', 'vws.played = 1']; + const conditions = ['vws.video_id = videos.id', 'vws.played = 1']; const replacements = {}; if (config.watchStatusWatchedRule === 'primary') { conditions.push( diff --git a/server/modules/playlistModule.js b/server/modules/playlistModule.js index 343fbb61..d8450320 100644 --- a/server/modules/playlistModule.js +++ b/server/modules/playlistModule.js @@ -551,15 +551,15 @@ class PlaylistModule { // then the upload date. const downloaded = await sequelize.query( `SELECT - Videos.youtubeId, - Videos.channel_id, - Videos.youTubeChannelName, - COALESCE(Videos.last_downloaded_at, MAX(Jobs.timeCreated), STR_TO_DATE(Videos.originalDate, '%Y%m%d')) AS downloadedAt - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - WHERE Videos.youtubeId IN (:youtubeIds) - GROUP BY Videos.id`, + videos.youtube_id AS "youtubeId", + videos.channel_id, + videos.youtube_channel_name AS "youTubeChannelName", + COALESCE(videos.last_downloaded_at, MAX(jobs.time_created), STR_TO_DATE(videos.original_date, '%Y%m%d')) AS downloadedAt + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + WHERE videos.youtube_id IN (:youtubeIds) + GROUP BY videos.id`, { replacements: { youtubeIds }, type: Sequelize.QueryTypes.SELECT } ); if (!downloaded || !downloaded.length) return; diff --git a/server/modules/videoDeletionModule.js b/server/modules/videoDeletionModule.js index c63bc147..e8d0a26b 100644 --- a/server/modules/videoDeletionModule.js +++ b/server/modules/videoDeletionModule.js @@ -313,27 +313,27 @@ class VideoDeletionModule { try { const excludeClause = excludeIds && excludeIds.length > 0 - ? ' AND Videos.id NOT IN (:excludeIds)\n' + ? ' AND videos.id NOT IN (:excludeIds)\n' : ''; // Use raw SQL query to match the timeCreated calculation in videosModule.js const query = ` SELECT DISTINCT - Videos.id, - Videos.youtubeId, - Videos.youTubeVideoName, - Videos.youTubeChannelName, - Videos.fileSize, - COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1 - WHERE Videos.removed = 0 - AND Videos.protected = 0 - AND COALESCE(ProtChannel.auto_removal_protected, 0) = 0 - AND COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) IS NOT NULL - AND COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) < DATE_SUB(NOW(), INTERVAL :ageInDays DAY) + videos.id, + videos.youtube_id AS "youtubeId", + videos.youtube_video_name AS "youTubeVideoName", + videos.youtube_channel_name AS "youTubeChannelName", + videos.file_size AS "fileSize", + COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) AS timeCreated + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1 + WHERE videos.removed = 0 + AND videos.protected = 0 + AND COALESCE(protchannel.auto_removal_protected, 0) = 0 + AND COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) IS NOT NULL + AND COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) < DATE_SUB(NOW(), INTERVAL :ageInDays DAY) ${excludeClause} ORDER BY timeCreated ASC `; @@ -366,25 +366,25 @@ ${excludeClause} ORDER BY timeCreated ASC try { const excludeClause = excludeIds && excludeIds.length > 0 - ? ' AND Videos.id NOT IN (:excludeIds)\n' + ? ' AND videos.id NOT IN (:excludeIds)\n' : ''; const query = ` SELECT DISTINCT - Videos.id, - Videos.youtubeId, - Videos.youTubeVideoName, - Videos.youTubeChannelName, - Videos.fileSize, - COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id - LEFT JOIN channels AS ProtChannel ON ProtChannel.channel_id = Videos.channel_id AND ProtChannel.enabled = 1 - WHERE Videos.removed = 0 - AND Videos.protected = 0 - AND COALESCE(ProtChannel.auto_removal_protected, 0) = 0 - AND COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) IS NOT NULL + videos.id, + videos.youtube_id AS "youtubeId", + videos.youtube_video_name AS "youTubeVideoName", + videos.youtube_channel_name AS "youTubeChannelName", + videos.file_size AS "fileSize", + COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) AS timeCreated + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id + LEFT JOIN channels AS protchannel ON protchannel.channel_id = videos.channel_id AND protchannel.enabled = 1 + WHERE videos.removed = 0 + AND videos.protected = 0 + AND COALESCE(protchannel.auto_removal_protected, 0) = 0 + AND COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) IS NOT NULL ${excludeClause} ORDER BY timeCreated ASC LIMIT :limit `; diff --git a/server/modules/videosModule.js b/server/modules/videosModule.js index b5dc7645..6c31594d 100644 --- a/server/modules/videosModule.js +++ b/server/modules/videosModule.js @@ -49,35 +49,35 @@ class VideosModule { const replacements = {}; if (search) { - whereConditions.push('(Videos.youTubeVideoName LIKE :search OR Videos.youTubeChannelName LIKE :search)'); + whereConditions.push('(videos.youtube_video_name LIKE :search OR videos.youtube_channel_name LIKE :search)'); replacements.search = `%${search}%`; } if (channelFilter) { - whereConditions.push('Videos.youTubeChannelName = :channelFilter'); + whereConditions.push('videos.youtube_channel_name = :channelFilter'); replacements.channelFilter = channelFilter; } if (dateFrom) { - whereConditions.push('Videos.originalDate >= :dateFrom'); + whereConditions.push('videos.original_date >= :dateFrom'); replacements.dateFrom = dateFrom.replace(/-/g, ''); } if (dateTo) { - whereConditions.push('Videos.originalDate <= :dateTo'); + whereConditions.push('videos.original_date <= :dateTo'); replacements.dateTo = dateTo.replace(/-/g, ''); } if (protectedFilter === 'only') { - whereConditions.push('Videos.protected = 1'); + whereConditions.push('videos.protected = 1'); } else if (protectedFilter === 'exclude') { - whereConditions.push('Videos.protected = 0'); + whereConditions.push('videos.protected = 0'); } if (missingFilter === 'only') { - whereConditions.push('Videos.removed = 1'); + whereConditions.push('videos.removed = 1'); } else if (missingFilter === 'exclude') { - whereConditions.push('Videos.removed = 0'); + whereConditions.push('videos.removed = 0'); } if (watchedFilter === 'only' || watchedFilter === 'exclude') { @@ -91,18 +91,18 @@ class VideosModule { // Build ORDER BY let orderByColumn; if (sortBy === 'published') { - orderByColumn = 'Videos.originalDate'; + orderByColumn = 'videos.original_date'; } else { - orderByColumn = 'COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, \'%Y%m%d\'))'; + orderByColumn = 'COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, \'%Y%m%d\'))'; } const orderByClause = `ORDER BY ${orderByColumn} ${sortOrder.toUpperCase()}`; // Get total count const countQuery = ` - SELECT COUNT(DISTINCT Videos.id) as total - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id + SELECT COUNT(DISTINCT videos.id) as total + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id ${whereClause} `; @@ -116,30 +116,30 @@ class VideosModule { // Get paginated videos const query = ` SELECT - Videos.id, - Videos.youtubeId, - Videos.youTubeChannelName, - Videos.youTubeVideoName, - Videos.duration, - Videos.originalDate, - Videos.description, - Videos.channel_id, - Videos.filePath, - Videos.fileSize, - Videos.audioFilePath, - Videos.audioFileSize, - Videos.removed, - Videos.youtube_removed, - Videos.youtube_removed_checked_at, - Videos.media_type, - Videos.normalized_rating, - Videos.rating_source, - Videos.protected, - Videos.video_resolution, - COALESCE(Videos.last_downloaded_at, Jobs.timeCreated, STR_TO_DATE(Videos.originalDate, '%Y%m%d')) AS timeCreated - FROM Videos - LEFT JOIN JobVideos ON Videos.id = JobVideos.video_id - LEFT JOIN Jobs ON Jobs.id = JobVideos.job_id + videos.id, + videos.youtube_id AS "youtubeId", + videos.youtube_channel_name AS "youTubeChannelName", + videos.youtube_video_name AS "youTubeVideoName", + videos.duration, + videos.original_date AS "originalDate", + videos.description, + videos.channel_id, + videos.file_path AS "filePath", + videos.file_size AS "fileSize", + videos.audio_file_path AS "audioFilePath", + videos.audio_file_size AS "audioFileSize", + videos.removed, + videos.youtube_removed, + videos.youtube_removed_checked_at, + videos.media_type, + videos.normalized_rating, + videos.rating_source, + videos.protected, + videos.video_resolution, + COALESCE(videos.last_downloaded_at, jobs.time_created, STR_TO_DATE(videos.original_date, '%Y%m%d')) AS timeCreated + FROM videos + LEFT JOIN jobvideos ON videos.id = jobvideos.video_id + LEFT JOIN jobs ON jobs.id = jobvideos.job_id ${whereClause} ${orderByClause} LIMIT :limit OFFSET :offset @@ -215,7 +215,7 @@ class VideosModule { } } - // Bulk update Videos table for removed videos + // Bulk update videos table for removed videos if (youtubeUpdates.length > 0) { await Video.update( { youtube_removed: true, youtube_removed_checked_at: new Date() }, @@ -223,7 +223,7 @@ class VideosModule { ); } - // Bulk update Videos table for timestamp-only updates + // Bulk update videos table for timestamp-only updates if (timestampUpdates.length > 0) { await Video.update( { youtube_removed_checked_at: new Date() }, @@ -339,10 +339,10 @@ class VideosModule { // Get all unique channel names from videos table const videoChannelsQuery = ` - SELECT DISTINCT youTubeChannelName - FROM Videos - WHERE youTubeChannelName IS NOT NULL - ORDER BY youTubeChannelName + SELECT DISTINCT youtube_channel_name AS "youTubeChannelName" + FROM videos + WHERE youtube_channel_name IS NOT NULL + ORDER BY youtube_channel_name `; const videoChannels = await sequelize.query(videoChannelsQuery, { @@ -467,19 +467,19 @@ class VideosModule { const replacements = []; if (update.filePath !== undefined) { - setClauses.push('filePath = ?'); + setClauses.push('file_path = ?'); replacements.push(update.filePath); } if (update.fileSize !== undefined) { - setClauses.push('fileSize = ?'); + setClauses.push('file_size = ?'); replacements.push(update.fileSize); } if (update.audioFilePath !== undefined) { - setClauses.push('audioFilePath = ?'); + setClauses.push('audio_file_path = ?'); replacements.push(update.audioFilePath); } if (update.audioFileSize !== undefined) { - setClauses.push('audioFileSize = ?'); + setClauses.push('audio_file_size = ?'); replacements.push(update.audioFileSize); } if (update.video_resolution !== undefined) { @@ -493,7 +493,7 @@ class VideosModule { if (setClauses.length > 0) { replacements.push(update.id); - const query = `UPDATE Videos SET ${setClauses.join(', ')} WHERE id = ?`; + const query = `UPDATE videos SET ${setClauses.join(', ')} WHERE id = ?`; try { await sequelize.query(query, {