diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4dc3b3fa..43275a7eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,54 @@ jobs: coverage/coverage-summary.json coverage/lcov.info + test-external-api-database: + name: External API DB migrations (${{ matrix.engine }}) + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - engine: mariadb + image: mariadb:10.3 + - engine: mysql + image: mysql:8.0 + services: + database: + image: ${{ matrix.image }} + env: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: youtarr + ports: + - 3321:3306 + options: >- + --health-cmd="mysqladmin ping --protocol=tcp -h 127.0.0.1 -uroot -prootpass" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20.x' + cache: 'npm' + - name: Pin npm + run: npm install -g npm@11.15.0 --ignore-scripts + - name: Install dependencies + run: npm ci --ignore-scripts + - name: Run external API migration lifecycle + run: npm run test:backend -- migrations/__tests__/externalApiDatabase.integration.test.js --runInBand + env: + EXTERNAL_API_DATABASE_TEST: 'true' + DB_HOST: 127.0.0.1 + DB_PORT: 3321 + DB_USER: root + DB_PASSWORD: rootpass + DB_ADMIN_USER: root + DB_ADMIN_PASSWORD: rootpass + test-frontend: name: Frontend Tests runs-on: ubuntu-latest @@ -331,7 +379,7 @@ jobs: check-all: name: All Checks runs-on: ubuntu-latest - needs: [lint, test-backup-restore, test-cookie-validation, test-backend, test-frontend, test-storybook, security-audit, docs] + needs: [lint, test-backup-restore, test-cookie-validation, test-backend, test-external-api-database, test-frontend, test-storybook, security-audit, docs] if: always() steps: - name: Check all results @@ -341,6 +389,7 @@ jobs: echo "Backup/Restore Tests: ${{ needs.test-backup-restore.result }}" echo "Cookie Loader Tests: ${{ needs.test-cookie-validation.result }}" echo "Backend Tests: ${{ needs.test-backend.result }}" + echo "External API Database Tests: ${{ needs.test-external-api-database.result }}" echo "Frontend Tests: ${{ needs.test-frontend.result }}" echo "Storybook Tests: ${{ needs.test-storybook.result }}" echo "Security Audit: ${{ needs.security-audit.result }}" @@ -350,6 +399,7 @@ jobs: [ "${{ needs.test-backup-restore.result }}" != "success" ] || \ [ "${{ needs.test-cookie-validation.result }}" != "success" ] || \ [ "${{ needs.test-backend.result }}" != "success" ] || \ + [ "${{ needs.test-external-api-database.result }}" != "success" ] || \ [ "${{ needs.test-frontend.result }}" != "success" ] || \ [ "${{ needs.test-storybook.result }}" != "success" ] || \ [ "${{ needs.security-audit.result }}" != "success" ] || \ @@ -362,6 +412,7 @@ jobs: [ "${{ needs.test-backup-restore.result }}" != "success" ] && echo " - Backup/Restore Tests" [ "${{ needs.test-cookie-validation.result }}" != "success" ] && echo " - Cookie Loader Tests" [ "${{ needs.test-backend.result }}" != "success" ] && echo " - Backend Tests" + [ "${{ needs.test-external-api-database.result }}" != "success" ] && echo " - External API Database Tests" [ "${{ needs.test-frontend.result }}" != "success" ] && echo " - Frontend Tests" [ "${{ needs.test-storybook.result }}" != "success" ] && echo " - Storybook Tests" [ "${{ needs.security-audit.result }}" != "success" ] && echo " - Security Audit" diff --git a/docs/AUTHENTICATION.md b/docs/AUTHENTICATION.md index 789af0121..53351e6cb 100644 --- a/docs/AUTHENTICATION.md +++ b/docs/AUTHENTICATION.md @@ -121,6 +121,12 @@ For deployments behind external authentication or not exposed to the internet: API Keys provide persistent authentication for external integrations like bookmarklets, mobile shortcuts, and automation tools. +### External API policy foundation + +The database now stores policy metadata for future versioned external API access without enabling a public endpoint. Existing keys are backfilled with the `legacy_download` role and retain their current single-video behavior. External roles use explicit permissions and channel grants; `admin` is the broad role name. Keys may also carry approval, rating/media, quota, and revocation metadata. These fields are inert until the external API control plane is enabled. + +The schema and migration history are authoritative for field details. API-key values remain hashed and are never recoverable from the database; revocation is represented by `revoked_at` and inactive status. + ### Key Features - **Persistent**: No expiration (unlike session tokens) - **Scoped**: Limited to single video downloads only diff --git a/migrations/20260908100000-add-external-api-key-policy.js b/migrations/20260908100000-add-external-api-key-policy.js new file mode 100644 index 000000000..d63aee691 --- /dev/null +++ b/migrations/20260908100000-add-external-api-key-policy.js @@ -0,0 +1,40 @@ +'use strict'; +const { addColumnIfMissing, removeColumnIfExists } = require('./helpers'); +module.exports = { + async up(q, S) { + const cols = [ + ['role', { type: S.STRING(32), allowNull: false, defaultValue: 'legacy_download' }], + ['auto_approve_video_requests', { type: S.BOOLEAN, allowNull: false, defaultValue: false }], + ['auto_approve_channel_requests', { type: S.BOOLEAN, allowNull: false, defaultValue: false }], + ['auto_approve_delete_requests', { type: S.BOOLEAN, allowNull: false, defaultValue: false }], + ['max_rating_level', { type: S.INTEGER, allowNull: false, defaultValue: 4 }], + ['allow_unrated', { type: S.BOOLEAN, allowNull: false, defaultValue: false }], + ['allowed_media_types', { type: S.JSON, allowNull: true, defaultValue: null }], + ['revoked_at', { type: S.DATE, allowNull: true, defaultValue: null }], + ['allow_video_requests', { type: S.BOOLEAN, allowNull: true, defaultValue: null }], + ['allow_channel_requests', { type: S.BOOLEAN, allowNull: true, defaultValue: null }], + ['allow_delete_video_requests', { type: S.BOOLEAN, allowNull: true, defaultValue: null }], + ['max_active_jobs', { type: S.INTEGER, allowNull: false, defaultValue: 5 }], + ['hourly_write_limit', { type: S.INTEGER, allowNull: false, defaultValue: 30 }], + ['daily_write_limit', { type: S.INTEGER, allowNull: false, defaultValue: 200 }], + ]; + for (const [name, definition] of cols) await addColumnIfMissing(q, 'apikeys', name, definition); + await q.sequelize.query("UPDATE apikeys SET role = 'legacy_download' WHERE role IS NULL OR role = ''"); + await q.sequelize.query("UPDATE apikeys SET allowed_media_types = '[\"video\"]' WHERE allowed_media_types IS NULL"); + await q.changeColumn('apikeys', 'allowed_media_types', { type: S.JSON, allowNull: false }); + for (const [column, roles] of [['allow_video_requests', "'request', 'delete', 'admin'"], ['allow_channel_requests', "'request', 'delete', 'admin'"], ['allow_delete_video_requests', "'delete', 'admin'"]]) { + await q.sequelize.query(`UPDATE apikeys SET ${column} = CASE WHEN role IN (${roles}) THEN true ELSE false END WHERE ${column} IS NULL`); + await q.changeColumn('apikeys', column, { type: S.BOOLEAN, allowNull: false, defaultValue: false }); + } + }, + async down(q) { + const columns = await q.describeTable('apikeys'); + if (columns.role && columns.is_active) { + const revokedAt = columns.revoked_at + ? ', revoked_at = COALESCE(revoked_at, CURRENT_TIMESTAMP)' + : ''; + await q.sequelize.query(`UPDATE apikeys SET is_active = false${revokedAt} WHERE role IS NOT NULL AND role <> 'legacy_download'`); + } + for (const name of ['daily_write_limit', 'hourly_write_limit', 'max_active_jobs', 'allow_delete_video_requests', 'allow_channel_requests', 'allow_video_requests', 'revoked_at', 'allowed_media_types', 'allow_unrated', 'max_rating_level', 'auto_approve_delete_requests', 'auto_approve_channel_requests', 'auto_approve_video_requests', 'role']) await removeColumnIfExists(q, 'apikeys', name); + }, +}; diff --git a/migrations/20260908101000-create-api-key-channel-grants.js b/migrations/20260908101000-create-api-key-channel-grants.js new file mode 100644 index 000000000..41609f28f --- /dev/null +++ b/migrations/20260908101000-create-api-key-channel-grants.js @@ -0,0 +1,15 @@ +'use strict'; +const { createTableIfNotExists, dropTableIfExists, addIndexIfMissing } = require('./helpers'); +module.exports = { + async up(q, S) { + await createTableIfNotExists(q, 'api_key_channel_grants', { + id: { type: S.INTEGER, primaryKey: true, autoIncrement: true, allowNull: false }, + api_key_id: { type: S.INTEGER, allowNull: false, references: { model: 'apikeys', key: 'id' }, onUpdate: 'CASCADE', onDelete: 'CASCADE' }, + channel_id: { type: S.INTEGER, allowNull: false, references: { model: 'channels', key: 'id' }, onUpdate: 'CASCADE', onDelete: 'CASCADE' }, + created_at: { type: S.DATE, allowNull: false, defaultValue: S.NOW }, + }, { charset: 'utf8mb4', collate: 'utf8mb4_unicode_ci' }); + await addIndexIfMissing(q, 'api_key_channel_grants', ['api_key_id', 'channel_id'], { unique: true, name: 'api_key_channel_grants_key_channel_uq' }); + await addIndexIfMissing(q, 'api_key_channel_grants', ['channel_id'], { name: 'api_key_channel_grants_channel_idx' }); + }, + async down(q) { await dropTableIfExists(q, 'api_key_channel_grants'); }, +}; diff --git a/migrations/20260908102000-create-external-requests.js b/migrations/20260908102000-create-external-requests.js new file mode 100644 index 000000000..fea26cd2d --- /dev/null +++ b/migrations/20260908102000-create-external-requests.js @@ -0,0 +1,47 @@ +'use strict'; +const { createTableIfNotExists, dropTableIfExists, addIndexIfMissing } = require('./helpers'); +module.exports = { + async up(q, S) { + await createTableIfNotExists(q, 'external_requests', { + id: { type: S.UUID, primaryKey: true, allowNull: false }, + api_key_id: { + type: S.INTEGER, + allowNull: false, + references: { model: 'apikeys', key: 'id' }, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + }, + channel_id: { + type: S.INTEGER, + allowNull: true, + references: { model: 'channels', key: 'id' }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL', + }, + youtube_id: { type: S.STRING(32), allowNull: true }, + channel_url: { type: S.STRING(500), allowNull: true }, + grant_to_requesting_key: { type: S.BOOLEAN, allowNull: true }, + request_type: { type: S.STRING(20), allowNull: false, defaultValue: 'video' }, + status: { type: S.STRING(20), allowNull: false, defaultValue: 'pending' }, + active_dedupe_key: { type: S.STRING(191), allowNull: true }, + idempotency_hash: { type: S.STRING(64), allowNull: true }, + job_id: { + type: S.UUID, + allowNull: true, + references: { model: 'jobs', key: 'id' }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL', + }, + message: { type: S.STRING(500), allowNull: true }, + created_at: { type: S.DATE, allowNull: false, defaultValue: S.NOW }, + updated_at: { type: S.DATE, allowNull: false, defaultValue: S.NOW }, + decided_at: { type: S.DATE, allowNull: true }, + completed_at: { type: S.DATE, allowNull: true }, + }, { charset: 'utf8mb4', collate: 'utf8mb4_unicode_ci' }); + await addIndexIfMissing(q, 'external_requests', ['active_dedupe_key'], { unique: true, name: 'external_requests_active_dedupe_uq' }); + await addIndexIfMissing(q, 'external_requests', ['api_key_id', 'idempotency_hash'], { unique: true, name: 'external_requests_key_idempotency_uq' }); + await addIndexIfMissing(q, 'external_requests', ['api_key_id', 'created_at'], { name: 'external_requests_key_created_idx' }); + await addIndexIfMissing(q, 'external_requests', ['api_key_id', 'status'], { name: 'external_requests_key_status_idx' }); + }, + async down(q) { await dropTableIfExists(q, 'external_requests'); }, +}; diff --git a/migrations/20260908106000-create-external-api-usage-buckets.js b/migrations/20260908106000-create-external-api-usage-buckets.js new file mode 100644 index 000000000..a06ae3032 --- /dev/null +++ b/migrations/20260908106000-create-external-api-usage-buckets.js @@ -0,0 +1,24 @@ +'use strict'; +const { createTableIfNotExists, dropTableIfExists, addIndexIfMissing } = require('./helpers'); +module.exports = { + async up(q, S) { + await createTableIfNotExists(q, 'external_api_usage_buckets', { + id: { type: S.BIGINT, primaryKey: true, autoIncrement: true, allowNull: false }, + api_key_id: { + type: S.INTEGER, + allowNull: false, + references: { model: 'apikeys', key: 'id' }, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + }, + window_type: { type: S.STRING(8), allowNull: false }, + window_start: { type: S.DATE, allowNull: false }, + accepted_writes: { type: S.INTEGER, allowNull: false, defaultValue: 0 }, + created_at: { type: S.DATE, allowNull: false, defaultValue: S.NOW }, + updated_at: { type: S.DATE, allowNull: false, defaultValue: S.NOW }, + }); + await addIndexIfMissing(q, 'external_api_usage_buckets', ['api_key_id', 'window_type', 'window_start'], { unique: true, name: 'external_api_usage_key_window_uq' }); + await addIndexIfMissing(q, 'external_api_usage_buckets', ['window_start'], { name: 'external_api_usage_window_idx' }); + }, + async down(q) { await dropTableIfExists(q, 'external_api_usage_buckets'); }, +}; diff --git a/migrations/__tests__/externalApiDatabase.integration.test.js b/migrations/__tests__/externalApiDatabase.integration.test.js new file mode 100644 index 000000000..1fbf0c8fd --- /dev/null +++ b/migrations/__tests__/externalApiDatabase.integration.test.js @@ -0,0 +1,206 @@ +'use strict'; + +const path = require('path'); +const { QueryTypes, Sequelize } = require('sequelize'); +const Umzug = require('umzug'); +const { validateDatabaseSchema } = require('../../server/modules/databaseHealthModule'); +const ApiKey = require('../../server/models/apikey'); +const ApiKeyChannelGrant = require('../../server/models/apikeychannelgrant'); +const ExternalRequest = require('../../server/models/externalrequest'); +const ExternalApiUsageBucket = require('../../server/models/externalapiusagebucket'); +const policyMigration = require('../20260908100000-add-external-api-key-policy'); +const requestsMigration = require('../20260908102000-create-external-requests'); +const usageMigration = require('../20260908106000-create-external-api-usage-buckets'); + +const RUN_INTEGRATION = process.env.EXTERNAL_API_DATABASE_TEST === 'true'; +const describeDatabase = RUN_INTEGRATION ? describe : describe.skip; +const BASELINE = '20260830201917-lowercased-table-column-names.js'; +const UNRELATED_CHARSET_MIGRATION = '20250907000000-upgrade-to-utf8mb4-if-needed.js'; +const EXTERNAL_MIGRATIONS = [ + '20260908100000-add-external-api-key-policy.js', + '20260908101000-create-api-key-channel-grants.js', + '20260908102000-create-external-requests.js', + '20260908106000-create-external-api-usage-buckets.js', +]; +const DATABASE_NAME = `youtarr_external_api_${process.pid}`; +const dbOptions = { + dialect: 'mysql', + host: process.env.DB_HOST || '127.0.0.1', + port: Number(process.env.DB_PORT || 3321), + logging: false, + dialectOptions: { charset: 'utf8mb4', supportBigNumbers: true, bigNumberStrings: true }, +}; +const dbUser = process.env.DB_USER || 'root'; +const dbPassword = process.env.DB_PASSWORD || '123qweasd'; +const adminUser = process.env.DB_ADMIN_USER || dbUser; +const adminPassword = process.env.DB_ADMIN_PASSWORD || dbPassword; + +let admin; +let sequelize; +let queryInterface; +let migrator; +let databaseCreated = false; + +const query = (sql, replacements) => sequelize.query(sql, { + type: QueryTypes.SELECT, + replacements, +}); + +const tableNames = async () => query( + 'SELECT TABLE_NAME FROM information_schema.tables WHERE TABLE_SCHEMA = DATABASE()' +).then((rows) => rows.map((row) => row.TABLE_NAME)); + +const indexNames = async (tableName) => queryInterface.showIndex(tableName) + .then((indexes) => indexes.map((index) => index.name)); + +const foreignKeys = async () => query( + 'SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME ' + + 'FROM information_schema.KEY_COLUMN_USAGE ' + + 'WHERE CONSTRAINT_SCHEMA = DATABASE() AND REFERENCED_TABLE_NAME IS NOT NULL ' + + 'AND TABLE_NAME IN (\'external_requests\', \'api_key_channel_grants\', \'external_api_usage_buckets\')' +); + +const createDatabase = async () => { + // The generated name contains only a fixed prefix and numeric PID. + await admin.query(`DROP DATABASE IF EXISTS \`${DATABASE_NAME}\``); + await admin.query(`CREATE DATABASE \`${DATABASE_NAME}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`); + databaseCreated = true; +}; + +const expectFinalSchema = async () => { + await expect(tableNames()).resolves.toEqual(expect.arrayContaining([ + 'apikeys', 'channels', 'api_key_channel_grants', 'external_requests', 'external_api_usage_buckets', + ])); + const keyColumns = await queryInterface.describeTable('apikeys'); + expect(keyColumns).toEqual(expect.objectContaining({ + role: expect.any(Object), allowed_media_types: expect.any(Object), + allow_video_requests: expect.any(Object), allow_channel_requests: expect.any(Object), + allow_delete_video_requests: expect.any(Object), max_active_jobs: expect.any(Object), + hourly_write_limit: expect.any(Object), daily_write_limit: expect.any(Object), + })); + + const requestIndexes = await indexNames('external_requests'); + expect(requestIndexes).toEqual(expect.arrayContaining([ + 'external_requests_active_dedupe_uq', 'external_requests_key_idempotency_uq', + 'external_requests_key_created_idx', 'external_requests_key_status_idx', + ])); + expect(requestIndexes).not.toEqual(expect.arrayContaining([ + 'external_requests_catalog_status_idx', 'external_requests_management_idx', + ])); + expect(await indexNames('api_key_channel_grants')).toEqual(expect.arrayContaining([ + 'api_key_channel_grants_key_channel_uq', 'api_key_channel_grants_channel_idx', + ])); + expect(await indexNames('external_api_usage_buckets')).toEqual(expect.arrayContaining([ + 'external_api_usage_key_window_uq', 'external_api_usage_window_idx', + ])); + + const keys = await foreignKeys(); + expect(keys).toEqual(expect.arrayContaining([ + expect.objectContaining({ TABLE_NAME: 'external_requests', COLUMN_NAME: 'api_key_id', REFERENCED_TABLE_NAME: 'apikeys' }), + expect.objectContaining({ TABLE_NAME: 'external_requests', COLUMN_NAME: 'channel_id', REFERENCED_TABLE_NAME: 'channels' }), + expect.objectContaining({ TABLE_NAME: 'external_requests', COLUMN_NAME: 'job_id', REFERENCED_TABLE_NAME: 'jobs' }), + expect.objectContaining({ TABLE_NAME: 'api_key_channel_grants', COLUMN_NAME: 'api_key_id', REFERENCED_TABLE_NAME: 'apikeys' }), + expect.objectContaining({ TABLE_NAME: 'api_key_channel_grants', COLUMN_NAME: 'channel_id', REFERENCED_TABLE_NAME: 'channels' }), + expect.objectContaining({ TABLE_NAME: 'external_api_usage_buckets', COLUMN_NAME: 'api_key_id', REFERENCED_TABLE_NAME: 'apikeys' }), + ])); +}; + +describeDatabase('external API migration lifecycle on MySQL-compatible engines', () => { + jest.setTimeout(120000); + + beforeAll(async () => { + admin = new Sequelize('mysql', adminUser, adminPassword, dbOptions); + await admin.authenticate(); + await createDatabase(); + sequelize = new Sequelize(DATABASE_NAME, dbUser, dbPassword, dbOptions); + await sequelize.authenticate(); + queryInterface = sequelize.getQueryInterface(); + migrator = new Umzug({ + migrations: { path: path.join(__dirname, '..'), params: [queryInterface, Sequelize] }, + storage: 'sequelize', storageOptions: { sequelize }, logging: false, + }); + // The test database is already created with the target charset. Mark the + // unrelated charset conversion as complete so this suite isolates the + // external API migrations on both MySQL and MariaDB. + await migrator.storage.logMigration(UNRELATED_CHARSET_MIGRATION); + }); + + afterAll(async () => { + if (sequelize) await sequelize.close(); + if (admin) { + if (databaseCreated) { + await admin.query(`DROP DATABASE IF EXISTS \`${DATABASE_NAME}\``); + } + await admin.close(); + } + }); + + test('preserves legacy keys, leaves duplicate channels, and recovers from interrupted reruns', async () => { + await migrator.up({ to: BASELINE }); + await sequelize.query( + 'INSERT INTO apikeys (name, key_hash, key_prefix, usage_count, is_active) VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)', + { replacements: ['active legacy', 'hash-active', 'active', 17, 1, 'inactive legacy', 'hash-inactive', 'inactiv', 23, 0] } + ); + await sequelize.query( + 'INSERT INTO channels (channel_id, title, auto_download_enabled_tabs) ' + + 'VALUES (?, ?, ?), (?, ?, ?)', + { + replacements: [ + 'duplicate-channel', 'first', 'video', + 'duplicate-channel', 'second', 'video', + ], + } + ); + + // Simulate an interrupted policy migration that added only some nullable columns. + await queryInterface.addColumn('apikeys', 'revoked_at', { + type: Sequelize.DATE, + allowNull: true, + }); + await queryInterface.addColumn('apikeys', 'allow_video_requests', { + type: Sequelize.BOOLEAN, + allowNull: true, + }); + + // Apply only this feature's migrations so the regression stays isolated + // from unrelated post-baseline migrations on each database engine. + await migrator.up({ migrations: EXTERNAL_MIGRATIONS }); + await expectFinalSchema(); + const keys = await query('SELECT key_hash, key_prefix, usage_count, is_active, role, allowed_media_types FROM apikeys ORDER BY id'); + expect(keys.slice(-2)).toEqual([ + expect.objectContaining({ key_hash: 'hash-active', key_prefix: 'active', usage_count: 17, is_active: 1, role: 'legacy_download', allowed_media_types: expect.anything() }), + expect.objectContaining({ key_hash: 'hash-inactive', key_prefix: 'inactiv', usage_count: 23, is_active: 0, role: 'legacy_download', allowed_media_types: expect.anything() }), + ]); + const duplicateCount = await query('SELECT COUNT(*) AS count FROM channels WHERE channel_id = ?', ['duplicate-channel']); + expect(Number(duplicateCount[0].count)).toBe(2); + + // Recreate indexes after an interrupted post-table step, then rerun the full migrator. + await queryInterface.removeIndex('external_requests', 'external_requests_key_status_idx'); + await queryInterface.removeIndex('external_api_usage_buckets', 'external_api_usage_window_idx'); + await requestsMigration.up(queryInterface, Sequelize); + await usageMigration.up(queryInterface, Sequelize); + await expectFinalSchema(); + + await sequelize.query("UPDATE apikeys SET role = 'request' WHERE key_hash = 'hash-active'"); + await migrator.down({ migrations: EXTERNAL_MIGRATIONS.slice().reverse() }); + const rolledBack = await query( + 'SELECT key_hash, usage_count, is_active FROM apikeys WHERE key_hash IN (?, ?) ORDER BY key_hash', + ['hash-active', 'hash-inactive'] + ); + expect(rolledBack).toEqual([ + expect.objectContaining({ key_hash: 'hash-active', usage_count: 17, is_active: 0 }), + expect.objectContaining({ key_hash: 'hash-inactive', usage_count: 23, is_active: 0 }), + ]); + expect(await tableNames()).not.toEqual(expect.arrayContaining([ + 'api_key_channel_grants', 'external_requests', 'external_api_usage_buckets', + ])); + + await migrator.up({ migrations: EXTERNAL_MIGRATIONS }); + await policyMigration.up(queryInterface, Sequelize); + await expectFinalSchema(); + const validation = await validateDatabaseSchema(sequelize, { + ApiKey, ApiKeyChannelGrant, ExternalRequest, ExternalApiUsageBucket, + }); + expect(validation).toEqual({ valid: true, errors: [] }); + }); +}); diff --git a/migrations/__tests__/externalApiKeyPolicy.test.js b/migrations/__tests__/externalApiKeyPolicy.test.js new file mode 100644 index 000000000..4159e831d --- /dev/null +++ b/migrations/__tests__/externalApiKeyPolicy.test.js @@ -0,0 +1,68 @@ +'use strict'; + +const migration = require('../20260908100000-add-external-api-key-policy'); + +function queryInterface(columns = {}) { + const operations = []; + return { + operations, + describeTable: jest.fn().mockResolvedValue(columns), + addColumn: jest.fn(async (_table, column) => operations.push(['add', column])), + changeColumn: jest.fn(async (_table, column) => operations.push(['change', column])), + removeColumn: jest.fn(async (_table, column) => operations.push(['remove', column])), + sequelize: { query: jest.fn(async (sql) => operations.push(['query', sql])) }, + }; +} + +describe('external API key policy migration', () => { + test('backfills safe legacy defaults and is idempotent', async () => { + const qi = queryInterface(); + await migration.up(qi, { STRING: () => 'STRING', BOOLEAN: 'BOOLEAN', INTEGER: 'INTEGER', JSON: 'JSON', DATE: 'DATE' }); + expect(qi.operations.filter(([op]) => op === 'add')).toHaveLength(14); + expect(qi.operations.some(([, sql]) => /legacy_download/.test(sql))).toBe(true); + expect(qi.operations).toContainEqual([ + 'query', + "UPDATE apikeys SET allowed_media_types = '[\"video\"]' WHERE allowed_media_types IS NULL", + ]); + expect(qi.operations).toContainEqual(['change', 'allowed_media_types']); + expect(qi.operations.some(([, sql]) => /allow_video_requests/.test(sql))).toBe(true); + + const existing = Object.fromEntries(['role', 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', 'allowed_media_types', 'revoked_at', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', 'max_active_jobs', + 'hourly_write_limit', 'daily_write_limit'].map((key) => [key, {}])); + const repeat = queryInterface(existing); + await migration.up(repeat, { STRING: () => 'STRING', BOOLEAN: 'BOOLEAN', INTEGER: 'INTEGER', JSON: 'JSON', DATE: 'DATE' }); + expect(repeat.operations.filter(([op]) => op === 'add')).toEqual([]); + }); + + test('disables external roles before rollback drops their distinguishing fields', async () => { + const qi = queryInterface({ is_active: {}, role: {}, revoked_at: {}, allowed_media_types: {} }); + await migration.down(qi); + const disableAt = qi.operations.findIndex(([, sql]) => /SET is_active = false/.test(sql || '')); + const removeRoleAt = qi.operations.findIndex(([op, column]) => op === 'remove' && column === 'role'); + expect(disableAt).toBeGreaterThanOrEqual(0); + expect(removeRoleAt).toBeGreaterThan(disableAt); + expect(qi.operations[disableAt][1]).toContain("role <> 'legacy_download'"); + }); + + test('rolls back safely when revoked_at is not yet present', async () => { + const qi = queryInterface({ is_active: {}, role: {}, allowed_media_types: {} }); + await migration.down(qi); + expect(qi.operations[0][1]).toContain('SET is_active = false'); + expect(qi.operations[0][1]).not.toContain('revoked_at'); + expect(qi.operations.some(([operation, column]) => operation === 'remove' && column === 'role')).toBe(true); + }); + + test('skips fail-closed update when role or active state is incomplete', async () => { + for (const columns of [ + { is_active: {}, revoked_at: {}, allowed_media_types: {} }, + { role: {}, revoked_at: {}, allowed_media_types: {} }, + ]) { + const qi = queryInterface(columns); + await migration.down(qi); + expect(qi.operations.filter(([operation]) => operation === 'query')).toEqual([]); + expect(qi.operations.some(([operation]) => operation === 'remove')).toBe(true); + } + }); +}); diff --git a/migrations/__tests__/externalApiUsageBuckets.test.js b/migrations/__tests__/externalApiUsageBuckets.test.js new file mode 100644 index 000000000..790a1afa6 --- /dev/null +++ b/migrations/__tests__/externalApiUsageBuckets.test.js @@ -0,0 +1,49 @@ +'use strict'; + +const migration = require('../20260908106000-create-external-api-usage-buckets'); + +function queryInterface({ tables = ['apikeys'], columns = {} } = {}) { + const operations = []; + return { + operations, + showAllTables: jest.fn().mockImplementation(async () => [...tables]), + describeTable: jest.fn().mockResolvedValue(columns), + showIndex: jest.fn().mockResolvedValue([]), + addColumn: jest.fn(async (_table, column) => operations.push(['addColumn', column])), + removeColumn: jest.fn(async (_table, column) => operations.push(['removeColumn', column])), + createTable: jest.fn(async (table) => { + tables.push(table); + operations.push(['createTable', table]); + }), + dropTable: jest.fn(async (table) => operations.push(['dropTable', table])), + addIndex: jest.fn(async (_table, _fields, options) => + operations.push(['addIndex', options.name])), + }; +} + +describe('external API usage bucket migration', () => { + test('creates durable usage buckets and their indexes', async () => { + const qi = queryInterface(); + await migration.up(qi, { + INTEGER: 'INTEGER', + BIGINT: 'BIGINT', + STRING: jest.fn(() => 'STRING'), + DATE: 'DATE', + NOW: 'NOW', + }); + expect(qi.operations).toEqual(expect.arrayContaining([ + ['createTable', 'external_api_usage_buckets'], + ['addIndex', 'external_api_usage_key_window_uq'], + ['addIndex', 'external_api_usage_window_idx'], + ])); + }); + + test('drops usage storage before removing policy columns', async () => { + const qi = queryInterface({ + tables: ['apikeys', 'external_api_usage_buckets'], + }); + await migration.down(qi); + expect(qi.operations[0]).toEqual(['dropTable', 'external_api_usage_buckets']); + expect(qi.operations.slice(1)).toEqual([]); + }); +}); diff --git a/migrations/__tests__/externalRequests.test.js b/migrations/__tests__/externalRequests.test.js new file mode 100644 index 000000000..fff8fa553 --- /dev/null +++ b/migrations/__tests__/externalRequests.test.js @@ -0,0 +1,60 @@ +'use strict'; + +const migration = require('../20260908102000-create-external-requests'); + +function queryInterface(existing = false) { + const operations = []; + return { + operations, + showAllTables: jest.fn().mockResolvedValue(existing ? ['external_requests'] : []), + showIndex: jest.fn().mockResolvedValue([]), + createTable: jest.fn(async (table, columns) => operations.push(['create', table, columns])), + addIndex: jest.fn(async (_table, fields, options) => operations.push(['index', fields, options])), + dropTable: jest.fn(async (table) => operations.push(['drop', table])), + }; +} + +describe('external requests migration', () => { + const Sequelize = { + UUID: 'UUID', + INTEGER: 'INTEGER', + BOOLEAN: 'BOOLEAN', + STRING: jest.fn((length) => `STRING(${length})`), + DATE: 'DATE', + NOW: 'NOW', + }; + + test('creates scoped request storage and concurrency-safe unique indexes', async () => { + const qi = queryInterface(); + await migration.up(qi, Sequelize); + const create = qi.operations.find(([operation]) => operation === 'create'); + expect(create[2].api_key_id.references).toEqual({ model: 'apikeys', key: 'id' }); + expect(create[2].channel_id.references).toEqual({ model: 'channels', key: 'id' }); + expect(create[2].channel_id.allowNull).toBe(true); + expect(create[2].channel_id.onDelete).toBe('SET NULL'); + expect(create[2].youtube_id.allowNull).toBe(true); + expect(create[2].channel_url).toEqual(expect.objectContaining({ allowNull: true })); + expect(create[2].grant_to_requesting_key).toEqual(expect.objectContaining({ allowNull: true })); + expect(create[2].job_id.onDelete).toBe('SET NULL'); + expect(qi.operations).toContainEqual([ + 'index', + ['active_dedupe_key'], + { unique: true, name: 'external_requests_active_dedupe_uq' }, + ]); + expect(qi.operations).toContainEqual([ + 'index', + ['api_key_id', 'idempotency_hash'], + { unique: true, name: 'external_requests_key_idempotency_uq' }, + ]); + expect(qi.operations.map(([, , options]) => options?.name)).not.toEqual(expect.arrayContaining([ + 'external_requests_catalog_status_idx', + 'external_requests_management_idx', + ])); + }); + + test('rollback removes the feature table and therefore fails closed', async () => { + const qi = queryInterface(true); + await migration.down(qi); + expect(qi.operations).toEqual([['drop', 'external_requests']]); + }); +}); diff --git a/server/models/__tests__/externalApiModels.test.js b/server/models/__tests__/externalApiModels.test.js new file mode 100644 index 000000000..9f3addfbf --- /dev/null +++ b/server/models/__tests__/externalApiModels.test.js @@ -0,0 +1,30 @@ +'use strict'; + +const { ApiKey, ApiKeyChannelGrant, ExternalRequest, ExternalApiUsageBucket } = require('..'); + +describe('external API persistence models', () => { + test('expose safe defaults and constrained policy values', () => { + expect(ApiKey.rawAttributes.role.defaultValue).toBe('legacy_download'); + expect(ApiKey.rawAttributes.role.validate.isIn[0]).toContain('admin'); + expect(ApiKey.rawAttributes.allowed_media_types.defaultValue).toEqual(['video']); + expect(ApiKeyChannelGrant.options.indexes).toEqual(expect.arrayContaining([ + expect.objectContaining({ unique: true, fields: ['api_key_id', 'channel_id'] }), + ])); + expect(ExternalRequest.rawAttributes.status.validate.isIn[0]).toContain('pending'); + expect(ExternalApiUsageBucket.rawAttributes.window_type.validate.isIn[0]).toEqual(['hour', 'day']); + expect(ExternalRequest.options.indexes.map((index) => index.name)).toEqual(expect.arrayContaining([ + 'external_requests_key_created_idx', 'external_requests_key_status_idx', + ])); + expect(ExternalRequest.options.indexes.map((index) => index.name)).not.toEqual(expect.arrayContaining([ + 'external_requests_catalog_status_idx', 'external_requests_management_idx', + ])); + }); + + test('register associations for grants, requests, jobs, and quota buckets', () => { + expect(ApiKey.associations.channelGrants).toBeDefined(); + expect(ApiKey.associations.externalRequests).toBeDefined(); + expect(ApiKey.associations.usageBuckets).toBeDefined(); + expect(ExternalRequest.associations.channel).toBeDefined(); + expect(ExternalRequest.associations.job).toBeDefined(); + }); +}); diff --git a/server/models/apikey.js b/server/models/apikey.js index 05e860f6c..b830ee5fc 100644 --- a/server/models/apikey.js +++ b/server/models/apikey.js @@ -41,6 +41,40 @@ ApiKey.init( allowNull: false, defaultValue: 0, }, + role: { + type: DataTypes.STRING(32), + allowNull: false, + defaultValue: 'legacy_download', + validate: { + isIn: [['legacy_download', 'view', 'request', 'delete', 'admin']], + }, + }, + auto_approve_video_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + auto_approve_channel_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + auto_approve_delete_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + allow_video_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + allow_channel_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + allow_delete_video_requests: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + max_rating_level: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 4 }, + allow_unrated: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }, + allowed_media_types: { + type: DataTypes.JSON, + allowNull: false, + defaultValue: ['video'], + get() { + const stored = this.getDataValue('allowed_media_types'); + if (typeof stored !== 'string') return stored; + try { + return JSON.parse(stored); + } catch { + return stored; + } + }, + }, + max_active_jobs: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 5, validate: { min: 1, max: 5 } }, + hourly_write_limit: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 30, validate: { min: 1, max: 30 } }, + daily_write_limit: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 200, validate: { min: 1, max: 200 } }, + revoked_at: { type: DataTypes.DATE, allowNull: true }, }, { sequelize, @@ -51,4 +85,3 @@ ApiKey.init( ); module.exports = ApiKey; - diff --git a/server/models/apikeychannelgrant.js b/server/models/apikeychannelgrant.js new file mode 100644 index 000000000..8379f1dea --- /dev/null +++ b/server/models/apikeychannelgrant.js @@ -0,0 +1,15 @@ +const { DataTypes, Model } = require('sequelize'); +const { sequelize } = require('../db'); + +class ApiKeyChannelGrant extends Model {} +ApiKeyChannelGrant.init({ + id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true, allowNull: false }, + api_key_id: { type: DataTypes.INTEGER, allowNull: false }, + channel_id: { type: DataTypes.INTEGER, allowNull: false }, + created_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, +}, { sequelize, modelName: 'ApiKeyChannelGrant', tableName: 'api_key_channel_grants', timestamps: false, + indexes: [ + { unique: true, fields: ['api_key_id', 'channel_id'], name: 'api_key_channel_grants_key_channel_uq' }, + { fields: ['channel_id'], name: 'api_key_channel_grants_channel_idx' }, + ] }); +module.exports = ApiKeyChannelGrant; diff --git a/server/models/externalapiusagebucket.js b/server/models/externalapiusagebucket.js new file mode 100644 index 000000000..adbb44983 --- /dev/null +++ b/server/models/externalapiusagebucket.js @@ -0,0 +1,22 @@ +const { DataTypes, Model } = require('sequelize'); +const { sequelize } = require('../db'); +class ExternalApiUsageBucket extends Model {} +ExternalApiUsageBucket.init({ + id: { type: DataTypes.BIGINT, primaryKey: true, autoIncrement: true, allowNull: false }, + api_key_id: { type: DataTypes.INTEGER, allowNull: false }, + window_type: { + type: DataTypes.STRING(8), + allowNull: false, + validate: { isIn: [['hour', 'day']] }, + }, + window_start: { type: DataTypes.DATE, allowNull: false }, + accepted_writes: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 }, + created_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, + updated_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, +}, { sequelize, modelName: 'ExternalApiUsageBucket', tableName: 'external_api_usage_buckets', timestamps: false, + indexes: [ + { unique: true, fields: ['api_key_id', 'window_type', 'window_start'], name: 'external_api_usage_key_window_uq' }, + { fields: ['window_start'], name: 'external_api_usage_window_idx' }, + ], +}); +module.exports = ExternalApiUsageBucket; diff --git a/server/models/externalrequest.js b/server/models/externalrequest.js new file mode 100644 index 000000000..da0339cb0 --- /dev/null +++ b/server/models/externalrequest.js @@ -0,0 +1,30 @@ +const { DataTypes, Model } = require('sequelize'); +const { sequelize } = require('../db'); +const REQUEST_STATUSES = ['pending', 'approved', 'processing', 'completed', 'rejected', 'failed', 'cancelled']; +class ExternalRequest extends Model {} +ExternalRequest.init({ + id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true, allowNull: false }, + api_key_id: { type: DataTypes.INTEGER, allowNull: false }, + channel_id: { type: DataTypes.INTEGER, allowNull: true }, + youtube_id: { type: DataTypes.STRING(32), allowNull: true }, + channel_url: { type: DataTypes.STRING(500), allowNull: true }, + grant_to_requesting_key: { type: DataTypes.BOOLEAN, allowNull: true }, + request_type: { type: DataTypes.STRING(20), allowNull: false, defaultValue: 'video', validate: { isIn: [['video', 'channel', 'delete_video']] } }, + status: { type: DataTypes.STRING(20), allowNull: false, defaultValue: 'pending', validate: { isIn: [REQUEST_STATUSES] } }, + active_dedupe_key: { type: DataTypes.STRING(191), allowNull: true }, + idempotency_hash: { type: DataTypes.STRING(64), allowNull: true }, + job_id: { type: DataTypes.UUID, allowNull: true }, + message: { type: DataTypes.STRING(500), allowNull: true }, + created_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, + updated_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, + decided_at: { type: DataTypes.DATE, allowNull: true }, + completed_at: { type: DataTypes.DATE, allowNull: true }, +}, { sequelize, modelName: 'ExternalRequest', tableName: 'external_requests', timestamps: false, + indexes: [ + { unique: true, fields: ['active_dedupe_key'], name: 'external_requests_active_dedupe_uq' }, + { unique: true, fields: ['api_key_id', 'idempotency_hash'], name: 'external_requests_key_idempotency_uq' }, + { fields: ['api_key_id', 'created_at'], name: 'external_requests_key_created_idx' }, + { fields: ['api_key_id', 'status'], name: 'external_requests_key_status_idx' }, + ] }); +ExternalRequest.REQUEST_STATUSES = REQUEST_STATUSES; +module.exports = ExternalRequest; diff --git a/server/models/index.js b/server/models/index.js index 4710c5f9a..9bd394aed 100644 --- a/server/models/index.js +++ b/server/models/index.js @@ -14,6 +14,9 @@ const VideoWatchStatus = require('./videowatchstatus'); const MediaServerUser = require('./mediaserveruser'); const WatchStatusSyncCursor = require('./watchstatussynccursor'); const ScheduledTaskRun = require('./scheduledtaskrun'); +const ApiKeyChannelGrant = require('./apikeychannelgrant'); +const ExternalRequest = require('./externalrequest'); +const ExternalApiUsageBucket = require('./externalapiusagebucket'); Job.hasMany(JobVideo, { foreignKey: 'job_id', as: 'jobVideos' }); Job.hasMany(JobVideoDownload, { foreignKey: 'job_id', as: 'jobVideoDownloads' }); @@ -34,6 +37,19 @@ PlaylistSyncState.belongsTo(Playlist, { foreignKey: 'playlist_id', targetKey: 'i Video.hasMany(VideoWatchStatus, { foreignKey: 'video_id', as: 'watchStatuses' }); VideoWatchStatus.belongsTo(Video, { foreignKey: 'video_id', as: 'video' }); +ApiKey.hasMany(ApiKeyChannelGrant, { foreignKey: 'api_key_id', as: 'channelGrants' }); +ApiKeyChannelGrant.belongsTo(ApiKey, { foreignKey: 'api_key_id', as: 'apiKey' }); +Channel.hasMany(ApiKeyChannelGrant, { foreignKey: 'channel_id', as: 'apiKeyGrants' }); +ApiKeyChannelGrant.belongsTo(Channel, { foreignKey: 'channel_id', as: 'channel' }); +ApiKey.hasMany(ExternalRequest, { foreignKey: 'api_key_id', as: 'externalRequests' }); +ExternalRequest.belongsTo(ApiKey, { foreignKey: 'api_key_id', as: 'apiKey' }); +Channel.hasMany(ExternalRequest, { foreignKey: 'channel_id', as: 'externalRequests' }); +ExternalRequest.belongsTo(Channel, { foreignKey: 'channel_id', as: 'channel' }); +Job.hasMany(ExternalRequest, { foreignKey: 'job_id', as: 'externalRequests' }); +ExternalRequest.belongsTo(Job, { foreignKey: 'job_id', as: 'job' }); +ApiKey.hasMany(ExternalApiUsageBucket, { foreignKey: 'api_key_id', as: 'usageBuckets' }); +ExternalApiUsageBucket.belongsTo(ApiKey, { foreignKey: 'api_key_id', as: 'apiKey' }); + module.exports = { Job, JobVideo, @@ -50,4 +66,7 @@ module.exports = { MediaServerUser, WatchStatusSyncCursor, ScheduledTaskRun, + ApiKeyChannelGrant, + ExternalRequest, + ExternalApiUsageBucket, }; diff --git a/server/modules/__tests__/channelDownloadGrouper.test.js b/server/modules/__tests__/channelDownloadGrouper.test.js index 0d414b6a7..068cbc698 100644 --- a/server/modules/__tests__/channelDownloadGrouper.test.js +++ b/server/modules/__tests__/channelDownloadGrouper.test.js @@ -2,11 +2,9 @@ // Mock dependencies before requiring the module under test jest.mock('../../models/channel', () => { - const { Model } = require('sequelize'); - class MockChannel extends Model {} - MockChannel.findAll = jest.fn(); - MockChannel.init = jest.fn(() => MockChannel); - return MockChannel; + const Channel = jest.requireActual('../../models/channel'); + Channel.findAll = jest.fn(); + return Channel; }); jest.mock('../configModule', () => ({ diff --git a/server/modules/__tests__/channelPoster.test.js b/server/modules/__tests__/channelPoster.test.js index 34f78d9a1..3f092e881 100644 --- a/server/modules/__tests__/channelPoster.test.js +++ b/server/modules/__tests__/channelPoster.test.js @@ -51,10 +51,12 @@ describe('Channel Poster Functionality', () => { jest.doMock('node-cron', () => ({ schedule: jest.fn() })); - jest.doMock('../../models/channel', () => ({ - findAll: jest.fn(), - findOne: jest.fn() - })); + jest.doMock('../../models/channel', () => { + const Channel = jest.requireActual('../../models/channel'); + Channel.findAll = jest.fn(); + Channel.findOne = jest.fn(); + return Channel; + }); jest.doMock('../../models/channelvideo', () => ({})); jest.doMock('../messageEmitter', () => ({ emitMessage: jest.fn() diff --git a/server/modules/__tests__/downloadModule.test.js b/server/modules/__tests__/downloadModule.test.js index 1918afbd5..07d977692 100644 --- a/server/modules/__tests__/downloadModule.test.js +++ b/server/modules/__tests__/downloadModule.test.js @@ -34,9 +34,11 @@ jest.mock('../channelModule', () => ({ generateChannelsFile: jest.fn(), getEnabledChannelDownloadUrls: jest.fn(), })); -jest.mock('../../models/channel', () => ({ - findOne: jest.fn() -})); +jest.mock('../../models/channel', () => { + const Channel = jest.requireActual('../../models/channel'); + Channel.findOne = jest.fn(); + return Channel; +}); jest.mock('../../models/channelvideo', () => ({ findAll: jest.fn() }));