diff --git a/AGENTS.md b/AGENTS.md index 2feceacc..85e04bd9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,8 @@ url → detect platform → discover routes → capture each route in a browser - `src/lib/capture.ts` — orchestrates a run. - `src/lib/screenshot/` — the browser work: rendering, settling, DOM capture, CSS aggregation, interaction capture, fluid learning. - `src/lib/capture-export.ts` — turns captured routes into the portable `website/` tree: route paths, link rewriting, media localization, diagnostics. It consumes one responsive assembly result rather than classifying and assembling separately, and localizes one portable media plan rather than re-selecting families while copying. Publication of that tree and the export-owned root sidecars is one same-filesystem transaction (`src/lib/export-publication.ts`). Pre-commit failures restore the previous public generation; post-commit cleanup failures keep the new generation. A stale recoverer is not stolen, and a malformed journal fails closed. See `docs/export-publication.md`. -- `src/lib/portable-media-plan.ts` — the portable media plan. One pass over retained pages records which known reference strings meet the raw replacement-boundary check, then releases each page. Each family gets one eligibility, homepage-priority, byte-budget, and content-hash decision, in the original family order. The plan owns the existing rendition limits; output names, missing-media reasons, and asset evidence stay with the exporter. +- `src/lib/portable-media-plan.ts` — the portable media plan. One pass over retained pages records which known reference strings meet the raw replacement-boundary check, then releases each page. Each family gets one eligibility, homepage-priority, byte-budget, and content-hash decision, in the original family order. The plan owns the existing rendition limits. +- `src/lib/portable-media.ts` — admitted-media materialization. It consumes the plan and retained family references, copies/deduplicates files in family order, allocates names and projects exclusion/failed-media fallbacks. Its stage-owned indexes seed resource materialization; the returned media summary and diagnostics feed evidence projection. - `src/lib/portable-resources.ts` — captured-resource materialization. It owns recursive dependency copying, cycles, collision/deduplication state and captured-response media fallback. Media indexes are read-only seeds; the stage returns assets, local paths, replacements and diagnostics for page/evidence projection. Embedded HTML uses the caller's page-sanitization policy. Shared asset/path primitives live in `portable-assets.ts`; dependency and replacement semantics live in `portable-references.ts`. - `src/lib/capture-export-evidence.ts` — evidence ownership. It indexes bounded source asset references before localization, builds semantic shards, then projects geometry, source profiles, state summaries, cleanup coverage, diagnostics and the receipt into the private generation after rendering. Report/receipt decisions share one owner; the existing schema constants are re-exported from `capture-export.ts`. - `src/lib/responsive-assembly.ts` — one responsive assembly. The source pair is the raw captures: it decides the binding phone-only body-class gate and which documents are assembled. A raw collapse is rendered from that same analysis. A raw structural dual is assembled from the already-normalized portable pair, and that emitted analysis — not a second look at the source pair — is what the receipt records. Portable rendering stays ordered around the choice: dual inputs are already normalized; a raw collapse is normalized after assembly. A missing body ships the desktop document alone and records that, even when the source pair had a binding gate. diff --git a/src/lib/capture-export.ts b/src/lib/capture-export.ts index ebef9bd5..f3983623 100644 --- a/src/lib/capture-export.ts +++ b/src/lib/capture-export.ts @@ -1,6 +1,5 @@ import { createHash } from 'node:crypto'; import { - copyFileSync, existsSync, mkdirSync, readFileSync, @@ -56,7 +55,8 @@ import { isSourcePromotion } from './source-cleanup.js'; import { sameOriginPageAnchors } from './screenshot/unscheduled-anchors.js'; import { srcsetCandidates, srcsetReferences } from './srcset.js'; import { resolveDocumentReferences } from './document-resource-base.js'; -import { pathWithin, portableAssetUrl, uniqueAssetPath, TRANSPARENT_IMAGE_DATA_URL } from './portable-assets.js'; +import { pathWithin } from './portable-assets.js'; +import { materializePortableMedia, type FailedPortableMedia } from './portable-media.js'; import { isSrcsetShaped, elementSrcReferences, omitDegenerateReplacements, preparePortableReplacements } from './portable-references.js'; import { materializePortableResources } from './portable-resources.js'; import { collectAssetEvidenceReferences, buildSemanticEvidenceArtifacts, writeCaptureEvidence, UNCAPTURED_ROUTE_REASON, type CaptureFluidEvidence, type CaptureDocumentFluidEvidence, type SemanticEvidencePage } from './capture-export-evidence.js'; @@ -899,31 +899,6 @@ function mediaDimension( sourceUrl: string ): number { ); } -function portableMediaBasename( candidate: MediaCandidate ): string { - const localName = basename( candidate.localPath ); - if ( - /^\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg|wav|woff2?|ttf|otf)$/i.test( - extname( localName ) - ) - ) { - return localName; - } - - const cleanedUrl = candidate.sourceUrl.replace( /&(?:quot|apos|amp);?$/i, '' ); - const sourceExtension = extname( basename( new URL( cleanedUrl ).pathname ) ); - if ( - ! /^\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg|wav|woff2?|ttf|otf)$/i.test( - sourceExtension - ) - ) { - return localName; - } - return `${ localName.slice( - 0, - localName.length - extname( localName ).length - ) }${ sourceExtension.toLowerCase() }`; -} - function routeMatchesSourceOrigin( url: string, sourceUrl: string ): boolean { return sameHttpSite( url, sourceUrl ); } @@ -1480,9 +1455,6 @@ function buildExportCapture( } ); const semanticEvidence = semanticPages.length > 0 ? buildSemanticEvidenceArtifacts( semanticPages ) : undefined; - let mediaReplacements = new Map< string, string >(); - const unresolvedMedia: Array< { url: string; error: string } > = []; - const assets: Array< { sourceUrl: string; path: string } > = []; const mediaStubs = MediaStubStore.load( outputDir ); const assetReferenceLocations = collectAssetEvidenceReferences( retainedEntries, @@ -1493,7 +1465,7 @@ function buildExportCapture( const { rendered: renderedMediaReferences, retained: retainedMediaFamilies } = retainedMediaReferenceInventory( retainedEntries ); const mediaFamilies = new Map< string, MediaCandidate[] >(); - const failedMedia: Array< { sourceUrl: string; error: string; references: string[] } > = []; + const failedMedia: FailedPortableMedia[] = []; const capturedPages = new Set( [ options.sourceUrl, ...retainedEntries.map( ( entry ) => entry.url ) ].flatMap( ( url ) => { try { @@ -1534,7 +1506,7 @@ function buildExportCapture( ); const isReferenced = retainedMediaFamilies.has( family ) || exactReferences.length > 0; if ( stub.status === 'error' && isReferenced ) { - failedMedia.push( { sourceUrl, error: stub.error ?? 'media download failed', references } ); + failedMedia.push( { family, sourceUrl, error: stub.error ?? 'media download failed', references } ); continue; } if ( @@ -1562,93 +1534,12 @@ function buildExportCapture( portableMediaBudget, entrypointEntry.htmlPath, ); - let retainedExternalMediaCount = 0; - const localizedMediaFamilies = new Set< string >(); - const portableUrlByFamily = new Map< string, string >(); - const assetPathsByHash = new Map< string, string >(); - const assetHashesByPath = new Map< string, string >(); - let portablePathsBySource = new Map< string, string >(); - for ( const decision of portableMediaPlan.families ) { - const { family, candidates, eligible, admitted } = decision; - if ( decision.outcome === 'limit-excluded' ) { - for ( const reference of retainedMediaFamilies.get( family ) ?? [] ) - mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); - for ( const candidate of candidates ) { - for ( const reference of candidate.references ) { - mediaReplacements.set( reference, candidate.sourceUrl ); - } - unresolvedMedia.push( { - url: candidate.sourceUrl, - error: 'removed because media exceeds portable size or dimension limits', - } ); - retainedExternalMediaCount++; - } - continue; - } - if ( decision.outcome === 'budget-excluded' ) { - for ( const candidate of candidates ) { - for ( const reference of candidate.references ) { - mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); - } - } - unresolvedMedia.push( { - url: eligible[ 0 ].sourceUrl, - error: 'removed because the aggregate portable media limit was reached', - } ); - retainedExternalMediaCount++; - continue; - } - localizedMediaFamilies.add( family ); - let fallbackAssetPath = ''; - for ( const { candidate, contentHash } of admitted ) { - let assetPath = assetPathsByHash.get( contentHash ); - if ( assetPath === undefined ) { - assetPath = uniqueAssetPath( - join( 'media', portableMediaBasename( candidate ) ), - contentHash, - assetHashesByPath - ); - const destination = join( websiteDir, assetPath ); - mkdirSync( dirname( destination ), { recursive: true } ); - copyFileSync( candidate.localPath, destination ); - assetPathsByHash.set( contentHash, assetPath ); - assetHashesByPath.set( assetPath, contentHash ); - assets.push( { - sourceUrl: candidate.sourceUrl, - path: join( 'website', assetPath ).replace( /\\/g, '/' ), - } ); - } - portablePathsBySource.set( - candidate.sourceUrl, - `website/${ assetPath.replace( /\\/g, '/' ) }` - ); - fallbackAssetPath ||= assetPath; - for ( const reference of candidate.exactReferences ) { - mediaReplacements.set( reference, portableAssetUrl( assetPath ) ); - } - } - for ( const reference of retainedMediaFamilies.get( family ) ?? [] ) { - if ( ! mediaReplacements.has( reference ) ) - mediaReplacements.set( reference, portableAssetUrl( fallbackAssetPath ) ); - } - if ( fallbackAssetPath ) portableUrlByFamily.set( family, portableAssetUrl( fallbackAssetPath ) ); - } - const portableMedia = { - selected_count: assets.length, - selected_bytes: portableMediaPlan.selectedBytes, - retained_external_count: retainedExternalMediaCount, - max_bytes: portableMediaBudget, - reserved_bytes: 0, - }; - for ( const { sourceUrl, error, references } of failedMedia ) { - const family = mediaFamily( sourceUrl ); - if ( localizedMediaFamilies.has( family ) ) continue; - for ( const reference of retainedMediaFamilies.get( family ) ?? [] ) - mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); - for ( const reference of references ) - mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); - unresolvedMedia.push( { url: sourceUrl, error } ); - } + const mediaStage = materializePortableMedia( { + websiteDir, plan: portableMediaPlan, maxBytes: portableMediaBudget, + retainedReferences: retainedMediaFamilies, failedMedia, + } ); + let { mediaReplacements, portablePathsBySource } = mediaStage; + const { portableUrlByFamily, assetPathsByHash, assetHashesByPath, assets, unresolvedMedia, portableMedia } = mediaStage; const resourceStage = materializePortableResources( { sourceRoot: outputDir, diff --git a/src/lib/portable-media-plan.ts b/src/lib/portable-media-plan.ts index 266a9638..0a7d8538 100644 --- a/src/lib/portable-media-plan.ts +++ b/src/lib/portable-media-plan.ts @@ -4,7 +4,7 @@ import { readFileSync } from 'node:fs'; /** * Portable media selection plan. One pass records which known reference strings * occur in retained pages, then each family resolves eligibility, homepage - * priority, byte budget, and content-hash dedupe once. The exporter localizes + * priority, byte budget, and content-hash dedupe once. The media stage localizes * those decisions in original family order and does not reread staged HTML. */ diff --git a/src/lib/portable-media.test.ts b/src/lib/portable-media.test.ts new file mode 100644 index 00000000..9e897a05 --- /dev/null +++ b/src/lib/portable-media.test.ts @@ -0,0 +1,53 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { expect, it } from 'vitest'; +import { planPortableMediaFamilies, type PortableMediaCandidate } from './portable-media-plan.js'; +import { materializePortableMedia } from './portable-media.js'; +import { materializePortableResources } from './portable-resources.js'; +import { TRANSPARENT_IMAGE_DATA_URL } from './portable-assets.js'; + +it( 'hands collision/deduplication identities and failed-media replacements to resource fallback without mutating its seeds', () => { + const tempRoot = join( process.cwd(), '.tmp-test' ); + mkdirSync( tempRoot, { recursive: true } ); + const root = mkdtempSync( join( tempRoot, 'portable-media-stage-' ) ); + try { + const websiteDir = join( root, 'website' ); + mkdirSync( websiteDir ); + const candidate = ( family: string, content: string, dimension = 100 ): PortableMediaCandidate => { + const dir = join( root, family ); mkdirSync( dir ); + const localPath = join( dir, 'same.bin' ); writeFileSync( localPath, content ); + return { sourceUrl: `https://example.test/${ family }.png`, localPath, references: [ `/${ family }.png` ], exactReferences: [ `/${ family }.png` ], bytes: Buffer.byteLength( content ), dimension }; + }; + const candidates = [ candidate( 'a', 'AAA' ), candidate( 'b', 'BBBB' ), candidate( 'dedup', 'AAA' ), candidate( 'oversize', 'x', 4000 ), candidate( 'budget', 'not-admitted' ) ]; + const htmlPath = join( root, 'page.html' ); + writeFileSync( htmlPath, '' ); + const plan = planPortableMediaFamilies( candidates.map( value => ( { family: value.sourceUrl, candidates: [ value ] } ) ), 7, htmlPath ); + const planBefore = JSON.stringify( plan ); + const retainedReferences = new Map( candidates.map( value => [ value.sourceUrl, value.references ] ) ); + const media = materializePortableMedia( { + websiteDir, plan, maxBytes: 7, retainedReferences, + failedMedia: [ + { family: candidates[ 0 ].sourceUrl, sourceUrl: 'https://example.test/a.png?w=800', error: 'failed sibling', references: [ '/a.png?w=800' ] }, + { family: 'https://example.test/fallback.png', sourceUrl: 'https://example.test/fallback.png', error: 'download failed', references: [ '/fallback.png' ] }, + ], + } ); + expect( JSON.stringify( plan ) ).toBe( planBefore ); + expect( media.assets.map( value => value.path ) ).toEqual( [ 'website/media/same.png', 'website/media/same-4a8d8134f29b.png' ] ); + expect( readFileSync( join( websiteDir, 'media/same.png' ), 'utf8' ) ).toBe( 'AAA' ); + expect( readFileSync( join( websiteDir, 'media/same-4a8d8134f29b.png' ), 'utf8' ) ).toBe( 'BBBB' ); + expect( media.portablePathsBySource.get( candidates[ 2 ].sourceUrl ) ).toBe( 'website/media/same.png' ); + expect( media.portableMedia ).toEqual( { selected_count: 2, selected_bytes: 7, retained_external_count: 2, max_bytes: 7, reserved_bytes: 0 } ); + expect( media.unresolvedMedia.map( value => value.url ) ).toEqual( [ candidates[ 3 ].sourceUrl, candidates[ 4 ].sourceUrl, 'https://example.test/fallback.png' ] ); + expect( media.mediaReplacements.get( '/fallback.png' ) ).toBe( TRANSPARENT_IMAGE_DATA_URL ); + const resources = materializePortableResources( { + sourceRoot: root, websiteDir, entries: [ { url: 'https://example.test/', htmlPath } ], + resourceManifest: { version: 1, resources: { 'https://example.test/fallback.png': { path: 'a/same.bin', contentType: 'image/png' } }, failures: [] }, + ...media, embeddedSources: new Set(), projectEmbeddedHtml: html => html, + } ); + expect( resources.mediaReplacements.get( '/fallback.png' ) ).toBe( '/media/same.png' ); + expect( resources.portablePathsBySource.get( 'https://example.test/fallback.png' ) ).toBe( 'website/media/same.png' ); + expect( resources.assets ).toEqual( [] ); + expect( media.mediaReplacements.get( '/fallback.png' ) ).toBe( TRANSPARENT_IMAGE_DATA_URL ); + expect( media.portablePathsBySource.has( 'https://example.test/fallback.png' ) ).toBe( false ); + } finally { rmSync( root, { recursive: true, force: true } ); } +} ); diff --git a/src/lib/portable-media.ts b/src/lib/portable-media.ts new file mode 100644 index 00000000..3606d727 --- /dev/null +++ b/src/lib/portable-media.ts @@ -0,0 +1,156 @@ +import { copyFileSync, mkdirSync } from 'node:fs'; +import { basename, dirname, extname, join } from 'node:path'; +import type { PortableMediaCandidate, PortableMediaPlan } from './portable-media-plan.js'; +import { portableAssetUrl, uniqueAssetPath, TRANSPARENT_IMAGE_DATA_URL } from './portable-assets.js'; + +export interface FailedPortableMedia { + family: string; + sourceUrl: string; + error: string; + references: readonly string[]; +} + +export interface PortableMediaMaterializationInput { + websiteDir: string; + plan: PortableMediaPlan; + maxBytes: number; + retainedReferences: ReadonlyMap< string, readonly string[] >; + failedMedia: readonly FailedPortableMedia[]; +} + +export interface PortableMediaMaterialization { + mediaReplacements: Map< string, string >; + portableUrlByFamily: Map< string, string >; + assetPathsByHash: Map< string, string >; + assetHashesByPath: Map< string, string >; + portablePathsBySource: Map< string, string >; + assets: Array< { sourceUrl: string; path: string } >; + unresolvedMedia: Array< { url: string; error: string } >; + portableMedia: { + selected_count: number; + selected_bytes: number; + retained_external_count: number; + max_bytes: number; + reserved_bytes: number; + }; +} + +function portableMediaBasename( candidate: PortableMediaCandidate ): string { + const localName = basename( candidate.localPath ); + if ( + /^\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg|wav|woff2?|ttf|otf)$/i.test( + extname( localName ) + ) + ) { + return localName; + } + + const cleanedUrl = candidate.sourceUrl.replace( /&(?:quot|apos|amp);?$/i, '' ); + const sourceExtension = extname( basename( new URL( cleanedUrl ).pathname ) ); + if ( + ! /^\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg|wav|woff2?|ttf|otf)$/i.test( + sourceExtension + ) + ) { + return localName; + } + return `${ localName.slice( + 0, + localName.length - extname( localName ).length + ) }${ sourceExtension.toLowerCase() }`; +} + +/** Materialize admitted media in family order and return stage-owned resource seeds. */ +export function materializePortableMedia( input: PortableMediaMaterializationInput ): PortableMediaMaterialization { + const { websiteDir, plan, retainedReferences, failedMedia } = input; + const mediaReplacements = new Map< string, string >(); + const unresolvedMedia: PortableMediaMaterialization[ 'unresolvedMedia' ] = []; + const assets: PortableMediaMaterialization[ 'assets' ] = []; + let retainedExternalMediaCount = 0; + const localizedMediaFamilies = new Set< string >(); + const portableUrlByFamily = new Map< string, string >(); + const assetPathsByHash = new Map< string, string >(); + const assetHashesByPath = new Map< string, string >(); + const portablePathsBySource = new Map< string, string >(); + for ( const decision of plan.families ) { + const { family, candidates, eligible, admitted } = decision; + if ( decision.outcome === 'limit-excluded' ) { + for ( const reference of retainedReferences.get( family ) ?? [] ) + mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); + for ( const candidate of candidates ) { + for ( const reference of candidate.references ) { + mediaReplacements.set( reference, candidate.sourceUrl ); + } + unresolvedMedia.push( { + url: candidate.sourceUrl, + error: 'removed because media exceeds portable size or dimension limits', + } ); + retainedExternalMediaCount++; + } + continue; + } + if ( decision.outcome === 'budget-excluded' ) { + for ( const candidate of candidates ) { + for ( const reference of candidate.references ) { + mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); + } + } + unresolvedMedia.push( { + url: eligible[ 0 ].sourceUrl, + error: 'removed because the aggregate portable media limit was reached', + } ); + retainedExternalMediaCount++; + continue; + } + localizedMediaFamilies.add( family ); + let fallbackAssetPath = ''; + for ( const { candidate, contentHash } of admitted ) { + let assetPath = assetPathsByHash.get( contentHash ); + if ( assetPath === undefined ) { + assetPath = uniqueAssetPath( + join( 'media', portableMediaBasename( candidate ) ), + contentHash, + assetHashesByPath + ); + const destination = join( websiteDir, assetPath ); + mkdirSync( dirname( destination ), { recursive: true } ); + copyFileSync( candidate.localPath, destination ); + assetPathsByHash.set( contentHash, assetPath ); + assetHashesByPath.set( assetPath, contentHash ); + assets.push( { + sourceUrl: candidate.sourceUrl, + path: join( 'website', assetPath ).replace( /\\/g, '/' ), + } ); + } + portablePathsBySource.set( + candidate.sourceUrl, + `website/${ assetPath.replace( /\\/g, '/' ) }` + ); + fallbackAssetPath ||= assetPath; + for ( const reference of candidate.exactReferences ) { + mediaReplacements.set( reference, portableAssetUrl( assetPath ) ); + } + } + for ( const reference of retainedReferences.get( family ) ?? [] ) { + if ( ! mediaReplacements.has( reference ) ) + mediaReplacements.set( reference, portableAssetUrl( fallbackAssetPath ) ); + } + if ( fallbackAssetPath ) portableUrlByFamily.set( family, portableAssetUrl( fallbackAssetPath ) ); + } + const portableMedia = { + selected_count: assets.length, + selected_bytes: plan.selectedBytes, + retained_external_count: retainedExternalMediaCount, + max_bytes: input.maxBytes, + reserved_bytes: 0, + }; + for ( const { family, sourceUrl, error, references } of failedMedia ) { + if ( localizedMediaFamilies.has( family ) ) continue; + for ( const reference of retainedReferences.get( family ) ?? [] ) + mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); + for ( const reference of references ) + mediaReplacements.set( reference, TRANSPARENT_IMAGE_DATA_URL ); + unresolvedMedia.push( { url: sourceUrl, error } ); + } + return { mediaReplacements, portableUrlByFamily, assetPathsByHash, assetHashesByPath, portablePathsBySource, assets, unresolvedMedia, portableMedia }; +}