diff --git a/.vscodeignore b/.vscodeignore index 3f3d021..8cb78bd 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -77,3 +77,6 @@ node_modules/** # Exclude source maps from dist dist/**/*.map + +# The fdk-aac decoder is bundled into dist; its license must travel with the binary +!src/webview/vendor/fdk-aac/NOTICE diff --git a/rsbuild.webview.config.mjs b/rsbuild.webview.config.mjs index fc0ed73..66b1e96 100644 --- a/rsbuild.webview.config.mjs +++ b/rsbuild.webview.config.mjs @@ -63,5 +63,12 @@ export default defineConfig({ chunkSplit: { strategy: 'split-by-experience' } + }, + tools: { + rspack: (_config, { addRules }) => { + // The webview CSP has no `blob:`/`data:` in connect-src, so a wasm module cannot be fetched at + // runtime — inline it as a data URI and instantiate from bytes instead. + addRules([{ test: /\.wasm$/, type: 'asset/inline' }]); + } } }); diff --git a/src/types/modules.d.ts b/src/types/modules.d.ts index f5d2d61..3c37b57 100644 --- a/src/types/modules.d.ts +++ b/src/types/modules.d.ts @@ -23,6 +23,12 @@ declare module '*.jpg' { export default content; } +// Inlined as a data URI by rsbuild — the webview CSP forbids fetching wasm at runtime. +declare module '*.wasm' { + const content: string; + export default content; +} + declare module 'platform' { interface OS { family: string; diff --git a/src/webview/components/ResponsePane/QueryResult/QueryResultPreview/VideoPreview.tsx b/src/webview/components/ResponsePane/QueryResult/QueryResultPreview/VideoPreview.tsx index 09182c8..2fbe181 100644 --- a/src/webview/components/ResponsePane/QueryResult/QueryResultPreview/VideoPreview.tsx +++ b/src/webview/components/ResponsePane/QueryResult/QueryResultPreview/VideoPreview.tsx @@ -1,35 +1,194 @@ import React from 'react'; -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import ReactPlayer from 'react-player'; +import styled from 'styled-components'; +import { IconMaximize, IconMinimize, IconVolume, IconVolumeOff, IconLoader2 } from '@tabler/icons'; +import { useDecodedAudioTrack } from 'hooks/useDecodedAudioTrack'; +import { bindEvents } from 'utils/common/dom'; interface VideoPreviewProps { contentType: string; dataBuffer: string; } +// VS Code's webview iframe lacks the `fullscreen` permission, so Chromium disables its own control. +const nativeFullscreenAvailable = typeof document !== 'undefined' && document.fullscreenEnabled; + +const CONTROLS_IDLE_MS = 2500; + +const CONTROLS_HEIGHT = 48 + 24; + +const PlayerFrame = styled.div<{ $hideFullscreenControl: boolean; $hideMuteControl: boolean }>` + container-type: inline-size; + + video { + object-position: top; + } + + ${(props) => props.$hideFullscreenControl && 'video::-webkit-media-controls-fullscreen-button { display: none; }'} + ${(props) => props.$hideMuteControl && 'video::-webkit-media-controls-mute-button { display: none; }'} +`; + +const PlayerBox = styled.div` + position: relative; + width: 100%; + height: calc(100cqw / var(--video-aspect, 1.7778) + ${CONTROLS_HEIGHT}px); + max-height: 100%; +`; + const VideoPreview: React.FC = React.memo(({ contentType, dataBuffer }) => { const [videoUrl, setVideoUrl] = useState(null); + const [expanded, setExpanded] = useState(false); + const [video, setVideo] = useState(null); + const [controlsVisible, setControlsVisible] = useState(true); + const playerBoxRef = useRef(null); + const audioRef = useRef(null); + const playerRef = useRef(null); + + const mediaBytes = useMemo(() => { + const decoded = Buffer.from(dataBuffer, 'base64'); + return new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength); + }, [dataBuffer]); + const audio = useDecodedAudioTrack(video, mediaBytes, audioRef); useEffect(() => { const videoType = contentType.split(';')[0]; - const byteArray = Buffer.from(dataBuffer, 'base64'); - const blob = new Blob([byteArray], { type: videoType }); + const blob = new Blob([mediaBytes], { type: videoType }); const url = URL.createObjectURL(blob); setVideoUrl(url); return () => URL.revokeObjectURL(url); - }, [contentType, dataBuffer]); + }, [contentType, mediaBytes]); + + useEffect(() => { + if (!expanded) return; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') { + setExpanded(false); + } + }; + + document.addEventListener('keydown', onKeyDown); + return () => document.removeEventListener('keydown', onKeyDown); + }, [expanded]); + + useEffect(() => { + const box = playerBoxRef.current; + if (!video || !box) return; + + const applyAspect = () => { + if (video.videoWidth && video.videoHeight) { + box.style.setProperty('--video-aspect', String(video.videoWidth / video.videoHeight)); + } + }; + + applyAspect(); + video.addEventListener('loadedmetadata', applyAspect); + return () => video.removeEventListener('loadedmetadata', applyAspect); + }, [video]); + + useEffect(() => { + const box = playerBoxRef.current; + if (!video || !box) return; + + let idleTimer: number; + + const show = () => { + setControlsVisible(true); + window.clearTimeout(idleTimer); + if (!video.paused) { + idleTimer = window.setTimeout(() => setControlsVisible(false), CONTROLS_IDLE_MS); + } + }; + + const hideWhilePlaying = () => { + window.clearTimeout(idleTimer); + if (!video.paused) setControlsVisible(false); + }; + + const unbind = [ + bindEvents(box, { mousemove: show, mouseleave: hideWhilePlaying }), + bindEvents(video, { play: show, pause: show }) + ]; + show(); + + return () => { + window.clearTimeout(idleTimer); + unbind.forEach((dispose) => dispose()); + }; + }, [video]); if (!videoUrl) return
Loading video...
; + const audioLabel = { + unavailable: '', + idle: 'Play audio (decoded in Bruno — VS Code cannot decode this audio codec)', + decoding: 'Decoding audio…', + enabled: 'Mute audio', + failed: 'This audio track could not be decoded' + }[audio.state]; + + const controlsInset = nativeFullscreenAvailable ? 'right-24' : 'right-12'; + + const hideMuteControl = audio.state !== 'unavailable'; + return ( - console.error('Error loading video:', e)} - /> + + + setVideo((playerRef.current?.getInternalPlayer() as HTMLVideoElement) || null)} + onError={(e) => console.error('Error loading video:', e)} + /> + + ); }); diff --git a/src/webview/hooks/useDecodedAudioTrack/index.ts b/src/webview/hooks/useDecodedAudioTrack/index.ts new file mode 100644 index 0000000..4e674fd --- /dev/null +++ b/src/webview/hooks/useDecodedAudioTrack/index.ts @@ -0,0 +1,178 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { isDecodableAacTrack, parseMp4AudioTrack, type Mp4AudioTrack } from 'utils/media/mp4-audio'; +import { bindEvents } from 'utils/common/dom'; + +export type DecodedAudioState = 'unavailable' | 'idle' | 'decoding' | 'enabled' | 'failed'; + +/** Seconds. */ +const MAX_DRIFT = 0.12; + +const ignorePlaybackRejection = (): void => undefined; + +const audioDecodedBytes = (video: HTMLVideoElement): number => + (video as unknown as { webkitAudioDecodedByteCount?: number }).webkitAudioDecodedByteCount || 0; + +/** Plays the audio track of a video whose codec the host cannot decode. See `vendor/fdk-aac/README.md`. */ +export const useDecodedAudioTrack = ( + video: HTMLVideoElement | null, + mediaBytes: Uint8Array | null, + audioRef: React.RefObject +) => { + const [state, setState] = useState('unavailable'); + const trackRef = useRef(null); + const urlRef = useRef(null); + const enableIdRef = useRef(0); + const mutedByUsRef = useRef(false); + + const release = useCallback(() => { + const audio = audioRef.current; + if (audio) { + audio.pause(); + audio.removeAttribute('src'); + audio.load(); + } + if (urlRef.current) { + URL.revokeObjectURL(urlRef.current); + urlRef.current = null; + } + }, [audioRef]); + + const unmute = useCallback((element: HTMLVideoElement | null) => { + if (!element || !mutedByUsRef.current) return; + + element.muted = false; + mutedByUsRef.current = false; + }, []); + + const reset = useCallback( + (element: HTMLVideoElement | null) => { + enableIdRef.current++; + trackRef.current = null; + release(); + unmute(element); + setState('unavailable'); + }, + [release, unmute] + ); + + useEffect(() => { + reset(video); + if (!video || !mediaBytes) return; + + let cancelled = false; + + // canPlayType() and AudioDecoder.isConfigSupported() claim AAC support even with no decoder present. + // Only the decoded-byte counter tells the truth, and it reads 0 until the host has decoded something. + const detect = () => { + if (cancelled) return; + if (audioDecodedBytes(video)) { + reset(video); + return; + } + if (trackRef.current) return; + + const track = parseMp4AudioTrack(mediaBytes); + if (!isDecodableAacTrack(track)) return; + + trackRef.current = track; + setState('idle'); + }; + + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) detect(); + const unbind = bindEvents(video, { loadeddata: detect, playing: detect }); + + return () => { + cancelled = true; + unbind(); + }; + }, [video, mediaBytes, reset]); + + const disable = useCallback(() => { + enableIdRef.current++; + release(); + unmute(video); + setState((current) => (current === 'unavailable' ? current : 'idle')); + }, [release, unmute, video]); + + const enable = useCallback(async () => { + const audio = audioRef.current; + if (!video || !audio || !mediaBytes || !trackRef.current) return; + + const enableId = ++enableIdRef.current; + const superseded = () => enableId !== enableIdRef.current; + + setState('decoding'); + try { + const { decodeAacTrack } = await import('utils/media/aac-decoder'); + const wav = await decodeAacTrack(mediaBytes, trackRef.current); + if (superseded()) return; + if (!wav) { + setState('failed'); + return; + } + + const url = URL.createObjectURL(wav); + urlRef.current = url; + audio.src = url; + audio.currentTime = video.currentTime; + audio.playbackRate = video.playbackRate; + audio.volume = video.volume; + + video.muted = true; + mutedByUsRef.current = true; + if (!video.paused) await audio.play().catch(ignorePlaybackRejection); + if (superseded()) return; + + setState('enabled'); + } catch { + if (superseded()) return; + release(); + setState('failed'); + } + }, [audioRef, mediaBytes, release, video]); + + useEffect(() => { + const audio = audioRef.current; + if (state !== 'enabled' || !video || !audio) return; + + const resync = () => { + if (Math.abs(audio.currentTime - video.currentTime) > MAX_DRIFT) { + audio.currentTime = video.currentTime; + } + }; + + const onPlay = () => { + audio.currentTime = video.currentTime; + void audio.play().catch(ignorePlaybackRejection); + }; + const onPause = () => audio.pause(); + const onSeeked = () => { + audio.currentTime = video.currentTime; + }; + const onRateChange = () => { + audio.playbackRate = video.playbackRate; + }; + const onVolumeChange = () => { + audio.volume = video.volume; + }; + + return bindEvents(video, { + play: onPlay, + pause: onPause, + ended: onPause, + seeked: onSeeked, + ratechange: onRateChange, + volumechange: onVolumeChange, + timeupdate: resync + }); + }, [audioRef, state, video]); + + useEffect(() => { + return () => { + enableIdRef.current++; + release(); + }; + }, [release]); + + return { state, enable, disable }; +}; diff --git a/src/webview/utils/common/dom.ts b/src/webview/utils/common/dom.ts new file mode 100644 index 0000000..c6f45aa --- /dev/null +++ b/src/webview/utils/common/dom.ts @@ -0,0 +1,7 @@ +type Listener = (event: Event) => void; + +export const bindEvents = (target: EventTarget, listeners: Record): (() => void) => { + const entries = Object.entries(listeners); + entries.forEach(([type, listener]) => target.addEventListener(type, listener)); + return () => entries.forEach(([type, listener]) => target.removeEventListener(type, listener)); +}; diff --git a/src/webview/utils/media/aac-decoder.ts b/src/webview/utils/media/aac-decoder.ts new file mode 100644 index 0000000..aa9f430 --- /dev/null +++ b/src/webview/utils/media/aac-decoder.ts @@ -0,0 +1,127 @@ +import wasmDataUri from '../../vendor/fdk-aac/fdk-aac-decoder.wasm'; +import { + isDecodableAacTrack, + MAX_DECODED_BYTES, + NOMINAL_FRAME_SAMPLES, + type Mp4AudioTrack +} from './mp4-audio'; +import { wavBlobFromPcmChunks } from './wav'; + +interface FdkExports { + memory: WebAssembly.Memory; + _initialize: () => void; + fdkaac_open: () => number; + fdkaac_config: (decoder: number, ptr: number, len: number) => number; + fdkaac_decode: (decoder: number, ptr: number, len: number) => number; + fdkaac_pcm: (decoder: number) => number; + fdkaac_sample_rate: (decoder: number) => number; + fdkaac_channels: (decoder: number) => number; + fdkaac_close: (decoder: number) => void; + fdkaac_malloc: (size: number) => number; + fdkaac_free: (ptr: number) => void; +} + +const MAX_FRAME_BYTES = 8192; + +const FRAMES_PER_SLICE = 512; + +let instancePromise: Promise | null = null; + +const loadDecoder = (): Promise => { + if (!instancePromise) { + instancePromise = (async () => { + const base64 = wasmDataUri.slice(wasmDataUri.indexOf(',') + 1); + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + + // Unused: the module never touches stdio, but links wasi-libc. + const stub = () => 0; + const { instance } = await WebAssembly.instantiate(bytes, { + wasi_snapshot_preview1: { + fd_close: stub, + fd_prestat_get: () => 8, // EBADF + fd_prestat_dir_name: stub, + fd_seek: stub, + fd_write: stub, + proc_exit: stub + } + }); + const exports = instance.exports as unknown as FdkExports; + exports._initialize(); + return exports; + })().catch((error) => { + instancePromise = null; + throw error; + }); + } + return instancePromise; +}; + +/** An access unit that fails to decode contributes silence, so the audio keeps the video's timeline. */ +export const decodeAacTrack = async (bytes: Uint8Array, track: Mp4AudioTrack): Promise => { + if (!isDecodableAacTrack(track)) return null; + + const fdk = await loadDecoder(); + const decoder = fdk.fdkaac_open(); + if (!decoder) return null; + + const config = track.config as Uint8Array; + const configPtr = fdk.fdkaac_malloc(config.length); + const framePtr = fdk.fdkaac_malloc(MAX_FRAME_BYTES); + + try { + new Uint8Array(fdk.memory.buffer).set(config, configPtr); + if (fdk.fdkaac_config(decoder, configPtr, config.length) !== 0) return null; + + const chunks: Int16Array[] = []; + let channels = Math.max(track.channels, 1); + let sampleRate = track.sampleRate; + let frameSamples = NOMINAL_FRAME_SAMPLES; + let decoded = 0; + let total = 0; + + for (let i = 0; i < track.samples.length; i++) { + if (i > 0 && i % FRAMES_PER_SLICE === 0) await new Promise((resolve) => setTimeout(resolve)); + + const sample = track.samples[i]; + const readable = sample.size <= MAX_FRAME_BYTES && sample.offset + sample.size <= bytes.byteLength; + let frames = 0; + + // A short read would hand the decoder whatever the last frame left in its buffer. + if (readable) { + new Uint8Array(fdk.memory.buffer).set(bytes.subarray(sample.offset, sample.offset + sample.size), framePtr); + frames = fdk.fdkaac_decode(decoder, framePtr, sample.size); + } + + if (frames > 0) { + channels = fdk.fdkaac_channels(decoder) || channels; + sampleRate = fdk.fdkaac_sample_rate(decoder) || sampleRate; + frameSamples = frames; + decoded++; + const pcmPtr = fdk.fdkaac_pcm(decoder); + const count = frames * channels; + chunks.push(new Int16Array(fdk.memory.buffer, pcmPtr, count).slice()); + total += count; + } else { + const count = frameSamples * channels; + chunks.push(new Int16Array(count)); + total += count; + } + + // An SBR stream decodes to twice what the sample table suggested. + if (total * 2 > MAX_DECODED_BYTES) return null; + } + + if (!decoded || !sampleRate) return null; + + // trimStart is at the sample entry's rate, which SBR doubles on the way out. + const trimSamples = Math.round((track.trimStart * sampleRate) / track.sampleRate) * channels; + + return wavBlobFromPcmChunks({ chunks, totalSamples: total, trimSamples, sampleRate, channels }); + } finally { + fdk.fdkaac_free(configPtr); + fdk.fdkaac_free(framePtr); + fdk.fdkaac_close(decoder); + } +}; diff --git a/src/webview/utils/media/mp4-audio.spec.ts b/src/webview/utils/media/mp4-audio.spec.ts new file mode 100644 index 0000000..5cedcdd --- /dev/null +++ b/src/webview/utils/media/mp4-audio.spec.ts @@ -0,0 +1,328 @@ +import { isDecodableAacTrack, parseMp4AudioTrack, type Mp4AudioTrack } from './mp4-audio'; + +const ascii = (text: string) => Uint8Array.from(text, (c) => c.charCodeAt(0)); + +const concat = (parts: Uint8Array[]) => { + const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +}; + +const u32 = (value: number) => { + const out = new Uint8Array(4); + new DataView(out.buffer).setUint32(0, value); + return out; +}; + +const i32 = (value: number) => { + const out = new Uint8Array(4); + new DataView(out.buffer).setInt32(0, value); + return out; +}; + +const u16 = (value: number) => { + const out = new Uint8Array(2); + new DataView(out.buffer).setUint16(0, value); + return out; +}; + +const u64 = (value: number) => { + const out = new Uint8Array(8); + new DataView(out.buffer).setBigUint64(0, BigInt(value)); + return out; +}; + +/** 64-bit size: the 32-bit field holds 1 and the real size follows the type. */ +const largeBox = (type: string, ...payload: Uint8Array[]) => { + const body = concat(payload); + return concat([u32(1), ascii(type), u64(body.length + 16), body]); +}; + +const box = (type: string, ...payload: Uint8Array[]) => { + const body = concat(payload); + return concat([u32(body.length + 8), ascii(type), body]); +}; + +const zeros = (n: number) => new Uint8Array(n); +const fullBoxHeader = zeros(4); // version + flags + +const esds = (config: number[]) => + box( + 'esds', + fullBoxHeader, + Uint8Array.from([0x03, 3 + 15 + 2 + config.length]), + concat([u16(1), Uint8Array.from([0x00])]), // ES_ID + flags + Uint8Array.from([0x04, 13]), + Uint8Array.from([0x40]), // objectTypeIndication: AAC + zeros(12), + Uint8Array.from([0x05, config.length]), + Uint8Array.from(config) + ); + +const sampleEntry = ({ format, channels, sampleRate, config }: { + format: string; + channels: number; + sampleRate: number; + config?: number[]; +}) => + box( + format, + zeros(6), + u16(1), // data_reference_index + zeros(8), + u16(channels), + u16(16), // samplesize + zeros(4), + u32(sampleRate << 16), + ...(config ? [esds(config)] : []) + ); + +interface TrackSpec { + handler: string; + format?: string; + channels?: number; + sampleRate?: number; + timescale?: number; + config?: number[]; + sizes?: number[]; + chunkOffsets?: number[]; + sampleToChunk?: { firstChunk: number; samplesPerChunk: number }[]; + mediaTime?: number | number[] | null; + fixedSampleSize?: number; + sampleCount?: number; + use64BitOffsets?: boolean; +} + +const trak = (spec: TrackSpec) => { + const { + handler, + format = 'mp4a', + channels = 2, + sampleRate = 44100, + timescale = 44100, + config = [0x12, 0x10], + sizes = [100, 120], + chunkOffsets = [1000], + sampleToChunk = [{ firstChunk: 1, samplesPerChunk: 2 }], + mediaTime = null, + fixedSampleSize = 0, + sampleCount = sizes.length, + use64BitOffsets = false + } = spec; + + const stsz = fixedSampleSize + ? box('stsz', fullBoxHeader, u32(fixedSampleSize), u32(sampleCount)) + : box('stsz', fullBoxHeader, u32(0), u32(sampleCount), ...sizes.map(u32)); + + const chunkTable = use64BitOffsets + ? box('co64', fullBoxHeader, u32(chunkOffsets.length), ...chunkOffsets.map(u64)) + : box('stco', fullBoxHeader, u32(chunkOffsets.length), ...chunkOffsets.map(u32)); + + const stbl = box( + 'stbl', + box('stsd', fullBoxHeader, u32(1), sampleEntry({ format, channels, sampleRate, config })), + stsz, + box( + 'stsc', + fullBoxHeader, + u32(sampleToChunk.length), + ...sampleToChunk.flatMap((r) => [u32(r.firstChunk), u32(r.samplesPerChunk), u32(1)]) + ), + chunkTable + ); + + const parts = [ + box( + 'mdia', + box('mdhd', fullBoxHeader, zeros(8), u32(timescale), u32(1000)), + box('hdlr', fullBoxHeader, zeros(4), ascii(handler)), + box('minf', stbl) + ) + ]; + + if (mediaTime !== null) { + const entries = Array.isArray(mediaTime) ? mediaTime : [mediaTime]; + parts.unshift( + box( + 'edts', + box('elst', fullBoxHeader, u32(entries.length), ...entries.flatMap((time) => [u32(5000), i32(time), u32(1)])) + ) + ); + } + + return box('trak', ...parts); +}; + +const mp4 = (...traks: Uint8Array[]) => concat([box('ftyp', ascii('isom')), box('moov', ...traks)]); + +describe('parseMp4AudioTrack', () => { + it('reads the audio track config, sample table and edit-list trim', () => { + const track = parseMp4AudioTrack(mp4(trak({ handler: 'soun', mediaTime: 1024 }))); + + expect(track).not.toBeNull(); + expect(track!.format).toBe('mp4a'); + expect(track!.sampleRate).toBe(44100); + expect(track!.channels).toBe(2); + expect(Array.from(track!.config!)).toEqual([0x12, 0x10]); + expect(track!.trimStart).toBe(1024); + expect(track!.samples).toEqual([ + { offset: 1000, size: 100 }, + { offset: 1100, size: 120 } + ]); + }); + + it('ignores video tracks and returns null when there is no audio track', () => { + expect(parseMp4AudioTrack(mp4(trak({ handler: 'vide' })))).toBeNull(); + }); + + it('picks the audio track when other tracks come first', () => { + const file = mp4(trak({ handler: 'vide', format: 'avc1' }), trak({ handler: 'soun', sampleRate: 48000 })); + expect(parseMp4AudioTrack(file)!.sampleRate).toBe(48000); + }); + + it('spreads samples across chunks per the sample-to-chunk table', () => { + const track = parseMp4AudioTrack( + mp4( + trak({ + handler: 'soun', + sizes: [10, 20, 30, 40, 50], + chunkOffsets: [500, 900], + sampleToChunk: [ + { firstChunk: 1, samplesPerChunk: 2 }, + { firstChunk: 2, samplesPerChunk: 3 } + ] + }) + ) + ); + + expect(track!.samples).toEqual([ + { offset: 500, size: 10 }, + { offset: 510, size: 20 }, + { offset: 900, size: 30 }, + { offset: 930, size: 40 }, + { offset: 970, size: 50 } + ]); + }); + + it('treats a missing or empty edit list as no trim', () => { + expect(parseMp4AudioTrack(mp4(trak({ handler: 'soun' })))!.trimStart).toBe(0); + expect(parseMp4AudioTrack(mp4(trak({ handler: 'soun', mediaTime: -1 })))!.trimStart).toBe(0); + }); + + it('takes the trim from the first real edit when an empty one precedes it', () => { + const track = parseMp4AudioTrack(mp4(trak({ handler: 'soun', mediaTime: [-1, 1024] }))); + expect(track!.trimStart).toBe(1024); + }); + + it('returns null when the size table claims more entries than it holds', () => { + expect(parseMp4AudioTrack(mp4(trak({ handler: 'soun', sampleCount: 50 })))).toBeNull(); + }); + + it('returns null when the samples would outweigh the file', () => { + const file = mp4(trak({ handler: 'soun', fixedSampleSize: 64, sampleCount: 10_000 })); + expect(parseMp4AudioTrack(file)).toBeNull(); + }); + + it('returns null for a table too large to decode, without building it', () => { + // 17k stereo access units decode to ~70MB of PCM. + const padded = (sampleCount: number) => + concat([ + mp4(trak({ handler: 'soun', fixedSampleSize: 1, sampleCount })), + box('free', zeros(20_000)) + ]); + + expect(parseMp4AudioTrack(padded(17_000))).toBeNull(); + expect(parseMp4AudioTrack(padded(100))).not.toBeNull(); + }); + + it('returns null for a file whose boxes run past the end of the buffer', () => { + const file = mp4(trak({ handler: 'soun' })); + expect(parseMp4AudioTrack(file.subarray(0, file.length - 12))).toBeNull(); + }); + + it('returns null instead of throwing when a 64-bit box size is itself truncated', () => { + // A size of 1 promises 8 more length bytes; only 4 are here. + expect(parseMp4AudioTrack(concat([u32(1), ascii('moov'), u32(0)]))).toBeNull(); + }); + + it('returns null for a truncated file with no sample tables', () => { + expect(parseMp4AudioTrack(concat([box('ftyp', ascii('isom'))]))).toBeNull(); + }); + + it('reads 64-bit chunk offsets from co64', () => { + const track = parseMp4AudioTrack( + mp4(trak({ handler: 'soun', chunkOffsets: [8_589_934_592], use64BitOffsets: true })) + ); + + expect(track!.samples).toEqual([ + { offset: 8_589_934_592, size: 100 }, + { offset: 8_589_934_692, size: 120 } + ]); + }); + + it('applies a fixed sample size to every sample', () => { + const track = parseMp4AudioTrack( + mp4(trak({ handler: 'soun', fixedSampleSize: 64, sampleCount: 3, chunkOffsets: [200], sampleToChunk: [{ firstChunk: 1, samplesPerChunk: 3 }] })) + ); + + expect(track!.samples).toEqual([ + { offset: 200, size: 64 }, + { offset: 264, size: 64 }, + { offset: 328, size: 64 } + ]); + }); + + it('walks boxes that carry a 64-bit size', () => { + const file = concat([box('ftyp', ascii('isom')), largeBox('moov', trak({ handler: 'soun' }))]); + expect(parseMp4AudioTrack(file)!.format).toBe('mp4a'); + }); + + it('converts the edit-list trim from the media timescale to samples', () => { + // 500 ticks of a 1000-tick timescale is 0.5s, i.e. 22050 samples at 44.1kHz. + const track = parseMp4AudioTrack(mp4(trak({ handler: 'soun', timescale: 1000, mediaTime: 500 }))); + expect(track!.trimStart).toBe(22050); + }); +}); + +describe('isDecodableAacTrack', () => { + const track = (overrides: Partial = {}): Mp4AudioTrack => ({ + format: 'mp4a', + sampleRate: 44100, + channels: 2, + config: Uint8Array.from([0x12, 0x10]), + samples: [{ offset: 0, size: 100 }], + trimStart: 0, + ...overrides + }); + + it('accepts an AAC track with a config and samples', () => { + expect(isDecodableAacTrack(track())).toBe(true); + }); + + it('rejects a missing track, another codec, or a track with no config', () => { + expect(isDecodableAacTrack(null)).toBe(false); + expect(isDecodableAacTrack(track({ format: 'ac-3' }))).toBe(false); + expect(isDecodableAacTrack(track({ config: null }))).toBe(false); + expect(isDecodableAacTrack(track({ samples: [] }))).toBe(false); + }); + + it('rejects a track whose decoded audio would not fit the memory budget', () => { + const samples = new Array(26_000).fill({ offset: 0, size: 100 }); + expect(isDecodableAacTrack(track({ samples }))).toBe(false); + }); + + it('accepts a track that fits the budget', () => { + const samples = new Array(6_000).fill({ offset: 0, size: 100 }); + expect(isDecodableAacTrack(track({ samples }))).toBe(true); + }); + + it('measures the budget per channel, so mono allows twice the length of stereo', () => { + const samples = new Array(20_000).fill({ offset: 0, size: 100 }); + expect(isDecodableAacTrack(track({ samples, channels: 1 }))).toBe(true); + expect(isDecodableAacTrack(track({ samples, channels: 2 }))).toBe(false); + }); +}); diff --git a/src/webview/utils/media/mp4-audio.ts b/src/webview/utils/media/mp4-audio.ts new file mode 100644 index 0000000..7e8f38f --- /dev/null +++ b/src/webview/utils/media/mp4-audio.ts @@ -0,0 +1,256 @@ +export interface Mp4AudioSample { + offset: number; + size: number; +} + +export interface Mp4AudioTrack { + format: string; + sampleRate: number; + channels: number; + /** AudioSpecificConfig, from the `esds` box. */ + config: Uint8Array | null; + samples: Mp4AudioSample[]; + /** Encoder delay, from the edit-list `media_time`. */ + trimStart: number; +} + +/** AAC-LC per access unit; SBR emits twice as many. */ +export const NOMINAL_FRAME_SAMPLES = 1024; + +export const MAX_DECODED_BYTES = 64 * 1024 * 1024; + +export const withinDecodeBudget = (sampleCount: number, channels: number): boolean => + sampleCount * NOMINAL_FRAME_SAMPLES * Math.max(channels, 1) * 2 <= MAX_DECODED_BYTES; + +export const isDecodableAacTrack = (track: Mp4AudioTrack | null): boolean => { + if (!track || track.format !== 'mp4a' || !track.config || !track.samples.length) return false; + if (!track.sampleRate) return false; + return withinDecodeBudget(track.samples.length, track.channels); +}; + +const CONTAINERS = new Set(['moov', 'trak', 'mdia', 'minf', 'stbl', 'edts']); + +interface TrackBuilder { + handler: string | null; + format: string | null; + sampleRate: number; + channels: number; + config: Uint8Array | null; + timescale: number; + trimStart: number; + tables: Record; +} + +export const parseMp4AudioTrack = (bytes: Uint8Array): Mp4AudioTrack | null => { + try { + return readAudioTrack(bytes); + } catch { + return null; + } +}; + +const readAudioTrack = (bytes: Uint8Array): Mp4AudioTrack | null => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const u32 = (o: number) => view.getUint32(o); + const u16 = (o: number) => view.getUint16(o); + const u64 = (o: number) => Number(view.getBigUint64(o)); + const type = (o: number) => String.fromCharCode(bytes[o], bytes[o + 1], bytes[o + 2], bytes[o + 3]); + + const walk = (start: number, end: number, visit: (t: string, s: number, e: number) => void) => { + let offset = start; + while (offset + 8 <= end) { + let size = u32(offset); + let header = 8; + if (size === 1) { + size = u64(offset + 8); + header = 16; + } else if (size === 0) { + size = end - offset; + } + if (size < header || offset + size > end) return; + visit(type(offset + 4), offset + header, offset + size); + offset += size; + } + }; + + const tracks: TrackBuilder[] = []; + let track: TrackBuilder | null = null; + + const visit = (boxType: string, start: number, end: number) => { + if (boxType === 'trak') { + track = { + handler: null, + format: null, + sampleRate: 0, + channels: 0, + config: null, + timescale: 0, + trimStart: 0, + tables: {} + }; + tracks.push(track); + } + + if (CONTAINERS.has(boxType)) { + walk(start, end, visit); + return; + } + if (!track) return; + + switch (boxType) { + case 'hdlr': + track.handler = type(start + 8); + break; + case 'mdhd': + track.timescale = bytes[start] === 1 ? u32(start + 20) : u32(start + 12); + break; + case 'elst': { + // A negative media_time marks an empty edit, written ahead of the real trim. + const version = bytes[start]; + const entrySize = version === 1 ? 20 : 12; + const entries = u32(start + 4); + for (let i = 0; i < entries; i++) { + const entry = start + 8 + i * entrySize; + if (entry + entrySize > end) break; + + const mediaTime = version === 1 ? Number(view.getBigInt64(entry + 8)) : view.getInt32(entry + 4); + if (mediaTime >= 0) { + track.trimStart = mediaTime; + break; + } + } + break; + } + case 'stsd': { + const entry = start + 8; + track.format = type(entry + 4); + // AudioSampleEntry: 8 header, 6 reserved, 2 data_reference_index, 8 reserved, 2 channelcount, + // 2 samplesize, 2 predefined, 2 reserved, 4 samplerate as 16.16 fixed point. + track.channels = u16(entry + 24); + track.sampleRate = u32(entry + 32) >>> 16; + walk(entry + 36, end, (childType, childStart, childEnd) => { + if (childType === 'esds') { + track!.config = parseEsds(bytes, childStart, childEnd); + } + }); + break; + } + case 'stsz': + case 'stsc': + case 'stco': + case 'co64': + track.tables[boxType] = { start, end }; + break; + default: + break; + } + }; + + walk(0, bytes.byteLength, visit); + + const audio = tracks.find((t) => t.handler === 'soun'); + if (!audio) return null; + + const { stsz, stsc, stco, co64 } = audio.tables; + if (!stsz || !stsc || !(stco || co64)) return null; + + const entryCount = (table: { start: number; end: number }, countOffset: number, entryBytes: number): number => { + const count = u32(table.start + countOffset); + return table.start + countOffset + 4 + count * entryBytes <= table.end ? count : 0; + }; + + if (stsz.start + 12 > stsz.end) return null; + + const fixedSize = u32(stsz.start + 4); + const sampleCount = fixedSize ? u32(stsz.start + 8) : entryCount(stsz, 8, 4); + if (!sampleCount || sampleCount * Math.max(fixedSize, 1) > bytes.byteLength) return null; + if (!withinDecodeBudget(sampleCount, audio.channels)) return null; + + const sizes = new Array(sampleCount); + for (let i = 0; i < sampleCount; i++) { + sizes[i] = fixedSize || u32(stsz.start + 12 + i * 4); + } + + const chunkOffsets: number[] = []; + if (stco) { + const count = entryCount(stco, 4, 4); + for (let i = 0; i < count; i++) chunkOffsets.push(u32(stco.start + 8 + i * 4)); + } else { + const count = entryCount(co64!, 4, 8); + for (let i = 0; i < count; i++) chunkOffsets.push(u64(co64!.start + 8 + i * 8)); + } + + const runs: { firstChunk: number; samplesPerChunk: number }[] = []; + const runCount = entryCount(stsc, 4, 12); + for (let i = 0; i < runCount; i++) { + const o = stsc.start + 8 + i * 12; + runs.push({ firstChunk: u32(o), samplesPerChunk: u32(o + 4) }); + } + if (!runs.length) return null; + + const samples: Mp4AudioSample[] = []; + let index = 0; + for (let chunk = 0; chunk < chunkOffsets.length && index < sampleCount; chunk++) { + let perChunk = runs[0].samplesPerChunk; + for (let r = runs.length - 1; r >= 0; r--) { + if (chunk + 1 >= runs[r].firstChunk) { + perChunk = runs[r].samplesPerChunk; + break; + } + } + let offset = chunkOffsets[chunk]; + for (let s = 0; s < perChunk && index < sampleCount; s++) { + samples.push({ offset, size: sizes[index] }); + offset += sizes[index]; + index++; + } + } + + // media_time is in the media timescale, usually but not always the sample rate. + const trimStart = audio.timescale + ? Math.round((audio.trimStart * audio.sampleRate) / audio.timescale) + : audio.trimStart; + + return { + format: audio.format || '', + sampleRate: audio.sampleRate, + channels: audio.channels, + config: audio.config, + samples, + trimStart + }; +}; + +/** esds -> ES_Descriptor -> DecoderConfigDescriptor -> DecoderSpecificInfo. */ +const parseEsds = (bytes: Uint8Array, start: number, end: number): Uint8Array | null => { + let offset = start + 4; // version + flags + + const readLength = () => { + let length = 0; + for (let i = 0; i < 4; i++) { + const byte = bytes[offset++]; + length = (length << 7) | (byte & 0x7f); + if (!(byte & 0x80)) break; + } + return length; + }; + + while (offset < end) { + const tag = bytes[offset++]; + const length = readLength(); + if (tag === 0x03) { + offset += 2; // ES_ID + const flags = bytes[offset++]; + if (flags & 0x80) offset += 2; // dependsOn_ES_ID + if (flags & 0x40) offset += 1 + bytes[offset]; // URL + if (flags & 0x20) offset += 2; // OCR_ES_Id + } else if (tag === 0x04) { + offset += 13; // objectTypeIndication + stream type + buffer/bitrate fields + } else if (tag === 0x05) { + return bytes.slice(offset, Math.min(offset + length, end)); + } else { + offset += length; + } + } + return null; +}; diff --git a/src/webview/utils/media/wav.spec.ts b/src/webview/utils/media/wav.spec.ts new file mode 100644 index 0000000..de21992 --- /dev/null +++ b/src/webview/utils/media/wav.spec.ts @@ -0,0 +1,108 @@ +import { wavBlobFromPcmChunks } from './wav'; + +const header = async (blob: Blob) => new DataView(await blob.arrayBuffer()); + +const ascii = (view: DataView, offset: number, length: number) => + Array.from({ length }, (_, i) => String.fromCharCode(view.getUint8(offset + i))).join(''); + +const wav = (chunks: Int16Array[], sampleRate: number, channels: number) => + wavBlobFromPcmChunks({ + chunks, + totalSamples: chunks.reduce((n, chunk) => n + chunk.length, 0), + sampleRate, + channels + }); + +describe('wavBlobFromPcmChunks', () => { + const pcm = Int16Array.from([0, 1000, -1000, 32767]); + + it('writes a 16-bit PCM header matching the audio format', async () => { + const view = await header(wav([pcm], 44100, 2)); + + expect(ascii(view, 0, 4)).toBe('RIFF'); + expect(ascii(view, 8, 4)).toBe('WAVE'); + expect(ascii(view, 12, 4)).toBe('fmt '); + expect(view.getUint32(16, true)).toBe(16); // fmt chunk size + expect(view.getUint16(20, true)).toBe(1); // PCM + expect(view.getUint16(22, true)).toBe(2); // channels + expect(view.getUint32(24, true)).toBe(44100); + expect(view.getUint32(28, true)).toBe(44100 * 2 * 2); // byte rate + expect(view.getUint16(32, true)).toBe(4); // block align + expect(view.getUint16(34, true)).toBe(16); // bits per sample + }); + + it('sizes the RIFF and data chunks from the sample count', async () => { + const blob = wav([pcm], 8000, 1); + const view = await header(blob); + + expect(blob.size).toBe(44 + pcm.length * 2); + expect(view.getUint32(4, true)).toBe(36 + pcm.length * 2); + expect(ascii(view, 36, 4)).toBe('data'); + expect(view.getUint32(40, true)).toBe(pcm.length * 2); + }); + + it('writes the samples little-endian after the header', async () => { + const view = await header(wav([pcm], 8000, 1)); + + for (let i = 0; i < pcm.length; i++) { + expect(view.getInt16(44 + i * 2, true)).toBe(pcm[i]); + } + }); + + it('joins the chunks in order and releases each one', async () => { + const chunks = [Int16Array.from([1, 2]), Int16Array.from([3]), Int16Array.from([4, 5])]; + const view = await header(wav(chunks, 8000, 1)); + + expect([0, 1, 2, 3, 4].map((i) => view.getInt16(44 + i * 2, true))).toEqual([1, 2, 3, 4, 5]); + expect(chunks.every((chunk) => chunk.length === 0)).toBe(true); + }); + + it('drops the trimmed samples from the front, across chunk boundaries', async () => { + const blob = wavBlobFromPcmChunks({ + chunks: [Int16Array.from([1, 2]), Int16Array.from([3, 4, 5])], + totalSamples: 5, + trimSamples: 3, + sampleRate: 8000, + channels: 1 + }); + const view = await header(blob); + + expect(blob.size).toBe(44 + 2 * 2); + expect(view.getInt16(44, true)).toBe(4); + expect(view.getInt16(46, true)).toBe(5); + }); + + it('drops everything when the trim covers the whole track', async () => { + const blob = wavBlobFromPcmChunks({ + chunks: [Int16Array.from([1, 2])], + totalSamples: 2, + trimSamples: 9, + sampleRate: 8000, + channels: 1 + }); + + expect(blob.size).toBe(44); + }); + + it('stops at the stated sample count when a chunk would overrun it', async () => { + const blob = wavBlobFromPcmChunks({ + chunks: [Int16Array.from([1, 2]), Int16Array.from([3, 4])], + totalSamples: 2, + sampleRate: 8000, + channels: 1 + }); + const view = await header(blob); + + expect(blob.size).toBe(44 + 4); + expect(view.getInt16(44, true)).toBe(1); + expect(view.getInt16(46, true)).toBe(2); + }); + + it('handles mono 8kHz with no samples', async () => { + const blob = wav([], 8000, 1); + const view = await header(blob); + + expect(blob.size).toBe(44); + expect(view.getUint32(40, true)).toBe(0); + }); +}); diff --git a/src/webview/utils/media/wav.ts b/src/webview/utils/media/wav.ts new file mode 100644 index 0000000..b716c22 --- /dev/null +++ b/src/webview/utils/media/wav.ts @@ -0,0 +1,63 @@ +export interface PcmChunks { + /** Emptied as it is written. */ + chunks: Int16Array[]; + totalSamples: number; + /** Encoder delay, from the mp4 edit list. */ + trimSamples?: number; + sampleRate: number; + channels: number; +} + +const EMPTY = new Int16Array(0); + +/** Chunks are dropped as they are copied; holding both copies exhausts webview memory on a long track. */ +export const wavBlobFromPcmChunks = ({ + chunks, + totalSamples, + trimSamples = 0, + sampleRate, + channels +}: PcmChunks): Blob => { + const trim = Math.min(Math.max(trimSamples, 0), totalSamples); + const samples = totalSamples - trim; + const dataBytes = samples * 2; + const buffer = new ArrayBuffer(44 + dataBytes); + const view = new DataView(buffer); + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i)); + }; + + ascii(0, 'RIFF'); + view.setUint32(4, 36 + dataBytes, true); + ascii(8, 'WAVE'); + ascii(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, channels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * channels * 2, true); + view.setUint16(32, channels * 2, true); + view.setUint16(34, 16, true); + ascii(36, 'data'); + view.setUint32(40, dataBytes, true); + + const pcm = new Int16Array(buffer, 44); + let dropped = 0; + let written = 0; + for (let i = 0; i < chunks.length; i++) { + let chunk = chunks[i]; + chunks[i] = EMPTY; + + if (dropped < trim) { + const drop = Math.min(trim - dropped, chunk.length); + dropped += drop; + chunk = chunk.subarray(drop); + } + if (written + chunk.length > samples) break; + + pcm.set(chunk, written); + written += chunk.length; + } + + return new Blob([buffer], { type: 'audio/wav' }); +}; diff --git a/src/webview/vendor/fdk-aac/NOTICE b/src/webview/vendor/fdk-aac/NOTICE new file mode 100644 index 0000000..05b32bd --- /dev/null +++ b/src/webview/vendor/fdk-aac/NOTICE @@ -0,0 +1,92 @@ +Software License for The Fraunhofer FDK AAC Codec Library for Android + +© Copyright 1995 - 2018 Fraunhofer-Gesellschaft zur Förderung der angewandten +Forschung e.V. All rights reserved. + + 1. INTRODUCTION +The Fraunhofer FDK AAC Codec Library for Android ("FDK AAC Codec") is software +that implements the MPEG Advanced Audio Coding ("AAC") encoding and decoding +scheme for digital audio. This FDK AAC Codec software is intended to be used on +a wide variety of Android devices. + +AAC's HE-AAC and HE-AAC v2 versions are regarded as today's most efficient +general perceptual audio codecs. AAC-ELD is considered the best-performing +full-bandwidth communications codec by independent studies and is widely +deployed. AAC has been standardized by ISO and IEC as part of the MPEG +specifications. + +Patent licenses for necessary patent claims for the FDK AAC Codec (including +those of Fraunhofer) may be obtained through Via Licensing +(www.vialicensing.com) or through the respective patent owners individually for +the purpose of encoding or decoding bit streams in products that are compliant +with the ISO/IEC MPEG audio standards. Please note that most manufacturers of +Android devices already license these patent claims through Via Licensing or +directly from the patent owners, and therefore FDK AAC Codec software may +already be covered under those patent licenses when it is used for those +licensed purposes only. + +Commercially-licensed AAC software libraries, including floating-point versions +with enhanced sound quality, are also available from Fraunhofer. Users are +encouraged to check the Fraunhofer website for additional applications +information and documentation. + +2. COPYRIGHT LICENSE + +Redistribution and use in source and binary forms, with or without modification, +are permitted without payment of copyright license fees provided that you +satisfy the following conditions: + +You must retain the complete text of this software license in redistributions of +the FDK AAC Codec or your modifications thereto in source code form. + +You must retain the complete text of this software license in the documentation +and/or other materials provided with redistributions of the FDK AAC Codec or +your modifications thereto in binary form. You must make available free of +charge copies of the complete source code of the FDK AAC Codec and your +modifications thereto to recipients of copies in binary form. + +The name of Fraunhofer may not be used to endorse or promote products derived +from this library without prior written permission. + +You may not charge copyright license fees for anyone to use, copy or distribute +the FDK AAC Codec software or your modifications thereto. + +Your modified versions of the FDK AAC Codec must carry prominent notices stating +that you changed the software and the date of any change. For modified versions +of the FDK AAC Codec, the term "Fraunhofer FDK AAC Codec Library for Android" +must be replaced by the term "Third-Party Modified Version of the Fraunhofer FDK +AAC Codec Library for Android." + +3. NO PATENT LICENSE + +NO EXPRESS OR IMPLIED LICENSES TO ANY PATENT CLAIMS, including without +limitation the patents of Fraunhofer, ARE GRANTED BY THIS SOFTWARE LICENSE. +Fraunhofer provides no warranty of patent non-infringement with respect to this +software. + +You may use this FDK AAC Codec software or modifications thereto only for +purposes that are authorized by appropriate patent licenses. + +4. DISCLAIMER + +This FDK AAC Codec software is provided by Fraunhofer on behalf of the copyright +holders and contributors "AS IS" and WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, +including but not limited to the implied warranties of merchantability and +fitness for a particular purpose. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE for any direct, indirect, incidental, special, exemplary, +or consequential damages, including but not limited to procurement of substitute +goods or services; loss of use, data, or profits, or business interruption, +however caused and on any theory of liability, whether in contract, strict +liability, or tort (including negligence), arising in any way out of the use of +this software, even if advised of the possibility of such damage. + +5. CONTACT INFORMATION + +Fraunhofer Institute for Integrated Circuits IIS +Attention: Audio and Multimedia Departments - FDK AAC LL +Am Wolfsmantel 33 +91058 Erlangen, Germany + +www.iis.fraunhofer.de/amm +amm-info@iis.fraunhofer.de + diff --git a/src/webview/vendor/fdk-aac/README.md b/src/webview/vendor/fdk-aac/README.md new file mode 100644 index 0000000..b9c65b2 --- /dev/null +++ b/src/webview/vendor/fdk-aac/README.md @@ -0,0 +1,59 @@ +# fdk-aac decoder (WebAssembly) + +`fdk-aac-decoder.wasm` is a decoder-only build of the Fraunhofer FDK AAC library, used to play the +audio of AAC responses in the response pane. + +## Why this exists + +VS Code ships Electron with the non-proprietary ffmpeg build, which contains no AAC (or MP3) decoder. +An mp4 response therefore plays video but has **no audio track at all** from the webview's point of +view: `webkitAudioDecodedByteCount` stays `0`, `AudioContext.decodeAudioData()` fails with +`EncodingError`, and Chromium disables the mute/unmute control because the element reports no audio. +Note that `canPlayType()`, `MediaSource.isTypeSupported()` and `AudioDecoder.isConfigSupported()` all +report AAC as supported in that environment — they answer from a compile-time table, not from the +decoders actually present, so they cannot be used to detect this. + +Bruno desktop is unaffected: it runs as a top-level Electron document with a full-codec ffmpeg. + +## Licensing and patents + +The FDK AAC license (`NOTICE`) permits redistribution in source and binary form with attribution. +It is **not** GPL, unlike FAAD2-based decoders (`@audio/decode-aac` and everything that wraps it), +which is why it was chosen for this MIT-licensed extension. + +Section 3 of that license disclaims any patent grant: *"the use of this software may be subject to +third party patent rights"*. AAC is patent-encumbered, which is precisely why VS Code omits the +decoder. Keep `NOTICE` alongside the binary in any redistribution. + +## Provenance + +| | | +|---|---| +| Source | https://github.com/mstorsjo/fdk-aac | +| Revision | `d8e6b1a3aa606c450241632b64b703f21ea31ce3` | +| Toolchain | wasi-sdk 33.0 (clang 22.1.0-wasi-sdk), target `wasm32-wasi` | +| Size | 630 KB | +| SHA-256 | `743ef1e78c93d4731034421d03129871c85e583b6a670a25917def816d7dc68a` | + +Encoder libraries are excluded from the build; only the decoder is compiled. + +## Rebuilding + +```bash +FDK_AAC_SRC=/path/to/fdk-aac WASI_SDK=/path/to/wasi-sdk ./build.sh +``` + +`fdkaac_wrap.c` is the wrapper compiled into the module. It exposes a small C ABI +(`fdkaac_open`, `fdkaac_config`, `fdkaac_decode`, `fdkaac_pcm`, `fdkaac_sample_rate`, +`fdkaac_channels`, `fdkaac_close`, `fdkaac_malloc`, `fdkaac_free`) over `aacdecoder_lib.h`. +The module is a WASI reactor: call `_initialize()` once after instantiating. Its six +`wasi_snapshot_preview1` imports are stdio/exit stubs that are never exercised — see +`utils/media/aac-decoder.ts`. + +## Verifying a rebuild + +The committed binary was checked against Chromium's own AAC decoder by decoding a 5s AAC-LC clip +both ways and comparing PCM: identical sample count and **0.29% residual RMS** (max absolute sample +difference 0.0025), consistent with fixed-point vs float rounding. If you rebuild, re-run that +comparison rather than only checking that audio is audible — a wrong channel or sample-rate +configuration still produces plausible-sounding output. diff --git a/src/webview/vendor/fdk-aac/build.sh b/src/webview/vendor/fdk-aac/build.sh new file mode 100755 index 0000000..501b59f --- /dev/null +++ b/src/webview/vendor/fdk-aac/build.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Rebuilds fdk-aac-decoder.wasm. Run from this directory: +# +# FDK_AAC_SRC=/path/to/fdk-aac WASI_SDK=/path/to/wasi-sdk ./build.sh +# +# See README.md for the source revision and toolchain version the committed binary was built with. +# +set -euo pipefail + +: "${FDK_AAC_SRC:?set FDK_AAC_SRC to a checkout of https://github.com/mstorsjo/fdk-aac}" +: "${WASI_SDK:?set WASI_SDK to an extracted wasi-sdk release}" + +OUT="$(cd "$(dirname "$0")" && pwd)" + +DEC_LIBS="libAACdec libFDK libSYS libMpegTPDec libSBRdec libPCMutils libArithCoding libDRCdec libSACdec" + +INCLUDES="" +for d in libAACdec libArithCoding libDRCdec libSACdec libSBRdec libMpegTPDec libSYS libFDK libPCMutils; do + INCLUDES="$INCLUDES -I$FDK_AAC_SRC/$d/include -I$FDK_AAC_SRC/$d/src" +done + +SOURCES="" +for d in $DEC_LIBS; do + SOURCES="$SOURCES $(ls "$FDK_AAC_SRC/$d"/src/*.cpp)" +done + +# Decoder-only build: the encoder libraries (libAACenc, libSBRenc, libMpegTPEnc, libSACenc) are +# deliberately excluded. -w silences fdk-aac's "set architecture characterization defines" warning, +# which is expected for a generic (non-SIMD) target. +"$WASI_SDK/bin/clang++" \ + --target=wasm32-wasi \ + --sysroot="$WASI_SDK/share/wasi-sysroot" \ + -mexec-model=reactor \ + -Oz -w -DNDEBUG -fno-exceptions -fno-rtti -fno-threadsafe-statics \ + -fvisibility=hidden -ffunction-sections -fdata-sections \ + $INCLUDES \ + -I"$OUT" \ + -x c++ $SOURCES \ + -x c "$OUT/fdkaac_wrap.c" \ + -Wl,--no-entry -Wl,--gc-sections -Wl,--strip-all -Wl,--allow-undefined \ + -o "$OUT/fdk-aac-decoder.wasm" + +ls -la "$OUT/fdk-aac-decoder.wasm" diff --git a/src/webview/vendor/fdk-aac/fdk-aac-decoder.wasm b/src/webview/vendor/fdk-aac/fdk-aac-decoder.wasm new file mode 100755 index 0000000..8919393 Binary files /dev/null and b/src/webview/vendor/fdk-aac/fdk-aac-decoder.wasm differ diff --git a/src/webview/vendor/fdk-aac/fdkaac_wrap.c b/src/webview/vendor/fdk-aac/fdkaac_wrap.c new file mode 100644 index 0000000..bf56725 --- /dev/null +++ b/src/webview/vendor/fdk-aac/fdkaac_wrap.c @@ -0,0 +1,95 @@ +/* + * Minimal WebAssembly wrapper around the FDK AAC decoder. + * + * Exposes just enough of aacdecoder_lib.h to decode raw AAC access units (as stored in an + * mp4 `mdat`) into interleaved 16-bit PCM. Built for wasm32-wasi with wasi-sdk; see build.sh. + */ +#include +#include + +#include "aacdecoder_lib.h" + +/* 8 channels * 2048 samples covers AAC-LC (1024) and HE-AAC/SBR (2048) frame sizes. */ +#define MAX_OUT_SAMPLES (8 * 2048) + +typedef struct { + HANDLE_AACDECODER dec; + INT_PCM pcm[MAX_OUT_SAMPLES]; + int sample_rate; + int channels; + int frame_size; +} Decoder; + +#define EXPORT(name) __attribute__((export_name(name))) + +EXPORT("fdkaac_open") +Decoder *fdkaac_open(void) { + Decoder *d = (Decoder *)calloc(1, sizeof(Decoder)); + if (!d) return 0; + /* TT_MP4_RAW: packet-based access units, configured out of band via ConfigRaw. */ + d->dec = aacDecoder_Open(TT_MP4_RAW, 1); + if (!d->dec) { + free(d); + return 0; + } + return d; +} + +/* Feeds the AudioSpecificConfig from the mp4 `esds` box. Returns 0 on success. */ +EXPORT("fdkaac_config") +int fdkaac_config(Decoder *d, unsigned char *asc, int len) { + if (!d || !d->dec || len <= 0) return -1; + UCHAR *conf[1] = {asc}; + UINT conf_len[1] = {(UINT)len}; + return (int)aacDecoder_ConfigRaw(d->dec, conf, conf_len); +} + +/* + * Decodes one access unit. Returns the number of samples per channel on success, + * or a negative fdk-aac error code. + */ +EXPORT("fdkaac_decode") +int fdkaac_decode(Decoder *d, unsigned char *in, int len) { + if (!d || !d->dec || len <= 0) return -1; + + UCHAR *buf[1] = {in}; + UINT buf_size[1] = {(UINT)len}; + UINT valid = (UINT)len; + + AAC_DECODER_ERROR err = aacDecoder_Fill(d->dec, buf, buf_size, &valid); + if (err != AAC_DEC_OK) return -(int)err; + + err = aacDecoder_DecodeFrame(d->dec, d->pcm, MAX_OUT_SAMPLES, 0); + if (err != AAC_DEC_OK) return -(int)err; + + CStreamInfo *info = aacDecoder_GetStreamInfo(d->dec); + if (!info) return -1; + + d->sample_rate = info->sampleRate; + d->channels = info->numChannels; + d->frame_size = info->frameSize; + return info->frameSize; +} + +EXPORT("fdkaac_pcm") +INT_PCM *fdkaac_pcm(Decoder *d) { return d ? d->pcm : 0; } + +EXPORT("fdkaac_sample_rate") +int fdkaac_sample_rate(Decoder *d) { return d ? d->sample_rate : 0; } + +EXPORT("fdkaac_channels") +int fdkaac_channels(Decoder *d) { return d ? d->channels : 0; } + +EXPORT("fdkaac_close") +void fdkaac_close(Decoder *d) { + if (!d) return; + if (d->dec) aacDecoder_Close(d->dec); + free(d); +} + +/* Buffer allocation for the JS side. */ +EXPORT("fdkaac_malloc") +void *fdkaac_malloc(int size) { return malloc((size_t)size); } + +EXPORT("fdkaac_free") +void fdkaac_free(void *p) { free(p); }