Skip to content

Commit cae2567

Browse files
committed
fix(macos): decode 24-bit integer microphone audio
The ScreenCaptureKit microphone output delivers the input device's own format, not the 48 kHz stereo Float32 the stream is configured for. On a MacBook Pro (M1, macOS 26.6) that is 48 kHz mono 24-bit integer, for both the built-in and a USB microphone. The mixer only read Float32, Int16 and Int32, so it rejected every microphone buffer with audio-source-undecodable and each take recorded a silent microphone (#709). Integers of any width up to 32 bits are now read from their real container (mBytesPerFrame, not bits/8), packed or padded, high- or low-aligned. Big-endian input is still rejected rather than misread.
1 parent a90a7f1 commit cae2567

2 files changed

Lines changed: 155 additions & 14 deletions

File tree

‎electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -470,11 +470,13 @@ public final class AudioTrackMixer {
470470

471471
/// Decodes one capture buffer into gain-applied 48 kHz interleaved-stereo Float.
472472
///
473-
/// Both SCStream audio outputs are configured for 48 kHz stereo, so in practice this is a
474-
/// straight Float32 de-interleave. The format-adaptive paths (Int16/Int32, interleaved or
475-
/// not, off-rate sources) exist because the format is the stream's to choose, not ours —
476-
/// and because a resampled source's rounding drift is absorbed by timeline placement
477-
/// rather than accumulating, unlike in a FIFO mixer.
473+
/// Both SCStream audio outputs are configured for 48 kHz stereo, but only system audio
474+
/// honours that reliably: the microphone output can arrive in the input device's own format
475+
/// — 48 kHz mono 24-bit integer on a MacBook Pro's built-in and USB microphones (#709). So
476+
/// the format-adaptive paths (integers of any width up to 32 bits, interleaved or not,
477+
/// off-rate sources) are not defensive extras; the format is the stream's to choose, not
478+
/// ours. A resampled source's rounding drift is absorbed by timeline placement rather than
479+
/// accumulating, unlike in a FIFO mixer.
478480
private func decodeInterleavedStereo(_ sampleBuffer: CMSampleBuffer, gain: Float) -> [Float]? {
479481
guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
480482
let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)
@@ -487,11 +489,24 @@ public final class AudioTrackMixer {
487489
let sourceChannels = Int(asbd.mChannelsPerFrame)
488490
let bitsPerChannel = Int(asbd.mBitsPerChannel)
489491
let isFloat = asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0
492+
let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0
493+
// The container each sample sits in, which is not always `bitsPerChannel / 8`: a
494+
// microphone commonly arrives as 24-bit integer, either packed into 3 bytes or held in 4.
495+
// Measured on a MacBook Pro (M1, macOS 26.6): the ScreenCaptureKit microphone output
496+
// delivered 48 kHz mono 24-bit interleaved, which the Float32/Int16/Int32-only decoder
497+
// rejected buffer by buffer — every take's microphone was silent (issue #709).
498+
let bytesPerSample = sourceChannels > 0 && asbd.mBytesPerFrame > 0
499+
? Int(asbd.mBytesPerFrame) / (isNonInterleaved ? 1 : sourceChannels)
500+
: (bitsPerChannel + 7) / 8
490501
guard asbd.mFormatID == kAudioFormatLinearPCM,
502+
asbd.mFormatFlags & kAudioFormatFlagIsBigEndian == 0,
491503
sourceChannels > 0,
492504
asbd.mSampleRate > 0,
493505
sourceFrames > 0,
494-
isFloat ? bitsPerChannel == 32 : (bitsPerChannel == 16 || bitsPerChannel == 32)
506+
isFloat
507+
? bitsPerChannel == 32 && bytesPerSample == 4
508+
: (8...32).contains(bitsPerChannel) && (2...4).contains(bytesPerSample)
509+
&& bitsPerChannel <= bytesPerSample * 8
495510
else {
496511
return nil
497512
}
@@ -503,7 +518,6 @@ public final class AudioTrackMixer {
503518
// outputs disagree here: system audio arrives non-interleaved, the microphone
504519
// interleaved, so sizing this off the channel count alone silently drops every
505520
// microphone buffer.
506-
let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0
507521
let bufferCount = isNonInterleaved ? sourceChannels : 1
508522
let bufferList = AudioBufferList.allocate(maximumBuffers: bufferCount)
509523
defer { free(bufferList.unsafeMutablePointer) }
@@ -524,7 +538,6 @@ public final class AudioTrackMixer {
524538
}
525539

526540
return withExtendedLifetime(blockBuffer) { () -> [Float]? in
527-
let bytesPerChannelSample = bitsPerChannel / 8
528541
// `bufferList` is subscripted against its own count, not the format's channel
529542
// count: indexing past `count` would trap rather than degrade.
530543
guard bufferList.count > 0 else {
@@ -545,11 +558,15 @@ public final class AudioTrackMixer {
545558
readers.append(
546559
ChannelReader(
547560
base: UnsafeRawPointer(data),
548-
sampleCount: Int(buffer.mDataByteSize) / bytesPerChannelSample,
561+
sampleCount: Int(buffer.mDataByteSize) / bytesPerSample,
549562
stride: isNonInterleaved ? 1 : sourceChannels,
550563
start: isNonInterleaved ? 0 : sourceChannel,
551-
bytesPerSample: bytesPerChannelSample,
552-
isFloat: isFloat
564+
bytesPerSample: bytesPerSample,
565+
isFloat: isFloat,
566+
// Where the significant bits sit when they do not fill the container.
567+
// Irrelevant to packed formats, where the two coincide.
568+
padBits: bytesPerSample * 8 - bitsPerChannel,
569+
isAlignedHigh: asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh != 0
553570
)
554571
)
555572
}
@@ -691,6 +708,8 @@ public final class AudioTrackMixer {
691708
let start: Int
692709
let bytesPerSample: Int
693710
let isFloat: Bool
711+
let padBits: Int
712+
let isAlignedHigh: Bool
694713

695714
func value(at frame: Int) -> Float {
696715
let index = start + frame * stride
@@ -702,10 +721,18 @@ public final class AudioTrackMixer {
702721
if isFloat {
703722
return base.loadUnaligned(fromByteOffset: offset, as: Float.self)
704723
}
705-
if bytesPerSample == 2 {
706-
return Float(base.loadUnaligned(fromByteOffset: offset, as: Int16.self)) / 32_768
724+
// Integer of any width up to 32 bits: assemble the little-endian bytes at the top of
725+
// a 32-bit word, so the sign bit lands on bit 31 and one full scale serves every width.
726+
var word: UInt32 = 0
727+
for byte in 0..<bytesPerSample {
728+
let value = UInt32(base.load(fromByteOffset: offset + byte, as: UInt8.self))
729+
word |= value << UInt32(8 * (4 - bytesPerSample + byte))
730+
}
731+
if padBits > 0 && !isAlignedHigh {
732+
// Low-aligned: the padding is on top, so shift it out to put the sign bit on 31.
733+
word <<= UInt32(padBits)
707734
}
708-
return Float(base.loadUnaligned(fromByteOffset: offset, as: Int32.self)) / 2_147_483_648
735+
return Float(Int32(bitPattern: word)) / 2_147_483_648
709736
}
710737
}
711738

‎electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,47 @@ final class AudioTrackMixerTests: XCTestCase {
455455
XCTAssertLessThan(loudest, 16_500)
456456
}
457457

458+
/// A microphone does not have to arrive as Float32. A MacBook Pro (M1, macOS 26.6) handed
459+
/// the ScreenCaptureKit microphone output over as 48 kHz mono 24-bit integer, which the
460+
/// decoder rejected — Float32, Int16 and Int32 were the only formats it read — so every take
461+
/// recorded there had a silent microphone and one `audio-source-undecodable` warning to show
462+
/// for it (issue #709). Every integer container CoreAudio describes must decode, at the right
463+
/// level and with the right sign.
464+
func testIntegerMicrophoneFormatsAreHeardAtTheirLevel() {
465+
let layouts: [(bits: Int, bytes: Int, alignedHigh: Bool, label: String)] = [
466+
(24, 3, false, "24-bit packed (the reported format)"),
467+
(24, 4, false, "24-bit in 4 bytes, low-aligned"),
468+
(24, 4, true, "24-bit in 4 bytes, high-aligned"),
469+
(16, 2, false, "16-bit"),
470+
(32, 4, false, "32-bit"),
471+
]
472+
for layout in layouts {
473+
let sink = RecordingSink()
474+
let clock = TestClock(.zero)
475+
let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1)
476+
mixer.beginTimeline(at: clock.now)
477+
mixer.ingest(
478+
makeIntegerMonoBuffer(
479+
alternating: 0.5, frames: sampleRate / 5, at: clock.now,
480+
bits: layout.bits, bytes: layout.bytes, alignedHigh: layout.alignedHigh
481+
),
482+
from: .microphone
483+
)
484+
clock.advance(seconds: 0.2)
485+
mixer.finish(atSourceTime: clock.now)
486+
487+
let frames = mixedFrames(sink)
488+
XCTAssertEqual(seconds(ofFrames: frames), 0.2, accuracy: 0.011, layout.label)
489+
// Mono feeds both sides at half scale, and +0.5 stays positive: a byte-order or
490+
// sign-extension mistake shows up as the wrong level or a flipped first sample.
491+
for lane in 0..<channelCount {
492+
XCTAssertEqual(Int(peak(frames, from: 0, to: 0.2, channel: lane)), 16_384, accuracy: 2, layout.label)
493+
XCTAssertGreaterThan(frames[lane], 16_000, layout.label)
494+
XCTAssertLessThan(frames[channelCount + lane], -16_000, layout.label)
495+
}
496+
}
497+
}
498+
458499
// MARK: - Fixtures
459500

460501
private func makeMixer(
@@ -588,6 +629,79 @@ final class AudioTrackMixerTests: XCTestCase {
588629

589630
// MARK: - Readback
590631

632+
/// A mono, interleaved integer capture buffer, `bits` significant bits in a `bytes`-wide
633+
/// little-endian container, alternating +amplitude / -amplitude frame by frame.
634+
private func makeIntegerMonoBuffer(
635+
alternating amplitude: Float,
636+
frames frameCount: Int,
637+
at presentationTime: CMTime,
638+
bits: Int,
639+
bytes: Int,
640+
alignedHigh: Bool
641+
) -> CMSampleBuffer {
642+
var asbd = AudioStreamBasicDescription(
643+
mSampleRate: Float64(sampleRate),
644+
mFormatID: kAudioFormatLinearPCM,
645+
mFormatFlags: kAudioFormatFlagIsSignedInteger
646+
| (bits == bytes * 8 ? kAudioFormatFlagIsPacked : 0)
647+
| (alignedHigh ? kAudioFormatFlagIsAlignedHigh : 0),
648+
mBytesPerPacket: UInt32(bytes),
649+
mFramesPerPacket: 1,
650+
mBytesPerFrame: UInt32(bytes),
651+
mChannelsPerFrame: 1,
652+
mBitsPerChannel: UInt32(bits),
653+
mReserved: 0
654+
)
655+
var formatDescription: CMAudioFormatDescription?
656+
XCTAssertEqual(
657+
CMAudioFormatDescriptionCreate(
658+
allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil,
659+
magicCookieSize: 0, magicCookie: nil, extensions: nil,
660+
formatDescriptionOut: &formatDescription
661+
),
662+
noErr
663+
)
664+
var sampleBuffer: CMSampleBuffer?
665+
XCTAssertEqual(
666+
CMAudioSampleBufferCreateWithPacketDescriptions(
667+
allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false,
668+
makeDataReadyCallback: nil, refcon: nil, formatDescription: formatDescription!,
669+
sampleCount: frameCount, presentationTimeStamp: presentationTime,
670+
packetDescriptions: nil, sampleBufferOut: &sampleBuffer
671+
),
672+
noErr
673+
)
674+
675+
let byteCount = frameCount * bytes
676+
let memory = UnsafeMutableRawPointer.allocate(byteCount: byteCount, alignment: 16)
677+
defer { memory.deallocate() }
678+
let fullScale = Double(Int64(1) << (bits - 1))
679+
for frame in 0..<frameCount {
680+
let sample = Double(frame % 2 == 0 ? amplitude : -amplitude)
681+
var value = Int64((sample * fullScale).rounded())
682+
value = min(max(value, -Int64(fullScale)), Int64(fullScale) - 1)
683+
// High-aligned puts the padding below the value; low-aligned sign-extends above it.
684+
let word = UInt32(truncatingIfNeeded: alignedHigh ? value << Int64(bytes * 8 - bits) : value)
685+
for byte in 0..<bytes {
686+
memory.storeBytes(of: UInt8(truncatingIfNeeded: word >> UInt32(8 * byte)),
687+
toByteOffset: frame * bytes + byte, as: UInt8.self)
688+
}
689+
}
690+
let bufferList = AudioBufferList.allocate(maximumBuffers: 1)
691+
defer { free(bufferList.unsafeMutablePointer) }
692+
bufferList[0] = AudioBuffer(mNumberChannels: 1, mDataByteSize: UInt32(byteCount), mData: memory)
693+
XCTAssertEqual(
694+
CMSampleBufferSetDataBufferFromAudioBufferList(
695+
sampleBuffer!, blockBufferAllocator: kCFAllocatorDefault,
696+
blockBufferMemoryAllocator: kCFAllocatorDefault, flags: 0,
697+
bufferList: bufferList.unsafePointer
698+
),
699+
noErr
700+
)
701+
XCTAssertEqual(CMSampleBufferSetDataReady(sampleBuffer!), noErr)
702+
return sampleBuffer!
703+
}
704+
591705
private func mixedFrames(_ sink: RecordingSink) -> [Int16] {
592706
var samples = [Int16]()
593707
for buffer in sink.buffers {

0 commit comments

Comments
 (0)