Skip to content

Commit 36ebf97

Browse files
committed
fix(macos): realign a microphone stamped on a foreign clock
ScreenCaptureKit does not promise the microphone output is stamped on the host clock the writer runs on. The mixer placed every buffer by its raw presentation timestamp, so a microphone whose time base sits seconds or hours away from the session landed wholly before frame zero (trimmed as pre-roll) or past the end of the take (never reached): a silent mic track, with no error. It went unnoticed because the only test Mac has no input device, and there the microphone output mirrors system audio, clock included. The first buffer of each source now decides its offset: within 1 s of the clock it is left alone (ordinary latency); beyond, it is mapped so that buffer ends where the clock stands and later buffers keep their spacing. The offset is reported in the audio-timeline summary and warned about once.
1 parent 520f6e5 commit 36ebf97

2 files changed

Lines changed: 149 additions & 3 deletions

File tree

‎electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift‎

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ public final class AudioTrackMixer {
9595
static let maxPendingChunks = 500
9696
/// How long the final flush waits for the input to accept the tail before giving up.
9797
static let finalFlushTimeout = 5.0
98+
/// How far a source's first buffer may sit from the clock before its timestamps are
99+
/// taken to be on some other clock entirely. A live ScreenCaptureKit output hands
100+
/// over audio tens of milliseconds after capturing it; a whole second apart is not
101+
/// latency, it is a different time base. See `clockOffsets`.
102+
static let foreignClockThreshold = CMTime(value: 1, timescale: 1)
98103
}
99104

100105
private let input: MixedAudioSink
@@ -124,6 +129,22 @@ public final class AudioTrackMixer {
124129
private var pending: [CMSampleBuffer] = []
125130
private var didWarnAboutBacklog = false
126131
private var didWarnAboutDecode: Set<Int> = []
132+
/// Per source, what is added to every presentation timestamp before placement — fixed on
133+
/// that source's first delivery, nil until then.
134+
///
135+
/// Zero for a source whose timestamps are on the writer's clock, which is what placement
136+
/// by timestamp assumes. ScreenCaptureKit does not promise that for the microphone output
137+
/// (Apple: the microphone and app audio are on independent clocks), and on a Mac with no
138+
/// input device at all the microphone output mirrors system audio, clock included — so
139+
/// the only machine the mixer had been proven on was the one place the assumption held.
140+
/// Placed raw, a microphone stamped seconds or hours off the session lands entirely
141+
/// before frame zero (all trimmed) or beyond the end of the take (never reached), and the
142+
/// track comes out silent with every buffer accounted as delivered.
143+
///
144+
/// A foreign time base is recognised on the first buffer and mapped onto the clock once:
145+
/// that buffer is taken to end where the clock stands, and every later one keeps its own
146+
/// spacing from it, so the source's timing is preserved and only its origin moves.
147+
private var clockOffsets = [CMTime?](repeating: nil, count: Source.allCases.count)
127148

128149
public init(
129150
input: MixedAudioSink,
@@ -164,8 +185,8 @@ public final class AudioTrackMixer {
164185
guard includes(source), let anchor else {
165186
return
166187
}
167-
let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
168-
guard presentationTime.isValid, presentationTime.isNumeric else {
188+
let capturedTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
189+
guard capturedTime.isValid, capturedTime.isNumeric else {
169190
return
170191
}
171192
guard let frames = decodeInterleavedStereo(sampleBuffer, gain: gain(for: source)),
@@ -178,6 +199,15 @@ public final class AudioTrackMixer {
178199
}
179200

180201
let now = clock()
202+
let presentationTime = CMTimeAdd(
203+
capturedTime,
204+
clockOffset(
205+
for: source,
206+
capturedAt: capturedTime,
207+
frameCount: frames.count / MixFormat.channelCount,
208+
now: now
209+
)
210+
)
181211
let startFrame = frameIndex(of: presentationTime, from: anchor)
182212
noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount))
183213
sources[source.rawValue].ingest(frames, atFrame: startFrame)
@@ -212,6 +242,41 @@ public final class AudioTrackMixer {
212242
flushPending(force: true)
213243
}
214244

245+
/// The offset `clockOffsets` documents, fixed on the source's first delivery.
246+
private func clockOffset(
247+
for source: Source,
248+
capturedAt capturedTime: CMTime,
249+
frameCount: Int,
250+
now: CMTime
251+
) -> CMTime {
252+
if let offset = clockOffsets[source.rawValue] {
253+
return offset
254+
}
255+
guard now.isValid, now.isNumeric else {
256+
return .zero
257+
}
258+
259+
// Judged on where the buffer starts: a live buffer is a few milliseconds long and
260+
// arrives just after it, so start and arrival differ only by latency.
261+
let skew = CMTimeSubtract(now, capturedTime)
262+
guard CMTimeCompare(CMTimeAbsoluteValue(skew), MixFormat.foreignClockThreshold) > 0 else {
263+
clockOffsets[source.rawValue] = .zero
264+
return .zero
265+
}
266+
// Realigned so that the buffer ends where the clock stands — it was just captured.
267+
let offset = CMTimeSubtract(
268+
skew,
269+
CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(MixFormat.sampleRate))
270+
)
271+
clockOffsets[source.rawValue] = offset
272+
emit([
273+
"event": "warning",
274+
"code": "audio-source-clock-rebased",
275+
"message": "\(source == .system ? "System" : "Microphone") audio is timestamped \(String(format: "%.3f", CMTimeGetSeconds(skew))) s off the capture clock; it was realigned to arrival.",
276+
])
277+
return offset
278+
}
279+
215280
private func warnAboutDecodeFailure(_ source: Source, _ sampleBuffer: CMSampleBuffer) {
216281
guard !didWarnAboutDecode.contains(source.rawValue) else {
217282
return
@@ -308,6 +373,7 @@ public final class AudioTrackMixer {
308373
"longestHoleSeconds": report.longestHoleSeconds,
309374
"droppedSeconds": report.droppedSeconds,
310375
"trimmedSeconds": report.trimmedSeconds,
376+
"clockOffsetSeconds": report.clockOffsetSeconds,
311377
]
312378
}
313379
emit(fields)
@@ -321,6 +387,9 @@ public final class AudioTrackMixer {
321387
public let longestHoleSeconds: Double
322388
public let droppedSeconds: Double
323389
public let trimmedSeconds: Double
390+
/// What `clockOffsets` added to this source's timestamps; 0 when it was already on the
391+
/// capture clock (or never delivered).
392+
public let clockOffsetSeconds: Double
324393
}
325394

326395
public func deliveryReport(for source: Source) -> DeliveryReport {
@@ -329,7 +398,8 @@ public final class AudioTrackMixer {
329398
undeliveredSeconds: seconds(undeliveredFrames[source.rawValue]),
330399
longestHoleSeconds: seconds(longestHoleFrames[source.rawValue]),
331400
droppedSeconds: seconds(sources[source.rawValue].droppedFrames),
332-
trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames)
401+
trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames),
402+
clockOffsetSeconds: clockOffsets[source.rawValue].map(CMTimeGetSeconds) ?? 0
333403
)
334404
}
335405

‎electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,82 @@ final class AudioTrackMixerTests: XCTestCase {
218218
XCTAssertEqual((microphoneOnset ?? 0) - (systemOnset ?? 0), 0.090, accuracy: 0.002)
219219
}
220220

221+
/// The microphone output is not guaranteed to be stamped on the writer's clock. On this
222+
/// project's only test Mac it was — with no input device, ScreenCaptureKit's microphone
223+
/// output mirrors system audio, clock included — while on a MacBook with a real microphone
224+
/// every take came out with a silent track. Placed raw, audio stamped far behind the
225+
/// session is trimmed as pre-roll and audio stamped far ahead is never reached; either way
226+
/// the file holds silence. Both directions must come out audible, where they were spoken.
227+
func testAMicrophoneOnAForeignClockIsStillHeard() {
228+
for foreignOffset in [-3_600.0, -47.5, 9_000.0] {
229+
let (frames, mixer) = recordLiveTake(microphoneClockOffset: foreignOffset)
230+
let label = "offset \(foreignOffset)"
231+
XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011, label)
232+
// Heard from its first delivery on (the span before that is the ordinary head trim
233+
// every source gets) and still heard at the end of the take.
234+
XCTAssertLessThanOrEqual(firstAudibleSecond(frames, channel: 1) ?? 99, 0.111, label)
235+
XCTAssertGreaterThan(peak(frames, from: 1.7, to: 1.9, channel: 1), audibleThreshold, label)
236+
// System audio, on the right clock, is untouched by the realignment.
237+
XCTAssertGreaterThan(peak(frames, from: 0.1, to: 1.9, channel: 0), audibleThreshold, label)
238+
XCTAssertEqual(mixer.deliveryReport(for: .system).clockOffsetSeconds, 0, label)
239+
// The microphone was moved by its foreign offset and nothing else: its first buffer
240+
// ends where the clock stood when it arrived, which is where it was captured.
241+
XCTAssertEqual(
242+
mixer.deliveryReport(for: .microphone).clockOffsetSeconds, -foreignOffset, accuracy: 0.001, label)
243+
XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, label)
244+
}
245+
}
246+
247+
/// The realignment must not touch a microphone that is on the clock: its buffers arrive a
248+
/// little after the audio they describe, and that latency is not a foreign time base.
249+
func testAMicrophoneOnTheCaptureClockIsNotRealigned() {
250+
let (frames, mixer) = recordLiveTake(microphoneClockOffset: 0, microphoneLatency: 0.3)
251+
XCTAssertEqual(mixer.deliveryReport(for: .microphone).clockOffsetSeconds, 0)
252+
XCTAssertGreaterThan(peak(frames, from: 1.2, to: 1.4, channel: 1), audibleThreshold)
253+
}
254+
255+
/// Two seconds of live capture, delivered the way ScreenCaptureKit does: each 100 ms
256+
/// buffer describes the 100 ms that just elapsed. System audio on the left, stamped on the
257+
/// writer's clock; the microphone on the right, stamped `microphoneClockOffset` seconds off
258+
/// it and handed over `microphoneLatency` seconds after its span ends.
259+
private func recordLiveTake(
260+
microphoneClockOffset: Double,
261+
microphoneLatency: Double = 0
262+
) -> ([Int16], AudioTrackMixer) {
263+
let sink = RecordingSink()
264+
let clock = TestClock(CMTime(value: 50, timescale: 1))
265+
let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1)
266+
mixer.beginTimeline(at: clock.now)
267+
let sessionStart = clock.now
268+
let step = 0.1
269+
var pendingMicrophone: [(deliverAt: Double, buffer: CMSampleBuffer)] = []
270+
271+
for index in 0..<20 {
272+
let spanStart = CMTimeAdd(sessionStart, CMTime(seconds: Double(index) * step, preferredTimescale: 48_000))
273+
clock.advance(seconds: step)
274+
mixer.ingest(
275+
makeSourceBuffer(burst(seconds: step, left: 0.5, right: 0), at: spanStart, nonInterleaved: true),
276+
from: .system
277+
)
278+
pendingMicrophone.append((
279+
deliverAt: Double(index + 1) * step + microphoneLatency,
280+
buffer: makeSourceBuffer(
281+
burst(seconds: step, left: 0, right: 0.5),
282+
at: CMTimeAdd(spanStart, CMTime(seconds: microphoneClockOffset, preferredTimescale: 48_000)),
283+
nonInterleaved: false
284+
)
285+
))
286+
let elapsed = Double(index + 1) * step
287+
while let next = pendingMicrophone.first, next.deliverAt <= elapsed + 1e-9 {
288+
mixer.ingest(next.buffer, from: .microphone)
289+
pendingMicrophone.removeFirst()
290+
}
291+
mixer.tick()
292+
}
293+
mixer.finish(atSourceTime: clock.now)
294+
return (mixedFrames(sink), mixer)
295+
}
296+
221297
/// A dead microphone must not hold the track back — the whole reason the mixer marks a
222298
/// source stalled rather than waiting for it. Now that lateness is measured against the
223299
/// clock, this works even when the *other* source is silent too, which is the case the old

0 commit comments

Comments
 (0)