@@ -95,6 +95,11 @@ public final class AudioTrackMixer {
9595 static let maxPendingChunks = 500
9696 /// How long the final flush waits for the input to accept the tail before giving up.
9797 static let finalFlushTimeout = 5.0
98+ /// How far a source's first buffer may sit from the clock before its timestamps are
99+ /// taken to be on some other clock entirely. A live ScreenCaptureKit output hands
100+ /// over audio tens of milliseconds after capturing it; a whole second apart is not
101+ /// latency, it is a different time base. See `clockOffsets`.
102+ static let foreignClockThreshold = CMTime ( value: 1 , timescale: 1 )
98103 }
99104
100105 private let input : MixedAudioSink
@@ -124,6 +129,22 @@ public final class AudioTrackMixer {
124129 private var pending : [ CMSampleBuffer ] = [ ]
125130 private var didWarnAboutBacklog = false
126131 private var didWarnAboutDecode : Set < Int > = [ ]
132+ /// Per source, what is added to every presentation timestamp before placement — fixed on
133+ /// that source's first delivery, nil until then.
134+ ///
135+ /// Zero for a source whose timestamps are on the writer's clock, which is what placement
136+ /// by timestamp assumes. ScreenCaptureKit does not promise that for the microphone output
137+ /// (Apple: the microphone and app audio are on independent clocks), and on a Mac with no
138+ /// input device at all the microphone output mirrors system audio, clock included — so
139+ /// the only machine the mixer had been proven on was the one place the assumption held.
140+ /// Placed raw, a microphone stamped seconds or hours off the session lands entirely
141+ /// before frame zero (all trimmed) or beyond the end of the take (never reached), and the
142+ /// track comes out silent with every buffer accounted as delivered.
143+ ///
144+ /// A foreign time base is recognised on the first buffer and mapped onto the clock once:
145+ /// that buffer is taken to end where the clock stands, and every later one keeps its own
146+ /// spacing from it, so the source's timing is preserved and only its origin moves.
147+ private var clockOffsets = [ CMTime? ] ( repeating: nil , count: Source . allCases. count)
127148
128149 public init (
129150 input: MixedAudioSink ,
@@ -164,8 +185,8 @@ public final class AudioTrackMixer {
164185 guard includes ( source) , let anchor else {
165186 return
166187 }
167- let presentationTime = CMSampleBufferGetPresentationTimeStamp ( sampleBuffer)
168- guard presentationTime . isValid, presentationTime . isNumeric else {
188+ let capturedTime = CMSampleBufferGetPresentationTimeStamp ( sampleBuffer)
189+ guard capturedTime . isValid, capturedTime . isNumeric else {
169190 return
170191 }
171192 guard let frames = decodeInterleavedStereo ( sampleBuffer, gain: gain ( for: source) ) ,
@@ -178,6 +199,15 @@ public final class AudioTrackMixer {
178199 }
179200
180201 let now = clock ( )
202+ let presentationTime = CMTimeAdd (
203+ capturedTime,
204+ clockOffset (
205+ for: source,
206+ capturedAt: capturedTime,
207+ frameCount: frames. count / MixFormat. channelCount,
208+ now: now
209+ )
210+ )
181211 let startFrame = frameIndex ( of: presentationTime, from: anchor)
182212 noteDelivery ( source, from: startFrame, frameCount: Int64 ( frames. count / MixFormat. channelCount) )
183213 sources [ source. rawValue] . ingest ( frames, atFrame: startFrame)
@@ -212,6 +242,41 @@ public final class AudioTrackMixer {
212242 flushPending ( force: true )
213243 }
214244
245+ /// The offset `clockOffsets` documents, fixed on the source's first delivery.
246+ private func clockOffset(
247+ for source: Source ,
248+ capturedAt capturedTime: CMTime ,
249+ frameCount: Int ,
250+ now: CMTime
251+ ) -> CMTime {
252+ if let offset = clockOffsets [ source. rawValue] {
253+ return offset
254+ }
255+ guard now. isValid, now. isNumeric else {
256+ return . zero
257+ }
258+
259+ // Judged on where the buffer starts: a live buffer is a few milliseconds long and
260+ // arrives just after it, so start and arrival differ only by latency.
261+ let skew = CMTimeSubtract ( now, capturedTime)
262+ guard CMTimeCompare ( CMTimeAbsoluteValue ( skew) , MixFormat . foreignClockThreshold) > 0 else {
263+ clockOffsets [ source. rawValue] = . zero
264+ return . zero
265+ }
266+ // Realigned so that the buffer ends where the clock stands — it was just captured.
267+ let offset = CMTimeSubtract (
268+ skew,
269+ CMTime ( value: CMTimeValue ( frameCount) , timescale: CMTimeScale ( MixFormat . sampleRate) )
270+ )
271+ clockOffsets [ source. rawValue] = offset
272+ emit ( [
273+ " event " : " warning " ,
274+ " code " : " audio-source-clock-rebased " ,
275+ " message " : " \( source == . system ? " System " : " Microphone " ) audio is timestamped \( String ( format: " %.3f " , CMTimeGetSeconds ( skew) ) ) s off the capture clock; it was realigned to arrival. " ,
276+ ] )
277+ return offset
278+ }
279+
215280 private func warnAboutDecodeFailure( _ source: Source , _ sampleBuffer: CMSampleBuffer ) {
216281 guard !didWarnAboutDecode. contains ( source. rawValue) else {
217282 return
@@ -308,6 +373,7 @@ public final class AudioTrackMixer {
308373 " longestHoleSeconds " : report. longestHoleSeconds,
309374 " droppedSeconds " : report. droppedSeconds,
310375 " trimmedSeconds " : report. trimmedSeconds,
376+ " clockOffsetSeconds " : report. clockOffsetSeconds,
311377 ]
312378 }
313379 emit ( fields)
@@ -321,6 +387,9 @@ public final class AudioTrackMixer {
321387 public let longestHoleSeconds : Double
322388 public let droppedSeconds : Double
323389 public let trimmedSeconds : Double
390+ /// What `clockOffsets` added to this source's timestamps; 0 when it was already on the
391+ /// capture clock (or never delivered).
392+ public let clockOffsetSeconds : Double
324393 }
325394
326395 public func deliveryReport( for source: Source ) -> DeliveryReport {
@@ -329,7 +398,8 @@ public final class AudioTrackMixer {
329398 undeliveredSeconds: seconds ( undeliveredFrames [ source. rawValue] ) ,
330399 longestHoleSeconds: seconds ( longestHoleFrames [ source. rawValue] ) ,
331400 droppedSeconds: seconds ( sources [ source. rawValue] . droppedFrames) ,
332- trimmedSeconds: seconds ( sources [ source. rawValue] . trimmedFrames)
401+ trimmedSeconds: seconds ( sources [ source. rawValue] . trimmedFrames) ,
402+ clockOffsetSeconds: clockOffsets [ source. rawValue] . map ( CMTimeGetSeconds) ?? 0
333403 )
334404 }
335405
0 commit comments