import Foundation import LiveKitWebRTC import os // Where the audio stack's slow work runs: never on the main thread. // // Paul, 0.3.8, MacBook Air, AirPods Pro: "after I open the Test window, // there's a pause, then a beachball, then it works." Measured on an M1 Max // with a main-thread watchdog (now `CallsInterop --audio-start-stall`), // 0.3.8's test held the main thread three times on its way to a running // microphone: // // 1. Building the audio (CallAudio): 166–200 ms. WebRTC's peer connection // factory with LiveKit's engine module, 100–135 ms; the module's first // device list, read only to be logged, 61–68 ms. // 2. The in-memory loopback's answer, applied: 188–229 ms. Applying it // starts the engine's playout on WebRTC's worker thread, and // `setLocalDescription` waits for that before it returns, completion // handler or not — every RTCPeerConnection call is a blocking hop to // WebRTC's own threads. // 3. `initRecording`, which waits on the worker thread while the engine // starts with the microphone. Here, with no microphone permission, 25 // ms. With AirPods it is as long as the headset takes to move from A2DP // to its hands-free profile, often one to two seconds: with that start // simulated at 1.5 s the main thread stalled for 1.5–1.7 s. That is the // beachball; 1 and 2 are the pause before it. // // A call's join had the same shape (1.3–1.5 s with the switch simulated), // and more: while the worker thread is starting the engine, *every* call // into the engine module waits for it — `isRecording`, `isPlaying`, the // device lists, the factory's processing state, a track's // `isEnabled = false` (mute), its processing options — 0.9 to 1.2 s each, // and a call's meters read some of them ten times a second. // // None of it is the main thread's business. It all runs here, on one // serial queue, in the order it was asked for; the main actor awaits the // result and only ever sets what the window shows. What the main thread // needs to know about the module it reads from ModuleState. /// The audio stack's serial queue: the factory and its engine module, the /// engine's start and stop, and the peer connections' calls. /// /// Serial on purpose. A test closed while its microphone is still /// starting queues its stop behind the start, so the stop always sees the /// microphone it has to close; nothing is left recording. enum AudioWork { static let queue = DispatchQueue(label: "net.amadan.oficina.calls.audio", qos: .userInitiated) /// Runs `body` on the audio queue and hands its result back. static func run(_ body: @escaping () -> T) async -> T { await withCheckedContinuation { (c: CheckedContinuation) in queue.async { c.resume(returning: body()) } } } static func run(_ body: @escaping () throws -> T) async throws -> T { try await withCheckedThrowingContinuation { (c: CheckedContinuation) in queue.async { do { c.resume(returning: try body()) } catch { c.resume(throwing: error) } } } } /// Queues `body` and does not wait for it: a stop, a close, a log line /// that has to read the module. static func post(_ body: @escaping () -> Void) { queue.async(execute: body) } /// Waits for everything queued so far. For tests and the harness. static func drain() async { await run {} } } /// A peer connection's calls, made from the audio queue. /// /// Every RTCPeerConnection method is a blocking hop to one of WebRTC's /// own threads, completion handler or not, and applying a description /// waits for the worker thread — which may be busy starting the audio /// engine for seconds. So a call makes them from here and awaits the /// answer; the main thread never waits on WebRTC. enum PeerCalls { static func describe( _ pc: RTCPeerConnection, answer: Bool, _ constraints: RTCMediaConstraints ) async throws -> RTCSessionDescription { try await withCheckedThrowingContinuation { c in AudioWork.queue.async { let completion: @Sendable (RTCSessionDescription?, Error?) -> Void = { desc, error in if let desc { c.resume(returning: desc) } else { c.resume(throwing: error ?? CancellationError()) } } if answer { pc.answer(for: constraints, completionHandler: completion) } else { pc.offer(for: constraints, completionHandler: completion) } } } } static func set(_ pc: RTCPeerConnection, local: Bool, _ desc: RTCSessionDescription) async throws { try await withCheckedThrowingContinuation { (c: CheckedContinuation) in AudioWork.queue.async { let done: @Sendable (Error?) -> Void = { error in if let error { c.resume(throwing: error) } else { c.resume() } } if local { pc.setLocalDescription(desc, completionHandler: done) } else { pc.setRemoteDescription(desc, completionHandler: done) } } } } static func add(_ pc: RTCPeerConnection, _ candidate: RTCIceCandidate) async throws { try await withCheckedThrowingContinuation { (c: CheckedContinuation) in AudioWork.queue.async { pc.add(candidate) { error in if let error { c.resume(throwing: error) } else { c.resume() } } } } } static func statistics(_ pc: RTCPeerConnection) async -> RTCStatisticsReport { await withCheckedContinuation { c in AudioWork.queue.async { pc.statistics { c.resume(returning: $0) } } } } static func statistics(_ pc: RTCPeerConnection, for receiver: RTCRtpReceiver) async -> RTCStatisticsReport { await withCheckedContinuation { c in AudioWork.queue.async { pc.statistics(for: receiver) { c.resume(returning: $0) } } } } static func statistics(_ pc: RTCPeerConnection, for sender: RTCRtpSender) async -> RTCStatisticsReport { await withCheckedContinuation { c in AudioWork.queue.async { pc.statistics(for: sender) { c.resume(returning: $0) } } } } /// Anything else, read or done on the audio queue. static func read(_ pc: RTCPeerConnection, _ body: @escaping (RTCPeerConnection) -> T) async -> T { await AudioWork.run { body(pc) } } /// Closes it after whatever is queued for it; not waited for. Closing /// stops the audio engine when it is the last connection, and that can /// take as long as a start. static func close(_ pc: RTCPeerConnection) { AudioWork.post { pc.close() } } } #if canImport(DeepFilter) /// DeepFilterNet's model, loaded before anyone needs it. /// /// Loading takes about a third of a second on an M1 Max (and longer on /// an Air) of one core, and a call or test that has to load it starts on /// RNNoise and moves to DeepFilterNet when it is ready. So one is loaded /// in the background a few seconds after launch, at utility priority, /// and the first test or call takes it ready-made. /// /// Memory is the cost: a loaded model is about 88 MB resident. So only /// one is kept spare; a spare nobody has taken for fifteen minutes is /// let go, and the next user loads its own as before (off the main /// thread, starting on RNNoise). Taking the spare does not replace it at /// once — a call would then hold two — but when its user is done, a new /// spare is loaded for the next one. A model is never handed from one /// user to the next: its state holds the last audio it heard, and there /// is no way to clear it. public enum DeepFilterPreload { private struct State { var spare: DeepFilterSuppressor? var loading = false var generation = 0 } // The suppressor is only ever touched by whoever holds it; the lock // guards the handing over. nonisolated(unsafe) private static var state = State() private static let lock = NSLock() private static let background = DispatchQueue(label: "net.amadan.oficina.calls.deepfilter", qos: .utility) static let idleRelease: TimeInterval = 15 * 60 /// After launch: loads a spare in `delay` seconds, if there is a model /// and no spare yet. public static func preload(after delay: TimeInterval = 3) { background.asyncAfter(deadline: .now() + delay) { load() } } /// The spare, if one is ready; nil otherwise, and the caller loads. static func take() -> DeepFilterSuppressor? { lock.lock() defer { lock.unlock() } let spare = state.spare state.spare = nil state.generation += 1 if spare != nil { AudioLog.info("DeepFilterNet: using the preloaded model") } return spare } /// A user of a model is done with it: a fresh spare, a little later. static func finished() { preload(after: 5) } /// Whether a spare is loaded (for the tests). static var ready: Bool { lock.lock() defer { lock.unlock() } return state.spare != nil } private static func load() { lock.lock() guard state.spare == nil, !state.loading, let model = DeepFilterSuppressor.model else { lock.unlock() return } state.loading = true lock.unlock() let started = CaptureDenoiser.now() let loaded = DeepFilterSuppressor(model: model) lock.lock() state.loading = false state.spare = loaded state.generation += 1 let generation = state.generation lock.unlock() guard loaded != nil else { return } AudioLog.info( "DeepFilterNet: preloaded in \(Int((CaptureDenoiser.now() - started) * 1000)) ms, kept for the next test or call") background.asyncAfter(deadline: .now() + idleRelease) { release(ifStill: generation) } } private static func release(ifStill generation: Int) { lock.lock() guard state.generation == generation, state.spare != nil else { lock.unlock() return } let spare = state.spare state.spare = nil lock.unlock() _ = spare AudioLog.info("DeepFilterNet: preloaded model unused for \(Int(idleRelease / 60)) minutes; released") } } #endif /// What the app does for its audio once it has launched: loads the /// noise suppressor's model in the background (DeepFilterPreload), so the /// first test or call does not wait for it. Nothing on iPhone, which runs /// RNNoise (made in well under a millisecond). @MainActor public enum AudioWarmUp { private static var done = false /// Once per launch; later calls (a window shown again) do nothing. public static func afterLaunch() { guard !done else { return } done = true #if canImport(DeepFilter) DeepFilterPreload.preload(after: 3) #endif } }