import AVFoundation import Foundation import LiveKitWebRTC /// Something other than a microphone to capture from: the interop /// harness's synthetic voice, as SyntheticVideo is its camera. Called on /// the audio engine's render thread with mono float samples in -1...1. /// On Mac only, where the engine module can run without any device. public protocol AudioInputSource: AnyObject, Sendable { func render(_ samples: UnsafeMutablePointer, count: Int, sampleRate: Double) } /// Which of WebRTC's processing stages run on the microphone, decided in /// one place so the rule can be read and tested apart from WebRTC. /// /// The rule the plan holds: our suppressor never runs beside another one. /// Not beside WebRTC's own noise suppressor, and not beside Apple's Voice /// Isolation. And WebRTC's gain control, which runs *before* our /// suppressor, is off whenever ours is on; SpeechGain lifts the voice /// after the noise has gone instead. public struct ProcessingPlan: Equatable, Sendable { public enum Where: Equatable, Sendable { case off, software, platform } /// Our neural suppressor, in the capture post-processing hook. public var suppressor: Bool public var echoCancellation: Where public var noiseSuppression: Where public var gainControl: Where /// WebRTC's high-pass filter, always: it is cheap and only takes out /// rumble below the voice. public var highPass = true /// - Parameters: /// - iPhone: the phone runs on Apple's voice-processing unit, whose /// echo canceller and noise suppressor are one switch; the Mac runs /// WebRTC's software processing. /// - reduceNoise: the person's toggle. /// - rate: the capture rate WebRTC reported, or nil before it has. /// - voiceIsolation: Apple's Voice Isolation is the active mic mode. public static func plan(iPhone: Bool, reduceNoise: Bool, rate: Int?, voiceIsolation: Bool) -> ProcessingPlan { let rateOK = rate.map { $0 == CaptureDenoiser.supportedRate } ?? true let ours = reduceNoise && rateOK && !voiceIsolation if iPhone { // Apple's echo canceller cannot be had without Apple's suppressor // beside it; that pairing is what the phone has always run. Its gain // control is separate, and is what comes off. return ProcessingPlan( suppressor: ours, echoCancellation: .platform, noiseSuppression: .platform, gainControl: ours ? .off : .platform) } let classicNS = reduceNoise && !rateOK return ProcessingPlan( suppressor: ours, echoCancellation: .software, noiseSuppression: classicNS ? .software : .off, gainControl: ours ? .off : .software) } } /// What the audio readout says about processing. public struct ProcessingReport: Equatable, Sendable { public var suppression: SuppressionState public var echoCancellation: String public var noiseSuppression: String public var gainControl: String /// Set when WebRTC refused the options asked of it. public var refused: String? /// False until the microphone is actually capturing. WebRTC opens it /// only once there is someone to send to, so a person alone in a call /// has no microphone running yet, and what the processing module says /// before then is its construction-time default (16 kHz, everything /// off) rather than anything about their microphone. public var started = true /// One or two plain sentences. public var summary: String { guard started else { return "The microphone has not started yet. It starts when someone else is in the call, and this will say what is running on it then." } var lines: [String] = [] switch suppression { case .off: lines.append("Background noise reduction is off.") case .running(let engine, let cpu): lines.append( "Reducing background noise with \(engine) (\(String(format: "%.1f", cpu * 100))% of real time on the audio thread).") case .idle(let engine): lines.append("Reducing background noise with \(engine) when you speak.") case .unsupportedRate(let rate): lines.append( "The microphone runs at \(rate / 1000) kHz, which the noise reducer cannot; WebRTC's own is standing in.") case .standingAside: lines.append("Voice Isolation is on, so Calls leaves noise reduction to it.") } lines.append( "Echo cancellation: \(echoCancellation). Noise suppression: \(noiseSuppression). Gain: \(gainControl).") if let refused { lines.append("WebRTC refused the requested processing: \(refused)") } return lines.joined(separator: " ") } } /// How a call's audio is built: which audio device module WebRTC runs on, /// what processing runs on the microphone, and — on Mac — what the app is /// told about the devices it can use. /// /// Kept out of Media.swift so that file only changes where the factory is /// made. @MainActor final class CallAudio { let factory: RTCPeerConnectionFactory /// The factory's audio device module, fetched once, off the main /// thread. Fetching it, and every property and call on it, is a blocking /// hop to WebRTC's worker thread, which is held for the whole of an /// engine start, a Bluetooth switch included (1,544 ms measured on the /// main thread with the switch simulated). So it is only ever asked on /// the audio queue; the main thread reads `state`. let module: RTCAudioDeviceModule /// The module's state for the main thread to read: never by asking the /// module, whose every property waits out an engine start (ModuleState). let state: ModuleState /// Nil for the iPhone tests' silent device, which takes the constructor /// that has no processing module. let processing: RTCDefaultAudioProcessingModule? let denoiser: CaptureDenoiser private weak var track: RTCAudioTrack? private var rate: Int? private var voiceIsolation = false private var refused: String? private(set) var plan: ProcessingPlan private var buffersAtStart: UInt64 = 0 var reduceNoise: Bool { didSet { if reduceNoise != oldValue { apply() } } } #if os(macOS) private let devices: EngineWatch /// Where the engine is put (AudioDevices.swift); read by the tests. var enginePin: AudioEnginePin { devices.pin } #endif /// What building a CallAudio takes, made off the main thread (`make`): /// the peer connection factory with its audio device module and /// processing module, and the capture hook. Making the factory alone /// took 100–135 ms on an M1 Max. struct Parts: @unchecked Sendable { let factory: RTCPeerConnectionFactory let module: RTCAudioDeviceModule let state: ModuleState let processing: RTCDefaultAudioProcessingModule? let denoiser: CaptureDenoiser #if os(macOS) let devices: EngineWatch #endif } /// Builds the parts. Nonisolated, and slow: `make` runs it on the audio /// queue. `routes` is only held (weakly) by the module's observer, which /// reaches it on the main queue. nonisolated static func parts( routes: AudioRoutes, defaults: UserDefaults, audioDevice: RTCAudioDevice?, input: AudioInputSource?, denoiser: CaptureDenoiser? ) -> Parts { let encoder = RTCDefaultVideoEncoderFactory() // WebRTC's own, except in the gate's decoder-wedge scenarios. let decoder: RTCVideoDecoderFactory = DecoderWedge.enabled ? DecoderWedge.factory() : RTCDefaultVideoDecoderFactory() let denoiser = denoiser ?? CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) #if os(iOS) if let audioDevice { let factory = RTCPeerConnectionFactory( encoderFactory: encoder, decoderFactory: decoder, audioDevice: audioDevice) let module = factory.audioDeviceModule return Parts( factory: factory, module: module, state: ModuleState(factory: factory, module: module), processing: nil, denoiser: denoiser) } let processing = RTCDefaultAudioProcessingModule( config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) // The platform default on iPhone is the voice-processing I/O unit it // has always been. let factory = RTCPeerConnectionFactory( audioDeviceModuleType: .platformDefault, bypassVoiceProcessing: false, encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) let module = factory.audioDeviceModule return Parts( factory: factory, module: module, state: ModuleState(factory: factory, module: module), processing: processing, denoiser: denoiser) #else let processing = RTCDefaultAudioProcessingModule( config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) // AVAudioEngine rather than the CoreAudio module: it is the one with a // device list and a way to choose from it. let factory = RTCPeerConnectionFactory( audioDeviceModuleType: .audioEngine, bypassVoiceProcessing: false, encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) let module = factory.audioDeviceModule // Apple's voice processing stays off on Mac. Left allowed, the engine // module resolves echo cancellation, noise suppression and gain to // Apple's coupled voice-processing unit instead of WebRTC's own — // a different pipeline from the one the Mac app has always had, and // one whose noise suppression cannot be separated from its echo // canceller. WebRTC's software processing is what ran before this // module, and is what runs now. _ = module.setPlatformVoiceProcessingAllowed(false) let pin = AudioEnginePin() let devices = EngineWatch( routes: routes, module: module, input: input, pin: pin, channels: InputChannelMapper( defaults: defaults, layout: pin.inputLayout, explicitInput: { pin.choices.input })) module.observer = devices if input != nil { _ = module.setManualRenderingMode(true) } return Parts( factory: factory, module: module, state: ModuleState(factory: factory, module: module), processing: processing, denoiser: denoiser, devices: devices) #endif } /// Builds a CallAudio with its slow half on the audio queue: the main /// thread only attaches the result. What a call and the audio test use. static func make( routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, reduceNoise: Bool = true, denoiser: CaptureDenoiser? = nil ) async -> CallAudio { #if os(macOS) let defaults = routes.defaults #else let defaults = UserDefaults.standard #endif let parts = await AudioWork.run { CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: denoiser) } return CallAudio(parts: parts, routes: routes, reduceNoise: reduceNoise) } /// `audioDevice` is the iPhone tests' silent device. It takes the old /// constructor, which has no audio device module to speak of. /// /// `input`, on Mac, replaces the microphone and the speaker both: the /// engine runs in manual rendering mode, touching no device at all. /// /// Builds on the calling thread; the app uses `make`. convenience init( routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, reduceNoise: Bool = true ) { #if os(macOS) let defaults = routes.defaults #else let defaults = UserDefaults.standard #endif self.init( parts: CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: nil), routes: routes, reduceNoise: reduceNoise) } /// Attaches built parts: cheap, and on the main actor. init(parts: Parts, routes: AudioRoutes, reduceNoise: Bool) { factory = parts.factory module = parts.module state = parts.state processing = parts.processing denoiser = parts.denoiser self.reduceNoise = reduceNoise #if os(iOS) plan = ProcessingPlan.plan(iPhone: true, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) #else plan = ProcessingPlan.plan(iPhone: false, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) devices = parts.devices // Attached in manual rendering too, so the interop harness drives the // same routing code as the app; no device is opened either way. let watch = devices routes.attach(module, state: state, pin: devices.pin, inputRate: { watch.lastInputRate }) #endif guard let processing else { return } // The listener first, then the delegate: attaching the delegate can // initialise it at once, and that report used to go nowhere. denoiser.rateChanged = { [weak self] rate in DispatchQueue.main.async { MainActor.assumeIsolated { guard let self, self.rate != rate else { return } self.rate = rate self.apply() } } } // Set after the factory exists: handed to the module's initialiser, the // delegate is never called (seen in the spike, M150). The property is // weak; `denoiser` is what keeps it alive. processing.capturePostProcessingDelegate = denoiser upgraded = CallAudio.upgrade(denoiser) denoiser.enabled = plan.suppressor } /// Whether this CallAudio's suppressor is (or is becoming) DeepFilterNet, /// so a fresh spare is loaded for the next one when it is done. private var upgraded = false deinit { #if canImport(DeepFilter) if upgraded { DeepFilterPreload.finished() } #endif // The factory goes on the audio queue, after everything queued for // it: a stop or a close queued there must never find its module gone // (it did, in the harness: a crash in `stopRecording`). Its teardown, // which stops the engine, then happens there too, not on the main // thread. let held: (RTCPeerConnectionFactory, RTCAudioDeviceModule) = (factory, module) AudioWork.post { withExtendedLifetime(held) {} } } /// Whether the microphone is capturing. The module opens it when the /// first connection starts sending, not when the track is made. Either /// the module says it is recording, or the capture hook has been handed /// audio since this call's track was made (which is also how the /// interop gate's manual rendering shows up). var capturing: Bool { #if os(macOS) return state.read().isRecording || denoiser.buffersSeen > buffersAtStart #else // Not asked on iPhone: its module is the platform one, whose // recording state has not been checked against a device here. return true #endif } /// Whether the capture hook has been handed audio since this call's /// track was made: the microphone meter's "capturing" on iPhone, where /// `capturing` cannot tell a microphone that has not started (alone in a /// call) from one that has, and a warning about silence from a /// microphone nobody has opened would be wrong. var hookFlowing: Bool { denoiser.buffersSeen > buffersAtStart } /// Set while this CallAudio asked the module to record by itself (the /// audio test before a call); undone by `stopRecordingAlone`. private var recordingAlone = false /// Whether the module really was started by this CallAudio's ask, as /// opposed to already recording. Only read and written on the audio /// queue, where the start and the stop run in order. private final class Started: @unchecked Sendable { var byUs = false } private var started = Started() /// Opens the microphone outside a call, as a call's first sending stream /// does: the module's `initRecording` then `startRecording`, which is /// exactly what WebRTC's AudioState::AddSendingStream calls. /// /// The audio test before a call needs this. Its in-memory loopback is /// never connected, and WebRTC only starts a send stream on a /// connection whose transport has been writable (BaseChannel:: /// IsReadyToSendMedia_w, `was_ever_writable_`), so up to 0.3.7 the test /// never opened the microphone on a real device: an empty meter and /// "We can't hear anything", for every microphone (Paul's AirPods, /// 2026-09-26). The interop harness did not see it: in manual rendering /// the module runs its input whenever its output runs. /// /// On the audio queue, never the main thread (0.3.9): `initRecording` /// waits on WebRTC's worker thread while the engine starts, and with a /// Bluetooth headset that includes its move to the hands-free profile, /// one to two seconds. In 0.3.8 the main thread waited with it. /// /// - Parameter ask: false only for the harness's control run, which /// shows that without it the module is never asked to record. /// - Returns: 0, or the module's first error (-9000: no microphone /// permission, as the module sees it). @discardableResult func startRecordingAlone(ask: Bool = true) async -> Int { guard ask else { AudioLog.info("test: not asking the module to record (control run)") return 0 } let module = self.module // Set before the wait, so a stop asked for during it is queued, and // runs after the start, and closes what it opened. recordingAlone = true let started = self.started let began = CaptureDenoiser.now() let (initialised, result, recording, already) = await AudioWork.run { () -> (Int, Int, Bool, Bool) in guard !module.isRecording else { return (0, 0, true, true) } let initialised = module.initRecording() let result = initialised == 0 ? module.startRecording() : 0 started.byUs = true return (initialised, result, module.isRecording, false) } if already { AudioLog.info("test: the module is already recording") } else { AudioLog.info( "test: asked the module to record (init \(initialised), start \(result)) in \(Int((CaptureDenoiser.now() - began) * 1000)) ms off the main thread; recording \(recording)" ) } return initialised != 0 ? initialised : result } /// A module error, in words, for the audio test. static func recordingError(_ code: Int) -> String { switch code { case -9000, -4005, -2003: return AudioSelfTest.noPermission case -4010, -2000, -2001, -2002: return "The microphone would not start: macOS says it isn’t available (error \(code))." case -2004: return "The microphone would not start: another app is using it (error \(code))." default: return "The microphone would not start (error \(code))." } } /// Closes what `startRecordingAlone` opened, on the audio queue after /// it: never waited for here (the engine's stop can take as long as its /// start, a headset moving back to A2DP). func stopRecordingAlone() { guard recordingAlone else { return } recordingAlone = false let module = self.module let factory = self.factory let started = self.started self.started = Started() AudioWork.post { // The module is only as alive as its factory. withExtendedLifetime(factory) {} guard started.byUs else { return } started.byUs = false let result = module.stopRecording() AudioLog.info("test: stopped the module's recording (\(result))") } } /// Whether the module has been asked to enable its microphone since /// this CallAudio was made (its "will enable" hook ran with recording /// on). Asked, not granted: without microphone permission the module /// refuses right after. The harness's proof that the test opens the /// microphone on a real device. var recordingRequested: Bool { #if os(macOS) return devices.recordingRequested #else return true #endif } /// The suppressor a call starts on. See `upgrade`. nonisolated static func makeSuppressor() -> NoiseSuppressor { RNNoiseSuppressor() } /// DeepFilterNet 3 on Mac, where the app carries its model: loaded off /// the main thread, then swapped in. RNNoise otherwise, and always on /// iPhone (DeepFilterNet's cost there has not been measured on a phone). /// /// The model preloaded after launch is taken when there is one /// (DeepFilterPreload), so the suppressor is DeepFilterNet from the /// first buffer. Returns whether it is, or will be, DeepFilterNet. private static func upgrade(_ denoiser: CaptureDenoiser) -> Bool { #if canImport(DeepFilter) if denoiser.engineName == "DeepFilterNet 3" { return true } if let ready = DeepFilterPreload.take() { denoiser.replace(with: ready) return true } guard let model = DeepFilterSuppressor.model else { return false } DispatchQueue.global(qos: .userInitiated).async { if let deep = DeepFilterSuppressor(model: model) { denoiser.replace(with: deep) } } return true #else return false #endif } /// The microphone track, made here so its processing options are set /// before it is ever sent. func makeTrack() -> RTCAudioTrack { let track = factory.audioTrack(withTrackId: "microphone") self.track = track buffersAtStart = denoiser.buffersSeen apply() return track } /// Re-reads Apple's mic mode and reapplies the plan if anything moved. /// Called on the audio-health tick. func refresh() { let isolating = AVCaptureDevice.activeMicrophoneMode == .voiceIsolation if isolating != voiceIsolation { voiceIsolation = isolating apply() } } private func apply() { #if os(iOS) let iPhone = true #else let iPhone = false #endif plan = ProcessingPlan.plan( iPhone: iPhone, reduceNoise: reduceNoise, rate: rate, voiceIsolation: voiceIsolation) denoiser.enabled = reduceNoise denoiser.standAside = voiceIsolation guard let track, processing != nil else { return } // Set on the audio queue: setting a track's processing waits out an // engine start (1,061 ms measured on the main thread with a Bluetooth // switch simulated), and a rate change — which calls this — arrives // exactly then. Queued ahead of anything that sends the track, so the // options are still in place before it is ever sent. let plan = self.plan let factory = self.factory applied += 1 let generation = applied AudioWork.post { withExtendedLifetime(factory) {} var refused: String? let result = track.setAudioProcessingOptions(CallAudio.options(plan)) refused = result.isSuccess ? nil : result.message // Platform modes can be refused where Apple's unit cannot give them // (a simulator, a route without it). Then let WebRTC choose where each // enabled stage runs; what was off stays off, so the plan's rule — // never two suppressors, never gain before ours — still holds. if !result.isSuccess { let fallback = track.setAudioProcessingOptions(CallAudio.options(plan, automatic: true)) if !fallback.isSuccess { refused = fallback.message } } DispatchQueue.main.async { MainActor.assumeIsolated { [weak self] in guard let self, self.applied == generation else { return } self.refused = refused } } } } private var applied = 0 static func options(_ plan: ProcessingPlan, automatic: Bool = false) -> RTCAudioProcessingOptions { func component(_ place: ProcessingPlan.Where) -> RTCAudioProcessingComponentOptions { switch place { case .off: return RTCAudioProcessingComponentOptions(enabled: false) case .software: return RTCAudioProcessingComponentOptions(enabled: true, mode: automatic ? .automatic : .software) case .platform: return RTCAudioProcessingComponentOptions(enabled: true, mode: automatic ? .automatic : .platform) } } return RTCAudioProcessingOptions( echoCancellationOptions: component(plan.echoCancellation), noiseSuppressionOptions: component(plan.noiseSuppression), autoGainControlOptions: component(plan.gainControl), highPassFilterOptions: component(plan.highPass ? .software : .off)) } /// For "Copy audio diagnostics": the module, the engine, the hook. func diagnostics() -> [String] { var lines: [String] = [] let module = state.read() #if os(macOS) lines.append( "module: manual rendering \(module.isManualRenderingMode), engine running \(module.isEngineRunning), playout initialised \(module.isPlayoutInitialized), playing \(module.isPlaying), recording initialised \(module.isRecordingInitialized), recording \(module.isRecording), microphone muted \(module.isMicrophoneMuted)\(module.stale ? " (as last read: the module is busy)" : "")" ) let choices = enginePin.choices lines.append( "engine pin: chosen input \(choices.input ?? "default"), output \(choices.output ?? "default"); last applied \(enginePin.lastApplied?.description ?? "none yet"); would apply now \(enginePin.target().description)" ) lines.append("recording requested since made: \(devices.recordingRequested)") lines.append(contentsOf: devices.history()) #else lines.append("module: playing \(module.isPlaying), recording \(module.isRecording)") #endif let hook = denoiser.format lines.append( "capture hook: \(hook.initialised ? "initialised" : "not initialised") at \(hook.rate) Hz, \(hook.channels) ch; buffers \(denoiser.buffersSeen) in all, \(denoiser.buffersSeen - buffersAtStart) since this microphone track; frames suppressed \(denoiser.framesProcessed)" ) lines.append("suppressor: \(denoiser.engineName), \(String(describing: denoiser.state)); reduce background noise \(reduceNoise)") lines.append("plan: \(String(describing: plan))") lines.append("processing: \(report().summary)") return lines } /// What is actually running, as WebRTC reports it — not what was asked. func report() -> ProcessingReport { guard let state = state.read().processing else { return ProcessingReport( suppression: denoiser.state, echoCancellation: "unknown", noiseSuppression: "unknown", gainControl: "unknown", refused: refused, started: capturing) } func says(_ component: RTCAudioProcessingComponentState) -> String { switch component.effective { case .disabled: return "off" case .software: return "WebRTC" case .platform: return "Apple" case .softwareAndPlatform: return "WebRTC and Apple" default: return "unknown" } } var gain = says(state.autoGainControl) if plan.suppressor, gain == "off" { gain = "Calls, after noise reduction" } return ProcessingReport( suppression: denoiser.state, echoCancellation: says(state.echoCancellation), noiseSuppression: says(state.noiseSuppression), gainControl: gain, refused: refused, started: capturing) } } #if os(macOS) /// The module's observer. The device-list callback keeps the pickers /// current; "will enable" is where the engine is put on the chosen /// devices, before the module reads a format from it (AudioDevices.swift); /// configuring the input is where a synthetic source is wired in, or — /// for a real microphone — where the chosen input of a multi-channel one /// is mapped (InputChannel.swift). Starting and stopping are watched so a /// device the engine cannot start on is noticed rather than left silent. final class EngineWatch: NSObject, RTCAudioDeviceModuleDelegate, @unchecked Sendable { weak var routes: AudioRoutes? weak var module: RTCAudioDeviceModule? let input: AudioInputSource? let pin: AudioEnginePin let channels: InputChannelMapper private var node: AVAudioSourceNode? private let lock = NSLock() /// Starts and stops seen, so a start that is followed by neither a /// running engine nor a deliberate stop is known to have failed. private var starts = 0 private var stops = 0 private var requested = false /// The input format of the last engine start, for the rate watch. private var inputRate: Double? /// The last few engine events, for the diagnostics. private var events: [String] = [] var recordingRequested: Bool { lock.lock() defer { lock.unlock() } return requested } /// The input rate the engine last started with; nil when it last /// started for playout only. var lastInputRate: Double? { lock.lock() defer { lock.unlock() } return inputRate } func history() -> [String] { lock.lock() defer { lock.unlock() } return events.isEmpty ? ["engine: no events yet"] : events.map { "engine: \($0)" } } private func note(_ event: String) { lock.lock() events.append("\(AudioDiagnosticsStore.stamp(Date())) \(event)") if events.count > 12 { events.removeFirst(events.count - 12) } lock.unlock() } init( routes: AudioRoutes, module: RTCAudioDeviceModule, input: AudioInputSource?, pin: AudioEnginePin, channels: InputChannelMapper ) { self.routes = routes self.module = module self.input = input self.pin = pin self.channels = channels } func audioDeviceModuleDidUpdateDevices(_ audioDeviceModule: RTCAudioDeviceModule) { DispatchQueue.main.async { [weak self] in MainActor.assumeIsolated { self?.routes?.refresh() } } } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool, isVoiceProcessingEnabled: Bool ) -> Int { if isRecordingEnabled { lock.lock() requested = true lock.unlock() // The harness's stand-in for a Bluetooth headset changing profile, // which holds up the engine's start here, on WebRTC's worker // thread. Zero in the app. let delay = AudioStartProbe.engineStartDelay if delay > 0, input == nil { AudioLog.info("harness: holding the engine's start \(Int(delay * 1000)) ms, as a Bluetooth switch does") Thread.sleep(forTimeInterval: delay) } } note("will enable: playout \(isPlayoutEnabled), recording \(isRecordingEnabled), voice processing \(isVoiceProcessingEnabled)") guard input == nil, !engine.isInManualRenderingMode else { AudioLog.info( "engine will enable (playout \(isPlayoutEnabled), recording \(isRecordingEnabled)): manual rendering, no device") return 0 } let outcome = pin.apply(to: engine, recording: isRecordingEnabled) switch outcome { case .applied(let target): AudioLog.info( "engine will enable (playout \(isPlayoutEnabled), recording \(isRecordingEnabled)): on \(target.description)") case .failed(let target, let why): AudioLog.error( "engine will enable: could not put it on \(target.description): \(why)") DispatchQueue.main.async { [weak self] in MainActor.assumeIsolated { self?.routes?.engineFailed(target, reason: why) } } } return 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent ) {} func audioDeviceModule(_ audioDeviceModule: RTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) -> Int { 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool ) -> Int { // By now the engine has connected the input and set the unit's // client format, so the map is sized to what it will run with. // Only with the microphone in use: asking for `inputNode` makes one. guard input == nil, !engine.isInManualRenderingMode else { return 0 } if isRecordingEnabled { channels.apply(to: engine.inputNode) } let unit = isRecordingEnabled ? engine.inputNode.audioUnit : engine.outputNode.audioUnit if isPlayoutEnabled, let unit { pin.mapOutput(on: unit) } let actual = unit.flatMap(pin.actualInput) let target = pin.lastApplied ?? .systemDefault let output = engine.outputNode.outputFormat(forBus: 0) let device = unit.flatMap(CoreAudioDevices.current(of:)) let on = device.flatMap(CoreAudioDevices.uid) ?? "unknown" let rate = device.map(CoreAudioDevices.sampleRate) ?? 0 let line: String if isRecordingEnabled { let format = engine.inputNode.inputFormat(forBus: 0) line = "engine starting on \(target.description) (unit on \(on), \(Int(rate)) Hz): microphone \(actual ?? "unknown"), \(format.channelCount) ch at \(Int(format.sampleRate)) Hz; output \(output.channelCount) ch at \(Int(output.sampleRate)) Hz" lock.lock() inputRate = format.sampleRate lock.unlock() } else { line = "engine starting on \(target.description) (unit on \(on), \(Int(rate)) Hz) for playout only; output \(output.channelCount) ch at \(Int(output.sampleRate)) Hz" lock.lock() inputRate = nil lock.unlock() } AudioLog.info(line) note(line) lock.lock() starts += 1 let seen = (starts, stops) lock.unlock() DispatchQueue.main.async { [weak self] in MainActor.assumeIsolated { if isRecordingEnabled { self?.routes?.engineStarted(input: actual) } } } // The module retries a start a few times and then gives up without // telling its observer. Looked at once it has had time: an engine // that is not running, with no stop since, did not start. // Asked on the audio queue: the module's answer waits out any start // in progress. AudioWork.queue.asyncAfter(deadline: .now() + 2) { [weak self] in guard let self, let module = self.module else { return } self.lock.lock() let now = (self.starts, self.stops) self.lock.unlock() guard now == seen, !module.isEngineRunning else { return } AudioLog.error("engine did not start on \(target.description)") self.note("did not start on \(target.description)") DispatchQueue.main.async { [weak self] in MainActor.assumeIsolated { self?.routes?.engineFailed(target, reason: "the audio engine would not start on it") } } } return 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool ) -> Int { note("stopped (playout \(isPlayoutEnabled), recording \(isRecordingEnabled) next)") stopped() return 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool ) -> Int { 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine ) -> Int { note("released") stopped() return 0 } private func stopped() { lock.lock() stops += 1 lock.unlock() } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource source: AVAudioNode?, toDestination destination: AVAudioNode, format: AVAudioFormat, context: [AnyHashable: Any] ) -> Int { // A real microphone: map the chosen input and connect nothing. With // nothing connected to the mixer the module makes its own default // connection, input node to mixer in mono (audio_engine_device.mm, // "Nothing connected to input mixer, connecting input node"). guard let input else { channels.apply(to: source) return 0 } guard let mono = AVAudioFormat(standardFormatWithSampleRate: format.sampleRate, channels: 1) else { return 0 } if let node { engine.detach(node) } let rate = format.sampleRate let node = AVAudioSourceNode(format: mono) { _, _, frames, buffers in for buffer in UnsafeMutableAudioBufferListPointer(buffers) { guard let data = buffer.mData else { continue } input.render(data.assumingMemoryBound(to: Float.self), count: Int(frames), sampleRate: rate) } return noErr } self.node = node engine.attach(node) let mixer = engine.isInManualRenderingMode ? engine.mainMixerNode : (context[kLKRTCAudioEngineInputMixerNodeKey] as? AVAudioMixerNode) ?? engine.mainMixerNode engine.connect(node, to: mixer, format: mono) return 0 } func audioDeviceModule( _ audioDeviceModule: RTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource source: AVAudioNode, toDestination destination: AVAudioNode?, format: AVAudioFormat, context: [AnyHashable: Any] ) -> Int { 0 } } #endif /// The audio device module's state, for the main thread to read. /// /// Every property of LiveKit's module is a blocking hop to WebRTC's worker /// thread, and the worker holds on to it for the whole of an engine start /// or rebuild — a Bluetooth headset's profile switch included. Measured /// with that switch simulated at 1.5 s: `isRecording`, `isEngineRunning`, /// `isPlaying`, the device lists and the factory's processing state each /// held the main thread for 0.9 to 1.2 s. The meters read it ten times a /// second. /// /// So the main thread never asks the module. It asks this, which reads the /// module on a queue of its own and waits at most a few milliseconds for /// the answer; if the module is busy it gets the last answer, marked stale, /// and the fresh one is there for the next look. final class ModuleState: @unchecked Sendable { struct Snapshot: @unchecked Sendable { var isRecording = false var isPlaying = false var isEngineRunning = false var isManualRenderingMode = false var isPlayoutInitialized = false var isRecordingInitialized = false var isMicrophoneMuted = false var processing: RTCAudioProcessingState? /// The module did not answer in time: this is its last answer. var stale = false } private weak var factory: RTCPeerConnectionFactory? private weak var module: RTCAudioDeviceModule? private let queue = DispatchQueue(label: "net.amadan.oficina.calls.audio-state", qos: .userInitiated) private let lock = NSLock() private var latest = Snapshot() private var reading: DispatchGroup? /// How long a read on the main thread may wait for the module. static let patience: TimeInterval = 0.005 /// Reads once, here and now: made off the main thread with the parts. init(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) { self.factory = factory self.module = module latest = ModuleState.snapshot(factory: factory, module: module) } func read(patience: TimeInterval = ModuleState.patience) -> Snapshot { lock.lock() let group: DispatchGroup if let reading { group = reading } else { group = DispatchGroup() group.enter() reading = group queue.async { [self] in let fresh = factory.flatMap { f in module.map { ModuleState.snapshot(factory: f, module: $0) } } lock.lock() if let fresh { latest = fresh } reading = nil lock.unlock() group.leave() } } lock.unlock() let answered = group.wait(timeout: .now() + patience) == .success lock.lock() defer { lock.unlock() } var out = latest out.stale = !answered return out } private static func snapshot(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) -> Snapshot { var s = Snapshot() s.isRecording = module.isRecording s.isPlaying = module.isPlaying #if os(macOS) s.isEngineRunning = module.isEngineRunning s.isManualRenderingMode = module.isManualRenderingMode s.isPlayoutInitialized = module.isPlayoutInitialized s.isRecordingInitialized = module.isRecordingInitialized s.isMicrophoneMuted = module.isMicrophoneMuted #endif s.processing = factory.audioProcessingState return s } }