import AVFoundation import Foundation import LiveKitWebRTC #if os(macOS) import AudioToolbox import CoreAudio #endif // "Test audio": the microphone, the speaker, and a recording played back, // all through the path a call uses. // // Where the microphone is measured. The meter and the verdict read the // capture hook's own meters (CaptureDenoiser.level and .inputLevel), and // the recording is taken from its sink: the audio after WebRTC's audio // processing (echo canceller, high-pass filter) and after our noise // reduction and gain — the audio a call encodes and sends, and nothing // nearer the microphone than that. // // How the microphone is started outside a call. A CallAudio is built // exactly as a call builds one (LiveKit's AVAudioEngine module on Mac, the // platform module on iPhone, WebRTC's processing module with the // CaptureDenoiser as its capture post-processing delegate, the same // ProcessingPlan and the person's Reduce background noise setting), a // microphone track is made on it as a call makes one, and the track is // sent from one peer connection to another in this process // (SelfTestLoopback): negotiated in memory, never connected, nothing // leaving the device. That starts the microphone the way a call does, and // the capture runs through the processing module and the hook exactly as // it would in a call. Asking the module to record on its own // (`initAndStartRecording`) was tried first and is not enough: the module // runs, but the hook sees no buffers. The path is checked end to end by // `CallsInterop --audio-selftest`, which runs it in manual rendering with // the synthetic voice and requires the recording to be non-silent (and a // silent input to be called silent). // // During a call nothing is started or stopped: the test reads the running // call's meters and does not touch its audio. It does not play anything // either — a sound played beside a call is heard by its microphone and // sent to everyone, so the speaker test and the recording are for before // a call, and say so. /// Where the test's microphone audio comes from. The standalone capture /// outside a call, the call's own inside one, and a fake in the tests. @MainActor protocol AudioSelfTestCapture: AnyObject { var denoiser: CaptureDenoiser { get } var microphoneName: String { get } /// The device the microphone actually is, with how it is connected and /// its rate now: "Paul's AirPods Pro (Bluetooth, 24 kHz)". Empty where /// that is not known. var microphoneDetail: String { get } var speakerName: String { get } /// The chosen speaker, as the module lists it (a CoreAudio UID on Mac). var speakerID: String? { get } var inCall: Bool { get } /// False in a call whose microphone WebRTC has not opened yet. var captureExpected: Bool { get } var muted: Bool { get } var reduceNoise: Bool { get } /// Whether the capture must stop while the test plays something. On /// iPhone the voice-processing unit ducks other audio while it runs. var stopsForPlayback: Bool { get } func permitted() async -> Bool /// A Bluetooth microphone that starting will move to its hands-free /// profile, by name: said while it switches. Nil for anything else. func switchingMicrophone() async -> String? func start() async throws func stop() /// The meters since the last read: before the suppressor, and after. func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) /// For "Copy audio diagnostics": titled blocks of lines. func diagnostics() -> [(String, [String])] } extension AudioSelfTestCapture { func switchingMicrophone() async -> String? { nil } } /// Plays mono audio to the chosen speaker. Public so the interop harness /// can stand in a player that only checks what it was given. @MainActor public protocol AudioSelfTestPlayer: AnyObject { /// `samples` in -1...1. Returns once they have played, or on `stop`. func play(_ samples: [Float], sampleRate: Double, speaker: String?, meter: OutputMeter) async throws func stop() } public enum SpeakerTestState: Equatable, Sendable { case idle, playing, played case failed(String) case unavailableInCall } public enum RecordingTestState: Equatable, Sendable { case idle, recording, playing /// `silent`: the recording itself was below the silence line. case played(silent: Bool) case failed(String) case unavailableInCall } /// What the last playback of a recording was, without the audio: for the /// interop harness, which asserts on it. public struct RecordingSummary: Equatable, Sendable { public var seconds: Double public var rate: Int public var rms: Float } @MainActor public final class AudioSelfTest: ObservableObject { @Published public private(set) var micStep = 0 @Published public private(set) var micVerdict: MicVerdict = .listening /// Said instead of the verdict when the microphone cannot be tested at /// all: no permission, muted in the call, or it would not start. @Published public private(set) var micProblem: String? @Published public private(set) var outputStep = 0 @Published public private(set) var speaker: SpeakerTestState @Published public private(set) var recording: RecordingTestState /// 0...1 in tenths while recording. @Published public private(set) var recordingProgress = 0.0 @Published public private(set) var microphoneName = "" /// "Paul's AirPods Pro (Bluetooth, 24 kHz)": what the verdict names. @Published public private(set) var microphoneDetail = "" @Published public private(set) var speakerName = "" /// Where the microphone is on its way to running: the window opens at /// once and says "Starting microphone…" until it is. @Published public private(set) var startPhase: MicStartPhase = .idle public private(set) var lastRecording: RecordingSummary? public let recordSeconds: Double /// The standalone test's devices, for its pickers. Nil in a call, whose /// device panel already has them. public let routes: AudioRoutes? public var inCall: Bool { capture.inCall } public var reduceNoise: Bool { capture.reduceNoise } /// Whether a recording exists right now. The harness checks it is gone /// after playback. public var holdsRecording: Bool { recorder.recorded > 0 } public var suppressorName: String { capture.denoiser.engineName } public var framesSuppressed: UInt64 { capture.denoiser.framesProcessed } private let capture: AudioSelfTestCapture private let player: AudioSelfTestPlayer private let clock: () -> Double private let recorder: SelfTestRecorder private let outputMeter = OutputMeter() private var judge = MicJudge() private var start = MicStart() private var capturing = false private var closed: Bool { start.phase == .cancelled } /// Set while this test is playing something, when the microphone's /// readings are its own sound and are not judged. private var playingSince: Double? private var ticker: Task? private var ticks = 0 private var lastVerdict: MicVerdict? init( capture: AudioSelfTestCapture, player: AudioSelfTestPlayer, routes: AudioRoutes?, recordSeconds: Double = 5, clock: @escaping () -> Double = CaptureDenoiser.now ) { self.capture = capture self.player = player self.routes = routes self.recordSeconds = recordSeconds self.clock = clock recorder = SelfTestRecorder(seconds: recordSeconds) speaker = capture.inCall ? .unavailableInCall : .idle recording = capture.inCall ? .unavailableInCall : .idle refreshNames() } /// Before a call: a capture of its own, on the person's remembered /// devices and their Reduce background noise setting. public static func standalone() -> AudioSelfTest { let routes = AudioRoutes() let capture = StandaloneCapture(routes: routes, input: nil, reduceNoise: Call.rememberedReduceNoise) return AudioSelfTest(capture: capture, player: EnginePlayer(), routes: routes) } #if os(macOS) /// For the interop harness: the standalone test's capture on the real /// audio device module (device rendering, no synthetic voice), started /// as the test starts it, and whether that asked the module to record. /// Asking is what is checked, because a test process has no microphone /// permission and the module refuses right after being asked; without /// the ask (0.3.7) the microphone was never opened on any Mac. /// /// - Parameter askToRecord: false is the control: the 0.3.7 start, /// which must come back not asked. public static func deviceStartProbe(askToRecord: Bool) async -> (asked: Bool, detail: String) { let routes = AudioRoutes(defaults: UserDefaults(suiteName: "calls-selftest-probe-\(UUID().uuidString)")!) let capture = StandaloneCapture(routes: routes, input: nil, reduceNoise: true, askToRecord: askToRecord) do { try await capture.start() } catch { // Refused (in a test process, for want of permission): what // matters is whether it was asked. let asked = capture.audio?.recordingRequested ?? false return (asked, "recording requested \(asked); the module refused: \(error.localizedDescription)") } let deadline = Date().addingTimeInterval(3) while capture.audio?.recordingRequested != true, Date() < deadline { try? await Task.sleep(for: .milliseconds(100)) } let asked = capture.audio?.recordingRequested ?? false let recording = capture.audio?.module.isRecording ?? false capture.stop() await AudioWork.drain() return (asked, "recording requested \(asked), module recording \(recording) (no permission refuses it)") } #endif /// In a call: its meters, read and nothing else. Standalone if the call /// has no media up. public static func during(_ call: Call) -> AudioSelfTest { guard let capture = LiveCallCapture(call) else { return standalone() } return AudioSelfTest(capture: capture, player: EnginePlayer(), routes: nil) } #if os(macOS) /// The interop harness's: the module in manual rendering with `input` /// as the microphone, and devices remembered nowhere real. public static func harness( input: AudioInputSource, player: AudioSelfTestPlayer, reduceNoise: Bool, recordSeconds: Double = 5 ) -> AudioSelfTest { let routes = AudioRoutes(defaults: UserDefaults(suiteName: "calls-selftest-\(UUID().uuidString)")!) let capture = StandaloneCapture(routes: routes, input: input, reduceNoise: reduceNoise) return AudioSelfTest(capture: capture, player: player, routes: routes, recordSeconds: recordSeconds) } #endif // MARK: Lifecycle /// Starts the microphone (outside a call) and the meters. /// /// Nothing here waits on the main thread: the audio is built, the /// microphone started and any Bluetooth headset switched on the audio /// queue (AudioWork), and the window says what is happening meanwhile /// (`startPhase`). Closing it partway cancels cleanly: a start that /// finishes after the close is stopped at once, so the microphone is /// never left open. public func open() async { guard start.open() else { return } startPhase = start.phase let began = clock() let granted = await capture.permitted() guard start.permission(granted) else { startPhase = start.phase if !closed { micProblem = AudioSelfTest.noPermission } return } startPhase = start.phase ticker = Task { [weak self] in while !Task.isCancelled { try? await Task.sleep(for: .milliseconds(100)) guard let self, !Task.isCancelled else { return } self.tick() } } await startCapture() guard !closed else { return } refreshNames() AudioLog.info( "test opened \(inCall ? "in a call" : "before a call") in \(Int((clock() - began) * 1000)) ms: microphone \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail); capturing \(capturing)") } /// Stops everything this test started and throws any recording away. /// Called when the view closes; safe to call twice, and at any point of /// the start. public func close() { guard !closed else { return } let live = start.close() startPhase = start.phase ticker?.cancel() ticker = nil player.stop() if !capture.inCall { capture.denoiser.sink = nil } if capturing || live { capture.stop() } capturing = false recorder.discard() } private func startCapture() async { let switching = await capture.switchingMicrophone() start.microphone(bluetoothSwitching: switching) startPhase = start.phase var failure: Error? do { try await capture.start() } catch { failure = error } switch start.finished(ok: failure == nil) { case .keep: startPhase = start.phase capturing = true micProblem = nil judge.start(at: clock()) // Whatever was measured before this start is not this microphone's. _ = capture.levels() case .stop: // Closed while it started: what it opened is closed now. capture.stop() AudioLog.info("test: closed while the microphone started; stopped it") case .failed: startPhase = start.phase capturing = false guard !closed, let failure else { return } let said = failure.localizedDescription micProblem = said.hasPrefix("The microphone") || said == AudioSelfTest.noPermission ? said : "The microphone would not start: \(said)" AudioLog.error("test: \(micProblem ?? said)") } } /// A different microphone was chosen: judge it afresh. public func microphoneChanged() { judge.start(at: clock()) refreshNames() } private func refreshNames() { if microphoneName != capture.microphoneName { microphoneName = capture.microphoneName } let detail = capture.microphoneDetail if microphoneDetail != detail { microphoneDetail = detail } if speakerName != capture.speakerName { speakerName = capture.speakerName } } /// What "Copy audio diagnostics" copies from the test. public func diagnosticsText() -> String { var test = [ "where: \(inCall ? "during a call (reads the call's microphone)" : "before a call (its own microphone)")", "microphone: \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail)", "speaker: \(speakerName)", "capturing: \(capturing); verdict \(String(describing: micVerdict)); meter step \(micStep)", "says: \(micSentence)", "reduce background noise: \(reduceNoise)", "recording test: \(String(describing: recording)); speaker test: \(String(describing: speaker))", ] if let problem = micProblem { test.append("problem: \(problem)") } return AudioDiagnosticsReport.make( context: inCall ? "Test audio in a call" : "Test audio before a call", sections: [("Test audio", test)] + capture.diagnostics()) } // MARK: The 10 Hz step /// One read of the meters. Publishes only what changed, so a view is /// redrawn at most ten times a second and usually less. func tick() { let (input, output) = capture.levels() let out = outputMeter.take() let playing = playingSince != nil set(\.outputStep, playing ? MeterSteps.step(out) : 0) if case .recording = recording { set(\.recordingProgress, (min(recorder.recorded / recordSeconds, 1) * 10).rounded(.down) / 10) } guard capturing, !playing else { set(\.micStep, 0) return } if capture.inCall, capture.muted { set(\.micStep, 0) set(\.micProblem, AudioSelfTest.mutedInCall) judge.start(at: clock()) return } if micProblem == AudioSelfTest.mutedInCall { micProblem = nil } set(\.micStep, output.buffers > 0 ? MeterSteps.step(output.rms) : 0) let verdict = judge.observe( input: input, output: output, at: clock(), captureExpected: capture.captureExpected) set(\.micVerdict, verdict) AudioDiagnosticsStore.shared.record( .init( at: clock(), input: input.peak, inputBuffers: input.buffers, output: output.peak, outputBuffers: output.buffers, source: "test")) if verdict != lastVerdict { lastVerdict = verdict AudioLog.info("test verdict: \(String(describing: verdict)) — \(MicJudge.sentence(verdict, device: verdictName))") } // The device's name is cheap; its rate means reading CoreAudio, so // once a second (a Bluetooth headset changes rate as its microphone // opens). ticks += 1 if ticks % 10 == 0 { refreshNames() } } /// The microphone as the verdict names it: the device it resolves to, /// with its connection and rate, where that is known. var verdictName: String { microphoneDetail.isEmpty ? microphoneName : microphoneDetail } private func set(_ key: ReferenceWritableKeyPath, _ value: Value) { if self[keyPath: key] != value { self[keyPath: key] = value } } // MARK: Speaker public func playTestSound() async { guard !capture.inCall, !closed, !busy else { return } speaker = .playing do { try await playing { try await player.play(TestChime.samples(), sampleRate: TestChime.rate, speaker: capture.speakerID, meter: outputMeter) } if !closed { speaker = .played } } catch { if !closed { speaker = .failed(error.localizedDescription) } } } // MARK: Record and play back /// Records `recordSeconds` from the capture hook's sink — what a call /// would send — and plays it back once. The audio is held in memory /// only, handed to the player, and gone when this returns. public func recordAndPlayBack() async { guard !capture.inCall, !closed, !busy else { return } guard capturing else { recording = .failed(micProblem ?? "The microphone is not running.") return } let recorder = self.recorder recorder.arm() recordingProgress = 0 recording = .recording capture.denoiser.sink = { samples, frames, rate in recorder.append(samples, frames: frames, rate: rate) } let deadline = clock() + recordSeconds + 3 while !recorder.full, clock() < deadline, !closed { try? await Task.sleep(for: .milliseconds(100)) } capture.denoiser.sink = nil guard !closed else { return recorder.discard() } var taken = recorder.take() guard !taken.samples.isEmpty, taken.rate > 0 else { recording = .failed("Nothing was recorded. \(MicJudge.sentence(.noAudio, device: verdictName))") return } let rms = SampleLevel.rms(taken.samples) lastRecording = RecordingSummary( seconds: Double(taken.samples.count) / Double(taken.rate), rate: taken.rate, rms: rms) recordingProgress = 1 recording = .playing do { let samples = taken.samples let rate = Double(taken.rate) taken.samples = [] try await playing { try await player.play(samples, sampleRate: rate, speaker: capture.speakerID, meter: outputMeter) } if !closed { recording = .played(silent: rms < MicJudge.silenceRMS) } } catch { if !closed { recording = .failed(error.localizedDescription) } } } private var busy: Bool { speaker == .playing || recording == .recording || recording == .playing } /// Runs `body` with the microphone's readings set aside (they would be /// this test's own sound), stopping the capture where it must stop, and /// judging on from where it left off. private func playing(_ body: () async throws -> Void) async throws { let started = clock() playingSince = started let restart = capturing && capture.stopsForPlayback if restart { capture.stop() capturing = false } var failure: Error? do { try await body() } catch { failure = error } playingSince = nil outputStep = 0 if restart, start.restart() { await startCapture() } judge.resume(after: clock() - started) if let failure { throw failure } } // MARK: Words static let noPermission = "Calls doesn’t have permission to use the microphone. Allow it in Settings, under Privacy & Security, Microphone." static let mutedInCall = "You’re muted. Unmute to test your microphone." public static let inCallPlayback = "Not during a call: everyone in it would hear the sound too. Test your speaker and hear yourself from the home screen before a call." public var micSentence: String { micProblem ?? start.sentence ?? MicJudge.sentence(micVerdict, device: verdictName) } /// Said beside a spinner while the microphone starts; nil once it has. public var startingSentence: String? { micProblem == nil ? start.sentence : nil } public var micAdvice: String? { micProblem == nil && !start.isStarting ? MicJudge.advice(micVerdict) : nil } public var speakerSentence: String? { let name = speakerName.isEmpty ? "your speaker" : speakerName switch speaker { case .idle: return nil case .playing: return "Playing a test sound through \(name)." case .played: return "Played a test sound through \(name). If you didn’t hear it, check the volume and the speaker chosen here." case .failed(let why): return "The test sound didn’t play: \(why)" case .unavailableInCall: return AudioSelfTest.inCallPlayback } } public var recordingSentence: String? { switch recording { case .idle: return nil case .recording: return "Recording. Speak as you would in a call." case .playing: return "Playing back what others would hear." case .played(let silent): return silent ? "The recording was silent. \(MicJudge.sentence(.silent, device: verdictName))" : "That is how you sound to others. The recording has been deleted." case .failed(let why): return why case .unavailableInCall: return nil } } } extension Call { /// The person's Reduce background noise setting, as a call reads it. static var rememberedReduceNoise: Bool { UserDefaults.standard.object(forKey: Call.reduceNoiseKey) as? Bool ?? true } } // MARK: - Captures /// The microphone before a call: a CallAudio of the test's own, recording /// on its own. See the top of this file. /// /// The CallAudio is built when the test starts, not when its window is /// made, and on the audio queue (CallAudio.make): the window appears at /// once. The capture hook is made first, cheaply, so the test can read it /// from the start. @MainActor final class StandaloneCapture: AudioSelfTestCapture { let routes: AudioRoutes /// Nil until the start has built it. private(set) var audio: CallAudio? let denoiser: CaptureDenoiser let reduceNoise: Bool private let input: AudioInputSource? private let askToRecord: Bool private var track: RTCAudioTrack? private var loopback: SelfTestLoopback? /// Bumped by every stop, so a start in flight knows it was stopped. private var attempt = 0 #if os(iOS) private var session: SavedSession? #endif /// Set by the stall harness only, which has no microphone permission to /// ask for (a test process), and measures the start behind the question. private let assumePermitted: Bool init( routes: AudioRoutes, input: AudioInputSource?, reduceNoise: Bool, askToRecord: Bool = true, assumePermitted: Bool = false ) { self.routes = routes self.reduceNoise = reduceNoise self.askToRecord = askToRecord self.assumePermitted = assumePermitted self.input = input denoiser = CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) } private var synthetic: Bool { input != nil } var inCall: Bool { false } var captureExpected: Bool { true } var muted: Bool { false } #if os(iOS) var stopsForPlayback: Bool { true } #else var stopsForPlayback: Bool { false } #endif var microphoneName: String { AudioSelfTest.inputName(routes) } var microphoneDetail: String { AudioSelfTest.inputDetail(routes) } var speakerName: String { routes.outputName } var speakerID: String? { routes.currentOutput } /// Read from CoreAudio on the audio queue: the process's first CoreAudio /// read sets up its connection to the audio server, 80–180 ms measured. func switchingMicrophone() async -> String? { guard !synthetic else { return nil } #if os(macOS) let (preference, failed) = (AudioPreference.load(input: true, routes.defaults), routes.failed) return await AudioWork.run { AudioRoutes.bluetoothSwitching(preference: preference, failed: failed) } #else return nil #endif } func diagnostics() -> [(String, [String])] { #if os(macOS) let routeLines = routes.diagnostics() #else let routeLines = ["microphone: \(microphoneName)", "speaker: \(speakerName)"] #endif return [ ("Devices", routeLines), ("Audio engine and processing", audio?.diagnostics() ?? ["the audio has not been built yet"]), ] } func permitted() async -> Bool { if synthetic || assumePermitted { return true } #if os(macOS) switch AVCaptureDevice.authorizationStatus(for: .audio) { case .authorized: return true case .notDetermined: return await AVCaptureDevice.requestAccess(for: .audio) default: return false } #else switch AVAudioApplication.shared.recordPermission { case .granted: return true case .undetermined: return await AVAudioApplication.requestRecordPermission() default: return false } #endif } /// Builds the audio (the first time), sends the microphone track over /// the loopback and opens the microphone: every slow step on the audio /// queue. A stop during any of it wins: what was opened is closed, in /// order, behind it. func start() async throws { let attempt = self.attempt #if os(iOS) if session == nil { session = try SavedSession.configureForCall() } #endif let audio: CallAudio if let built = self.audio { audio = built } else { #if os(macOS) audio = await CallAudio.make( routes: routes, audioDevice: nil, input: input, reduceNoise: reduceNoise, denoiser: denoiser) #else audio = await CallAudio.make(routes: routes, audioDevice: nil, reduceNoise: reduceNoise, denoiser: denoiser) #endif self.audio = audio } guard attempt == self.attempt else { throw CancellationError() } // Made here, as a call makes its own: making it applies the call's // processing options (CallAudio.apply). CallAudio holds it weakly. let track = audio.makeTrack() self.track = track let loop = try await SelfTestLoopback.make(factory: audio.factory, track: track) guard attempt == self.attempt else { loop.close() throw CancellationError() } loopback = loop // The loopback is never connected, and WebRTC opens the microphone // only for a connection that has been (CallAudio.startRecordingAlone): // so the microphone is opened here, as a call's first send stream // opens it. let result = await audio.startRecordingAlone(ask: askToRecord) guard attempt == self.attempt else { throw CancellationError() } routes.refresh() if result != 0, !synthetic { stop() throw NSError( domain: "Calls", code: result, userInfo: [NSLocalizedDescriptionKey: CallAudio.recordingError(result)]) } } /// Closes the loopback and the microphone, on the audio queue behind /// anything still starting; never waits. func stop() { attempt += 1 loopback?.close() loopback = nil audio?.stopRecordingAlone() track = nil #if os(iOS) session?.restore() session = nil #endif } func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { (denoiser.inputLevel.take(), denoiser.level.take()) } } /// The microphone during a call: the call's own hook, read only. /// /// The hook's two meters are the call's to read — its own microphone /// meter takes from them, and a peak-hold read by two readers gives each /// half the peaks — so the test measures from the hook's sink instead, /// which the call does not use. That is the processed audio, so here the /// "can't hear anything" line is drawn after the suppressor too. It is /// still -90 dBFS: the suppressor takes at most 30 dB /// (noiseAttenuationLimitDb), so a room above -60 dBFS stays above it. @MainActor final class LiveCallCapture: AudioSelfTestCapture { private weak var call: Call? let denoiser: CaptureDenoiser let reduceNoise: Bool private let tapped = CaptureLevel() init?(_ call: Call) { guard let denoiser = call.captureDenoiser else { return nil } self.call = call self.denoiser = denoiser reduceNoise = call.reduceNoise } var inCall: Bool { true } var captureExpected: Bool { call?.captureStarted ?? false } var muted: Bool { call?.muted ?? false } var stopsForPlayback: Bool { false } var microphoneName: String { call.map { AudioSelfTest.inputName($0.audio) } ?? "" } var microphoneDetail: String { call.map { AudioSelfTest.inputDetail($0.audio) } ?? "" } var speakerName: String { call?.audio.outputName ?? "" } func diagnostics() -> [(String, [String])] { call?.audioDiagnosticSections() ?? [] } var speakerID: String? { call?.audio.currentOutput } func permitted() async -> Bool { true } func start() async throws { let tapped = self.tapped denoiser.sink = { samples, frames, _ in tapped.record(samples, frames: frames, at: 0) } } func stop() { denoiser.sink = nil } func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { let reading = tapped.take() return (reading, reading) } } extension AudioSelfTest { /// The chosen microphone's name as the pickers list it. static func inputName(_ routes: AudioRoutes) -> String { routes.inputs.first { $0.id == routes.currentInput }?.name ?? "" } /// The device the microphone resolves to, with connection and rate. static func inputDetail(_ routes: AudioRoutes) -> String { #if os(macOS) return routes.inputDescription() #else let session = AVAudioSession.sharedInstance() guard let port = session.currentRoute.inputs.first else { return "" } let kind = port.portType == .bluetoothHFP ? "Bluetooth, " : "" return "\(port.portName) (\(kind)\(Int(session.sampleRate / 1000)) kHz)" #endif } } #if os(iOS) /// The audio session as the test found it, put back when it is done. /// Outside a call nothing else is using the session, so the test sets /// it up as a call would (play and record, video chat, no hands-free /// Bluetooth) and, at the end, restores the category and deactivates it /// if it was not active before. @MainActor struct SavedSession { let category: AVAudioSession.Category let mode: AVAudioSession.Mode let options: AVAudioSession.CategoryOptions let wasActive: Bool static func configureForCall() throws -> SavedSession { let session = RTCAudioSession.sharedInstance() let saved = SavedSession( category: AVAudioSession.Category(rawValue: session.category), mode: AVAudioSession.Mode(rawValue: session.mode), options: session.categoryOptions, wasActive: session.isActive) session.lockForConfiguration() defer { session.unlockForConfiguration() } try session.setCategory( AVAudioSession.Category.playAndRecord, mode: AVAudioSession.Mode.videoChat, options: AudioRoutes.options(bluetoothInput: false)) return saved } func restore() { let session = RTCAudioSession.sharedInstance() session.lockForConfiguration() defer { session.unlockForConfiguration() } try? session.setCategory(category, mode: mode, options: options) if !wasActive, session.isActive { try? session.setActive(false) } } } #endif // MARK: - Playing /// Plays through an AVAudioEngine of its own, apart from WebRTC's. /// /// On Mac the engine's output unit is pointed at the chosen speaker (the /// UID the call's module lists it by; the system default if it cannot be /// found). It only plays outside a call, so it never shares a device with /// the call's engine. On iPhone it plays on the session the test set up, /// to whatever the route is — the speaker a call would use. @MainActor final class EnginePlayer: AudioSelfTestPlayer { private var engine: AVAudioEngine? private var node: AVAudioPlayerNode? private var waiting: CheckedContinuation? /// Which playback a completion belongs to: one from a sound that was /// stopped must not end the next. private var generation = 0 #if os(iOS) /// RTCAudioSession counts activations, and only the last balanced /// deactivation lets the session go; each play is balanced in `finish`. private var activated = false #endif func play(_ samples: [Float], sampleRate: Double, speaker: String?, meter: OutputMeter) async throws { stop() guard !samples.isEmpty, let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1), let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)), let channel = buffer.floatChannelData?[0] else { return } buffer.frameLength = AVAudioFrameCount(samples.count) samples.withUnsafeBufferPointer { channel.update(from: $0.baseAddress!, count: samples.count) } defer { channel.update(repeating: 0, count: samples.count) } let engine = AVAudioEngine() #if os(macOS) if let speaker, let device = EnginePlayer.device(uid: speaker), let unit = engine.outputNode.audioUnit { var id = device AudioUnitSetProperty( unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &id, UInt32(MemoryLayout.size)) } #else let session = RTCAudioSession.sharedInstance() session.lockForConfiguration() activated = (try? session.setActive(true)) != nil session.unlockForConfiguration() #endif let node = AVAudioPlayerNode() engine.attach(node) engine.connect(node, to: engine.mainMixerNode, format: format) engine.mainMixerNode.installTap(onBus: 0, bufferSize: 1024, format: nil) { tapped, _ in guard let data = tapped.floatChannelData?[0] else { return } meter.record(data, count: Int(tapped.frameLength)) } self.engine = engine self.node = node generation += 1 let playback = generation // Started on the audio queue: an engine start on a Bluetooth headset // waits for its route, and the main thread must not wait with it. do { try await AudioWork.run { try engine.start() } } catch { if generation == playback { finish() } throw error } // Stopped while it started: `finish` has queued its stop. guard generation == playback, self.engine === engine else { return } try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in waiting = continuation node.scheduleBuffer(buffer, at: nil, options: [], completionCallbackType: .dataPlayedBack) { [weak self] _ in DispatchQueue.main.async { MainActor.assumeIsolated { guard let self, self.generation == playback else { return } self.finish() } } } node.play() } } private func finish() { let waiting = self.waiting self.waiting = nil // Stopped on the audio queue, behind its start if that is still // running. if let engine, let node { AudioWork.post { engine.mainMixerNode.removeTap(onBus: 0) node.stop() engine.stop() } } node = nil engine = nil #if os(iOS) if activated { activated = false let session = RTCAudioSession.sharedInstance() session.lockForConfiguration() try? session.setActive(false) session.unlockForConfiguration() } #endif waiting?.resume() } func stop() { generation += 1 finish() } #if os(macOS) /// A CoreAudio device from its UID, or nil. static func device(uid: String) -> AudioDeviceID? { guard !uid.isEmpty else { return nil } var address = AudioObjectPropertyAddress( mSelector: kAudioHardwarePropertyTranslateUIDToDevice, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMain) var device = AudioDeviceID(kAudioObjectUnknown) var size = UInt32(MemoryLayout.size) let cf = uid as CFString let status = withUnsafePointer(to: cf) { qualifier in AudioObjectGetPropertyData( AudioObjectID(kAudioObjectSystemObject), &address, UInt32(MemoryLayout.size), qualifier, &size, &device) } guard status == noErr, device != kAudioObjectUnknown else { return nil } return device } #endif }