import Combine import Foundation #if os(iOS) import AVFoundation #endif // Go serializes this API with a mutex; Objective-C bindings carry no Sendable annotation. @preconcurrency import CallsBridge import LiveKitWebRTC // The Go callback only enqueues. The main actor owns the interface and media; // an epoch prevents callbacks from a closed room reaching a later call. private final class Events: NSObject, BridgeListenerProtocol { weak var call: Call? let epoch: UUID init(call: Call, epoch: UUID) { self.call = call self.epoch = epoch } func onEvent(_ event: String?) { guard let event, let data = event.data(using: .utf8) else { return } guard let decoded = try? JSONDecoder().decode(WireEvent.self, from: data) else { DispatchQueue.main.async { [weak self] in self?.call?.invalidEvents += 1 } return } DispatchQueue.main.async { [weak self] in guard let self, let call = self.call, call.epoch == self.epoch else { return } call.receive(decoded) } } } @MainActor public final class Call: ObservableObject { @Published public private(set) var state: RoomState? { didSet { arrangement.roster(state) } } /// This viewer's layout, pin and order of people (0.4.0). public let arrangement = CallArrangement() @Published public private(set) var title = "Calls" /// Whether `title` is one someone chose, shown over the call, rather /// than one Calls wrote, which is not (CallTitle). @Published public private(set) var named = false @Published public private(set) var standing = "" @Published public private(set) var status = "Ready to join" @Published public private(set) var remoteTracks: [String: RTCVideoTrack] = [:] /// §5.4: the peers showing a screen here, oldest first, and the screens /// that have arrived. Two things and not one, because the `share` /// message usually beats the media by a renegotiation, and the tile /// carries its caption before there is anything to show in it. @Published public private(set) var screens: [String] = [] @Published public private(set) var remoteScreens: [String: RTCVideoTrack] = [:] /// Whether this app is sending a screen, and what the last attempt to /// had to say — which is usually nothing. @Published public private(set) var sharingScreen = false @Published public private(set) var screenNote = "" private var screenPending = false /// How this app's own screen is going out, while it is sharing: the /// slowest connection's, read once a second. See ScreenSendReport. @Published public private(set) var screenReport: ScreenSendReport? /// "Optimise for": what a shared screen gives up first. Remembered, and /// applied at once to a share in progress (see ScreenSendPolicy). @Published public var screenMode: ScreenShareMode = .remembered { didSet { guard screenMode != oldValue else { return } ScreenShareMode.remembered = screenMode built?.screenMode = screenMode } } private var screenReports: [String: ScreenSendReport] = [:] private var screenWatch: Task? /// The sharer's watchdog (ScreenWatchdog) and what it is doing, for the /// panel's screen line. private var screenWatchdog = ScreenWatchdog() @Published public private(set) var screenWatchStatus = "" /// The panel's second screen line: encoded and sent rates, keyframes, /// and the watchdog (ScreenSendReport.detail). @Published public private(set) var screenDetail = "" /// The quiet notice while someone's view of this app's screen is being /// put right: "Vicky's view of your screen stalled — recovering". @Published public private(set) var screenRecovery = "" /// The receiver's watchdog, and the list last reported as /// `screenStalled` (spec §15.21). private var screenReceiveWatch = ScreenReceiveWatch() private var reportedStalled: [String] = [] /// Each arriving screen's statistics at the last look, for the stall /// log's "what moved" and for "Copy call diagnostics". private var screenReceived: [String: ScreenReceiveReport] = [:] /// Where each arriving screen stood when it last moved, so a stall can /// be described against it. private var screenReceivedMoving: [String: ScreenReceiveReport] = [:] private var screenReattaches = 0 /// "Reconnecting…", said only for a problem that lasts (spec §15.23): /// the whole call's, and the people whose tiles say it on their own. private var reconnectWatch = ReconnectWatch() @Published public private(set) var reconnecting = false @Published public private(set) var reconnectingPeers: Set = [] /// " is connecting and can't hear you yet" (spec §15.24): the /// watch, and the people it is said for now — on their tiles, and as a /// dot in the mini window when they are not the one it shows. private var connectingWatch = ConnectingWatch() @Published public private(set) var connectingPeers: Set = [] /// This app's own half: whose audio path is established (HearingRule), /// per connection generation; whose receiver reports have shown this /// app's audio reaching them; and the list last reported as `hearing`. private var heardFrom: [String: (epoch: Int64, heard: Bool)] = [:] private var ackedBy: [String: Int64] = [:] private var reportedHearing: [String] = [] /// Quality reports (spec §15.26): the notices tallied over the window, /// the running totals the last report was taken from, the loop that /// sends one every `qualityInterval`, and whether the server has said to /// stop (its kill switch) or the final one has gone. private var qualityTally = QualityEventTally() private var qualityBuilder = QualityReportBuilder() private var qualityWatch: Task? private var qualityStopped = false private var qualityFinalSent = false /// Thirty seconds, as the ruling has it; the gate shortens it /// (CALLS_QUALITY_INTERVAL, read by the interop harness). @_spi(Fixtures) public static var qualityInterval: Double = 30 /// "Report a problem" (spec §15.26): the sheet, while it is open. @Published public var problemReport: ProblemReportModel? /// Until when the last call can still be reported: ten minutes after it /// ended, which is as long as the server keeps its report token. The /// closed bridge client is kept that long — its transport and token /// outlive the room — with the call's diagnostics as they were at the end. @Published public private(set) var reportableUntil: Date? private var reportClient: BridgeClient? private var reportDiagnostics = "" static let reportWindow: TimeInterval = 600 /// A fixture call answers a report itself, for the UI test of the sheet. private var fixtureReports = false /// The phase the notices are drawn for: the core's, except that its /// `reconnecting` is only said once ReconnectWatch says so, and is said /// for a second after it is over. public var noticePhase: String? { if reconnecting { return "reconnecting" } if state?.phase == "reconnecting" { return "admitted" } return state?.phase } private var screenWatchFrozenUntil = 0.0 @Published public private(set) var localTrack: RTCVideoTrack? @Published public private(set) var muted = false @Published public private(set) var cameraOff = false @Published public private(set) var floor = false @Published public private(set) var chat: [String] = [] @Published public private(set) var receivedChatCount = 0 @Published public private(set) var invitationURL: URL? @Published public private(set) var invitationError = "" @Published public private(set) var loadingInvitation = false /// A brief message over the stage, such as "Invite link copied". @Published public internal(set) var toast: String? var toastSerial = 0 /// What the last device change had to say, and usually nothing. @Published public private(set) var deviceNote = "" /// The audio-health readout, once a call has been up long enough for /// one. Nil means no sample yet, which is not the same as healthy. @Published public private(set) var audioHealth: AudioHealth? /// The camera in use. Published rather than read through the capture on /// demand, because a picker bound to a plain computed property never /// learns that a switch finished. @Published public private(set) var currentCamera: String? /// What the camera is doing (see CameraStatus), and a fallback worth /// saying. Nil status for a capture that does not say, and outside a /// call. @Published public private(set) var cameraStatus: CameraStatus? @Published public private(set) var cameraNote = "" /// The working camera in use when it is not the remembered one, which /// the panel offers to remember. @Published public private(set) var cameraToKeep: CameraDevice? /// A real camera, so the panel shows its row even with nothing in it. @Published public private(set) var offersCameras = false /// Audio routing belongs to this call. public let audio = AudioRoutes() /// "Reduce background noise": the neural suppressor in place of WebRTC's /// own. On by default, and remembered. @Published public var reduceNoise: Bool = UserDefaults.standard.object(forKey: Call.reduceNoiseKey) as? Bool ?? true { didSet { UserDefaults.standard.set(reduceNoise, forKey: Call.reduceNoiseKey) guard let built else { return } built.audio.reduceNoise = reduceNoise audioProcessing = built.audio.report() } } static let reduceNoiseKey = "reduceBackgroundNoise" /// Which processing is running on the microphone, for the readout. @Published public private(set) var audioProcessing: ProcessingReport? /// The meters (spec §15.20): your microphone and everyone's sound, in /// objects of their own so a meter moving does not redraw the call. public let levels = AudioLevels() private var levelWatch: Task? /// What was last reported as `noSoundFrom`, and the queue that reports /// it: the bridge's send is a request, and not for the main thread. private var reportedNoSound: [String] = [] private let noSoundQueue = DispatchQueue(label: "calls.no-sound-from") /// Received bytes per person at the last statistics read, and whether /// they had risen: "arriving" for the no-sound rule. private var audioBytes: [String: UInt64] = [:] private var audioArriving: [String: Bool] = [:] /// Some connection is up, and so carrying the microphone to someone /// (every connection is given the track when it is made). Read from the /// peer sampling, five times a second; the microphone warning waits on /// it (MicSample.sending). private var sendingAudio = false /// The loudest remote sound since the gate last asked (sampleStats). private var remotePeakSinceSample: Float = 0 private var micPeakSinceSample: Float = 0 private var cameraWork: Task? private var cameraSwitches = 0 private let knocks = KnockWatch() fileprivate var epoch = UUID() private var bridge: BridgeClient? private var events: Events? /// The call's media, made by the first join (MediaEngine.make, off the /// main thread) and kept for the next. private var built: MediaEngine? /// The media, made here and now only if something reaches for it /// before any join has — which nothing in the app does. private var media: MediaEngine { if let built { return built } AudioLog.info("call media made on the main thread, before a join") let made = MediaEngine(call: self) built = made return made } /// The audio test's view of this call's microphone (AudioSelfTestModel): /// the capture hook, nil outside a call, and whether WebRTC has opened it. var captureDenoiser: CaptureDenoiser? { bridge == nil ? nil : built?.audio.denoiser } var captureStarted: Bool { bridge != nil && built?.audio.capturing == true } /// For "Copy audio diagnostics" in the panel's ⓘ and in Test audio /// during a call: this call's microphone, devices, engine and hook. func audioDiagnosticSections() -> [(String, [String])] { var lines = [ "in a call: \(bridge != nil); muted \(muted); microphone track \(bridge != nil && built?.localAudio != nil)", "people here: \(self.state?.peers.count ?? 0)", "microphone meter: \(levels.mic.silent ? "silent" : "step \(levels.mic.step)"), speaking \(levels.mic.speaking), sending from \(levels.mic.source.isEmpty ? "-" : levels.mic.source)", ] if let health = audioHealth { lines.append("audio health: \(health.summary)") } #if os(macOS) let routeLines = audio.diagnostics() #else let routeLines = audio.inputs.map { "input: \($0.name) [\($0.id)]\($0.id == audio.currentInput ? " (current)" : "")" } + ["output: \(audio.outputName)"] #endif var sections: [(String, [String])] = [("Call", lines), ("Devices", routeLines)] if bridge != nil, let built { sections.append(("Audio engine and processing", built.audio.diagnostics())) } return sections } public func audioDiagnosticsText() -> String { diagnosticsText { self.name(of: $0) } } /// "Copy call diagnostics" as "Report a problem" sends it (spec §15.26): /// everyone in it named "peer abc123", the first six characters of their /// peer id, rather than by name. The server turns those into its tags. /// Device names stay, as the spec says. public func anonymousDiagnosticsText() -> String { diagnosticsText(label: Self.anonymousLabel) } nonisolated static func anonymousLabel(_ peer: String) -> String { "peer " + peer.prefix(6) } private func diagnosticsText(label: (String) -> String) -> String { AudioDiagnosticsReport.make( context: "a call's details", sections: audioDiagnosticSections() + peopleDiagnosticSections(label: label) + screenDiagnosticSections(label: label)) } /// Everyone else in the call, as this app sees them: what they report /// about their own media and hearing, and what this app is saying /// about them. private func peopleDiagnosticSections(label: (String) -> String) -> [(String, [String])] { guard let state else { return [] } var lines: [String] = [] for person in state.peers where person.peer != state.peer { var facts = ["connection \(person.gen ?? 0)"] if let media = person.media { facts.append(media.muted ? "muted" : "unmuted") facts.append(media.cameraOff ? "camera off" : "camera on") if let hearing = media.hearing { facts.append(hearing.contains(state.peer) ? "hears you" : "does not hear you yet") } if media.noSoundFrom?.contains(state.peer) == true { facts.append("hears no sound from you") } if media.audioBlocked == true { facts.append("their browser blocked sound") } } else { facts.append("no media report") } if levels.noSoundFrom.contains(person.peer) { facts.append("no sound from them here") } if reconnectingPeers.contains(person.peer) { facts.append("reconnecting") } if connectingPeers.contains(person.peer) { facts.append("connecting, can't hear you yet") } lines.append("\(label(person.peer)): " + facts.joined(separator: ", ")) } return lines.isEmpty ? [] : [("People", lines)] } /// The screen's part of "Copy call diagnostics": how it is leaving, what /// the watchdog has done, and whose screens have stalled here. private func screenDiagnosticSections(label name: (String) -> String) -> [(String, [String])] { var lines: [String] = [] if sharingScreen { lines.append("Sharing (\(screenMode.rawValue))") for (peer, report) in screenReports.sorted(by: { $0.key < $1.key }) { lines.append("to \(name(peer)): \(report.summary)") lines.append(" frames captured \(report.framesCaptured), encoded \(report.framesEncoded), sent \(report.framesSent), keyframes \(report.keyFramesEncoded), asked for \(report.keyframeRequests), limitation \(report.limitation.isEmpty ? "none" : report.limitation)") } if let capture = media.screenCaptureCounts { lines.append("capture: \(capture.heartbeats ?? 0) reports, \(capture.delivered) frames delivered") } lines.append("watchdog: \(screenWatchdog.status)") } else { lines.append("Not sharing") } if sharingScreen { for peer in (state?.peers ?? []).map(\.peer) where screenWatchdog.escalation(of: peer) != .none { lines.append("recovering \(name(peer))'s view: \(screenWatchdog.escalation(of: peer))") } lines.append( "recoveries: \(screenWatchdog.keyframes) keyframes, \(screenWatchdog.encoderRecoveries) fresh encoders, \(screenWatchdog.reAdds) re-adds, \(screenWatchdog.rebuilds) rebuilds, \(screenWatchdog.reshares) shares again, \(screenWatchdog.failures) given up") } // The receiver's half: every screen arriving here, as its statistics // have it, whether or not it has stalled. for (peer, report) in screenReceived.sorted(by: { $0.key < $1.key }) { lines.append("from \(name(peer)): \(report.summary)") } if !reportedStalled.isEmpty { lines.append("stalled here: " + reportedStalled.map { name($0) }.joined(separator: ", ")) } if screenReattaches > 0 { lines.append("pictures attached again here: \(screenReattaches)") } let log = AudioDiagnosticsStore.screen.lines(last: AudioDiagnosticsStore.lineLimit) let connection = AudioDiagnosticsStore.connection.lines(last: 50) return [ ("Screen", lines), ("Screen log (last \(AudioDiagnosticsStore.lineLimit) lines)", log.isEmpty ? ["(nothing logged)"] : log), ("Connection log (last 50 lines)", connection.isEmpty ? ["(nothing logged)"] : connection), ] } private var ice: ICEConfig? private var pendingMedia: [Message] = [] fileprivate var invalidEvents: UInt64 = 0 let playAudio: Bool #if os(iOS) let customAudioDevice: RTCAudioDevice? public init(audioDevice: RTCAudioDevice? = nil, playAudio: Bool = true) { customAudioDevice = audioDevice self.playAudio = playAudio } #else let audioInput: AudioInputSource? /// `audioInput` stands in for the microphone (the interop harness). public init(playAudio: Bool = true, audioInput: AudioInputSource? = nil) { self.playAudio = playAudio self.audioInput = audioInput } #endif public func join(origin: String, room: String, cookies: String, capture: VideoCapture? = nil) async throws { let old = bridge bridge = nil close() let joiningEpoch = epoch if let old { await Task.detached { old.leave() }.value } guard epoch == joiningEpoch else { throw CancellationError() } let listener = Events(call: self, epoch: epoch) var error: NSError? guard let client = BridgeNewClient(origin, room, cookies, listener, &error) else { throw error ?? NSError(domain: "Calls", code: 1) } events = listener bridge = client arrangement.begin(meeting: origin + "/" + room) muted = false cameraOff = false chat = [] receivedChatCount = 0 title = "Calls" standing = "" // The factory and its audio device module, built on the audio queue // the first time: the call's window is up while it is made. if built == nil { let made = await MediaEngine.make(call: self) if built == nil { built = made } } guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } try await media.prepare(capture: capture) guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } localTrack = media.localVideo cameraChanged() reportMedia() media.audio.reduceNoise = reduceNoise audioProcessing = media.audio.report() watchLevels() startQualityWatch() KnockNotice.requestPermission() // Asking Apple at join, once there is a session to register the // answer against. Apple replies on its own schedule, at the app // delegate, and may never reply at all. RemoteKnocks.register() try client.start() } public func close() { epoch = UUID() let old = bridge bridge = nil events = nil built?.close() // This call's microphone and speaker choices end with it (see AudioRoutes). #if os(macOS) audio.reset() #endif cameras = [] remoteTracks = [:] screens = [] remoteScreens = [:] sharingScreen = false screenNote = "" endScreenWatch() localTrack = nil ice = nil state = nil pendingMedia = [] invitationURL = nil invitationError = "" loadingInvitation = false deviceNote = "" audioHealth = nil audioProcessing = nil endLevelWatch() qualityWatch?.cancel() qualityWatch = nil currentCamera = nil cameraStatus = nil cameraNote = "" cameraToKeep = nil knocks.reset() cameraSwitches += 1 cameraWork?.cancel() cameraWork = nil if let old { DispatchQueue.global().async { old.close() } } } public func leave() async { // The final quality report reads the connections' statistics, so it is // built before they close, and sent beside the leave rather than after // it: quitting gives the two together three seconds. let final = await finalQualityReport() let old = bridge if let old { keepForReports(old) } bridge = nil close() status = "You left the meeting" if let old { await Task.detached { await withTaskGroup(of: Void.self) { group in if let final { group.addTask { _ = old.sendQuality(final) } } group.addTask { old.leave() } } }.value } } public func mute() { muted.toggle() // On the audio queue: switching a track waits out an engine start, // 1,136 ms measured on the main thread with a Bluetooth switch // simulated — and the first thing someone does as they join is often // to mute. let track = media.localAudio let on = !muted AudioWork.post { track?.isEnabled = on } reportMedia() } public func toggleCamera() { cameraOff.toggle() let track = media.localVideo let on = !cameraOff AudioWork.post { track?.isEnabled = on } reportMedia() } // --------------------------------------------------------------- // Media state (§15.19) // --------------------------------------------------------------- /// Tells the core what this app is sending. The core reports it once /// admitted, again whenever a roster says the server has not got it — /// a resume, a restart — and at once on a change, so this is called on /// each toggle and once the tracks exist, and nowhere else. No track at /// all is muted, or off, as far as anyone listening can tell. private func reportMedia() { try? bridge?.setMedia(muted || media.localAudio == nil, cameraOff: cameraOff || media.localVideo == nil) } /// A remote peer's reported state, or nil when it has not reported. public func peerMedia(_ peer: String) -> PeerMedia? { Self.peerMedia(peer, in: state) } /// Whether a remote peer has said it is muted. Unknown is not muted. public func isMuted(_ peer: String) -> Bool { peerMedia(peer)?.muted ?? false } /// Whether a remote peer has said its camera is off. Unknown is not off. public func isCameraOff(_ peer: String) -> Bool { peerMedia(peer)?.cameraOff ?? false } nonisolated static func peerMedia(_ peer: String, in state: RoomState?) -> PeerMedia? { state?.peers.first { $0.peer == peer }?.media } // --------------------------------------------------------------- // Devices // --------------------------------------------------------------- /// The cameras available, and the one in use. Read through the running /// capture rather than from AVFoundation directly, so what the picker /// offers is what the call could actually switch to. @Published public private(set) var cameras: [CameraDevice] = [] public func refreshDevices() { let available = built?.cameras ?? [] if cameras != available { cameras = available } if let currentCamera, !available.contains(where: { $0.id == currentCamera }) { self.currentCamera = nil } audio.refresh() } /// The capture said something changed on its own: a camera came or /// went, a fallback, the first frame. Everything camera-shaped is read /// back from it. func cameraChanged() { offersCameras = media.offersCameras refreshDevices() let now = media.currentCamera if currentCamera != now { currentCamera = now } let status = media.cameraStatus if cameraStatus != status { cameraStatus = status } let note = media.cameraNote if cameraNote != note { cameraNote = note } let keep = media.cameraToKeep if cameraToKeep != keep { cameraToKeep = keep } } /// What your own tile says in place of your picture, or nil to show it. /// Never an open-ended "Waiting for video": it says what it is waiting /// for, or why there is nothing. public var selfPlaceholder: String? { if state?.phase == "ended" { return "Not in the call" } if localTrack == nil { return "Starting camera…" } return cameraStatus?.tileText } /// The panel's "Try again", for a call with no working camera. public func retryCamera() async { await media.retryCamera() cameraChanged() } /// Switches camera mid-call. Serialised: two picks are two capture /// restarts on one device, and run at once the survivor is whichever /// session happened to open last, which is not the one the person /// chose. The counter decides who wins; the queue decides that only /// one runs. public func useCamera(_ id: String) async { cameraSwitches += 1 let mine = cameraSwitches let previous = cameraWork let work = Task { @MainActor [weak self] in await previous?.value guard let self, mine == self.cameraSwitches, !Task.isCancelled else { return } do { try await self.media.use(camera: id) guard mine == self.cameraSwitches, !Task.isCancelled else { return } self.deviceNote = "" } catch is CancellationError { } catch { guard mine == self.cameraSwitches, !Task.isCancelled else { return } self.deviceNote = error.localizedDescription } // Read back rather than assume: a refused switch left the capture // wherever it actually is, and the picker has to follow the camera // and not the intention. guard mine == self.cameraSwitches, !Task.isCancelled else { return } self.cameraChanged() } cameraWork = work await work.value } /// Records something a device change had to say. The panel has no /// place of its own to put an error, and a route that would not change /// belongs beside the control that would not change it. public func noteDevice(_ text: String) { deviceNote = text } /// Polls the audio-health counters while a call is up. /// /// This exists because of a call on 2026-09-17 that came back as an /// unintelligible growl, and because nothing in this app recorded /// anything about it — no log, no counters, no artefact. A reproduction /// would have produced nothing either. (Until the move to LiveKit's /// WebRTC build it was also the only instrument on macOS, which had no /// way to switch away from a suspect device.) /// /// Five seconds because this is a readout a person glances at when /// something sounds wrong, not a time series. public func watchAudio() async { while !Task.isCancelled { try? await Task.sleep(nanoseconds: 5_000_000_000) guard !Task.isCancelled, bridge != nil else { return } refreshDevices() audioHealth = AudioHealth(await media.stats()) media.audio.refresh() audioProcessing = media.audio.report() } } // --------------------------------------------------------------- // Levels (§15.20) // --------------------------------------------------------------- /// What a tile shows about its sound: yours, or a remote person's. public var selfAudio: TileAudio { .you(levels.mic) } public func tileAudio(_ peer: String) -> TileAudio { .peer(levels.peer(peer), name: name(of: peer)) } /// Reads the meters ten times a second while the call is up: your /// microphone every tick, everyone else's every other (5 Hz), and the /// received byte counts once a second. What is published only changes /// when a meter step or a line does (see AudioLevels). private func watchLevels() { levelWatch?.cancel() levels.reset() resetConnecting() reportedNoSound = [] audioBytes = [:] audioArriving = [:] levelWatch = Task { @MainActor [weak self] in var tick = 0 while !Task.isCancelled { try? await Task.sleep(nanoseconds: 100_000_000) guard let self, !Task.isCancelled, self.bridge != nil else { return } tick += 1 let now = ProcessInfo.processInfo.systemUptime self.sampleMic(now: now) if tick % 2 == 0 { await self.samplePeers(stats: tick % 10 == 0, now: now) } if tick % 10 == 5 { await self.watchRemoteScreens(now: now) } } } } private func endLevelWatch() { levelWatch?.cancel() levelWatch = nil screenReceiveWatch = ScreenReceiveWatch() reportedStalled = [] screenReceived = [:] screenReceivedMoving = [:] levels.reset() resetConnecting() reportedNoSound = [] audioBytes = [:] audioArriving = [:] sendingAudio = false } private func resetConnecting() { connectingWatch = ConnectingWatch() if !connectingPeers.isEmpty { connectingPeers = [] } heardFrom = [:] ackedBy = [:] // The core starts every report with `hearing: []` (spec §15.24). reportedHearing = [] } private func sampleMic(now: Double) { let denoiser = media.audio.denoiser let sent = denoiser.level.take() let heard = denoiser.inputLevel.take() micPeakSinceSample = max(micPeakSinceSample, sent.peak) #if os(macOS) let source = audio.inputSource let channels = audio.inputChannels.count > 1 let capturing = media.audio.capturing #else let source = audio.inputs.first { $0.id == audio.currentInput }?.name ?? "" let channels = false let capturing = media.audio.hookFlowing #endif levels.updateMic( MicSample( muted: muted || media.localAudio == nil, capturing: capturing, inputPeak: heard.peak, inputBuffers: heard.buffers, sentPeak: sent.peak, sending: sendingAudio), now: now, source: source, channels: channels) AudioDiagnosticsStore.shared.record( .init( at: now, input: heard.peak, inputBuffers: heard.buffers, output: sent.peak, outputBuffers: sent.buffers, source: muted ? "call (muted)" : capturing ? "call" : "call (mic not open)")) } private func samplePeers(stats: Bool, now: Double) async { // §15.24: the people whose audio path is not established yet are read // closely, five times a second, until it is; the people in their first // moments who do not say who they hear are asked about this app's audio. let mine = state?.peer ?? "" let joining = connectingWatch.joining var detail = Set() var acks = Set() for person in state?.peers ?? [] where person.peer != mine { let epoch = person.gen ?? 0 if heardFrom[person.peer].map({ $0.epoch != epoch || !$0.heard }) ?? true { detail.insert(person.peer) } if joining.contains(person.peer), HearingRule.hearsYou(person.media, me: mine) == nil, ackedBy[person.peer] != epoch { acks.insert(person.peer) } } let readings = await media.peerAudio(stats: stats, detail: detail, acks: acks) guard levelWatch != nil, let state else { return } sendingAudio = readings.values.contains { $0.connected } let me = state.peer var present = Set() var people: [String: ReconnectWatch.Peer] = [:] var arrivals: [String: ConnectingWatch.Peer] = [:] for person in state.peers where person.peer != me { guard let reading = readings[person.peer] else { continue } present.insert(person.peer) if stats, let bytes = reading.bytes { audioArriving[person.peer] = bytes > (audioBytes[person.peer] ?? 0) audioBytes[person.peer] = bytes } let epoch = person.gen ?? 0 var was = heardFrom[person.peer] ?? (epoch, false) if was.epoch != epoch { was = (epoch, false) } let heard = HearingRule.heard( was: was.heard, .init( connected: reading.connected, bad: reading.gone, hasReceiver: reading.hasReceiver, receiverLive: reading.receiverLive, packets: reading.packets ?? 0, decoded: reading.buffers, senderMuted: person.media?.muted == true)) heardFrom[person.peer] = (epoch, heard) if reading.acked == true { ackedBy[person.peer] = epoch } arrivals[person.peer] = ConnectingWatch.Peer( epoch: epoch, hearsMe: HearingRule.hearsYou(person.media, me: me), reported: person.media != nil, connected: reading.connected, acked: ackedBy[person.peer] == epoch) } for id in Array(heardFrom.keys) where !present.contains(id) { heardFrom.removeValue(forKey: id) } for id in Array(ackedBy.keys) where !present.contains(id) { ackedBy.removeValue(forKey: id) } for id in connectingWatch.observe(at: now, people: arrivals) { CallLog.info("\(id.prefix(6)) can hear us") AudioAnnouncer.say(ConnectingWords.canHear(name(of: id))) } let nowJoining = connectingWatch.joining if connectingPeers != connectingWatch.shown { connectingPeers = connectingWatch.shown } for person in state.peers where person.peer != me { guard let reading = readings[person.peer] else { continue } // Someone in their first moments has not lost anything yet: §15.24 // speaks for them until it gives up. people[person.peer] = nowJoining.contains(person.peer) ? ReconnectWatch.Peer() : ReconnectWatch.Peer( connectionBad: reading.connectionBad, audioMissing: audioArriving[person.peer] == false && person.media?.muted != true) let said = ConnectingWatch.tile( name: person.name, media: person.media, me: me, muted: muted, joining: nowJoining.contains(person.peer), shown: connectingPeers.contains(person.peer), reconnecting: reconnectingPeers.contains(person.peer)) remotePeakSinceSample = max(remotePeakSinceSample, reading.peak) levels.updatePeer( person.peer, name: person.name, sample: PeerAudioSample( reported: person.media, connected: reading.connected, arriving: audioArriving[person.peer] ?? false, level: reading.peak), notices: said.notices, connecting: said.connecting, now: now) } watchReconnect(now: now, people: people) observeQuality(now: now, state: state) levels.settle(present: present) for id in Array(audioBytes.keys) where !present.contains(id) { audioBytes.removeValue(forKey: id) audioArriving.removeValue(forKey: id) } reportNoSound(levels.noSoundFrom) reportHearing(heardFrom.filter { $0.value.heard }.keys.sorted(), present: present) } /// §15.24: whose audio this app is receiving and playing. Sent only on a /// change, in order, off the main thread, as `noSoundFrom` is. private func reportHearing(_ peers: [String], present: Set) { guard peers != reportedHearing, let client = bridge else { return } let moved = HearingRule.moved(from: reportedHearing, to: peers, present: present) reportedHearing = peers guard moved else { return } let body = json(peers) ?? "[]" noSoundQueue.async { try? client.setHearing(body) } } /// One look by ReconnectWatch (spec §15.23), and the words it says. private func watchReconnect(now: Double, people: [String: ReconnectWatch.Peer]) { reconnectWatch.observe(at: now, streamDown: state?.phase == "reconnecting", people: people) if reconnectingPeers != reconnectWatch.peers { for id in reconnectWatch.peers.subtracting(reconnectingPeers) { CallLog.info("reconnecting to \(id.prefix(6)): shown") } reconnectingPeers = reconnectWatch.peers } guard reconnecting != reconnectWatch.call else { return } reconnecting = reconnectWatch.call CallLog.info(reconnecting ? "reconnecting: shown (a problem that lasted)" : "reconnecting: cleared") switch state?.phase { case "reconnecting": status = reconnecting ? ReconnectWords.call : "In the meeting" case "admitted": status = reconnecting ? ReconnectWords.call : "In the meeting" default: break } } /// §15.20's receiving half: who is reaching this app as silence. The /// core keeps only roster peers and reports on a change; this sends only /// on a change too, in order, off the main thread. private func reportNoSound(_ peers: [String]) { guard peers != reportedNoSound, let client = bridge else { return } reportedNoSound = peers let body = json(peers) ?? "[]" noSoundQueue.async { try? client.setNoSoundFrom(body) } } public func retryJoin() { send(type: "join", to: "", gen: 0, payload: "") } public func loadInvitation() async { guard let client = bridge, !loadingInvitation else { return } let currentEpoch = epoch invitationURL = nil invitationError = "" loadingInvitation = true defer { if epoch == currentEpoch { loadingInvitation = false } } do { let link = try await Task.detached { var error: NSError? let value = client.shareLink(&error) if let error { throw error } return value }.value guard epoch == currentEpoch else { return } invitationURL = URL(string: link) } catch { guard epoch == currentEpoch else { return } invitationError = "Could not get the invitation. Try again." } } public func admit(_ peer: String) { send(type: "admit", to: "", gen: 0, payload: json(["peer": peer]) ?? "") } @discardableResult public func sendChat(_ text: String) -> Bool { guard !text.isEmpty else { return false } guard let payload = json(["text": text]), payload.utf8.count <= 2059 else { status = "Keep the message within 2 KB." return false } guard let bridge else { return false } do { try bridge.send("chat", to: "", generation: 0, payload: payload) return true } catch { status = "Could not send the message. Try again." return false } } func send(type: String, to: String, gen: Int64, payload: String) { do { try bridge?.send(type, to: to, generation: gen, payload: payload) } catch { status = "Could not send to the meeting. Try again." } } fileprivate func receive(_ event: WireEvent) { switch event.type { case "sent", "send-failed": guard let message = event.signal, message.type == "chat", let text = message.payload?.text else { return } appendChat("\(event.type == "sent" ? "You" : "Not sent"): \(text)") case "info": if let info = event.info { title = info.title named = info.named ?? false standing = info.standing floor = info.floor } case "state": guard let state = event.state else { return } self.state = state // Only a member is sent a waiting list at all (spec §4.3), so this // needs no check of its own for who is allowed to be told. KnockNotice.announce( knocks.arrivals(waiting: state.waiting, visible: KnockNotice.appIsFrontmost)) switch state.phase { case "admitted": status = reconnecting ? ReconnectWords.call : "In the meeting" case "waiting": status = "Waiting to be let in" case "full": status = "The meeting is full. Try joining again shortly." // Said only once it has lasted (ReconnectWatch, spec §15.23): a // stream renewal is a reconnect of half a second nobody should see. case "reconnecting": if reconnecting { status = ReconnectWords.call } case "ended": status = [ "left": "You left the meeting", "ended": "This meeting has ended", "replaced": "This meeting is open elsewhere", "removed": "You were removed from this meeting", "standing": "You no longer have access", "expired": "Your invitation has expired", "signin": "Sign in again to join", "unavailable": "This meeting is unavailable", ][state.reason ?? ""] ?? "The call has ended" endedForReports() media.close() remoteTracks = [:] screens = [] remoteScreens = [:] sharingScreen = false endScreenWatch() endLevelWatch() localTrack = nil cameraChanged() default: status = "Connecting…" } if state.peer.isEmpty { media.closePeers() remoteTracks = [:] screens = [] remoteScreens = [:] ice = nil } if let ice { media.reconcile(state: state, ice: ice) } case "ice": guard let config = event.ice else { return } let renewing = ice != nil ice = config floor = config.iceServers.isEmpty if let state { media.reconcile(state: state, ice: config) } if renewing { media.updateICE(config) } let pending = pendingMedia pendingMedia = [] for message in pending { media.receive(message) } case "ice-unavailable": status = "Could not refresh the call connection. Retrying…" case "message": guard let message = event.message else { return } if message.type == "chat", let text = message.payload?.text { appendChat("\(message.name ?? "Participant"): \(text)") receivedChatCount += 1 } if message.type == "share" { media.receiveShare(message) return } if ice == nil, ["offer", "answer", "candidate"].contains(message.type) { if pendingMedia.count < 256 { pendingMedia.append(message) } } else { media.receive(message) } default: break } } private func appendChat(_ line: String) { chat.append(line) if chat.count > 200 { chat.removeFirst(chat.count - 200) } } func setRemote(_ track: RTCVideoTrack, peer: String) { remoteTracks[peer] = track } func removeRemote(_ peer: String) { remoteTracks.removeValue(forKey: peer) dropScreen(peer) } // --------------------------------------------------------------- // Screens (§5.4) // --------------------------------------------------------------- /// The name the roster gives a peer, which is the only place a name /// comes from: the hub put it there, and nothing a peer sends can. public func name(of peer: String) -> String { state?.peers.first { $0.peer == peer }?.name ?? "Participant" } /// Shares a screen (§5.4): the capture asks the person what to share, /// every connection gets it on a transceiver of its own, and then the /// room is told — the label, not the media, so peers can caption the /// tile before the renegotiation lands. /// /// A refusal at the picker says one sentence, the browser's, and does /// not diagnose which kind of refusal it was. public func shareScreen(_ capture: ScreenCapture) async { // The picker stays open for as long as a person takes to answer it, // and the button is live the whole time. Without this a second press // starts a second capture that nothing would ever stop. guard bridge != nil, !sharingScreen, !screenPending else { return } screenPending = true defer { screenPending = false } let joined = epoch screenNote = "" media.screenMode = screenMode do { try await media.startScreen(capture) { [weak self] in // The capture ended on its own: the system's stop control, or // the shared window closing. The same ending as the button. guard let self, self.epoch == joined else { return } self.stopSharingScreen() } } catch is CancellationError { return } catch { guard epoch == joined else { return } screenNote = "Screen sharing was not allowed." return } guard epoch == joined, bridge != nil else { media.stopScreen() return } sharingScreen = true screenWatchdog.reset() ScreenLog.info("sharing started (\(screenMode.rawValue))") send(type: "share", to: "", gen: 0, payload: json(["on": true]) ?? "") watchScreen() } /// Reads the screen's outbound statistics once a second while sharing, /// for the device panel's line and the gate. private func watchScreen() { screenWatch?.cancel() screenWatch = Task { @MainActor [weak self] in while !Task.isCancelled { try? await Task.sleep(nanoseconds: 1_000_000_000) guard let self, !Task.isCancelled, self.sharingScreen else { return } let fresh = await self.media.screenReports() guard !Task.isCancelled, self.sharingScreen else { return } var measured: [String: ScreenSendReport] = [:] for (peer, report) in fresh { measured[peer] = report.measured(since: self.screenReports[peer]) } let previous = self.screenReports self.screenReports = measured let worst = ScreenSendReport.worst(Array(measured.values)) self.screenReport = worst // Motion mode scales from the source's own size, which only the // statistics know (a window can be any shape, and re-picked). if let worst { self.media.screenSource(width: worst.sourceWidth, height: worst.sourceHeight) } await self.runScreenWatchdog(measured, previous: previous) } } } /// Off only for the gate's measurement of what happens without it. @_spi(Fixtures) public static var screenWatchdogEnabled = true /// One look by the sharer's watchdog, and whatever it asks for. private func runScreenWatchdog( _ reports: [String: ScreenSendReport], previous previousReports: [String: ScreenSendReport] ) async { guard sharingScreen, let capture = media.screenCaptureCounts else { return } let me = state?.peer ?? "" let stalledBy = Set( (state?.peers ?? []).filter { $0.peer != me && ($0.media?.screenStalled ?? []).contains(me) }.map(\.peer)) let sending = reports.mapValues { ScreenSendCounts( encoded: $0.framesEncoded, sent: $0.framesSent, keyframes: $0.keyFramesEncoded, bandwidthLimited: $0.limitation == "bandwidth") } let failuresBefore = screenWatchdog.failures var actions = screenWatchdog.observe( at: ProcessInfo.processInfo.systemUptime, capture: capture, sending: sending, stalledBy: stalledBy) if !Self.screenWatchdogEnabled { actions = [] } countScreenSteps(actions, failed: screenWatchdog.failures - failuresBefore, stalledBy: stalledBy) for action in actions { switch action { case .restartCapture: ScreenLog.info("restarting the capture") let ok = await media.restartScreenCapture() if !ok { ScreenLog.error("the capture could not be restarted") } case .keyframe(let peer): ScreenLog.info("keyframe for \(ScreenWatchdog.short(peer))") media.forceScreenKeyframe(peer) case .recreateEncoder(let peer): ScreenLog.info("fresh encoder for \(ScreenWatchdog.short(peer))") media.recreateScreenEncoder(peer) case .reAdd(let peer): ScreenLog.info("screen re-added for \(ScreenWatchdog.short(peer))") media.reAddScreen(peer) sayShareAgain() case .rebuild(let peer): ScreenLog.info("screen rebuilt for \(ScreenWatchdog.short(peer)): new transceiver, same size and mode") media.reAddScreen(peer) sayShareAgain() case .reshare: ScreenLog.info("sharing again for everyone, from the same capture") await media.reshareScreen() sayShareAgain() } } screenWatchStatus = screenWatchdog.status if let (peer, worst) = reports.min(by: { $0.value.framesPerSecond < $1.value.framesPerSecond }) { screenDetail = worst.detail(since: previousReports[peer], watchdog: screenWatchdog.status) } var recovering: [String] = [] var restarting: [String] = [] var failed: [String] = [] for peer in stalledBy.sorted(by: { name(of: $0) < name(of: $1) }) { switch screenWatchdog.escalation(of: peer) { case .rebuilt, .reshared: restarting.append(name(of: peer)) case .failed: failed.append(name(of: peer)) case .none, .keyframes: recovering.append(name(of: peer)) } } screenRecovery = ScreenWords.notice(recovering: recovering, restarting: restarting, failed: failed) } /// A new transceiver's track arrives unlabelled; say it again. private func sayShareAgain() { Task { @MainActor [weak self] in try? await Task.sleep(nanoseconds: 2_000_000_000) guard let self, self.sharingScreen else { return } self.send(type: "share", to: "", gen: 0, payload: json(["on": true]) ?? "") } } private func endScreenWatch() { screenWatch?.cancel() screenWatch = nil screenReports = [:] screenReport = nil screenWatchdog.reset() screenWatchStatus = "" screenRecovery = "" screenDetail = "" } /// The receiver's watchdog, once a second from the level watch: whose /// screens have stopped moving here, reported as `screenStalled` when /// that changes (spec §15.21), each stall logged with what the /// statistics say is still arriving, and the picture attached again /// four seconds in (spec §15.22). private func watchRemoteScreens(now: Double) async { var stalled: [String] = [] if !screens.isEmpty { let reports = await media.remoteScreenReports() var frames = reports.mapValues(\.framesDecoded) // The gate's frozen picture: the counts held where they were. if now < screenWatchFrozenUntil { frames = frames.mapValues { _ in 0 } } let sharing = Set(screens) let before = screenReceived screenReceived = reports.filter { sharing.contains($0.key) } stalled = screenReceiveWatch.observe(at: now, decoded: frames.filter { sharing.contains($0.key) }) for (peer, report) in screenReceived where !stalled.contains(peer) { if before[peer].map({ $0.framesDecoded != report.framesDecoded || $0.mid != report.mid }) ?? true { screenReceivedMoving[peer] = report } } for peer in stalled where !reportedStalled.contains(peer) { let report = screenReceived[peer] ?? ScreenReceiveReport() let moving = screenReceivedMoving[peer] ScreenLog.info( "screen from \(ScreenWatchdog.short(peer)) stalled here: reporting it; since it last moved \(report.moved(since: moving)) — \(report.diagnosis(since: moving))") ScreenLog.info("screen from \(ScreenWatchdog.short(peer)): \(report.summary)") } for peer in screenReceiveWatch.reattach { let report = screenReceived[peer] ?? ScreenReceiveReport() ScreenLog.info( "screen from \(ScreenWatchdog.short(peer)) still stalled: attaching its picture again; \(report.moved(since: screenReceivedMoving[peer]))") reattachScreen(peer) } } else { screenReceiveWatch = ScreenReceiveWatch() screenReceived = [:] screenReceivedMoving = [:] } guard stalled != reportedStalled, let client = bridge else { return } for peer in reportedStalled where !stalled.contains(peer) { let report = screenReceived[peer] ScreenLog.info( "screen from \(ScreenWatchdog.short(peer)) moving here again\(report.map { ": " + $0.summary } ?? "")") } reportedStalled = stalled let body = json(stalled) ?? "[]" noSoundQueue.async { try? client.setScreenStalled(body) } } /// A stalled screen's picture dropped and attached again: a new renderer /// on the same track, for a picture stopped on this side. private func reattachScreen(_ peer: String) { guard let track = remoteScreens[peer] else { return } screenReattaches += 1 qualityTally.count(.reattach, for: peer) remoteScreens.removeValue(forKey: peer) Task { @MainActor [weak self] in try? await Task.sleep(nanoseconds: 150_000_000) guard let self, self.screens.contains(peer), self.remoteScreens[peer] == nil else { return } self.remoteScreens[peer] = track } } /// Stops sharing, from every way out: the button, the system's own stop /// control, and the capture ending under us. public func stopSharingScreen() { guard sharingScreen else { return } sharingScreen = false ScreenLog.info("sharing stopped") endScreenWatch() // What was never built has no screen to stop. built?.stopScreen() send(type: "share", to: "", gen: 0, payload: json(["on": false]) ?? "") } /// A peer has said it is sharing, or its screen has arrived. Either one /// is enough to draw the tile. func announceScreen(_ peer: String) { if !screens.contains(peer) { screens.append(peer) } } func setScreen(_ track: RTCVideoTrack, peer: String) { announceScreen(peer) remoteScreens[peer] = track } /// Both of §5.4's endings meet here — the `share {on:false}` message /// and the receiver going away with its stopped transceiver — and /// whichever comes first wins. The second finds nothing to do. func dropScreen(_ peer: String) { screens.removeAll { $0 == peer } remoteScreens.removeValue(forKey: peer) } func mediaFailed() { status = "A participant’s media could not connect." } /// Caps the call's uplink estimate at `bps` (nil lifts it): the gate's /// stand-in for a thin connection, which it has no other way to get. public func limitUplink(_ bps: Int?) { media.limitUplink(bps) } /// For the gate: this app's receiver watch sees every screen frozen for /// `seconds`, so its stall report and the sharer's answer can be driven /// end to end. @_spi(Fixtures) public func freezeScreenWatchForTest(seconds: Double) { screenWatchFrozenUntil = ProcessInfo.processInfo.systemUptime + seconds } /// For the gate's encoder stall (see MediaEngine). @_spi(Fixtures) public func stallScreenEncoderForTest() { media.stallScreenEncoderForTest() } /// For the gate: the watchdog's share again (spec §15.22), on demand. @_spi(Fixtures) public func reshareScreenForTest() async { guard sharingScreen else { return } ScreenLog.info("sharing again for everyone, from the same capture (asked for by the gate)") await media.reshareScreen() sayShareAgain() } public func sampleStats() async -> [String: UInt64] { var values = await media.stats() values["hasICE"] = ice == nil ? 0 : 1 values["remoteTracks"] = UInt64(remoteTracks.count) values["remoteScreens"] = UInt64(remoteScreens.count) values["screens"] = UInt64(screens.count) values["invalidEvents"] = invalidEvents // §15.24: people still in their first moments, people whose tiles say // they cannot hear you yet, and how many this app reports it hears. values["connectingJoining"] = UInt64(connectingWatch.joining.count) values["connectingShown"] = UInt64(connectingPeers.count) values["hearing"] = UInt64(reportedHearing.count) // This app's own screen, as it leaves: the slowest connection's. if let screen = screenReport { values["screenSendWidth"] = UInt64(screen.width) values["screenSendHeight"] = UInt64(screen.height) values["screenSendFpsTenths"] = UInt64((screen.framesPerSecond * 10).rounded()) values["screenCapturedFpsTenths"] = UInt64((screen.capturedPerSecond * 10).rounded()) values["screenFramesSent"] = screen.framesSent values["screenFramesEncoded"] = screen.framesEncoded values["screenFramesCaptured"] = screen.framesCaptured values["screenLimitation"] = screen.limitationCode values["screenLimitedBandwidthMs"] = UInt64(((screen.limitationSeconds["bandwidth"] ?? 0) * 1000).rounded()) values["screenLimitedCpuMs"] = UInt64(((screen.limitationSeconds["cpu"] ?? 0) * 1000).rounded()) values["screenTargetBitrate"] = screen.targetBitrate values["screenSentBitrate"] = screen.sentBitrate values["screenEncodeMsTenths"] = UInt64((screen.encodeMsPerFrame * 10).rounded()) values["screenHardwareEncoder"] = screen.hardwareEncoder ? 1 : 0 values["screenCodecH264"] = screen.codec.lowercased().hasSuffix("h264") ? 1 : 0 values["screenPath"] = screen.pathCode values["screenRttMs"] = UInt64((screen.roundTripTime * 1000).rounded()) values["screenPacketsLost"] = UInt64(max(0, screen.packetsLost)) } // The screen watchdogs (ScreenWatchdog): what the sharer's has done, // how many stall reports it has heard, and whose screens this app has // reported stalled. values["screenWatchdogCaptureRestarts"] = UInt64(screenWatchdog.captureRestarts) values["screenWatchdogKeyframes"] = UInt64(screenWatchdog.keyframes) values["screenWatchdogEncoderRecoveries"] = UInt64(screenWatchdog.encoderRecoveries) values["screenWatchdogReAdds"] = UInt64(screenWatchdog.reAdds) values["screenStallReports"] = UInt64(screenWatchdog.stallReports) values["screenStalledHere"] = UInt64(reportedStalled.count) values["screenWatchdogRebuilds"] = UInt64(screenWatchdog.rebuilds) values["screenWatchdogReshares"] = UInt64(screenWatchdog.reshares) values["screenWatchdogFailures"] = UInt64(screenWatchdog.failures) values["screenReattaches"] = UInt64(screenReattaches) values["iceRestarts"] = media.iceRestarts values["screenKeyframes"] = screenReport?.keyFramesEncoded ?? 0 if let capture = media.screenCaptureCounts { values["screenCaptureHeartbeats"] = capture.heartbeats ?? 0 values["screenCaptureDelivered"] = capture.delivered } values["rosterPeers"] = UInt64(state?.peers.count ?? 0) values["noiseFramesProcessed"] = media.audio.denoiser.framesProcessed values["noiseSuppressor"] = media.audio.denoiser.engineName == "RNNoise" ? 1 : 2 if case .running(_, let cpu) = media.audio.denoiser.state { values["noiseCpuPermille"] = UInt64((cpu * 1000).rounded()) } // What WebRTC reports running, not what was asked of it. let running = media.audio.report() values["webrtcEchoCancellation"] = running.echoCancellation == "off" ? 0 : 1 values["webrtcNoiseSuppression"] = running.noiseSuppression == "off" ? 0 : 1 values["webrtcGainControl"] = running.gainControl.hasPrefix("Calls") || running.gainControl == "off" ? 0 : 1 // Whether the readout would say the microphone has started. values["audioReadoutStarted"] = running.started ? 1 : 0 // §15.20's meters, as the tiles see them: the loudest remote sound and // the loudest sent sound since the last sample, in thousandths of full // scale; who is in "No sound from"; and whether your own tile says // the microphone is picking nothing up. values["remoteAudioPeakPermille"] = UInt64((remotePeakSinceSample * 1000).rounded()) values["micPeakPermille"] = UInt64((micPeakSinceSample * 1000).rounded()) remotePeakSinceSample = 0 micPeakSinceSample = 0 values["noSoundFrom"] = UInt64(levels.noSoundFrom.count) values["micSilent"] = levels.mic.silent ? 1 : 0 // The speaking glow behind your own name (0.3.7): lit while what is // sent is speech that has kept up. values["micGlow"] = levels.mic.glow.glow.lit ? 1 : 0 return values } } // --------------------------------------------------------------- // Quality and problem reports (§15.26) // --------------------------------------------------------------- extension Call { /// Sends a quality report every `qualityInterval` while admitted: the /// window starts at admission, and each report covers the time since /// the last. Read and built on the main actor, as the statistics reads /// for the meters are; sent off it, because the bridge's send is a /// request. fileprivate func startQualityWatch() { qualityWatch?.cancel() qualityTally = QualityEventTally() qualityBuilder = QualityReportBuilder() qualityStopped = false qualityFinalSent = false qualityWatch = Task { @MainActor [weak self] in while !Task.isCancelled { try? await Task.sleep(nanoseconds: 1_000_000_000) guard let self, !Task.isCancelled, let client = self.bridge else { return } let now = ProcessInfo.processInfo.systemUptime guard self.qualityBuilder.started else { if self.admittedForReports { self.qualityBuilder.start(at: now) self.qualityTally = QualityEventTally() } continue } guard self.qualityBuilder.elapsed(at: now) >= Self.qualityInterval else { continue } guard self.admittedForReports, !self.qualityStopped, QualityReporting.enabled() else { // Off, stopped, or between participations: the window moves on // with nothing sent, so a report never covers time it did not see. self.qualityBuilder.start(at: now) self.qualityTally = QualityEventTally() continue } let samples = await self.built?.qualitySamples() ?? [:] guard !Task.isCancelled, self.bridge === client else { return } guard let body = self.qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: samples, final: false) else { continue } Task.detached { [weak self] in let answer = client.sendQuality(body) if answer == "stop" { await MainActor.run { [weak self] in self?.qualityStopped = true CallLog.info("quality reports: the server asked for no more this call") } } } } } } private var admittedForReports: Bool { guard let phase = state?.phase else { return false } return phase == "admitted" || phase == "reconnecting" } private func qualityReport(at now: Double, samples: [String: QualityPeerSample], final: Bool) -> String? { qualityBuilder.build( at: now, samples: samples, events: &qualityTally, client: QualityClientInfo.current(), devices: qualityDevices(), final: final ).encoded() } /// The report sent at leave, built while the connections are still up; /// nil when there is none to send. fileprivate func finalQualityReport() async -> String? { guard let client = bridge, qualityBuilder.started, !qualityStopped, !qualityFinalSent, QualityReporting.enabled() else { return nil } let samples = await built?.qualitySamples() ?? [:] guard bridge === client, !qualityFinalSent else { return nil } qualityFinalSent = true return qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: samples, final: true) } /// The call ended under this app (ended, removed, open elsewhere): the /// final report goes now, with the notices but without statistics — the /// connections are closing as it is built — and the call stays /// reportable from here. fileprivate func endedForReports() { guard let client = bridge else { return } keepForReports(client) guard qualityBuilder.started, !qualityStopped, !qualityFinalSent, QualityReporting.enabled() else { return } qualityFinalSent = true guard let body = qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: [:], final: true) else { return } Task.detached { _ = client.sendQuality(body) } } /// The kinds of microphone and speaker in use. Never their names. private func qualityDevices() -> QualityDevices? { #if os(macOS) let kinds = audio.deviceKinds() return QualityDevices(input: kinds.input, output: kinds.output) #else let route = AVAudioSession.sharedInstance().currentRoute return QualityDevices( input: DeviceKind.ios(port: route.inputs.first?.portType.rawValue), output: DeviceKind.ios(port: route.outputs.first?.portType.rawValue)) #endif } /// One look by the level watch, tallied for the window's events. fileprivate func observeQuality(now: Double, state: RoomState) { guard qualityBuilder.started else { return } let me = state.peer var fromUs = Set() var blocked = Set() for person in state.peers where person.peer != me { if person.media?.noSoundFrom?.contains(me) == true { fromUs.insert(person.peer) } if person.media?.audioBlocked == true { blocked.insert(person.peer) } } qualityTally.observe( at: now, call: reconnecting, reconnecting: reconnectingPeers, cantHear: connectingPeers, noSound: Set(levels.noSoundFrom), noSoundFromUs: fromUs, audioBlocked: blocked, screenStalled: Set(reportedStalled)) } /// The sharer's watchdog's steps, toward the peers they were for. fileprivate func countScreenSteps(_ actions: [ScreenRecovery], failed: Int, stalledBy: Set) { let me = state?.peer ?? "" for action in actions { switch action { case .keyframe(let peer): qualityTally.count(.keyframe, for: peer) case .reAdd(let peer), .rebuild(let peer): qualityTally.count(.rebuild, for: peer) case .reshare: for person in state?.peers ?? [] where person.peer != me { qualityTally.count(.reshare, for: person.peer) } case .restartCapture, .recreateEncoder: break } } guard failed > 0 else { return } for peer in stalledBy where screenWatchdog.escalation(of: peer) == .failed { qualityTally.count(.failure, for: peer) } } /// Keeps a call's closed client, and its diagnostics as they are now, /// for "Report a problem" during the ten minutes after it. fileprivate func keepForReports(_ client: BridgeClient) { guard reportClient !== client else { return } reportClient = client reportDiagnostics = anonymousDiagnosticsText() let until = Date().addingTimeInterval(Self.reportWindow) reportableUntil = until Task { @MainActor [weak self] in try? await Task.sleep(nanoseconds: UInt64(Self.reportWindow * 1_000_000_000)) guard let self, self.reportableUntil == until else { return } self.reportableUntil = nil self.reportClient = nil self.reportDiagnostics = "" } } /// Whether "Report a problem" has anything to report: a call in /// progress, or one that ended less than ten minutes ago. public var canReportProblem: Bool { bridge != nil || fixtureReports || (reportableUntil.map { $0 > Date() } ?? false) } /// Opens the sheet. In a call it reports this call, its diagnostics read /// now; after one, the call that ended, as it was when it ended. public func reportProblem() { let info = QualityClientInfo.current() let deliver: @Sendable (String) async -> ProblemOutcome if let client = bridge ?? (canReportProblem ? reportClient : nil) { let diagnostics = bridge != nil ? anonymousDiagnosticsText() : reportDiagnostics deliver = { note in let body = ProblemReportBody(note: note, diagnostics: diagnostics, client: info).encoded() return await Task.detached { ProblemOutcome(bridgeAnswer: client.sendProblem(body)) }.value } } else if fixtureReports { // The UI test's call has no server: it is answered here, after a // moment, with a reference of the server's shape. deliver = { _ in try? await Task.sleep(nanoseconds: 300_000_000) return .sent(ref: "K7Q2M9PD") } } else { return } problemReport = ProblemReportModel(deliver: deliver) } } // --------------------------------------------------------------- // Fixtures // --------------------------------------------------------------- extension Call { /// Puts the call into a described state without a server, for the /// offline snapshots of the call views and the iPhone's UI test of its /// controls. `state` is a roster in the wire's own JSON. Never used by /// a call that has joined anything. @_spi(Fixtures) public func showFixture( title: String, standing: String = "member", status: String = "In the meeting", state json: String, screens: [String] = [], sharingScreen: Bool = false, muted: Bool = false, floor: Bool = false, named: Bool = false ) { self.title = title self.named = named self.standing = standing self.state = try? JSONDecoder().decode(RoomState.self, from: Data(json.utf8)) self.status = status // A fixture's reconnecting is one that has lasted (spec §15.23). self.reconnecting = self.state?.phase == "reconnecting" self.screens = screens self.sharingScreen = sharingScreen self.muted = muted self.floor = floor self.fixtureReports = true } /// Your own microphone, camera and screen, set over a fixture for the /// snapshots of the controls in each state. @_spi(Fixtures) public func showFixtureSelf(muted: Bool, cameraOff: Bool, sharingScreen: Bool) { self.muted = muted self.cameraOff = cameraOff self.sharingScreen = sharingScreen } /// Spec §15.24 for the snapshots: these people have just joined and the /// grace has passed, so each tile says what it would — "connecting", or /// a report of theirs that replaces it. @_spi(Fixtures) public func showFixtureConnecting(_ peers: [String]) { let me = state?.peer ?? "" var shown: Set = [] for person in state?.peers ?? [] where peers.contains(person.peer) { let said = ConnectingWatch.tile( name: person.name, media: person.media, me: me, muted: muted, joining: true, shown: true, reconnecting: false) if said.connecting != nil { shown.insert(person.peer) } levels.peer(person.peer).show( state: person.media?.muted == true ? .muted : .meter(0), speaking: false, notices: said.notices, connecting: said.connecting) } connectingPeers = shown } /// Everyone's glow, and yours, held at one level, for the snapshots. @_spi(Fixtures) public func showFixtureSpeaking(level: Float) { levels.mic.glow.showFixture(level: level) // Nobody muted glows: muted, the speaking gate is shut. if muted { levels.mic.glow.reset() } for peer in state?.peers ?? [] where peer.peer != state?.peer && !isMuted(peer.peer) { levels.peer(peer.peer).glow.showFixture(level: level) } levels.othersGlow.showFixture(level: level) levels.anyoneSpeaking.set(levels.othersGlow.glow.lit) } }