import AVFoundation import Foundation import LiveKitWebRTC import os #if os(macOS) import AppKit #else import UIKit #endif /// One camera the person can choose, named the way a menu wants it. /// /// The id is `AVCaptureDevice.uniqueID`, which survives unplugging and /// replugging the same camera, so a remembered choice means the same /// device next time rather than whatever now sits at that index. public struct CameraDevice: Identifiable, Hashable, Sendable { public let id: String public let name: String /// The Mac's own camera (the iPhone's front one): where a call goes /// when the remembered camera is not connected. public let builtIn: Bool /// `AVCaptureDevice.modelID`: the same for the same kind of camera /// wherever it is plugged in. A USB camera's `uniqueID` is built from /// the port it is on, so an ATEM Mini moved to another port (or through /// a different hub) comes back under a new id; the model, and failing /// that the name, is how a remembered choice still finds it. public let model: String /// `AVCaptureDevice.isSuspended`: listed and connected, but the system /// will not let it capture. On a Mac this is the built-in camera with /// the lid closed, where it opens and never sends a frame. public let suspended: Bool public init(id: String, name: String, builtIn: Bool = false, model: String = "", suspended: Bool = false) { self.id = id self.name = name self.builtIn = builtIn self.model = model self.suspended = suspended } } // Capture is injected separately from signaling and rendering. App targets // provide the platform camera; the interop executable provides synthetic frames. @MainActor public protocol VideoCapture: AnyObject { func start(source: RTCVideoSource) async throws func stop() /// The cameras this capture can offer, and which one it is on. A capture /// with nothing to choose between answers an empty list, and the picker /// does not appear — which is what the interop executable's synthetic /// frames want, and why these carry defaults rather than being required. var cameras: [CameraDevice] { get } var currentCamera: String? { get } func use(camera id: String) async throws /// Whether the call should always show a Camera row, even with no /// camera to list: true for real cameras, where "No camera found" is /// something a person needs to be told. var offersCameras: Bool { get } /// What the camera is doing, for the self tile and the panel. Nil for a /// capture that does not say. var cameraStatus: CameraStatus? { get } /// A fallback worth saying ("ATEM Mini disconnected; using FaceTime HD /// Camera"), and usually nothing. var cameraNote: String { get } /// The camera in use when it is working and is not the remembered one: /// what the panel offers to remember ("Use Blackmagic Design from now /// on"). Nil otherwise. var cameraToKeep: CameraDevice? { get } /// Called on the main actor whenever any of the above changes on its /// own — a camera connecting, a fallback, the first frame. func observe(_ changed: @escaping @MainActor () -> Void) /// The panel's "Try again". func retryCamera() async } extension VideoCapture { public var cameras: [CameraDevice] { [] } public var currentCamera: String? { nil } public func use(camera id: String) async throws {} public var offersCameras: Bool { false } public var cameraStatus: CameraStatus? { nil } public var cameraNote: String { "" } public var cameraToKeep: CameraDevice? { nil } public func observe(_ changed: @escaping @MainActor () -> Void) {} public func retryCamera() async {} } /// A screen, as §5.4 sends one: frames into a source of its own, which /// the call carries on a `sendonly` transceiver beside the camera's. /// /// `start` returns once the person has chosen what to share and frames /// are on their way; a refusal at the system's own picker is a throw, and /// nothing has been announced. `ended` is for the capture ending on its /// own — the system's stop control, a shared window closing, a broadcast /// stopped from Control Center — and is called at most once, on the main /// actor. `stop` is the call ending it, and calls nothing back. @MainActor public protocol ScreenCapture: AnyObject { func start(source: RTCVideoSource, ended: @escaping @MainActor () -> Void) async throws func stop() /// What the capture has produced so far, for the screen watchdog. var counts: ScreenCaptureCounts { get } /// Starts the same capture again — the same content into the same /// source — after the watchdog has seen it stall. False when this kind /// of capture cannot, or the attempt failed. func restart() async -> Bool /// The size the capture delivers frames at, when it knows before the /// first frame: the send policy's H.264 level cap (ScreenSendPolicy) /// then holds from the first frame rather than from the first /// statistics a second later, a second in which a browser's level-3.1 /// encoder refuses every frame of a Motion share (spec §15.22). var frameSize: (width: Int, height: Int)? { get } } extension ScreenCapture { public var counts: ScreenCaptureCounts { ScreenCaptureCounts() } public func restart() async -> Bool { false } public var frameSize: (width: Int, height: Int)? { nil } } /// A screen capture's running totals (see ScreenWatchdog). public struct ScreenCaptureCounts: Equatable, Sendable { /// Everything the capture has reported, still-screen reports included: /// ScreenCaptureKit reports "idle" at the frame rate while nothing on /// screen changes (measured 2026-09-28: 177 idle reports in six seconds /// of a still window), so a capture that reports nothing at all has /// stopped, not gone quiet. Nil for a capture that cannot tell the two /// apart — a ReplayKit broadcast sends nothing for a still screen — /// which turns the watchdog's capture check off for it. public var heartbeats: UInt64? /// Frames handed to the call's source, repeats of a still screen included. public var delivered: UInt64 = 0 public init(heartbeats: UInt64? = nil, delivered: UInt64 = 0) { self.heartbeats = heartbeats self.delivered = delivered } } // --------------------------------------------------------------------- // The managed camera: device choice, recovery and the watchdog // --------------------------------------------------------------------- /// Where frames actually come from, under the recovery rules: AVFoundation /// in the apps, synthetic frames in the interop gate. Everything about /// *which* camera and *when to try again* lives in `ManagedCamera` and /// `CameraDirector`; a backend only lists, opens and closes. @MainActor public protocol CameraBackend: AnyObject { /// Asks for camera (and microphone) permission. Throws when refused. func requestAccess() async throws /// The cameras connected now. var devices: [CameraDevice] { get } /// How many formats `open` can be asked for on this device, ranked. func formatCount(_ id: String) -> Int /// Opens the device in its ranked format, delivering frames to /// `delegate`. Throws when it will not open. func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws func close() async /// Set by the managed camera: the device list changed. var devicesChanged: (@MainActor () -> Void)? { get set } /// Set by the managed camera: the session reported an error or an /// interruption, or the machine woke. Capture should be restarted. var trouble: (@MainActor () -> Void)? { get set } } /// Counts frames on their way to the video source, which is all the /// watchdog needs to know whether a camera is working. The capturer holds /// its delegate weakly, so the managed camera holds this. final class FrameCounter: NSObject, RTCVideoCapturerDelegate, @unchecked Sendable { private let source: RTCVideoSource private let lock = OSAllocatedUnfairLock(initialState: UInt64(0)) init(source: RTCVideoSource) { self.source = source } var count: UInt64 { lock.withLock { $0 } } func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) { lock.withLock { $0 += 1 } source.capturer(capturer, didCapture: frame) } } /// A camera under the recovery rules (see `CameraDirector`): remembered /// between calls, followed as devices come and go, and watched for frames. /// /// Every open and close runs one at a time, and an open the director has /// since moved on from is skipped, so the last decision is the one that /// holds. @MainActor public final class ManagedCamera: VideoCapture { private let backend: CameraBackend private let director: CameraDirector private var counter: FrameCounter? private var work: Task? private var ticker: Task? private var running = false private var changed: (@MainActor () -> Void)? private(set) public var currentCamera: String? /// Opens performed, for the gate: a stall that recovered is one more. public private(set) var opens = 0 private let clock: () -> TimeInterval var tickInterval: Duration = .milliseconds(500) public init( backend: CameraBackend, defaults: UserDefaults = .standard, clock: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime } ) { self.backend = backend self.clock = clock director = CameraDirector(defaults: defaults, formats: { [weak backend] in backend?.formatCount($0) ?? 1 }) backend.devicesChanged = { [weak self] in self?.devicesChanged() } backend.trouble = { [weak self] in self?.trouble() } } public var cameras: [CameraDevice] { backend.devices } public var offersCameras: Bool { true } public var cameraStatus: CameraStatus? { running ? director.status : nil } public var cameraNote: String { running ? director.note : "" } public var cameraToKeep: CameraDevice? { running ? director.cameraToKeep : nil } public func observe(_ changed: @escaping @MainActor () -> Void) { self.changed = changed } public func start(source: RTCVideoSource) async throws { try await backend.requestAccess() counter = FrameCounter(source: source) running = true ticker = Task { @MainActor [weak self] in var slow = 0 while !Task.isCancelled { guard let self else { return } try? await Task.sleep(for: self.tickInterval) guard !Task.isCancelled else { return } self.tick() // Belt and braces for the device notifications: while there is no // working camera, look again every few seconds. slow += 1 if slow % 6 == 0, !(self.director.status.live) { self.devicesChanged() } } } // Returns once the first open has been tried, so a call starts with // its camera (or its reason for having none) already known. await perform(director.start(devices: backend.devices)) } public func stop() { running = false ticker?.cancel() ticker = nil director.stop() currentCamera = nil let backend = backend let previous = work work = Task { @MainActor in await previous?.value await backend.close() } counter = nil } public func use(camera id: String) async throws { guard running else { return } guard let command = director.choose(id) else { throw NSError( domain: "Calls", code: 4, userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."]) } await perform(command) } public func retryCamera() async { guard running else { return } await perform(director.retry()) } private func tick() { guard running, let counter else { return } #if os(iOS) // iOS stops every camera while the app is in the background; that // is not a stall to recover from. if UIApplication.shared.applicationState != .active { director.hold(frames: counter.count, at: clock()) return } #endif if let command = director.tick(frames: counter.count, at: clock()) { Task { await self.perform(command) } } notify() } private func devicesChanged() { guard running else { changed?() return } if let command = director.devicesChanged(backend.devices) { Task { await self.perform(command) } } notify() } private func trouble() { guard running else { return } if let command = director.restart(at: clock()) { Task { await self.perform(command) } } } private var published: (CameraStatus, String, String?, CameraDevice?)? private func notify() { let now = (director.status, director.note, currentCamera, director.cameraToKeep) if let published, published == now { return } published = now changed?() } /// Runs a command after whatever is running, and returns when it has. private func perform(_ command: CameraCommand?) async { guard let command else { notify() return } let previous = work let task = Task { @MainActor [weak self] in await previous?.value await self?.execute(command) } work = task await task.value } private func execute(_ command: CameraCommand) async { guard running else { return } switch command { case .stop: await backend.close() currentCamera = nil notify() case .open(let id, let format): // The director has moved on (a later pick, a fallback): skip. guard director.active == id, director.format == format, let counter else { return } await backend.close() currentCamera = nil notify() guard running, director.active == id else { return } do { try await backend.open(id, format: format, delegate: counter) opens += 1 guard running, director.active == id else { return } currentCamera = id director.opened(id, at: clock(), frames: counter.count) } catch { guard running, director.active == id else { return } if let next = director.openFailed(id) { notify() await execute(next) return } } notify() } } } // --------------------------------------------------------------------- // AVFoundation // --------------------------------------------------------------------- /// The cameras connected to this device, followed as they come and go: /// AVFoundation's connect and disconnect notifications, and on Mac the /// discovery session's device list by KVO. One for the process, so the /// panel can list cameras whether or not a capture is running. @MainActor public final class CameraWatch { public static let shared = CameraWatch() public private(set) var cameras: [CameraDevice] = [] private var listeners: [UUID: @MainActor () -> Void] = [:] private var tokens: [NSObjectProtocol] = [] /// Each listed camera's `isSuspended`, watched: the lid closing or /// opening changes it without any camera connecting or leaving. private var suspension: [String: NSKeyValueObservation] = [:] #if os(macOS) private let session = AVCaptureDevice.DiscoverySession( deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera], mediaType: .video, position: .unspecified) private var kvo: NSKeyValueObservation? #endif private init() { for name in [AVCaptureDevice.wasConnectedNotification, AVCaptureDevice.wasDisconnectedNotification] { tokens.append( NotificationCenter.default.addObserver(forName: name, object: nil, queue: .main) { _ in MainActor.assumeIsolated { CameraWatch.shared.refresh() } }) } #if os(macOS) kvo = session.observe(\.devices, options: [.new]) { _, _ in DispatchQueue.main.async { MainActor.assumeIsolated { CameraWatch.shared.refresh() } } } #endif cameras = Self.list(devices) watchSuspension() } private func watchSuspension() { let current = devices let ids = Set(current.map(\.uniqueID)) for id in suspension.keys where !ids.contains(id) { suspension.removeValue(forKey: id) } for device in current where suspension[device.uniqueID] == nil { suspension[device.uniqueID] = device.observe(\.isSuspended, options: [.new]) { _, _ in DispatchQueue.main.async { MainActor.assumeIsolated { CameraWatch.shared.refresh() } } } } } var devices: [AVCaptureDevice] { #if os(macOS) session.devices #else RTCCameraVideoCapturer.captureDevices() #endif } func device(_ id: String) -> AVCaptureDevice? { devices.first { $0.uniqueID == id } } static func list(_ devices: [AVCaptureDevice]) -> [CameraDevice] { devices.filter(\.isConnected).map { #if os(macOS) let builtIn = $0.deviceType == .builtInWideAngleCamera #else let builtIn = $0.position == .front #endif return CameraDevice( id: $0.uniqueID, name: $0.localizedName, builtIn: builtIn, model: $0.modelID, suspended: $0.isSuspended) } } public func refresh() { let now = Self.list(devices) guard now != cameras else { return } watchSuspension() cameras = now for listener in listeners.values { listener() } } func listen(_ listener: @escaping @MainActor () -> Void) -> UUID { let id = UUID() listeners[id] = listener return id } func unlisten(_ id: UUID) { listeners.removeValue(forKey: id) } } /// The formats a camera is opened in, best first, and the frame rate to /// ask for in each — held apart from AVFoundation so the rule can be /// tested with numbers. /// /// Nearest 640 wide first (a call does not need more, and the encoder /// scales down anyway), then nearest 30 frames a second without going /// over, then the device's own order. The rate asked for is always one /// the format supports: 30 where a range covers it, otherwise the /// fastest rate at or under 30, otherwise the slowest the format offers. /// An ATEM Mini offers 1080p at exactly 23.98, 24, 25, 29.97, 30, 50, /// 59.94 and 60 — ranges of one rate each — and asking one of those for a /// rate outside its range is an exception AVFoundation raises and WebRTC /// swallows, leaving the camera at whatever rate it was on. public struct CameraFormatChoice: Equatable, Sendable { public struct Candidate: Equatable, Sendable { public var width: Int public var height: Int /// Each supported range as (min, max) frames a second. public var ranges: [ClosedRange] public init(width: Int, height: Int, ranges: [ClosedRange]) { self.width = width self.height = height self.ranges = ranges } } /// The rate to run a format at, and whether it is a whole number WebRTC /// can be given as-is. public static func rate(for ranges: [ClosedRange], target: Double = 30) -> Double? { guard !ranges.isEmpty else { return nil } if ranges.contains(where: { $0.contains(target) }) { return target } let under = ranges.map(\.upperBound).filter { $0 <= target } if let best = under.max() { return best } return ranges.map(\.lowerBound).min() } /// The candidates' indexes, best first, one per distinct size and rate. public static func rank(_ candidates: [Candidate], width target: Int = 640) -> [Int] { var seen = Set() return candidates.indices.sorted { a, b in let ca = candidates[a], cb = candidates[b] let wa = abs(ca.width - target), wb = abs(cb.width - target) if wa != wb { return wa < wb } let ra = 30 - (rate(for: ca.ranges) ?? 0), rb = 30 - (rate(for: cb.ranges) ?? 0) if abs(ra) != abs(rb) { return (ra >= 0 ? ra : 1000 - ra) < (rb >= 0 ? rb : 1000 - rb) } return a < b }.filter { let c = candidates[$0] let key = "\(c.width)x\(c.height)@\(rate(for: c.ranges) ?? 0)" return seen.insert(key).inserted } } } /// AVFoundation's camera, through WebRTC's capturer. @MainActor final class AVCameraBackend: CameraBackend { private var capturer: RTCCameraVideoCapturer? private var observed: [NSObjectProtocol] = [] private var listener: UUID? #if os(macOS) private var woke: NSObjectProtocol? #endif var devicesChanged: (@MainActor () -> Void)? var trouble: (@MainActor () -> Void)? /// How long an open may take before it counts as refused. A camera /// that re-enumerates mid-open can leave the completion uncalled. static let openTimeout: TimeInterval = 6 init() { let watch = CameraWatch.shared listener = watch.listen { [weak self] in self?.devicesChanged?() } #if os(macOS) woke = ( NSWorkspace.shared.notificationCenter.addObserver( forName: NSWorkspace.didWakeNotification, object: nil, queue: .main ) { [weak self] _ in MainActor.assumeIsolated { CameraWatch.shared.refresh() self?.trouble?() } }) #endif } func requestAccess() async throws { for kind in [AVMediaType.video, .audio] { let granted = await AVCaptureDevice.requestAccess(for: kind) if !granted { throw NSError( domain: "Calls", code: 2, userInfo: [NSLocalizedDescriptionKey: "Allow camera and microphone access in Settings to join."]) } } } var devices: [CameraDevice] { CameraWatch.shared.refresh() return CameraWatch.shared.cameras } private func formats(_ device: AVCaptureDevice) -> [AVCaptureDevice.Format] { let all = RTCCameraVideoCapturer.supportedFormats(for: device) let candidates = all.map { format -> CameraFormatChoice.Candidate in let size = CMVideoFormatDescriptionGetDimensions(format.formatDescription) return CameraFormatChoice.Candidate( width: Int(size.width), height: Int(size.height), ranges: format.videoSupportedFrameRateRanges.map { $0.minFrameRate...$0.maxFrameRate }) } return CameraFormatChoice.rank(candidates).map { all[$0] } } func formatCount(_ id: String) -> Int { guard let device = CameraWatch.shared.device(id) else { return 0 } return formats(device).count } func open(_ id: String, format index: Int, delegate: RTCVideoCapturerDelegate) async throws { guard let device = CameraWatch.shared.device(id) else { throw NSError( domain: "Calls", code: 4, userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."]) } let ranked = formats(device) guard !ranked.isEmpty else { throw NSError( domain: "Calls", code: 3, userInfo: [NSLocalizedDescriptionKey: "No camera is available."]) } let format = ranked[min(index, ranked.count - 1)] let ranges = format.videoSupportedFrameRateRanges let rate = CameraFormatChoice.rate(for: ranges.map { $0.minFrameRate...$0.maxFrameRate }) ?? 30 let capture = capturer ?? makeCapturer(delegate: delegate) // WebRTC takes a whole number of frames a second. For a rate that is // not one (29.97), it is given the nearest, and the exact duration // from the format's own range is set on the device afterwards. let fps = max(1, Int(rate.rounded())) let once = OSAllocatedUnfairLock(initialState: false) do { try await start(capture, device: device, format: format, fps: fps, once: once) } catch { // A failed or abandoned open may still finish later on the // capturer's own queue and start delivering frames from this // device. The capturer is cut loose — no delegate, stopped, not // reused — so those frames reach nobody, and the next open gets a // fresh one. capture.delegate = nil capture.stopCapture(completionHandler: {}) capturer = nil throw error } if Double(fps) != rate, let range = ranges.first(where: { $0.minFrameRate <= rate && rate <= $0.maxFrameRate }) { let duration = rate == range.maxFrameRate ? range.minFrameDuration : range.maxFrameDuration if (try? device.lockForConfiguration()) != nil { device.activeVideoMinFrameDuration = duration device.activeVideoMaxFrameDuration = duration device.unlockForConfiguration() } } } private func start( _ capture: RTCCameraVideoCapturer, device: AVCaptureDevice, format: AVCaptureDevice.Format, fps: Int, once: OSAllocatedUnfairLock ) async throws { try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in let finish: @Sendable (Error?) -> Void = { error in let first = once.withLock { done -> Bool in defer { done = true } return !done } guard first else { return } if let error { continuation.resume(throwing: error) } else { continuation.resume() } } capture.startCapture(with: device, format: format, fps: fps) { finish($0) } DispatchQueue.main.asyncAfter(deadline: .now() + AVCameraBackend.openTimeout) { finish( NSError( domain: "Calls", code: 6, userInfo: [NSLocalizedDescriptionKey: "The camera did not start."])) } } } private func makeCapturer(delegate: RTCVideoCapturerDelegate) -> RTCCameraVideoCapturer { let capture = RTCCameraVideoCapturer(delegate: delegate) capturer = capture let session = capture.captureSession // A runtime error, or an interruption ending: WebRTC's capturer // retries a runtime error once itself; these restart it again, rate // limited by the director. for name in [ AVCaptureSession.runtimeErrorNotification, AVCaptureSession.interruptionEndedNotification, ] { observed.append( NotificationCenter.default.addObserver(forName: name, object: session, queue: .main) { [weak self] _ in MainActor.assumeIsolated { self?.trouble?() } }) } return capture } func close() async { guard let capturer else { return } await capturer.stopCapture() } deinit { for token in observed { NotificationCenter.default.removeObserver(token) } #if os(macOS) if let woke { NSWorkspace.shared.notificationCenter.removeObserver(woke) } #endif if let listener { Task { @MainActor in CameraWatch.shared.unlisten(listener) } } } } /// The platform camera: AVFoundation under the recovery rules. @MainActor final class CameraCapture { static func make() -> ManagedCamera { ManagedCamera(backend: AVCameraBackend()) } }