import Foundation /// What the camera is doing, said the way the self tile and the device /// panel say it. /// /// This exists because of a report against Calls 0.3.1: an ATEM Mini that /// had dropped off USB left the self tile on "Waiting for video" and the /// device panel with no Camera row at all, and nothing on screen said what /// had happened or offered a way back. Every state here has words of its /// own, and none of them is "waiting" without saying for what. public enum CameraStatus: Equatable, Sendable { /// Opening a camera, and no frame from it yet. case starting(String) /// Frames are arriving from the named camera. case live(String) /// The camera opened but sends nothing — an ATEM with no HDMI input, or /// one that is changing format. case noSignal(String) /// The camera in use went away and there is no other to use. case disconnected(String) /// There is no camera at all. case noCamera /// The camera would not open, and there is no other to use. case failed(String) /// What the self tile says in place of video. Nil while frames arrive. public var tileText: String? { switch self { case .starting: return "Starting camera…" case .live: return nil case .noSignal(let name): return "No signal from \(name) — check its input" case .disconnected: return "Camera disconnected" case .noCamera: return "No camera found" case .failed(let name): return "\(name) would not start" } } /// The plain-words line under the camera picker. Nil when all is well. public var panelText: String? { switch self { case .live: return nil case .starting(let name): return "Starting \(name)…" case .noSignal(let name): return "\(name) is connected but sending no picture. Check its input, or choose another camera." case .disconnected(let name): return "\(name) disconnected. Calls will use it again when it is back." case .noCamera: return "No camera found" case .failed(let name): return "\(name) would not start. Try again, or choose another camera." } } var live: Bool { if case .live = self { return true } return false } var disconnected: Bool { if case .disconnected = self { return true } return false } } /// The person's camera, remembered between calls and launches. /// /// Kept by `uniqueID`, with the model and name beside it. The uniqueID of /// a USB camera follows the port it is plugged into, so the same ATEM /// Mini on another port is a new id; `find` falls back to the model and /// then the name, so the remembered camera is still found. The name also /// lets the panel say which camera it is waiting for while it is gone. /// /// Only a person's own pick writes this (`CameraDirector.choose`, and the /// panel before a call). A fallback, a skip and a first-run default never /// do: which camera a call is on is not the same as which one you chose. public struct CameraPreference: Equatable, Sendable { public var id: String public var name: String /// The camera's model, for finding it again under a new id. Empty for a /// choice saved before 0.3.4, which is then found by name. public var model: String public init(id: String, name: String, model: String = "") { self.id = id self.name = name self.model = model } public init(_ device: CameraDevice) { self.init(id: device.id, name: device.name, model: device.model) } static let idKey = "preferredCameraID" static let nameKey = "preferredCameraName" static let modelKey = "preferredCameraModel" public static func load(_ defaults: UserDefaults = .standard) -> CameraPreference? { guard let id = defaults.string(forKey: idKey), !id.isEmpty else { return nil } return CameraPreference( id: id, name: defaults.string(forKey: nameKey) ?? "", model: defaults.string(forKey: modelKey) ?? "") } /// The connected camera this preference means: by id, then by model, /// then by name. public func find(in devices: [CameraDevice]) -> CameraDevice? { if let device = devices.first(where: { $0.id == id }) { return device } if !model.isEmpty, let device = devices.first(where: { $0.model == model }) { return device } if !name.isEmpty, let device = devices.first(where: { $0.name == name }) { return device } return nil } public func save(_ defaults: UserDefaults = .standard) { defaults.set(id, forKey: Self.idKey) defaults.set(name, forKey: Self.nameKey) defaults.set(model, forKey: Self.modelKey) } } /// The first-frame and stall watchdog, as a pure state machine over a /// frame counter and a clock. /// /// A camera that opens is not a camera that works: the ATEM Mini opens /// happily with nothing plugged into it and delivers no frame, ever. So /// the counter the capturer feeds is watched, and a camera with no first /// frame within `firstFrame`, or no frame for `stall` after it had been /// sending, is escalated one step at a time: /// /// 1. restart capture on the same device and format, /// 2. try the device's next format, /// 3. fall back to another camera. /// /// Frames arriving again at any point put it back at the bottom of the /// ladder. Each step waits for the capture to be opened again before /// the clock restarts, so a slow open is never counted as a stall. struct CameraWatchdog: Equatable { enum Step: Equatable { case restart, nextFormat, fallback } var firstFrame: TimeInterval = 3 var stall: TimeInterval = 3 /// How far up the ladder this device has gone since its last frame. private(set) var level = 0 /// Whether a frame has arrived since the capture was last opened. private(set) var flowing = false /// When the clock started: the open, or the last new frame. Nil while /// nothing is being watched (between a step and the next open). private var since: TimeInterval? private var count: UInt64 = 0 /// Top of the ladder with nowhere to go: no more steps, but a frame /// arriving still counts as the camera working again. private(set) var exhausted = false /// The capture has (re)opened. The clock starts for a first frame. mutating func opened(at now: TimeInterval, frames: UInt64) { since = now count = frames flowing = false exhausted = false } /// Nothing left to try: stop escalating, keep listening for frames. mutating func giveUp() { since = nil exhausted = true } /// A different camera, or the person's own choice: the ladder starts /// again from the bottom. mutating func reset() { level = 0 since = nil flowing = false exhausted = false } /// Stops watching until the next open. mutating func pause() { since = nil } /// Frames are not expected just now (an iPhone app in the background): /// the clock is held where it is, without climbing the ladder. mutating func hold(at now: TimeInterval, frames: UInt64) { guard since != nil else { return } since = now count = frames } /// Called on every tick with the frame counter. Answers the next step /// when one is due, and stops watching until `opened` is called again. mutating func observe(frames: UInt64, at now: TimeInterval) -> Step? { if frames != count { count = frames if since != nil || exhausted { since = now flowing = true level = 0 exhausted = false } return nil } guard let started = since else { return nil } guard now - started >= (flowing ? stall : firstFrame) else { return nil } since = nil let step: Step = level == 0 ? .restart : level == 1 ? .nextFormat : .fallback level = min(level + 1, 2) return step } } /// What the camera capture should do next. enum CameraCommand: Equatable { /// Open this device with the format at this rank (0 is the best). case open(String, format: Int) /// Stop capturing: there is nothing to capture from. case stop } /// The rules that decide which camera a call is on, and what it says /// about it — apart from AVFoundation, so they can be held by tests. /// /// - At join the remembered camera is used if it is connected, otherwise /// the built-in one, otherwise whatever there is. /// - A suspended camera (a Mac's built-in one with the lid closed, which /// is listed but never sends a frame) is passed over while there is /// another, and the panel says why ("MacBook Air Camera is off while /// the lid is closed; using Blackmagic Design"). It is still the /// remembered camera: opening the lid, mid-call or before the next one, /// goes back to it. Closing the lid mid-call falls back at once rather /// than waiting out the watchdog. /// - Choosing a camera remembers it, including the one the call is /// already on by fallback. Nothing else writes the preference. While /// the call is on a working camera other than the remembered one, the /// panel offers to remember it ("Use Blackmagic Design from now on"): /// one tap, never an automatic switch. /// - The camera in use disconnecting falls back to another at once, and /// says so ("ATEM Mini disconnected; using FaceTime HD Camera"). /// - The remembered camera reconnecting, or appearing later, is switched /// back to, without renegotiation (the capturer feeds the same source). /// - A camera that sends nothing is restarted, then tried in its next /// format, then fallen back from; with nothing to fall back to, it is /// named as having no signal and kept, in case its signal returns. /// - Session errors and waking from sleep restart capture, at most once /// every `restartSpacing` seconds. @MainActor final class CameraDirector { private(set) var devices: [CameraDevice] = [] private(set) var active: String? private(set) var format = 0 private(set) var status: CameraStatus = .noCamera /// A fallback the person should know about. Cleared by their own /// choice and by the remembered camera coming back. private(set) var note = "" private(set) var watchdog = CameraWatchdog() /// Cameras fallen back from for sending nothing, while they stay /// connected. Not returned to automatically until they reconnect. private var silent: Set = [] /// Cameras that would not open, this round. private var refused: Set = [] private var lastRestart: TimeInterval = -.infinity var restartSpacing: TimeInterval = 5 private let defaults: UserDefaults /// How many formats a device has to try, ranked. Supplied by the /// capture; the tests give numbers. private let formats: (String) -> Int init(defaults: UserDefaults = .standard, formats: @escaping (String) -> Int) { self.defaults = defaults self.formats = formats } var preference: CameraPreference? { CameraPreference.load(defaults) } /// The camera in use, when it is working and is not the remembered /// one: what the panel offers to remember. var cameraToKeep: CameraDevice? { guard status.live, let id = active, let device = devices.first(where: { $0.id == id }) else { return nil } guard let preference, preference.find(in: devices)?.id != id else { return nil } return device } /// "MacBook Air Camera is off while the lid is closed": what a /// suspended camera is called when it is passed over. nonisolated static func off(_ device: CameraDevice) -> String { #if os(macOS) if device.builtIn { return "\(device.name) is off while the lid is closed" } #endif return "\(device.name) is off" } private func name(_ id: String?) -> String { guard let id else { return "The camera" } if let device = devices.first(where: { $0.id == id }) { return device.name } if let preference, preference.id == id, !preference.name.isEmpty { return preference.name } return "The camera" } /// The camera a call should be on, given what is connected: the /// remembered one, then the built-in one, then the first — each only if /// it is not suspended, and a suspended one only when there is nothing /// else. Pure, and what `start` and every fallback ask. nonisolated static func choose( _ devices: [CameraDevice], preferred: CameraPreference?, excluding: Set = [] ) -> CameraDevice? { let usable = devices.filter { !excluding.contains($0.id) } let awake = usable.filter { !$0.suspended } if let device = preferred?.find(in: awake) { return device } if let device = awake.first(where: \.builtIn) ?? awake.first { return device } return preferred?.find(in: usable) ?? usable.first(where: \.builtIn) ?? usable.first } /// A call starts: pick the camera and open it. func start(devices: [CameraDevice]) -> CameraCommand? { self.devices = devices silent = [] refused = [] note = "" watchdog.reset() return pick(missingNote: true) } /// Chooses from scratch, excluding what has failed this round. private func pick(missingNote: Bool) -> CameraCommand? { let preferred = preference guard let device = Self.choose(devices, preferred: preferred, excluding: silent.union(refused)) else { active = nil status = devices.isEmpty ? .noCamera : status if devices.isEmpty { note = "" } return .stop } if missingNote, let preferred { if let found = preferred.find(in: devices) { if found.id != device.id, found.suspended { note = "\(Self.off(found)); using \(device.name)" } } else { let missing = preferred.name.isEmpty ? "Your camera" : preferred.name note = "\(missing) isn’t connected; using \(device.name)" } } return open(device.id) } private func open(_ id: String, format: Int = 0) -> CameraCommand { if active != id { watchdog.reset() } active = id self.format = format status = .starting(name(id)) watchdog.pause() return .open(id, format: format) } /// The capture opened `id`; the watchdog starts timing a first frame. func opened(_ id: String, at now: TimeInterval, frames: UInt64) { guard id == active else { return } refused.remove(id) watchdog.opened(at: now, frames: frames) } /// The capture could not open `id`. Another camera is tried; with none /// left, the panel says so and offers to try again. func openFailed(_ id: String) -> CameraCommand? { guard id == active else { return nil } let failedName = name(id) refused.insert(id) if let other = Self.choose(devices, preferred: preference, excluding: silent.union(refused)) { note = "\(failedName) would not start; using \(other.name)" return open(other.id) } status = .failed(failedName) watchdog.pause() return nil } /// A tick while frames are not expected — the iPhone app in the /// background, where the system stops every camera. func hold(frames: UInt64, at now: TimeInterval) { watchdog.hold(at: now, frames: frames) } /// A tick: the frame counter, read. func tick(frames: UInt64, at now: TimeInterval) -> CameraCommand? { guard let id = active else { return nil } let step = watchdog.observe(frames: frames, at: now) if watchdog.flowing, !status.live { status = .live(name(id)) silent.remove(id) } guard let step else { return nil } switch step { case .restart: return open(id, format: format) case .nextFormat: if format + 1 < formats(id) { return open(id, format: format + 1) } return fallBack(from: id) case .fallback: return fallBack(from: id) } } private func fallBack(from id: String) -> CameraCommand? { let silentName = name(id) if let other = Self.choose( devices, preferred: preference, excluding: silent.union(refused).union([id])) { silent.insert(id) if let device = devices.first(where: { $0.id == id }), device.suspended { note = "\(Self.off(device)); using \(other.name)" } else { note = "No signal from \(silentName); using \(other.name)" } return open(other.id) } // Nothing to fall back to. Stay on it, say so, and keep counting: an // ATEM whose input comes back starts sending on the same session. status = .noSignal(silentName) watchdog.giveUp() return nil } /// The connected cameras changed. func devicesChanged(_ now: [CameraDevice]) -> CameraCommand? { let before = Set(devices.map(\.id)) let wasSuspended = Set(devices.filter(\.suspended).map(\.id)) let previous = devices devices = now let present = Set(now.map(\.id)) // A camera that left is forgiven: when it comes back it is a new // session, and may be sending again. silent.formIntersection(present) refused.formIntersection(present) let preferred = preference if let id = active, !present.contains(id) { let gone = previous.first(where: { $0.id == id })?.name ?? name(id) if let other = Self.choose(now, preferred: preferred, excluding: silent.union(refused)) { note = "\(gone) disconnected; using \(other.name)" return open(other.id) } active = nil note = "" status = .disconnected(gone) watchdog.reset() return .stop } // The camera in use was suspended (the lid closed on it): fall back // now, while there is somewhere awake to go, rather than wait out the // watchdog's three tries on a camera that cannot send. if let id = active, let device = now.first(where: { $0.id == id }), device.suspended, let other = Self.choose(now, preferred: preferred, excluding: silent.union(refused).union([id])), !other.suspended { note = "\(Self.off(device)); using \(other.name)" return open(other.id) } // The remembered camera is back, has just appeared, or has woken (the // lid opened): go back to it. if let target = preferred?.find(in: now), active != target.id, !target.suspended, !before.contains(target.id) || wasSuspended.contains(target.id) || active == nil { silent.remove(target.id) note = "" return open(target.id) } // Nothing running and something to run on. if active == nil, !now.isEmpty { if case .disconnected = status { note = "" } return pick(missingNote: true) } // "Camera disconnected" stays said until something comes back. if active == nil, now.isEmpty, !status.disconnected { status = .noCamera } return nil } /// The person chose a camera: it is remembered and used. Choosing the /// camera the call is already on — there by fallback, say — is a choice /// like any other; when that camera is working it is only remembered, /// not reopened, so the picture does not blink. func choose(_ id: String) -> CameraCommand? { guard let device = devices.first(where: { $0.id == id }) else { return nil } CameraPreference(device).save(defaults) silent.remove(id) refused.remove(id) note = "" if active == id, status.live { return nil } watchdog.reset() return open(id) } /// "Try again": everything that failed is forgiven and the choice is /// made from scratch. func retry() -> CameraCommand? { silent = [] refused = [] note = "" watchdog.reset() return pick(missingNote: true) } /// A capture session error or interruption, or the Mac waking: restart /// on the same device, no more often than `restartSpacing`. func restart(at now: TimeInterval) -> CameraCommand? { guard let id = active, now - lastRestart >= restartSpacing else { return nil } lastRestart = now return open(id, format: format) } /// The call is over. func stop() { active = nil watchdog.reset() } }