import CoreVideo import Foundation import CallsKit import LiveKitWebRTC /// Two synthetic cameras behind the app's own camera management /// (`ManagedCamera`), so the gate drives the same choice, switching and /// recovery code the apps run — only the frames are made up. /// /// The cameras differ only in frame size, so that a switch is something /// the far side can *see* rather than something this side merely claims. /// There is no camera in CI; what the gate can still hold is the plumbing /// — that a switch is serialised, that the track and its m-line survive /// it, that frames keep arriving across it, and (`stall`) that a camera /// which stops sending is noticed and restarted. @MainActor final class SyntheticCameras: CameraBackend { private var delegate: RTCVideoCapturerDelegate? private var syntheticCapturer: RTCVideoCapturer? private var task: Task? private var chosen = SyntheticCameras.first /// Set by the gate: frames stop until the camera is next opened, as an /// ATEM's do when its input changes format. var stalled = false nonisolated static let first = "synthetic-320" nonisolated static let second = "synthetic-480" private var width: Int { chosen == SyntheticCameras.second ? 480 : 320 } private var height: Int { chosen == SyntheticCameras.second ? 360 : 240 } var devicesChanged: (@MainActor () -> Void)? var trouble: (@MainActor () -> Void)? func requestAccess() async throws {} var devices: [CameraDevice] { [ CameraDevice(id: SyntheticCameras.first, name: "Synthetic 320x240"), CameraDevice(id: SyntheticCameras.second, name: "Synthetic 480x360"), ] } func formatCount(_ id: String) -> Int { 1 } func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws { guard devices.contains(where: { $0.id == id }) else { throw NSError( domain: "Calls", code: 4, userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."]) } chosen = id stalled = false self.delegate = delegate syntheticCapturer = RTCVideoCapturer(delegate: delegate) task?.cancel() task = Task { @MainActor [weak self] in while !Task.isCancelled { self?.frame() try? await Task.sleep(nanoseconds: 66_666_667) } } } func close() async { task?.cancel() task = nil } private func frame() { guard !stalled, let delegate, let syntheticCapturer else { return } var pixel: CVPixelBuffer? CVPixelBufferCreate( kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixel) guard let pixel else { return } CVPixelBufferLockBaseAddress(pixel, []) if let base = CVPixelBufferGetBaseAddress(pixel) { memset( base, Int32(Int(Date().timeIntervalSince1970 * 30) % 200 + 30), CVPixelBufferGetDataSize(pixel)) } CVPixelBufferUnlockBaseAddress(pixel, []) delegate.capturer( syntheticCapturer, didCapture: RTCVideoFrame( buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, timeStampNs: Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000))) } } /// A screen with nothing on it but frames, for the gates: there is no /// display to share in CI and no person to answer a picker. What it can /// still hold is everything after the picker — the source, the sendonly /// transceiver, the `share` message, and both of §5.4's endings, the /// button's and the capture's own (`end`, which is the system's stop /// control as far as the call can tell). @MainActor final class SyntheticScreen: ScreenCapture { private var source: RTCVideoSource? private var capturer: RTCVideoCapturer? private var task: Task? private var ended: (@MainActor () -> Void)? /// A 1920×1080 page of text scrolling at 30 frames a second, for the /// motion gate: what someone scrolling a document looks like to the /// encoder, which is where a screencast source falls behind. private let page: ScrollingPage? init(motion: Bool = false) { page = motion ? ScrollingPage() : nil } func start(source: RTCVideoSource, ended: @escaping @MainActor () -> Void) async throws { self.source = source self.ended = ended let capturer = RTCVideoCapturer(delegate: source) self.capturer = capturer if let page { // Off the main actor and on a timer of its own, as a real capture // is: the gate measures WebRTC, not this process's main thread. page.start(source: source, capturer: capturer) return } task = Task { @MainActor [weak self] in while !Task.isCancelled { self?.frame() try? await Task.sleep(nanoseconds: 100_000_000) } } } /// Frames the moving page has produced, for the gate. var produced: Int { page?.produced ?? 0 } /// A capture that has stalled: no frames, and nothing else either, until /// the watchdog restarts it (`screen-stall`). var stalled = false private var made: UInt64 = 0 private(set) var restarts = 0 var counts: ScreenCaptureCounts { let n = page.map { UInt64($0.produced) } ?? made return ScreenCaptureCounts(heartbeats: n, delivered: n) } var frameSize: (width: Int, height: Int)? { page != nil ? (ScrollingPage.width, ScrollingPage.height) : (640, 360) } func restart() async -> Bool { guard source != nil else { return false } restarts += 1 stalled = false page?.stalled = false return true } func stall() { stalled = true page?.stalled = true } func stop() { page?.stop() task?.cancel() task = nil source = nil capturer = nil ended = nil } /// The capture ending on its own, as a shared window closing would. func end() { let ended = self.ended stop() ended?() } private func frame() { guard let source, let capturer, !stalled else { return } made += 1 // A 16:9 screen, wider than either synthetic camera, so the far side // can tell by size alone which one it is decoding. var pixel: CVPixelBuffer? CVPixelBufferCreate( kCFAllocatorDefault, 640, 360, kCVPixelFormatType_32BGRA, [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixel) guard let pixel else { return } CVPixelBufferLockBaseAddress(pixel, []) if let base = CVPixelBufferGetBaseAddress(pixel) { memset( base, Int32(Int(Date().timeIntervalSince1970 * 10) % 100 + 120), CVPixelBufferGetDataSize(pixel)) } CVPixelBufferUnlockBaseAddress(pixel, []) source.capturer( capturer, didCapture: RTCVideoFrame( buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, timeStampNs: Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000))) } } /// Lines of dark "glyphs" on white, twice a screen tall, shown through a /// 1920×1080 window that moves down 12 rows a frame and wraps: a fast /// scroll through a dense document, at 30 frames a second, with a window /// dragged across it. Built once; /// each frame is row copies into a pooled buffer, on a queue of its own. final class ScrollingPage: @unchecked Sendable { static let width = 1920 /// 1080 unless `CALLS_SYNTHETIC_SCREEN_HEIGHT` says otherwise: 1246 is a /// 16:10 Retina display as the Mac's capture fits it to 1920 wide, the /// size that meets H.264 level 3.1's block limit (spec §15.21, §15.22). static let height = Int(ProcessInfo.processInfo.environment["CALLS_SYNTHETIC_SCREEN_HEIGHT"] ?? "") ?? 1080 private let rows = ScrollingPage.height * 2 private let texture: [UInt8] private var offset = 0 private var frame = 0 private let queue = DispatchQueue(label: "calls.synthetic-screen", qos: .userInteractive) private var timer: DispatchSourceTimer? private var pool: CVPixelBufferPool? private let lock = NSLock() private var count = 0 var produced: Int { lock.withLock { count } } private var stalledFlag = false /// Set by the gate's `screen-stall`: no frames until cleared. var stalled: Bool { get { lock.withLock { stalledFlag } } set { lock.withLock { stalledFlag = newValue } } } init() { let stride = Self.width * 4 var texture = [UInt8](repeating: 0xF4, count: stride * rows) var seed: UInt64 = 0x9E37_79B9_7F4A_7C15 func random(_ n: Int) -> Int { seed = seed &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 return Int((seed >> 33) % UInt64(n)) } let rows = self.rows texture.withUnsafeMutableBytes { raw in guard let base = raw.baseAddress else { return } var line = 8 while line + 16 < rows { var x = 40 + random(40) let end = Self.width - 40 - random(600) while x < end { let glyph = 5 + random(9) let tall = 9 + random(6) let shade = Int32(20 + random(60)) for y in (line + 14 - tall)..<(line + 14) { memset(base + y * stride + x * 4, shade, min(glyph, end - x) * 4) } x += glyph + 2 + (random(6) == 0 ? 8 : 0) } line += 22 } } self.texture = texture CVPixelBufferPoolCreate( kCFAllocatorDefault, nil, [ kCVPixelBufferWidthKey: Self.width, kCVPixelBufferHeightKey: Self.height, kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, kCVPixelBufferIOSurfacePropertiesKey: [:], ] as CFDictionary, &pool) } func start(source: RTCVideoSource, capturer: RTCVideoCapturer) { // Strict: a lenient timer on a background process is coalesced to a // third of this rate, which measured the timer and not WebRTC. let timer = DispatchSource.makeTimerSource(flags: .strict, queue: queue) timer.schedule(deadline: .now(), repeating: .nanoseconds(33_333_333), leeway: .milliseconds(2)) timer.setEventHandler { [weak self] in guard let self, !self.stalled, let pixel = self.next() else { return } source.capturer( capturer, didCapture: RTCVideoFrame( buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, timeStampNs: Int64(DispatchTime.now().uptimeNanoseconds))) self.lock.withLock { self.count += 1 } } self.timer = timer timer.resume() } func stop() { timer?.cancel() timer = nil } private func bounce(_ travelled: Int, _ span: Int) -> Int { let lap = travelled % (2 * span) return lap < span ? lap : 2 * span - lap } private func next() -> CVPixelBuffer? { guard let pool else { return nil } var pixel: CVPixelBuffer? CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixel) guard let pixel else { return nil } CVPixelBufferLockBaseAddress(pixel, []) defer { CVPixelBufferUnlockBaseAddress(pixel, []) } guard let base = CVPixelBufferGetBaseAddress(pixel) else { return nil } let destination = CVPixelBufferGetBytesPerRow(pixel) let stride = Self.width * 4 let first = offset let rows = self.rows frame += 1 // A window being dragged across the page — 800×450, moving 10 rows // and 14 columns a frame and bouncing off the edges — whose content // changes every half second, as a page loading or a slide advancing // would. Together with the scroll beneath it, what someone moving // things around a busy screen sends the encoder. let span = (x: Self.width - 800, y: Self.height - 450) let wx = bounce(frame * 14, span.x) let wy = bounce(frame * 10, span.y) let scene = frame / 15 let jump = (scene &* 7919) % (rows - 450) let shift = ((scene &* 104_729) % 900) * 4 texture.withUnsafeBytes { raw in guard let source = raw.baseAddress else { return } for y in 0..= wy, y < wy + 450 { memcpy(base + y * destination + wx * 4, source + (jump + y - wy) * stride + shift, 800 * 4) } } } offset = (offset + 12) % rows return pixel } }