import CoreGraphics import Foundation import LiveKitWebRTC import SwiftUI /// What surrounds a shared screen on the stage (2026-10-05, Paul: "Should /// the video view stay fullscreen even here?"). A screen is fitted whole, /// never cropped, so a window of another shape leaves bars beside or /// above it. Rather than black, they show the same picture enlarged, soft /// and dimmed, as FaceTime does. /// /// Cheap by construction: twice a second, WebRTC scales one frame down to /// `width` × `height` (libyuv, from whatever buffer the decoder gave), it /// is converted and blurred at that size, and the picture is drawn /// enlarged, its smooth upscaling doing the rest of the blur. Nothing runs /// at the frame rate and nothing is blurred at the window's size. public struct ScreenBackdrop: View { let track: RTCVideoTrack @StateObject private var sampler = BackdropSampler() public init(track: RTCVideoTrack) { self.track = track } public var body: some View { ZStack { CallsAppearance.stageSurface if let image = sampler.image { Image(decorative: image, scale: 1) .resizable() .interpolation(.high) .aspectRatio(contentMode: .fill) // Past the edges a little: the blur's own edge is darker. .scaleEffect(1.15) .opacity(BackdropSampler.dimmed) .transition(.opacity) } } .clipped() .animation(.easeInOut(duration: 0.4), value: sampler.image != nil) .onAppear { sampler.attach(track) } .onChange(of: ObjectIdentifier(track)) { sampler.attach(track) } .onDisappear { sampler.detach() } .accessibilityHidden(true) } } /// Takes frames from a track at `interval` and keeps the latest as a small /// blurred picture. @MainActor final class BackdropSampler: ObservableObject { /// The picture's size: 16:9, enough for the colours and their places. nonisolated static let width = 64 nonisolated static let height = 36 /// How often a frame is taken. nonisolated static let interval: TimeInterval = 0.5 /// How much of the picture shows over the dark surface behind it. nonisolated static let dimmed: Double = 0.45 @Published private(set) var image: CGImage? private var track: RTCVideoTrack? private var renderer: Renderer? func attach(_ next: RTCVideoTrack) { guard track !== next else { return } detach() let renderer = Renderer { [weak self] image in Task { @MainActor in self?.image = image } } next.add(renderer) track = next self.renderer = renderer } func detach() { if let renderer { track?.remove(renderer) } track = nil renderer = nil image = nil } /// WebRTC's side: called on its decoding thread for every frame, it /// keeps one in `interval` and lets the rest go at once. final class Renderer: NSObject, RTCVideoRenderer, @unchecked Sendable { private let lock = NSLock() private var last: TimeInterval = 0 private let deliver: (CGImage) -> Void init(deliver: @escaping (CGImage) -> Void) { self.deliver = deliver } func setSize(_ size: CGSize) {} func renderFrame(_ frame: RTCVideoFrame?) { guard let frame else { return } let now = ProcessInfo.processInfo.systemUptime lock.lock() let due = now - last >= BackdropSampler.interval if due { last = now } lock.unlock() guard due, let image = BackdropSampler.picture(of: frame) else { return } deliver(image) } } /// One frame as the backdrop's picture: scaled down by WebRTC, then /// converted and blurred here. nonisolated static func picture(of frame: RTCVideoFrame) -> CGImage? { let source = frame.buffer guard source.width > 0, source.height > 0 else { return nil } let small = source.cropAndScale?( with: 0, offsetY: 0, cropWidth: source.width, cropHeight: source.height, scaleWidth: Int32(width), scaleHeight: Int32(height)) ?? source let planes = small.toI420() var rgba = rgbaFromI420( width: Int(planes.width), height: Int(planes.height), y: planes.dataY, strideY: Int(planes.strideY), u: planes.dataU, strideU: Int(planes.strideU), v: planes.dataV, strideV: Int(planes.strideV), outWidth: width, outHeight: height) boxBlur(&rgba, width: width, height: height, radius: 3, passes: 2) return image(rgba, width: width, height: height) } /// I420 to RGBA (BT.601, video range) at `outWidth` × `outHeight`, /// sampling the nearest source pixel when the sizes differ. Integer /// arithmetic (coefficients × 256) over raw buffers. nonisolated static func rgbaFromI420( width: Int, height: Int, y: UnsafePointer, strideY: Int, u: UnsafePointer, strideU: Int, v: UnsafePointer, strideV: Int, outWidth: Int, outHeight: Int ) -> [UInt8] { var out = [UInt8](repeating: 255, count: outWidth * outHeight * 4) guard width > 0, height > 0 else { return out } out.withUnsafeMutableBufferPointer { buffer in let o = buffer.baseAddress! var i = 0 for row in 0..> 1) * strideU let vRow = v + (sy >> 1) * strideV for col in 0..> 1]) - 128 let e = Int(vRow[sx >> 1]) - 128 o[i] = clamp8((c + 409 * e) >> 8) o[i + 1] = clamp8((c - 100 * d - 208 * e) >> 8) o[i + 2] = clamp8((c + 516 * d) >> 8) i += 4 } } } return out } @inline(__always) nonisolated private static func clamp8(_ value: Int) -> UInt8 { UInt8(truncatingIfNeeded: value < 0 ? 0 : value > 255 ? 255 : value) } /// A separable box blur, `passes` times (two is close to Gaussian), /// with running sums, so each pass is linear in the pixels. nonisolated static func boxBlur( _ pixels: inout [UInt8], width: Int, height: Int, radius: Int, passes: Int ) { guard radius > 0, width > 0, height > 0 else { return } var scratch = pixels pixels.withUnsafeMutableBufferPointer { a in scratch.withUnsafeMutableBufferPointer { b in for _ in 0.., _ out: UnsafeMutablePointer, count: Int, length: Int, step: Int, lineStep: Int, radius: Int ) { for line in 0..= 0 { sum -= Int(source[base + leaving * step + channel]) n -= 1 } if entering < length { sum += Int(source[base + entering * step + channel]) n += 1 } } } for at in 0.. CGImage? { guard let provider = CGDataProvider(data: Data(rgba) as CFData) else { return nil } return CGImage( width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipLast.rawValue), provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent) } } /// Where a picture goes in its tile, top-down: a screen fitted whole and /// centred; a camera filling the tile, cropped evenly from the sides but, /// from top and bottom, more from the bottom (`headroom`), since a face /// is above the middle of the picture and a crop through the forehead /// reads worse than one through the chest. The web's stage and tiles do /// the same with object-position (calls.css). public enum VideoFrame { /// The share of a vertical crop taken from the top: 0.5 would be even. public static let headroom: CGFloat = 0.35 public static func place(video: CGSize, in bounds: CGSize, fit: Bool) -> CGRect { guard video.width > 0, video.height > 0 else { return CGRect(origin: .zero, size: bounds) } let across = bounds.width / video.width let down = bounds.height / video.height let scale = fit ? min(across, down) : max(across, down) let size = CGSize(width: video.width * scale, height: video.height * scale) let spare = bounds.height - size.height // Spare room (a fitted screen) is shared evenly; a crop (negative) // takes headroom of it from the top. let top = spare >= 0 || fit ? spare / 2 : spare * headroom return CGRect(x: (bounds.width - size.width) / 2, y: top, width: size.width, height: size.height) } }