import Foundation import os // The screen watchdogs: the sharer's, which notices its own screen // stopping on the way out and puts it right, and the receiver's, which // notices someone else's screen stopping on the way in and says so. // // Written for Paul's report of 2026-09-28 (Mac 0.4.1): "When I was // screen-sharing, after a while, my screen froze on her end and she was // no longer able to see my screen. I had to stop and re-share again." A // stop and a share again is a fresh capture, a fresh transceiver, a fresh // encoder and a keyframe; the sharer's watchdog does each of those in // turn, cheapest first, without anyone noticing or pressing anything. /// `log show --last 1h --predicate 'subsystem == "net.amadan.oficina.calls" && category == "screen"' --info`. /// Every line is also kept in memory (the last 200) for "Copy call /// diagnostics". enum ScreenLog { static let logger = Logger(subsystem: "net.amadan.oficina.calls", category: "screen") static func info(_ message: String) { logger.info("\(message, privacy: .public)") AudioDiagnosticsStore.screen.log("info", message) } static func error(_ message: String) { logger.error("\(message, privacy: .public)") AudioDiagnosticsStore.screen.log("error", message) } } extension AudioDiagnosticsStore { /// The screen log's lines, kept apart from the audio log's so neither /// pushes the other out. public static let screen = AudioDiagnosticsStore() } /// One connection's screen on the way out, as the watchdog reads it. struct ScreenSendCounts: Equatable, Sendable { var encoded: UInt64 = 0 var sent: UInt64 = 0 var keyframes: UInt64 = 0 /// WebRTC says the link is what is holding the screen back. Frames stop /// being encoded for a while when the estimate collapses, and that is /// the link's to put right, not the encoder's: the watchdog leaves it /// alone (a receiver whose picture stops still reports it). var bandwidthLimited = false } /// What the sharer's watchdog asks for. Each is cheaper than the next. enum ScreenRecovery: Equatable, Sendable { /// A fresh capture of the same content into the same source. case restartCapture /// A keyframe on this connection's screen: the encoding switched off /// and on again, which WebRTC answers by starting the stream on one. case keyframe(String) /// A fresh encoder on this connection: the screen's size nudged, which /// makes WebRTC set the hardware encoder up again from nothing. case recreateEncoder(String) /// What stopping and sharing again did: this connection's screen /// transceiver stopped and a new one added, renegotiated. case reAdd(String) /// A receiver's report that keyframes did not answer: this connection's /// screen sending path built again — a new transceiver, so a new encoder /// and, at the far end, a new receiver and decoder — at the same size /// and mode. What stopping and sharing again does, for that one person. case rebuild(String) /// Still reported after that: every connection's screen stopped and /// added again from the same capture (no picker), which is exactly what /// Stop and Share did by hand. case reshare } /// Where the answer to one receiver's report has got to (the sharer's /// notice says it in words: see ScreenWords.notice). enum ScreenEscalation: Equatable, Sendable { case none /// Keyframes asked for since this time. case keyframes(since: Double) /// That peer's screen rebuilt at this time. case rebuilt(at: Double) /// Everyone's screen shared again at this time. case reshared(at: Double) /// Nothing more to try: the person is told to stop and share again. case failed } /// The sharer's watchdog: pure, fed once a second while sharing. /// /// Three things can freeze a screen on its way out, and each has its own /// sign: /// /// - **The capture stops.** ScreenCaptureKit reports even a still screen /// (as "idle"), so a capture that reports nothing for three seconds has /// stopped — a capture with every surface held by the encoder does /// exactly that. Asked for: a fresh capture, with a growing wait /// between tries. /// - **The encoder stops.** Frames go into the source and none come out /// encoded (or none are sent) for five seconds, on one connection, while /// the link is not what is holding it back. /// Asked for, three seconds apart while it lasts: a keyframe, a fresh /// encoder, then the transceiver itself; then a pause of thirty seconds /// before starting again. /// - **A receiver says so.** Someone's `screenStalled` names this peer /// (spec §15.21). A keyframe for that connection, four seconds apart /// (the report is a few seconds old when it lands), three times and then /// a rest — and the fuller steps only if the screen has stopped leaving /// here too. /// - **A receiver keeps saying so** (spec §15.22, Paul 2026-09-30: "still /// seeing issues with screen sharing after a while", the notice sat at /// "recovering"). Keyframes cannot mend a decoder that has wedged at the /// far end, or a receiver that has stopped taking this stream at all; /// a new transceiver can. Still reported six seconds after the first /// keyframe: that connection's screen is rebuilt, at the same size and /// mode (a report alone never shrinks a Text and detail share). Still /// reported ten seconds after that: everyone's screen is shared again /// from the same capture. Ten seconds more and the sharer is told it /// could not be restored. A rebuild at most once a minute per person, /// a share again at most once every five minutes. struct ScreenWatchdog: Equatable { static let captureStallAfter = 3.0 static let captureBackoffFirst = 5.0 static let captureBackoffMost = 60.0 /// Five seconds, not three: a link that has just been capped can hold a /// large keyframe back for that long in Text and detail, and a forced /// keyframe there only makes it longer (the motion gate, 2026-09-28). static let encoderStallAfter = 5.0 static let encoderStepEvery = 3.0 static let reportStepEvery = 4.0 static let holdOff = 30.0 /// No encoder verdict in a connection's first ten seconds: a screen /// kept at full size (Text and detail) on a link still ramping up from /// WebRTC's 300 kbit/s start encodes almost nothing for several seconds, /// and that is not a stall (the motion gate, 2026-09-28). A receiver's /// report is still answered then. static let warmUp = 10.0 /// §15.22's escalation: how long a report may outlast the first /// keyframe, then the rebuild, then the share again; and how often each /// of the last two may happen. static let rebuildAfter = 6.0 static let reshareAfter = 10.0 static let giveUpAfter = 10.0 static let rebuildEvery = 60.0 static let reshareEvery = 300.0 struct Peer: Equatable { var last = ScreenSendCounts() var movedAt: Double /// Delivered frames when `movedAt` was last set: a connection is only /// stalled if frames went in meanwhile. var deliveredAt: UInt64 var step = 0 var stepAt = 0.0 var reported = false var restingUntil = 0.0 var since = 0.0 var escalation = ScreenEscalation.none var lastRebuild: Double? /// Frames sent on this connection since the escalation last moved on: /// a receiver can only be failing to decode frames that were sent. A /// screen the link is holding back, or one with nothing new to send, /// is not taken further. var sentSinceStage: UInt64 = 0 } /// Frames that must leave for a receiver before its report is taken /// further than keyframes. static let escalateAfterSent: UInt64 = 3 private(set) var peers: [String: Peer] = [:] private var heartbeats: UInt64? private var heartbeatAt = 0.0 private var delivered: UInt64 = 0 private var captureRetryAt = 0.0 private var captureBackoff = ScreenWatchdog.captureBackoffFirst private(set) var captureStalled = false // For the readout and the gate. private(set) var captureRestarts = 0 private(set) var keyframes = 0 private(set) var encoderRecoveries = 0 private(set) var reAdds = 0 private(set) var stallReports = 0 private(set) var rebuilds = 0 private(set) var reshares = 0 private(set) var failures = 0 private var lastReshare: Double? /// One line on what the watchdog is doing, for the device panel. private(set) var status = "watching" init() {} /// Forgets everything: a share starting, or a restart that should be /// given a clean look. mutating func reset() { self = ScreenWatchdog() } /// One look. `capture` is the capture's running totals; `sending` each /// connection's screen on the way out, by peer id, for the connections /// that have one negotiated; `stalledBy` the peers whose latest report /// says our screen has stalled for them. Returns what to do now. mutating func observe( at now: Double, capture: ScreenCaptureCounts, sending: [String: ScreenSendCounts], stalledBy: Set ) -> [ScreenRecovery] { var out: [ScreenRecovery] = [] // The capture. let fresh = heartbeats == nil if let beats = capture.heartbeats { if fresh || beats != heartbeats { heartbeats = beats heartbeatAt = now if captureStalled { captureStalled = false captureBackoff = Self.captureBackoffFirst ScreenLog.info("capture reporting again") } } else if now - heartbeatAt >= Self.captureStallAfter { if !captureStalled { captureStalled = true ScreenLog.info("capture stalled: nothing reported for \(Int(now - heartbeatAt)) s") } if now >= captureRetryAt { out.append(.restartCapture) captureRestarts += 1 captureRetryAt = now + captureBackoff captureBackoff = min(captureBackoff * 2, Self.captureBackoffMost) } } } delivered = capture.delivered // Each connection. // A connection that has no screen negotiated for the moment — one the // watchdog has just re-added, above all — keeps its record, rest and // all, so the steps do not start again from nothing on the new one. for (id, counts) in sending { var peer = peers[id] ?? Peer(last: counts, movedAt: now, deliveredAt: delivered, since: now) peer.sentSinceStage += counts.sent >= peer.last.sent ? counts.sent - peer.last.sent : counts.sent if counts.encoded < peer.last.encoded || counts.sent < peer.last.sent { // A new transceiver's counters, starting again from nothing. peer.last = counts peer.movedAt = now peer.deliveredAt = delivered } let moved = counts.encoded > peer.last.encoded && counts.sent > peer.last.sent let wasReported = peer.reported let reported = stalledBy.contains(id) if reported && !wasReported { stallReports += 1 } peer.reported = reported peer.last = counts if moved && !reported { if peer.step > 0 { ScreenLog.info("screen to \(Self.short(id)) moving again after step \(peer.step)") peer.step = 0 } peer.movedAt = now peer.deliveredAt = delivered } else if moved { peer.movedAt = now peer.deliveredAt = delivered } let starved = !moved && !counts.bandwidthLimited && now - peer.since >= Self.warmUp && now - peer.movedAt >= Self.encoderStallAfter && delivered > peer.deliveredAt if (starved || reported) && now >= peer.restingUntil { let every = reported && !starved ? Self.reportStepEvery : Self.encoderStepEvery if peer.step == 0 || now - peer.stepAt >= every { let why = starved ? "not encoding or sending" : "reported stalled by the receiver" // A receiver's report while this side's screen is leaving // normally is not the encoder's fault — a slow link, a lost // packet — and a keyframe is all it can use: a smaller or a new // encoder would only take the picture down for nothing (Text and // detail after a thin link, the motion gate, 2026-09-28). Only a // screen that has stopped here as well goes further. switch starved ? peer.step : 0 { case 0: out.append(.keyframe(id)) keyframes += 1 case 1: out.append(.recreateEncoder(id)) encoderRecoveries += 1 default: out.append(.reAdd(id)) reAdds += 1 } peer.step += 1 peer.stepAt = now ScreenLog.info("screen to \(Self.short(id)) \(why): step \(peer.step)") if peer.step >= 3 { peer.step = 0 peer.restingUntil = now + Self.holdOff } } } else if !starved && !reported && peer.step > 0 && now - peer.stepAt >= Self.encoderStepEvery { peer.step = 0 } peers[id] = peer } out += escalate(at: now, stalledBy: stalledBy, negotiated: Set(sending.keys)) status = describe() return out } /// §15.22: a report that keyframes have not answered, taken further. Run /// over every peer with a record, whether or not its screen is /// negotiated this second — a rebuilt connection has none for a moment, /// and a rebuild that never comes back must still end in words. private mutating func escalate( at now: Double, stalledBy: Set, negotiated: Set ) -> [ScreenRecovery] { var out: [ScreenRecovery] = [] var reshare = false for id in peers.keys.sorted() { guard var peer = peers[id] else { continue } let reported = stalledBy.contains(id) // Frames went to this receiver since the last step, or there is no // screen negotiated with it at all (a rebuild that never came back). let owed = peer.sentSinceStage >= Self.escalateAfterSent || !negotiated.contains(id) let before = peer.escalation defer { if peers[id]?.escalation != before { peers[id]?.sentSinceStage = 0 } } if !reported { if peer.escalation != .none { switch peer.escalation { case .keyframes: ScreenLog.info("\(Self.short(id))'s view restored after keyframes") case .rebuilt: ScreenLog.info("\(Self.short(id))'s view restored after the rebuild") case .reshared: ScreenLog.info("\(Self.short(id))'s view restored after sharing again") case .failed: ScreenLog.info("\(Self.short(id))'s view came back after all") case .none: break } peer.escalation = .none } peers[id] = peer continue } let canRebuild = peer.lastRebuild.map { now - $0 >= Self.rebuildEvery } ?? true let canReshare = reshare || (lastReshare.map { now - $0 >= Self.reshareEvery } ?? true) switch peer.escalation { case .none: peer.escalation = .keyframes(since: now) case .keyframes(let since) where now - since >= Self.rebuildAfter && owed: if canRebuild { out.append(.rebuild(id)) rebuilds += 1 peer.lastRebuild = now peer.escalation = .rebuilt(at: now) ScreenLog.info( "\(Self.short(id)) still reports a stall \(Int(now - since)) s after keyframes: rebuilding its screen") } else if canReshare { reshare = true peer.escalation = .reshared(at: now) ScreenLog.info("\(Self.short(id)) still reports a stall, rebuilt under a minute ago: sharing again") } else { peer.escalation = .failed failures += 1 ScreenLog.error("\(Self.short(id)) still reports a stall and every step is resting: giving up") } case .rebuilt(let at) where now - at >= Self.reshareAfter && owed: if canReshare { reshare = true peer.escalation = .reshared(at: now) ScreenLog.info("\(Self.short(id)) still reports a stall \(Int(now - at)) s after the rebuild: sharing again") } else { peer.escalation = .failed failures += 1 ScreenLog.error("\(Self.short(id)) still reports a stall after the rebuild, shared again under five minutes ago: giving up") } case .reshared(let at) where now - at >= Self.giveUpAfter && owed: peer.escalation = .failed failures += 1 ScreenLog.error("\(Self.short(id)) still reports a stall \(Int(now - at)) s after sharing again: giving up") default: break } peers[id] = peer } if reshare { out.append(.reshare) reshares += 1 lastReshare = now // One share again answers every report in flight. for (id, peer) in peers where stalledBy.contains(id) { switch peer.escalation { case .keyframes, .rebuilt: peers[id]?.escalation = .reshared(at: now) peers[id]?.sentSinceStage = 0 default: break } } } return out } /// Where the answer to each reporting peer has got to, for the notice. func escalation(of peer: String) -> ScreenEscalation { peers[peer]?.escalation ?? .none } /// Whether anyone's view of the screen is being recovered right now. var recovering: Bool { captureStalled || peers.values.contains { $0.step > 0 || $0.reported } } private func describe() -> String { if captureStalled { return "capture stalled, restarting (\(captureRestarts))" } let stages = peers.values.map(\.escalation) if stages.contains(.failed) { return "a receiver's view could not be restored" } if stages.contains(where: { if case .reshared = $0 { return true } else { return false } }) { return "shared again for everyone" } if stages.contains(where: { if case .rebuilt = $0 { return true } else { return false } }) { return "rebuilt for a receiver" } if let (_, p) = peers.first(where: { $0.value.step > 0 }) { return ["", "keyframe asked for", "encoder recreated", "re-added"][min(p.step, 3)] } if peers.values.contains(where: \.reported) { return "a receiver reports a stall" } let total = captureRestarts + keyframes + encoderRecoveries + reAdds + rebuilds + reshares return total == 0 ? "watching" : "watching, \(total) recover\(total == 1 ? "y" : "ies") so far" } static func short(_ peer: String) -> String { String(peer.prefix(6)) } } /// The receiver's watchdog: whose screens have stopped moving here. /// /// A peer's screen is stalled when nothing new has been decoded from it /// for four seconds while it says it is sharing. Every sender repeats a /// still screen at least once a second (the iPhone's broadcast too, since /// 2026-09-30), so four seconds without a frame is not a still screen. The /// list is reported as `screenStalled` (spec §15.21) and the sharer /// recovers; it clears as soon as frames move. /// /// Any change in the count is movement, a smaller one included: a screen /// the sharer has rebuilt arrives on a new receiver whose count starts /// again from nothing (2026-09-30 — before this, a rebuilt screen that was /// moving again stayed "stalled" here until its new count passed the old /// one, and the sharer's notice with it). /// /// Four seconds into a stall the tile's picture is also dropped and /// attached again, once (spec §15.22): a renderer that has stopped, on /// this side, is this side's to mend. A report is held for at most three /// minutes from the last frame, long enough for the sharer to try /// everything and say so, and no longer. struct ScreenReceiveWatch: Equatable { static let stallAfter = 4.0 static let reattachAfter = 4.0 static let giveUpAfter = 180.0 struct Seen: Equatable { var decoded: UInt64 var movedAt: Double var reattached = false } private(set) var seen: [String: Seen] = [:] /// The peers whose picture should be attached again now: set by each /// look, and each stall's only once. private(set) var reattach: [String] = [] init() {} /// `decoded` is frames decoded so far from each peer's screen, for the /// peers that have one arriving. Returns the stalled peers, sorted. mutating func observe(at now: Double, decoded: [String: UInt64]) -> [String] { for id in seen.keys where decoded[id] == nil { seen.removeValue(forKey: id) } var stalled: [String] = [] reattach = [] for (id, count) in decoded { guard var s = seen[id], count == s.decoded else { seen[id] = Seen(decoded: count, movedAt: now) continue } let quiet = now - s.movedAt if quiet >= Self.stallAfter && quiet < Self.giveUpAfter { stalled.append(id) } if quiet >= Self.stallAfter + Self.reattachAfter && !s.reattached && quiet < Self.giveUpAfter { s.reattached = true reattach.append(id) } seen[id] = s } reattach.sort() return stalled.sorted() } /// How long each stalled peer's screen has gone without a frame. func quiet(at now: Double) -> [String: Double] { seen.mapValues { now - $0.movedAt } } } /// The words for the sharer's quiet notice. public enum ScreenWords { /// "Vicky’s view of your screen stalled — recovering", or nothing when /// nobody's has. public static func recovering(names: [String]) -> String { switch names.count { case 0: return "" case 1: return "\(names[0])\u{2019}s view of your screen stalled \u{2014} recovering" case 2: return "\(names[0])\u{2019}s and \(names[1])\u{2019}s views of your screen stalled \u{2014} recovering" default: return "\(names.count) people\u{2019}s views of your screen stalled \u{2014} recovering" } } /// The notice as the escalation moves on (spec §15.22): "recovering", /// then "Restarting your share for Samuel…", then either nothing or /// "Couldn’t restore Samuel’s view — try Stop and Share again". Worst /// news first when several people are at different steps. public static func notice(recovering: [String], restarting: [String], failed: [String]) -> String { var parts: [String] = [] if !failed.isEmpty { let whose: String switch failed.count { case 1: whose = "\(failed[0])\u{2019}s view" case 2: whose = "\(failed[0])\u{2019}s and \(failed[1])\u{2019}s views" default: whose = "\(failed.count) people\u{2019}s views" } parts.append("Couldn\u{2019}t restore \(whose) \u{2014} try Stop and Share again") } if !restarting.isEmpty { let who: String switch restarting.count { case 1: who = restarting[0] case 2: who = "\(restarting[0]) and \(restarting[1])" default: who = "\(restarting.count) people" } parts.append("Restarting your share for \(who)\u{2026}") } let stalled = Self.recovering(names: recovering) if !stalled.isEmpty { parts.append(stalled) } return parts.joined(separator: ". ") } } /// One peer's screen as it arrives here, from the receiver's statistics /// (spec §15.22): enough to tell the ways a screen stops apart. Packets /// arriving and frames assembled but none decoded is a decoder that has /// wedged; nothing arriving at all is the sender, or the path; frames /// decoded and a picture that does not move is the renderer. struct ScreenReceiveReport: Equatable, Sendable { var packetsReceived: UInt64 = 0 var packetsLost: Int64 = 0 var bytesReceived: UInt64 = 0 var framesReceived: UInt64 = 0 var framesDecoded: UInt64 = 0 var framesDropped: UInt64 = 0 var keyFramesDecoded: UInt64 = 0 var pliCount: UInt64 = 0 var firCount: UInt64 = 0 var nackCount: UInt64 = 0 var freezeCount: UInt64 = 0 var width = 0 var height = 0 var decoder = "" /// When the last packet arrived, in the statistics' own milliseconds. var lastPacketAt = 0.0 var mid = "" init() {} /// One inbound-rtp report's values added in (a screen has one). mutating func add(_ v: [String: NSObject]) { func n(_ key: String) -> UInt64 { (v[key] as? NSNumber)?.uint64Value ?? 0 } packetsReceived += n("packetsReceived") packetsLost += (v["packetsLost"] as? NSNumber)?.int64Value ?? 0 bytesReceived += n("bytesReceived") framesReceived += n("framesReceived") framesDecoded += n("framesDecoded") framesDropped += n("framesDropped") keyFramesDecoded += n("keyFramesDecoded") pliCount += n("pliCount") firCount += n("firCount") nackCount += n("nackCount") freezeCount += n("freezeCount") width = max(width, Int(n("frameWidth"))) height = max(height, Int(n("frameHeight"))) if let name = v["decoderImplementation"] as? String, !name.isEmpty { decoder = name } lastPacketAt = max(lastPacketAt, (v["lastPacketReceivedTimestamp"] as? NSNumber)?.doubleValue ?? 0) } /// What changed between two looks, in words: "packets +120, frames /// received +30, decoded +0, dropped +0, keyframes +0, PLI +2". func moved(since before: ScreenReceiveReport?) -> String { guard let b = before, b.mid == mid else { return summary } func d(_ a: UInt64, _ z: UInt64) -> String { a >= z ? "+\(a - z)" : "new \(a)" } return "packets \(d(packetsReceived, b.packetsReceived)), frames received \(d(framesReceived, b.framesReceived)), " + "decoded \(d(framesDecoded, b.framesDecoded)), dropped \(d(framesDropped, b.framesDropped)), " + "keyframes \(d(keyFramesDecoded, b.keyFramesDecoded)), PLI \(d(pliCount, b.pliCount)), " + "NACK \(d(nackCount, b.nackCount))" } /// The running totals, one line. var summary: String { "mid \(mid.isEmpty ? "-" : mid), \(width)x\(height), decoder \(decoder.isEmpty ? "unknown" : decoder); " + "packets \(packetsReceived) (lost \(packetsLost)), frames received \(framesReceived), " + "decoded \(framesDecoded), dropped \(framesDropped), keyframes \(keyFramesDecoded), " + "PLI \(pliCount), FIR \(firCount), NACK \(nackCount), freezes \(freezeCount)" } /// The verdict the numbers point to, for a screen that has stopped. func diagnosis(since before: ScreenReceiveReport?) -> String { guard let b = before, b.mid == mid else { return "no earlier reading" } if packetsReceived == b.packetsReceived { return "no packets arriving (the sender or the path)" } if framesReceived == b.framesReceived { return "packets arriving, no frames assembled (loss, or waiting on a keyframe)" } if framesDecoded == b.framesDecoded { return "frames arriving, none decoded (the decoder)" } return "decoding" } }