import XCTest @testable @_spi(Fixtures) import CallsKit /// Spec §15.26: the quality report built from fixture statistics — counts /// over the window, rates, the schema's enums and bounds — and nothing in /// it that names a person, a device or an address; the notices tallied; /// the device kinds; "Report a problem"'s anonymous diagnostics and its /// sheet's states. final class QualityReportTests: XCTestCase { // MARK: - Fixtures static let client = QualityClient(app: "mac", version: "0.4.8", os: "macos", osVersion: "26.0.1") /// One connection's statistics, as WebRTC's report has them, with the /// things that must never be reported planted in it: an address, a /// port, a name in a track's label, a device's name. static func sample( generation: Int64 = 1, scale: Double = 1, screen: Bool = false, lostOut: Double = 3, relayTLS: Bool = true, nan: Bool = false ) -> QualityPeerSample { var stats: [String: QualityStat] = [ "OA": QualityStat( type: "outbound-rtp", numbers: ["bytesSent": 120_000 * scale, "packetsSent": 1500 * scale], strings: ["kind": "audio", "mid": "0", "trackIdentifier": "Vicky Wallace's microphone"]), "RIA": QualityStat( type: "remote-inbound-rtp", numbers: ["packetsLost": lostOut * scale, "roundTripTime": 0.042, "jitter": 0.004], strings: ["kind": "audio", "localId": "OA"]), "OV": QualityStat( type: "outbound-rtp", numbers: [ "bytesSent": 1_500_000 * scale, "packetsSent": 1800 * scale, "frameWidth": 1280, "frameHeight": 720, "framesPerSecond": nan ? .nan : 29.97, ], strings: ["kind": "video", "mid": "1", "qualityLimitationReason": "bandwidth"]), "IA": QualityStat( type: "inbound-rtp", numbers: [ "bytesReceived": 110_000 * scale, "packetsReceived": 1490 * scale, "packetsLost": 10 * scale, "jitter": 0.012, "totalSamplesReceived": 1_440_000 * scale, "concealedSamples": 4800 * scale, "insertedSamplesForDeceleration": 960 * scale, "removedSamplesForAcceleration": 480 * scale, "jitterBufferDelay": 72_000 * 0.06 * scale, "jitterBufferEmittedCount": 72_000 * scale, ], strings: ["kind": "audio", "mid": "0"]), "IV": QualityStat( type: "inbound-rtp", numbers: [ "bytesReceived": 1_200_000 * scale, "packetsReceived": 1700 * scale, "packetsLost": 5 * scale, "jitter": 0.02, "freezeCount": 1 * scale, "totalFreezesDuration": 0.5 * scale, "framesDecoded": 890 * scale, "framesDropped": 2 * scale, "frameWidth": 640, "frameHeight": 480, "framesPerSecond": 30, ], strings: ["kind": "video", "mid": "1"]), "T": QualityStat(type: "transport", strings: ["selectedCandidatePairId": "CP"]), "CP": QualityStat( type: "candidate-pair", numbers: ["currentRoundTripTime": 0.038, "nominated": 1], strings: ["localCandidateId": "LC", "remoteCandidateId": "RC", "state": "succeeded"]), "LC": QualityStat( type: "local-candidate", numbers: ["port": 54321], strings: [ "candidateType": relayTLS ? "relay" : "host", "protocol": "udp", "relayProtocol": "tls", "address": "192.168.1.20", "url": "turns:relay.example.com:443", ]), "RC": QualityStat( type: "remote-candidate", numbers: ["port": 61000], strings: ["candidateType": "srflx", "protocol": "udp", "address": "203.0.113.7"]), "MS": QualityStat(type: "media-source", strings: ["kind": "audio", "deviceName": "Paul's AirPods Pro"]), ] if screen { stats["OS"] = QualityStat( type: "outbound-rtp", numbers: [ "bytesSent": 3_000_000 * scale, "packetsSent": 3000 * scale, "frameWidth": 1920, "frameHeight": 1080, "framesPerSecond": 8, ], strings: ["kind": "video", "mid": "3", "qualityLimitationReason": "cpu"]) stats["IS"] = QualityStat( type: "inbound-rtp", numbers: ["bytesReceived": 900_000 * scale, "framesDecoded": 300 * scale, "freezeCount": 2 * scale], strings: ["kind": "video", "mid": "4"]) } return QualityPeerSample( generation: generation, ownScreenMid: screen ? "3" : nil, screenMid: screen ? "4" : "", stats: stats) } func build( _ builder: inout QualityReportBuilder, at now: Double, _ samples: [String: QualityPeerSample], tally: QualityEventTally = QualityEventTally(), final: Bool = false ) -> QualityReport { var tally = tally return builder.build( at: now, samples: samples, events: &tally, client: Self.client, devices: QualityDevices(input: "bluetooth", output: "built-in"), final: final) } // MARK: - Counts over the window func testCountsAreTheWindowsAndRatesAreOverIt() throws { var builder = QualityReportBuilder() builder.start(at: 100) _ = build(&builder, at: 130, ["abc123def": Self.sample(scale: 1)]) // Thirty seconds later every counter has doubled: the window's counts // are the second half alone. let report = build(&builder, at: 160, ["abc123def": Self.sample(scale: 2)]) XCTAssertEqual(report.seconds, 30) XCTAssertNil(report.isFinal) let peer = try XCTUnwrap(report.peers?.first) XCTAssertEqual(peer.peer, "abc123def") let outAudio = try XCTUnwrap(peer.out?.audio) XCTAssertEqual(outAudio.packets, 1500) XCTAssertEqual(outAudio.kbps, 32, "120,000 bytes in 30 s is 32 kbit/s") XCTAssertEqual(outAudio.lost, 3) XCTAssertEqual(outAudio.rtt, 42) XCTAssertEqual(outAudio.jitter, 4) let outVideo = try XCTUnwrap(peer.out?.video) XCTAssertEqual(outVideo.kbps, 400) XCTAssertEqual(outVideo.limit, "bandwidth") XCTAssertEqual(outVideo.width, 1280) XCTAssertEqual(outVideo.fps, 30, "rounded to a tenth") let inAudio = try XCTUnwrap(peer.in?.audio) XCTAssertEqual(inAudio.packets, 1490) XCTAssertEqual(inAudio.lost, 10) XCTAssertEqual(inAudio.jitter, 12) XCTAssertEqual(inAudio.samples, 1_440_000) XCTAssertEqual(inAudio.concealed, 4800) XCTAssertEqual(inAudio.inserted, 960) XCTAssertEqual(inAudio.removed, 480) XCTAssertEqual(inAudio.jitterBufferMs, 60) let inVideo = try XCTUnwrap(peer.in?.video) XCTAssertEqual(inVideo.freezes, 1) XCTAssertEqual(inVideo.freezeSeconds, 0.5) XCTAssertEqual(inVideo.decoded, 890) XCTAssertEqual(inVideo.dropped, 2) XCTAssertNil(peer.in?.screen) XCTAssertNil(peer.out?.screen) XCTAssertEqual(peer.path, QualityPath(local: "relay", remote: "srflx", protocol: "tls")) XCTAssertEqual(peer.rtt, 38) } func testTheFirstWindowOfAConnectionCountsFromItsStart() throws { var builder = QualityReportBuilder() builder.start(at: 0) let report = build(&builder, at: 30, ["p1": Self.sample(scale: 1)]) XCTAssertEqual(report.peers?.first?.out?.audio?.packets, 1500) } func testANewGenerationOrACounterGoingBackStartsAgain() throws { var builder = QualityReportBuilder() builder.start(at: 0) _ = build(&builder, at: 30, ["p1": Self.sample(generation: 1, scale: 4)]) // A new connection: its counts are its own, not less than the old one's. var report = build(&builder, at: 60, ["p1": Self.sample(generation: 2, scale: 1)]) XCTAssertEqual(report.peers?.first?.in?.audio?.packets, 1490) // The same connection, a counter gone backwards (a receiver rebuilt). report = build(&builder, at: 90, ["p1": Self.sample(generation: 2, scale: 0.5)]) XCTAssertEqual(report.peers?.first?.in?.audio?.packets, 745) } func testScreensAreToldApartByMid() throws { var builder = QualityReportBuilder() builder.start(at: 0) let report = build(&builder, at: 30, ["p1": Self.sample(screen: true)]) let peer = try XCTUnwrap(report.peers?.first) XCTAssertEqual(peer.out?.screen?.width, 1920) XCTAssertEqual(peer.out?.screen?.limit, "cpu") XCTAssertEqual(peer.out?.video?.width, 1280, "the camera is not the screen") XCTAssertEqual(peer.in?.screen?.decoded, 300) XCTAssertEqual(peer.in?.screen?.freezes, 2) XCTAssertEqual(peer.in?.video?.decoded, 890) } func testBoundsNaNAndNegativesAndThePeerCeiling() throws { var builder = QualityReportBuilder() builder.start(at: 0) var samples: [String: QualityPeerSample] = [:] for i in 0..<6 { samples["peer\(i)"] = Self.sample(lostOut: -5, nan: true) } // A window longer than the schema's ten minutes. let report = build(&builder, at: 900, samples) XCTAssertEqual(report.seconds, 600) XCTAssertEqual(report.peers?.count, 4, "no more than the schema's four") XCTAssertEqual(report.peers?.map(\.peer), ["peer0", "peer1", "peer2", "peer3"]) let video = try XCTUnwrap(report.peers?.first?.out?.video) XCTAssertNil(video.fps, "NaN is left out, not sent") XCTAssertNil(report.peers?.first?.out?.audio?.lost, "a negative count is left out") XCTAssertEqual(QualityReportBuilder.bound(2e6, QualityBounds.kbps), 1e6) XCTAssertNil(QualityReportBuilder.bound(.infinity, 10)) XCTAssertNil(QualityReportBuilder.bound(-1, 10)) XCTAssertEqual(QualityReportBuilder.bound(1.26, 10), 1.3) } func testTheEnums() { XCTAssertEqual(QualityWords.limit("none"), "none") XCTAssertEqual(QualityWords.limit("cpu"), "cpu") XCTAssertEqual(QualityWords.limit("something new"), "other") XCTAssertNil(QualityWords.limit("")) XCTAssertEqual(QualityWords.candidate("prflx"), "prflx") XCTAssertEqual(QualityWords.candidate("mystery"), "unknown") XCTAssertEqual(QualityWords.candidate(nil), "unknown") XCTAssertEqual(QualityWords.transport("tcp", relayProtocol: nil, relay: false), "tcp") XCTAssertEqual(QualityWords.transport("udp", relayProtocol: "tls", relay: true), "tls") XCTAssertEqual(QualityWords.transport("udp", relayProtocol: "tls", relay: false), "udp") XCTAssertEqual(QualityWords.transport(nil, relayProtocol: nil, relay: false), "unknown") } // MARK: - Nothing that names anyone func testTheEncodedReportCarriesNoNamesAddressesOrDevices() throws { var builder = QualityReportBuilder() builder.start(at: 0) var tally = QualityEventTally() tally.observe(at: 1, call: false, reconnecting: ["p1"], cantHear: []) let report = build(&builder, at: 30, ["p1": Self.sample(screen: true)], tally: tally, final: true) let json = try XCTUnwrap(report.encoded()) for forbidden in [ "Vicky", "Wallace", "AirPods", "Paul", "192.168", "203.0.113", "54321", "61000", "relay.example.com", "turns:", "address", "trackIdentifier", "deviceName", ] { XCTAssertFalse(json.contains(forbidden), "the report carries \(forbidden): \(json)") } // And it is the schema's shape. let object = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(json.utf8)) as? [String: Any]) XCTAssertEqual(object["v"] as? Int, 1) XCTAssertEqual(object["final"] as? Bool, true) XCTAssertEqual(object["seconds"] as? Double, 30) XCTAssertEqual((object["client"] as? [String: String])?["app"], "mac") XCTAssertEqual((object["devices"] as? [String: String])?["input"], "bluetooth") let peer = try XCTUnwrap((object["peers"] as? [[String: Any]])?.first) XCTAssertNotNil(peer["in"], "\"in\" is the key, as the schema spells it") XCTAssertNotNil((peer["path"] as? [String: Any])?["protocol"]) XCTAssertEqual((peer["events"] as? [String: Any])?["reconnecting"] as? Double, 1) XCTAssertNil(object["isFinal"]) } @MainActor func testTheClientIsDigitsAndDots() { XCTAssertEqual(QualityClientInfo.version("0.4.8"), "0.4.8") XCTAssertNil(QualityClientInfo.version("0.4.8 (dev)")) XCTAssertNil(QualityClientInfo.version("dev")) XCTAssertNil(QualityClientInfo.version("1..2")) XCTAssertNil(QualityClientInfo.version(nil)) XCTAssertEqual( QualityClientInfo.osVersion(OperatingSystemVersion(majorVersion: 26, minorVersion: 0, patchVersion: 1)), "26.0.1") let current = QualityClientInfo.current() XCTAssertEqual(current.app, "mac") XCTAssertEqual(current.os, "macos") XCTAssertNotNil(current.osVersion) } // MARK: - Notices func testNoticesAreCountedWhenTheyBeginAndTimedWhileTheyShow() { var tally = QualityEventTally() tally.observe(at: 0, call: false, reconnecting: [], cantHear: ["b"]) tally.observe(at: 0.2, call: true, reconnecting: ["a"], cantHear: ["b"], noSound: ["a"]) tally.observe(at: 0.4, call: true, reconnecting: ["a"], cantHear: []) // A gap of a minute (the Mac asleep) counts as no more than a second. tally.observe(at: 60.4, call: false, reconnecting: [], cantHear: [], noSoundFromUs: ["c"], audioBlocked: ["c"]) tally.count(.keyframe, for: "a") tally.count(.reattach, for: "c") var builder = QualityReportBuilder() builder.start(at: 0) var mutable = tally let report = builder.build( at: 61, samples: [:], events: &mutable, client: Self.client, devices: nil, final: false) XCTAssertEqual(report.events?.reconnecting, 1) // Each look counts the time since the one before for what it sees. XCTAssertEqual(report.events?.reconnectingSeconds, 0.4) let peers = Dictionary(uniqueKeysWithValues: (report.peers ?? []).map { ($0.peer, $0) }) XCTAssertEqual(peers["a"]?.events?.reconnecting, 1) XCTAssertEqual(peers["a"]?.events?.reconnectingSeconds, 0.4) XCTAssertEqual(peers["a"]?.events?.noSound, true) XCTAssertEqual(peers["a"]?.out?.screen?.keyframes, 1) XCTAssertEqual(peers["b"]?.events?.cantHearSeconds, 0.2) XCTAssertEqual(peers["c"]?.events?.noSoundFromUs, true) XCTAssertEqual(peers["c"]?.events?.audioBlocked, true) XCTAssertEqual(peers["c"]?.in?.screen?.reattaches, 1) XCTAssertNil(peers["b"]?.events?.noSound) } func testANoticeStillShowingIsNotCountedAgainInTheNextWindow() { var tally = QualityEventTally() tally.observe(at: 0, call: true, reconnecting: ["a"], cantHear: []) tally.observe(at: 1, call: true, reconnecting: ["a"], cantHear: []) var builder = QualityReportBuilder() builder.start(at: 0) _ = builder.build(at: 1, samples: [:], events: &tally, client: Self.client, devices: nil, final: false) tally.observe(at: 2, call: true, reconnecting: ["a"], cantHear: []) let report = builder.build(at: 2, samples: [:], events: &tally, client: Self.client, devices: nil, final: false) XCTAssertNil(report.events?.reconnecting, "the same notice, carried over") XCTAssertEqual(report.events?.reconnectingSeconds, 1) XCTAssertNil(report.peers?.first?.events?.reconnecting) XCTAssertEqual(report.peers?.first?.events?.reconnectingSeconds, 1) } // MARK: - Devices func testDeviceKindsFromCoreAudioTransports() { XCTAssertEqual(DeviceKind.mac(transport: "built-in"), "built-in") XCTAssertEqual(DeviceKind.mac(transport: "bluetooth"), "bluetooth") XCTAssertEqual(DeviceKind.mac(transport: "usb"), "usb") XCTAssertEqual(DeviceKind.mac(transport: "airplay"), "airplay") XCTAssertEqual(DeviceKind.mac(transport: "virtual"), "virtual") XCTAssertEqual(DeviceKind.mac(transport: "aggregate"), "virtual") XCTAssertEqual(DeviceKind.mac(transport: "continuity"), "continuity") XCTAssertEqual(DeviceKind.mac(transport: "thunderbolt"), "other") XCTAssertEqual(DeviceKind.mac(transport: "0x12345678"), "other") XCTAssertEqual(DeviceKind.mac(transport: nil), "unknown") } func testDeviceKindsFromAudioSessionPorts() { // AVAudioSession.Port's raw values. for (port, kind) in [ ("MicrophoneBuiltIn", "built-in"), ("Speaker", "built-in"), ("Receiver", "built-in"), ("BluetoothHFP", "bluetooth"), ("BluetoothA2DPOutput", "bluetooth"), ("BluetoothLEOutput", "bluetooth"), ("USBAudio", "usb"), ("Headphones", "wired"), ("MicrophoneWired", "wired"), ("AirPlay", "airplay"), ("CarAudio", "other"), ("LineIn", "other"), ] { XCTAssertEqual(DeviceKind.ios(port: port), kind, port) } XCTAssertEqual(DeviceKind.ios(port: nil), "none") } func testEveryKindIsOneTheSchemaAllows() { let allowed: Set = ["built-in", "bluetooth", "usb", "wired", "airplay", "virtual", "continuity", "other", "none", "unknown"] for t in ["built-in", "bluetooth", "usb", "airplay", "virtual", "aggregate", "continuity", "display", "", "x"] { XCTAssertTrue(allowed.contains(DeviceKind.mac(transport: t))) } for p in ["Speaker", "BluetoothHFP", "USBAudio", "Headphones", "AirPlay", "CarAudio", "", "x"] { XCTAssertTrue(allowed.contains(DeviceKind.ios(port: p))) } } // MARK: - Report a problem static let roster = #""" {"phase":"admitted","peer":"me0000aaaa","boot":"b","waiting":[],"peers":[ {"peer":"me0000aaaa","name":"Paul Campbell","kind":"member","gen":1}, {"peer":"abc123zzzz","name":"Vicky Wallace","kind":"member","gen":2, "media":{"muted":true,"cameraOff":false,"noSoundFrom":["me0000aaaa"],"hearing":[]}} ]} """# @MainActor func testTheAnonymousDiagnosticsNameNobody() { let call = Call(playAudio: false) call.showFixture(title: "Planning with Vicky", state: Self.roster) let named = call.audioDiagnosticsText() XCTAssertTrue(named.contains("Vicky Wallace: "), "Copy call diagnostics still names people") let anonymous = call.anonymousDiagnosticsText() XCTAssertTrue(anonymous.contains("peer abc123: "), anonymous) XCTAssertTrue(anonymous.contains("muted")) XCTAssertTrue(anonymous.contains("hears no sound from you")) for forbidden in ["Vicky", "Wallace", "Paul Campbell", "Planning"] { XCTAssertFalse(anonymous.contains(forbidden), "the report names \(forbidden)") } } func testTheBridgesAnswers() { XCTAssertEqual(ProblemOutcome(bridgeAnswer: #"{"ref":"K7Q2M9PD"}"#), .sent(ref: "K7Q2M9PD")) XCTAssertEqual(ProblemOutcome(bridgeAnswer: #"{"outcome":"too-many"}"#), .tooMany) XCTAssertEqual(ProblemOutcome(bridgeAnswer: #"{"outcome":"gone"}"#), .gone) XCTAssertEqual(ProblemOutcome(bridgeAnswer: #"{"outcome":"failed"}"#), .failed) XCTAssertEqual(ProblemOutcome(bridgeAnswer: "nonsense"), .failed) XCTAssertEqual(ProblemOutcome(bridgeAnswer: #"{"ref":""}"#), .failed) } @MainActor func testTheSheetSendsAndShowsTheReference() async { let sent = Box() let model = ProblemReportModel { note in await sent.set(note) return .sent(ref: "K7Q2M9PD") } XCTAssertEqual(model.phase, .editing) XCTAssertTrue(model.canSend, "the note is optional") model.note = "Vicky sounded robotic" XCTAssertEqual(model.counter, "21/1000") await model.send() XCTAssertEqual(model.phase, .sent(ref: "K7Q2M9PD")) XCTAssertFalse(model.canSend, "sent once") let note = await sent.value XCTAssertEqual(note, "Vicky sounded robotic") XCTAssertEqual(ProblemWords.reference("K7Q2M9PD"), "Reference K7Q2M9PD") } @MainActor func testTheSheetSaysWhyNotAndCanTryAgain() async { let answers = Answers([.tooMany, .gone, .failed, .sent(ref: "ABCDEFGH")]) let model = ProblemReportModel { _ in await answers.next() } await model.send() XCTAssertEqual(model.phase, .refused("Too many reports from this call — try again in a minute.")) XCTAssertTrue(model.canSend) await model.send() XCTAssertEqual(model.phase, .refused("This call can no longer be reported.")) await model.send() XCTAssertEqual(model.phase, .refused("The report could not be sent.")) await model.send() XCTAssertEqual(model.phase, .sent(ref: "ABCDEFGH")) } @MainActor func testTheNoteIsAThousandCharactersAsTheServerCountsThem() { let model = ProblemReportModel { _ in .failed } model.note = String(repeating: "é", count: 1200) XCTAssertEqual(model.note.unicodeScalars.count, 1000) XCTAssertEqual(model.counter, "1000/1000") } func testTheWordsAreTheRulings() { XCTAssertEqual( ProblemWords.disclosure, "Sends this call's connection diagnostics to the Calls server log. No audio or video.") XCTAssertEqual(ProblemWords.sent, "Sent — thank you") XCTAssertEqual( QualityReporting.disclosure, "Calls sends anonymous connection quality reports to help fix call problems.") } func testAProblemReportFitsTheServersLimit() throws { let body = ProblemReportBody( note: "x", diagnostics: String(repeating: "line of diagnostics\n", count: 10_000), client: Self.client ).encoded() XCTAssertLessThanOrEqual(body.utf8.count, QualityBounds.problemBody) let object = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(body.utf8)) as? [String: Any]) XCTAssertEqual(object["note"] as? String, "x") XCTAssertTrue((object["diagnostics"] as? String)?.hasPrefix("(earlier lines cut)") == true) } func testTheReportingSwitchIsOnUnlessTurnedOff() throws { let defaults = try XCTUnwrap(UserDefaults(suiteName: "quality-\(UUID().uuidString)")) XCTAssertTrue(QualityReporting.enabled(defaults)) defaults.set(false, forKey: QualityReporting.enabledKey) XCTAssertFalse(QualityReporting.enabled(defaults)) } @MainActor func testAnEndedCallOffersReportAProblem() { let ended = CallNotices.notices( phase: "ended", status: "This meeting has ended", sharingScreen: false, screenNote: "", floor: false) XCTAssertEqual(ended.first?.action, .reportProblem) let call = Call(playAudio: false) XCTAssertFalse(call.canReportProblem, "no call, nothing to report") call.reportProblem() XCTAssertNil(call.problemReport) } } private actor Box { var value = "" func set(_ v: String) { value = v } } private actor Answers { var queue: [ProblemOutcome] init(_ queue: [ProblemOutcome]) { self.queue = queue } func next() -> ProblemOutcome { queue.removeFirst() } }