package hostlife import "testing" // The readiness arithmetic of moves spec §7.3 T11 (rev 7.7; R3-7, R5-5, // R6-3), in seconds. Rev 7.3's T11 required every one of three probe // attempts, each with its settle, to fit the window beside the fixed costs: // 150 s of probing alone, against a default window of 120 s, so no default // move was ever eligible. Rev 7.4 made the permitted attempt count depend // on the admitted budget. Rev 7.5 made the sealed transfer a full snapshot // but still priced the final mirror at the 16 MiB bar (R5-5). Rev 7.6 // built the snapshot on the target and priced three local passes, then // restored the generation cold a fourth time without pricing it (R6-3). // Rev 7.7 prices what the cut does, with regional S3 and the disk as // separate rates: the final delta mirror within the bar across regions; // on the target the restore of the staged replica (S3 read, disk write), // the snapshot write and the anchor write (disk read, S3 write each), and // the verification of the database the build produced (a disk read: its // sha256 against the install manifest), which is what the `restored` // phase does instead of a second cold restore. // Rev 7.11 prices what the new protocol does at every probe attempt (R10-8): // the admitted session purges the built database and restores the // prepared generation again before the app starts (RestoreAttempt, one S3 // read and one disk write of the whole database), plus the admission's // own overhead (the CAS, the purge, generation.json: Admit); a retried // probe is a fresh generation whose first upload is the whole database // again (FreshUpload, one disk read and one S3 write), so every attempt // after the first costs that too. type cutEstimate struct { Stop, Checkpoint, Mirror, Archive, Render, Open float64 RestoreStaged, SnapshotBuild, Anchor, Verify float64 // target-side build, whole database RestoreAttempt, Admit, FreshUpload float64 // per probe attempt (rev 7.11) Probe, Settle float64 // one attempt } const ( cutWindowDefault = 120.0 // the operator's default cut window probeAttemptsMax = 3 // §7.7: never more than three attempts probeSeconds = 30.0 settleSeconds = 20.0 admitSeconds = 2.0 // the admitting CAS, the purge and generation.json (rev 7.11) mirrorBarMiB = 16.0 // §7.3: the final round is at most this much ) func (e cutEstimate) fixed() float64 { return e.Stop + e.Checkpoint + e.Mirror + e.Archive + e.RestoreStaged + e.SnapshotBuild + e.Anchor + e.Verify + e.Render + e.Open } // attempt prices the i-th probe attempt (from 1): the restore of the // prepared generation and the admission overhead, the probe and its // settle; from the second attempt on, the fresh generation's first upload // as well (rev 7.11, R10-8). func (e cutEstimate) attempt(i int) float64 { c := e.RestoreAttempt + e.Admit + e.Probe + e.Settle if i > 1 { c += e.FreshUpload } return c } // attemptsAdmitted is T11 as rev 7.4 states it, repriced in rev 7.11: the // attempts permitted are those whose summed costs fit the window after // the fixed costs, at most three; the instance is eligible when that is // at least one, and the probe loop (§7.7) runs at most that many. func (e cutEstimate) attemptsAdmitted(window float64) int { left := window - e.fixed() n := 0 for n < probeAttemptsMax && left >= e.attempt(n+1) { left -= e.attempt(n + 1) n++ } return n } // fitsRev73 is T11 as rev 7.3 stated it: three attempts, always. func (e cutEstimate) fitsRev73(window float64) bool { return e.fixed()+probeAttemptsMax*(e.Probe+e.Settle) <= window } // rates are the measured rates T11 reads, in MiB/s: the cross-region copy // path, regional S3 (reads and writes from a box in the bucket's region) // and the box's disk. type rates struct{ cross, s3, disk float64 } // estimate prices a cut for a database of dbMiB at the given rates, as // rev 7.7 §7.3 does. Every target-side pass moves the whole database once // across S3 and once across the disk, bar the verification, which reads // the disk alone. func estimate(dbMiB float64, r rates) cutEstimate { return cutEstimate{ Stop: 2, Checkpoint: 1, Archive: 2, Render: 1, Open: 5, Mirror: mirrorBarMiB / r.cross, RestoreStaged: dbMiB/r.s3 + dbMiB/r.disk, SnapshotBuild: dbMiB/r.disk + dbMiB/r.s3, Anchor: dbMiB/r.disk + dbMiB/r.s3, Verify: dbMiB / r.disk, // rev 7.11: the probe's admission restores the prepared generation // again (purge, then the pinned snapshot from regional S3 to disk), // with the admission's overhead; a retried probe uploads the whole // database once more as its fresh generation's root RestoreAttempt: dbMiB/r.s3 + dbMiB/r.disk, Admit: admitSeconds, FreshUpload: dbMiB/r.disk + dbMiB/r.s3, Probe: probeSeconds, Settle: settleSeconds, } } // exampleRates are the rates the spec's worked example uses (§7.3 T11): // 10 MiB/s across regions, 250 MiB/s to regional S3, 500 MiB/s on disk. var exampleRates = rates{cross: 10, s3: 250, disk: 500} func TestReadinessBudget(t *testing.T) { // a tiny instance: every fixed cost at the floor of its measured rate tiny := estimate(1, exampleRates) if tiny.fitsRev73(cutWindowDefault) { t.Fatalf("rev 7.3's formula admits a tiny instance in the default window: R3-7 is stale") } if n := tiny.attemptsAdmitted(cutWindowDefault); n != 2 { t.Fatalf("rev 7.4's formula admits %d attempts for a tiny instance in %.0f s, want 2", n, cutWindowDefault) } // the boundary: fixed costs of 70 s leave exactly one attempt; 71 s none edge := cutEstimate{Stop: 70 - 5, Open: 5, Probe: probeSeconds, Settle: settleSeconds} if n := edge.attemptsAdmitted(cutWindowDefault); n != 1 { t.Fatalf("fixed costs of 70 s admit %d attempts, want 1", n) } edge.Stop++ if n := edge.attemptsAdmitted(cutWindowDefault); n != 0 { t.Fatalf("fixed costs of 71 s admit %d attempts, want 0 (refused)", n) } // a generous window never permits more than three if n := tiny.attemptsAdmitted(600); n != probeAttemptsMax { t.Fatalf("a 600 s window admits %d attempts, want %d", n, probeAttemptsMax) } // R5-5: the spec's example, a 2 GiB database at the example rates. Rev // 7.5's full-snapshot transfer alone takes 205 s and could never fit; // the delta within the bar and the target's build do, with one attempt big := estimate(2048, exampleRates) fullTransfer := 2048.0 / exampleRates.cross if fullTransfer < cutWindowDefault { t.Fatalf("the example no longer illustrates R5-5") } if n := big.attemptsAdmitted(cutWindowDefault); n != 1 { t.Fatalf("a 2 GiB database at the example rates admits %d attempts, want 1 (fixed %.1f s, first attempt %.1f s)", n, big.fixed(), big.attempt(1)) } if big.fixed() < 53 || big.fixed() > 54 { t.Fatalf("the example's fixed costs are %.1f s; the spec says 53.6 s", big.fixed()) } // rev 7.11 (R10-8): the first attempt restores the prepared generation // again and pays the admission: 12.3 + 2 + 30 + 20 = 64.3 s, so the // example runs to 117.9 s and admits one attempt by 2.1 s; a second // attempt would add 76.6 s (a fresh generation's full upload too) if a := big.attempt(1); a < 64 || a > 65 { t.Fatalf("the example's first attempt costs %.1f s; the spec says 64.3 s", a) } if tot := big.fixed() + big.attempt(1); tot < 117 || tot > 118.5 { t.Fatalf("the example runs to %.1f s with one attempt; the spec says 117.9 s", tot) } // the review's 2.5 GiB (R10-8): rev 7.10's arithmetic admitted it at // 113.8 s; with the probe's own restore it is 129.2 s, past the window, // and refused bigger := estimate(2560, exampleRates) old := bigger.fixed() + probeSeconds + settleSeconds if old < 113 || old > 114.5 { t.Fatalf("rev 7.10's arithmetic for 2.5 GiB is %.1f s; the review says 113.8 s", old) } if n := bigger.attemptsAdmitted(cutWindowDefault); n != 0 { t.Fatalf("2.5 GiB at the example rates admits %d attempts, want 0 (%.1f s with one attempt)", n, bigger.fixed()+bigger.attempt(1)) } // and a cross-region snapshot transfer, if a builder did that instead, // is refused by the same arithmetic naive := big naive.Mirror, naive.RestoreStaged, naive.SnapshotBuild = fullTransfer, 0, 0 if n := naive.attemptsAdmitted(cutWindowDefault); n != 0 { t.Fatalf("a full cross-region snapshot transfer admits %d attempts, want 0", n) } // R6-3: rev 7.6 priced three passes at one local rate and then restored // the generation cold a fourth time, unpriced. The review's workload: 3 // GiB, 200 MiB/s local, 10 MiB/s across regions, one probe. Three passes // admit one attempt (108.68 s with the probe); the fourth pass takes the // same workload to 124.04 s, past the window r := rates{cross: 10, s3: 200, disk: 200} three := estimate(3072, r) three.RestoreStaged, three.SnapshotBuild, three.Anchor, three.Verify = 3072/200.0, 3072/200.0, 3072/200.0, 0 // rev 7.6's arithmetic three.RestoreAttempt, three.Admit, three.FreshUpload = 0, 0, 0 // rev 7.6's arithmetic knew no per-attempt restore if n := three.attemptsAdmitted(cutWindowDefault); n != 1 { t.Fatalf("rev 7.6's three passes admit %d attempts for the review's workload, want 1 (the finding's premise; fixed %.2f s)", n, three.fixed()) } four := three four.Verify = 3072 / 200.0 // the unpriced cold restore, at the same rate if n := four.attemptsAdmitted(cutWindowDefault); n != 0 || four.fixed()+probeSeconds+settleSeconds < 124 { t.Fatalf("with the fourth pass priced the review's workload admits %d attempts at %.2f s with one probe, want 0 past 124 s", n, four.fixed()+probeSeconds+settleSeconds) } // rev 7.7 verifies the built database instead of restoring it again: // the same workload is refused honestly, by its own arithmetic if n := estimate(3072, r).attemptsAdmitted(cutWindowDefault); n != 0 { t.Fatalf("rev 7.7 admits %d attempts for the review's workload, want 0 (fixed %.1f s)", n, estimate(3072, r).fixed()) } }