package rollout import ( "os" "path/filepath" "slices" "testing" "time" ) var t0 = time.Unix(1_700_000_000, 0).UTC() // promotedAt is when the pointer the failures below happened under was signed. var promotedAt = t0.Add(-time.Hour) func rec(build, host string) FailureRecord { return FailureRecord{Account: "bab", App: "blog", Build: build, Host: host, Cause: "release", Mode: "overlap", Reason: "health check failed", PromotedAt: promotedAt, At: t0} } func TestFailureRecordBlocksOnlyInstancesThatHaveNotServedTheBuild(t *testing.T) { b := &FailureBook{} added, err := b.Record(rec("v2", "a.example.com")) if err != nil || added != Recorded { t.Fatalf("Record = %v, %v", added, err) } if !b.Blocks("bab", "blog", "v2", "b.example.com", false) { t.Fatal("an instance that has not served v2 must be held off it") } if b.Blocks("bab", "blog", "v2", "c.example.com", true) { t.Fatal("an instance already serving v2 must not be blocked (spec §8.4)") } if b.Blocks("bab", "blog", "v3", "b.example.com", false) { t.Fatal("a newer installed build no longer matches the record") } if b.Blocks("bab", "other", "v2", "b.example.com", false) { t.Fatal("the scope is (app, build, box), not the build name alone") } if added, _ := b.Record(rec("v2", "b.example.com")); added == Recorded { t.Fatal("the first failure stands; a second is not a second record") } } // Review focus 3: the record outlives an agent restart. func TestFailureBookSurvivesReopen(t *testing.T) { dir := t.TempDir() b, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } r := rec("v2", "a.example.com") r.HeldDown = []string{"a.example.com"} if _, err := b.Record(r); err != nil { t.Fatal(err) } again, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } if !again.Blocks("bab", "blog", "v2", "b.example.com", false) { t.Fatal("the reopened book lost the record") } if !again.HeldDown("bab", "blog", "v2", "a.example.com") { t.Fatal("the reopened book lost the held-down host") } } func TestACorruptFailureFileIsAnError(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, FailureFile), []byte("{nope"), 0o644); err != nil { t.Fatal(err) } if _, err := OpenFailureBook(dir); err == nil { t.Fatal("a book that cannot be read must refuse, not start empty") } } func TestRetryClearing(t *testing.T) { b := &FailureBook{} r := rec("v2", "a.example.com") r.HeldDown = []string{"a.example.com"} b.Record(r) // carlos instances retry: one host only. if ok, _ := b.RetryHost("bab", "blog", "v2", "a.example.com"); !ok { t.Fatal("a host retry clears that host") } if b.Blocks("bab", "blog", "v2", "a.example.com", false) || b.HeldDown("bab", "blog", "v2", "a.example.com") { t.Fatal("the retried host is still held") } if !b.Blocks("bab", "blog", "v2", "b.example.com", false) { t.Fatal("a host retry cleared another host") } // The retried host fails the build again: it is held again. b.Record(rec("v2", "a.example.com")) if !b.Blocks("bab", "blog", "v2", "a.example.com", false) { t.Fatal("a second failure must hold the retried host again") } // carlos promote --retry (spec rev 14 §8.4): a pointer naming the build // with a NEWER PromotedAt clears the record; the pointer it failed under, // seen again, does not. if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt); ok { t.Fatal("the pointer the build failed under must not clear the record") } if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(-time.Minute)); ok { t.Fatal("an older pointer must not clear the record") } if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(time.Second)); !ok { t.Fatal("a re-signed pointer for the same build clears the record") } if _, ok := b.Lookup("bab", "blog", "v2"); ok { t.Fatal("record still present after the retry") } } func TestPruneKeepsRecordsSomeRouteStillInstalls(t *testing.T) { b := &FailureBook{} b.Record(rec("v2", "a.example.com")) b.Record(rec("v3", "a.example.com")) if err := b.Prune(func(_, _, build string) bool { return build == "v3" }); err != nil { t.Fatal(err) } if _, ok := b.Lookup("bab", "blog", "v2"); ok { t.Fatal("v2 is installed nowhere and should have been pruned") } if _, ok := b.Lookup("bab", "blog", "v3"); !ok { t.Fatal("v3 is still installed and must stay held (spec §8.4)") } } // Review fix 1: a record handed out must not share its host lists with the // book, or a status read racing `carlos instances retry` sees a half-edited // list. func TestRecordsHandedOutAreCopies(t *testing.T) { b := &FailureBook{} in := rec("v2", "a.example.com") in.HeldDown = []string{"a.example.com", "b.example.com", "c.example.com"} if _, err := b.Record(in); err != nil { t.Fatal(err) } in.HeldDown[0] = "scribbled-by-the-caller" if got, _ := b.Lookup("bab", "blog", "v2"); got.HeldDown[0] != "a.example.com" { t.Fatalf("Record kept the caller's HeldDown array: %v", got.HeldDown) } look, _ := b.Lookup("bab", "blog", "v2") look.HeldDown[1] = "scribbled-on-lookup" list := b.List() list[0].HeldDown[2] = "scribbled-on-list" forApp := b.ForApp("bab", "blog") forApp[0].HeldDown[0] = "scribbled-on-forapp" want := []string{"a.example.com", "b.example.com", "c.example.com"} got, _ := b.Lookup("bab", "blog", "v2") if !slices.Equal(got.HeldDown, want) { t.Fatalf("book's HeldDown = %v, want %v: a handed-out record shares its array", got.HeldDown, want) } // Retried is shared the same way. b.RetryHost("bab", "blog", "v2", "b.example.com") look, _ = b.Lookup("bab", "blog", "v2") look.Retried[0] = "scribbled-retried" list = b.List() list[0].Retried[0] = "scribbled-retried" got, _ = b.Lookup("bab", "blog", "v2") if !slices.Equal(got.Retried, []string{"b.example.com"}) { t.Fatalf("book's Retried = %v", got.Retried) } } // The mutators edit copy-on-write, so a slice taken from the book before an // edit (as a reader holding the old record would) never changes under it. func TestMutatorsDoNotEditSlicesInPlace(t *testing.T) { b := &FailureBook{} in := rec("v2", "a.example.com") in.HeldDown = []string{"a.example.com", "b.example.com"} b.Record(in) b.mu.Lock() oldHeld := b.recs[0].HeldDown b.mu.Unlock() b.RetryHost("bab", "blog", "v2", "a.example.com") if !slices.Equal(oldHeld, []string{"a.example.com", "b.example.com"}) { t.Fatalf("RetryHost edited the HeldDown array in place: %v", oldHeld) } b.mu.Lock() oldRetried := b.recs[0].Retried b.mu.Unlock() b.Record(rec("v2", "a.example.com")) // spends a.example.com's retry if !slices.Equal(oldRetried, []string{"a.example.com"}) { t.Fatalf("Record edited the Retried array in place: %v", oldRetried) } } // failBook returns a book whose file cannot be written (the path is a // directory), plus a func that makes it writable again. func failBook(t *testing.T) (*FailureBook, string, func()) { t.Helper() dir := t.TempDir() b, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } p := filepath.Join(dir, FailureFile) if err := os.Mkdir(p, 0o755); err != nil { t.Fatal(err) } return b, dir, func() { if err := os.Remove(p); err != nil { t.Fatal(err) } } } // Review fix 2 (spec §8.4: the record survives agent restarts): a failed save // is retried by the next call, even one that adds nothing new. func TestFailedSaveIsRetriedByTheNextRecord(t *testing.T) { b, dir, heal := failBook(t) if _, err := b.Record(rec("v2", "a.example.com")); err == nil { t.Fatal("the first Record must report the failed save") } heal() if added, err := b.Record(rec("v2", "a.example.com")); err != nil || added != Unchanged { t.Fatalf("the repeat Record = %v, %v; want unchanged, nil", added, err) } again, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } if !again.Blocks("bab", "blog", "v2", "b.example.com", false) { t.Fatal("the no-op Record did not retry the save: a restart would lose the record") } } func TestFailedSaveIsRetriedByPrune(t *testing.T) { b, dir, heal := failBook(t) if _, err := b.Record(rec("v2", "a.example.com")); err == nil { t.Fatal("the first Record must report the failed save") } if err := b.Prune(func(_, _, _ string) bool { return true }); err == nil { t.Fatal("Prune must report that the save still fails") } heal() if err := b.Prune(func(_, _, _ string) bool { return true }); err != nil { t.Fatal(err) } again, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } if _, ok := again.Lookup("bab", "blog", "v2"); !ok { t.Fatal("a Prune that drops nothing did not retry the save") } } func TestFailedSaveIsRetriedAfterRetryHostAndClearOnPointer(t *testing.T) { b, dir, heal := failBook(t) b.Record(rec("v2", "a.example.com")) b.Record(rec("v3", "a.example.com")) if _, err := b.RetryHost("bab", "blog", "v2", "x.example.com"); err == nil { t.Fatal("RetryHost must report the failed save") } heal() // Nothing to clear here, but the pending save is still owed. if ok, err := b.ClearOnPointer("bab", "blog", "v3", promotedAt); ok || err != nil { t.Fatalf("ClearOnPointer = %v, %v", ok, err) } again, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } if got := again.List(); len(got) != 2 || !slices.Equal(got[0].Retried, []string{"x.example.com"}) { t.Fatalf("reopened book = %+v", got) } // And a failing ClearOnPointer is itself retried. if err := os.Remove(filepath.Join(dir, FailureFile)); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(dir, FailureFile), 0o755); err != nil { t.Fatal(err) } if ok, err := b.ClearOnPointer("bab", "blog", "v3", promotedAt.Add(time.Second)); !ok || err == nil { t.Fatalf("ClearOnPointer = %v, %v; want true and the save error", ok, err) } heal() if _, err := b.RetryHost("bab", "blog", "v2", "x.example.com"); err != nil { // no-op t.Fatal(err) } again, err = OpenFailureBook(dir) if err != nil { t.Fatal(err) } if _, ok := again.Lookup("bab", "blog", "v3"); ok { t.Fatal("the cleared record came back after a restart") } }