package host import ( "log" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // count is how many times s was logged (housekeeper_test.go's buffer). func (l *lockedBuffer) count(s string) int { return strings.Count(l.String(), s) } // lostLog sends the log to a buffer for the rest of the test; a pass may // log from goroutines, so the buffer is housekeeper_test.go's locked one. func lostLog(t *testing.T) *lockedBuffer { t.Helper() buf := &lockedBuffer{} out, flags := log.Writer(), log.Flags() log.SetOutput(buf) t.Cleanup(func() { log.SetOutput(out); log.SetFlags(flags) }) return buf } // rcLose lists box in the host claim's Lost, as a declaration leaves it. func rcLose(t *testing.T, h *phase1Harness, box string) { t.Helper() rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Lost = append(c.Lost, box) if c.LostRevisions == nil { c.LostRevisions = map[string]uint64{} } c.LostRevisions[box] = 1 }) } // Q9's gate (Plan 4 Task 10, witness wLostQuarantine; delta M15): a box // the host's claim lists in Lost runs no move step for the host, whatever // its role and the record's state, and logs one standing notice. The // quarantine's local retirement is the claim's (readmission requires its // receipt); moves adds only this gate. func TestReturningLostBoxRefusesEveryMoveStep(t *testing.T) { type snapshot struct { rec provision.Record claim hostclaim.Claim } take := func(t *testing.T, h *phase1Harness) snapshot { t.Helper() return snapshot{rec: h.read(t, p1Host), claim: *h.claim(t)} } t.Run("the source before the flip", func(t *testing.T) { buf := lostLog(t) h := newPhase1Harness(t) h.start(t, "auto") rcLose(t, h, "ie-box") before := take(t, h) h.pass(t) h.pass(t) if after := take(t, h); !reflect.DeepEqual(before, after) { t.Fatalf("a lost source moved the host:\n%+v\n%+v", before.rec.Move, after.rec.Move) } if r, err := readSourceReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { t.Fatalf("a lost source reported: %+v %v", r, err) } if n := buf.count("is lost on this host's claim"); n != 1 { t.Fatalf("notices %d:\n%s", n, buf.String()) } // The gate is this box's: the same record moves on once the claim // no longer lists it. rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Lost, c.LostRevisions = nil, nil }) h.pass(t) if r, err := readSourceReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r == nil { t.Fatalf("the readmitted source did not report: %v", err) } }) t.Run("the target before the flip", func(t *testing.T) { lostLog(t) h := newPhase1Harness(t) h.start(t, "auto") h.pass(t) rcLose(t, h, "uk-box") before := take(t, h) h.targetPass(t, h.targetCfg()) if after := take(t, h); !reflect.DeepEqual(before, after) { t.Fatalf("a lost target moved the host:\n%+v\n%+v", before.rec.Move, after.rec.Move) } if r, err := readTargetReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { t.Fatalf("a lost target reported: %+v %v", r, err) } }) t.Run("the source's recovery", func(t *testing.T) { buf := lostLog(t) h := newRecoverHarness(t, false) h.released(t) rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Lost, c.LostRevisions = append(c.Lost, "ie-box"), map[string]uint64{"ie-box": 1} }) rec, claim := h.read(t, p1Host), *h.claim(t) h.srcPass(t) if !reflect.DeepEqual(rec, h.read(t, p1Host)) || !reflect.DeepEqual(claim, *h.claim(t)) || len(h.probes) != 0 || h.srcRoute.HoldOp != p1ID { t.Fatalf("a lost source recovered: %+v", h.read(t, p1Host).Move.Rollback) } if n := buf.count("is lost on this host's claim"); n != 1 { t.Fatalf("notices %d", n) } }) } // rcCorruptClaim makes the host claim unreadable: its object no longer // decodes. func rcCorruptClaim(t *testing.T, s release.Store) { t.Helper() key, err := hostclaim.Key(p1Host) if err != nil { t.Fatal(err) } _, etag, err := s.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err := s.Put(t.Context(), key, []byte("{not a claim"), etag); err != nil { t.Fatal(err) } } // Fix round 1, item 3: the gate covers the whole pass. A host the gate // blocks, confirmed lost or with a claim that does not read, gets no target // report and no journal sweep: nothing is reported, committed or retired. func TestLostBoxGateCoversTheWholePass(t *testing.T) { t.Run("an unreadable claim blocks the target's report", func(t *testing.T) { buf := lostLog(t) h := newPhase1Harness(t) h.start(t, "auto") h.pass(t) rcCorruptClaim(t, h.s) h.targetPass(t, h.targetCfg()) if r, err := readTargetReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { t.Fatalf("a target with an unreadable claim reported: %+v %v", r, err) } if buf.count("does not read") == 0 { t.Fatalf("no notice:\n%s", buf.String()) } }) for name, block := range map[string]func(t *testing.T, h *revertHarness){ "a confirmed lost host": func(t *testing.T, h *revertHarness) { rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Lost, c.LostRevisions = append(c.Lost, "ie-box"), map[string]uint64{"ie-box": 1} }) }, "an unreadable claim": func(t *testing.T, h *revertHarness) { rcCorruptClaim(t, h.s) }, } { t.Run(name+" blocks the journal sweep", func(t *testing.T) { lostLog(t) h := newRevertHarness(t) h.fromStopped(t) h.arm(t) h.pass(t) h.reopened(t) h.rvAt = "after-close" o := h.laneChore(t) h.tPass(t) h.pastHorizon(t, o) h.tPass(t) h.pass(t) if !h.rvDone || h.read(t, p1Host).Move != nil || !moveJournalPresent(h.cfg.DataDir, p1Host, p1ID) { t.Fatalf("the setup did not leave the closed revert's journal: %v", h.rvHits) } journal, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID) if err != nil { t.Fatal(err) } block(t, h) h.freshWorker() h.pass(t) if !moveJournalPresent(h.cfg.DataDir, p1Host, p1ID) { t.Fatal("the sweep retired a blocked host's journal") } if after, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID); err != nil || !reflect.DeepEqual(after.Steps, journal.Steps) { t.Fatalf("the sweep committed a blocked host's journal: %v", err) } }) } } // Fix round 1, item 4: the gate names this box's region through the shared // placement rule (park.PlacementMatches under the verified home binding), // never raw string equality: on the home box (placement code "") a record // that spells the home by its roster name names this box, and is gated; // without the verified binding it names another region. func TestLostBoxGateReadsHomeSpellingsUnderTheBinding(t *testing.T) { s := release.NewDirStore(t.TempDir()) inc, err := hostclaim.Incarnation("bab", "app", p1Host, p1Created) if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(t.Context(), s, hostclaim.Observed{Host: p1Host, Absent: true}, func(c *hostclaim.Claim) error { c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", p1Host), Account: "bab", App: "app", Incarnation: inc, CreatedAt: p1Created, Location: "instances/" + p1Host, Generation: "cur", Stamp: 1} c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "home-box"} c.Lost, c.LostRevisions = []string{"home-box"}, map[string]uint64{"home-box": 1} return nil }); err != nil { t.Fatal(err) } for _, spelling := range []provision.Record{ {Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, From: "ie", To: "uk"}}, {Host: p1Host, Region: "uk", Move: &provision.MoveState{ID: p1ID, From: "uk", To: "ie"}}, {Host: p1Host, Region: "", Move: &provision.MoveState{ID: p1ID, From: "", To: "uk"}}, } { bound := &movesWorker{cfg: moveCfg{Box: "home-box", Region: "", ParkHomeRegion: "ie"}} if !movesNamesBox(bound.cfg, spelling) || !bound.lostHere(t.Context(), s, spelling) { t.Fatalf("the home box under the binding does not gate %+v", spelling.Move) } } unbound := moveCfg{Box: "home-box", Region: ""} if movesNamesBox(unbound, provision.Record{Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, From: "ie", To: "uk"}}) { t.Fatal("a roster name matched the home without the verified binding") } named := moveCfg{Box: "de-box", Region: "de", ParkHomeRegion: "ie"} if movesNamesBox(named, provision.Record{Host: p1Host, Region: "", Move: &provision.MoveState{ID: p1ID, From: "", To: "uk"}}) { t.Fatal("a named box matched the home") } }