package host import ( "context" "database/sql" "encoding/json" "errors" "fmt" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/registry" "path/filepath" "reflect" "testing" "time" ) func TestPlan12InventoryReplayChecksAllTimers(t *testing.T) { t.Parallel() for _, mode := range []string{"foreign-owner", "stale-incarnation", "route-changed", "ok"} { t.Run(mode, func(t *testing.T) { f, prior, h := plan12Handover(t, hostclaim.KindRestoring, "stop") path := filepath.Join(t.TempDir(), "registry.db") original, err := h.Registry.Get(prior.Host) if err != nil { t.Fatal(err) } h.Registry, err = registry.Open(path) if err != nil { t.Fatal(err) } defer h.Registry.Close() if err = h.Registry.Upsert(*original); err != nil { t.Fatal(err) } in := hostclaim.Admission{ObservedOpID: prior.Claim.Op.ID, ObservedPhase: prior.Claim.Op.Phase, ObservedSession: prior.Claim.Holder.Session, ObservedGeneration: prior.Claim.Holder.Generation, Box: h.Box, Mode: "cold", NewGeneration: "successor", Region: "ie", PredecessorRegion: "uk"} admitted, err := hostclaim.Update(t.Context(), f.claims, prior, hostclaim.AdmitWriter(in)) if err != nil { t.Fatal(err) } if err := h.After(t.Context(), prior, admitted); err != nil { t.Fatal(err) } row, err := h.Registry.Get(prior.Host) if err != nil { t.Fatal(err) } if mode == "route-changed" { row.Version = "changed" if err := h.Registry.Upsert(*row); err != nil { t.Fatal(err) } } if mode == "foreign-owner" || mode == "stale-incarnation" { owner := row.ClaimOwnership owner.ClaimRowID = "injected" if mode == "foreign-owner" { owner.ClaimOwner = "another-box" } else { owner.ClaimIncarnation = "stale" } if err := h.Registry.UpsertTimer(registry.Timer{Host: prior.Host, Name: "injected", Kind: "cron", Spec: "* * * * *"}); err != nil { t.Fatal(err) } raw, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } defer raw.Close() if _, err := raw.Exec("UPDATE timers SET claim_incarnation=?,claim_stamp=?,claim_owner=?,claim_row_id=? WHERE host=?", owner.ClaimIncarnation, owner.ClaimStamp, owner.ClaimOwner, owner.ClaimRowID, prior.Host); err != nil { t.Fatal(err) } } err = h.After(t.Context(), prior, admitted) if (err == nil) != (mode == "ok") { t.Fatalf("replay %s: %v", mode, err) } }) } } func TestPlan12InitialSourceRecoveryRequiresRootStop(t *testing.T) { t.Parallel() for _, session := range []uint64{0, 2} { t.Run(fmt.Sprint(session), func(t *testing.T) { f, o, _ := plan12Handover(t, hostclaim.KindRestoring, "stop") key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, o.Claim.Op.ID), "source") report, _ := json.Marshal(plan12Stopped(o, "source", "box-1")) if _, err := f.claims.Put(t.Context(), key, report, ""); err != nil { t.Fatal(err) } o, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Holder.Session = session c.Holder.WriterBox, c.Holder.Generation = "box-1", "src" c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: session, Box: "box-1", Generation: "src", Mode: "cold", Region: "ie"}} if session == 0 { c.Holder.WriterBox, c.Holder.Generation = "", "" c.Holder.Admissions = nil } c.Op.Journal[hostclaim.JournalCutSession] = fmt.Sprint(session) c.Op.Journal[hostclaim.JournalSrcGen] = c.Holder.Generation c.Op.Journal[hostclaim.JournalEvidenceSource] = key return nil }) if err != nil { t.Fatal(err) } c := f.controller("boot-1", "controller-1", 4101, "start-1") c.wake.Continue = hostclaim.SessionContinues c.assembly.Guard = f.guard c.assembly.bindClaimWakeStop() if err := c.wake.Stop(t.Context(), o); err == nil { t.Fatal("agent report substituted for root stop") } if len(f.requests) == 0 { t.Fatal("root was not asked to stop the cut installation") } }) } } type plan12RetirementEvidence struct { parkRegistrationEvidence seen []lease.ObjectVersion at time.Time err error } func (e *plan12RetirementEvidence) VerifyRetirement(_ context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) { e.seen = append([]lease.ObjectVersion(nil), v...) return o.Claim.Holder.Incarnation, e.at, e.err } func TestPlan12RetirementUsesOriginalEvidenceAndFullCensus(t *testing.T) { for _, refuses := range []bool{false, true} { t.Run(fmt.Sprint(refuses), func(t *testing.T) { f := newRecoveryFixture(t) o := f.read() versions := []lease.ObjectVersion{{Key: o.Claim.Holder.Location + "/g-old/db", VersionID: "one"}, {Key: "moves/" + o.Host + "/old/db", VersionID: "two"}} proof := &plan12RetirementEvidence{at: time.Unix(100, 0)} if refuses { proof.err = errors.New("move did not retire this generation") } p := productionClaimRegional{Now: func() time.Time { return time.Unix(200, 0) }, Parks: &claimMoveParkSources{Evidence: proof}} _, at, err := p.retirement(t.Context(), o, versions) if !reflect.DeepEqual(proof.seen, versions) { t.Fatalf("incomplete retirement census: %+v", proof.seen) } if refuses { if !errors.Is(err, proof.err) { t.Fatalf("ignored refusal: %v", err) } } else if err != nil || !at.Equal(proof.at) { t.Fatalf("retirement time=%v error=%v", at, err) } }) } } func TestPlan12RetirementRefusalAlertOncePerLocation(t *testing.T) { o := plan12Retired(t) loc := "instances/" + o.Host view := park.View{ClaimETag: o.ETag, Region: "ie", RefusedLocations: map[string]error{loc: claimRetirementDependency()}} versions := []lease.ObjectVersion{{Key: loc + "/0000/db", VersionID: "one"}, {Key: loc + "/0000/db", VersionID: "two"}} d, err := park.Decide(o, view, versions, "") if err != nil || len(d.Alerts) != 1 || len(d.Candidates) != 0 { t.Fatalf("decision=%+v err=%v", d, err) } } func TestPlan12PendingRecoveryRejectsChangedIdentity(t *testing.T) { t.Parallel() for _, field := range []string{"same", "session", "op", "writer", "incarnation", "stamp", "generation", "kind", "phase", "exec"} { t.Run(field, func(t *testing.T) { f, o, h := plan12Handover(t, hostclaim.KindRestoring, "stop") h.Launches = f.journal in := hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: h.Box, Mode: "cold", NewGeneration: "successor", Region: "ie", PredecessorRegion: "uk"} in, err := h.prepareRecovery(t.Context(), o, in) if err != nil { t.Fatal(err) } next, err := hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(in)) if err != nil { t.Fatal(err) } c := next.Claim holder := c.Holder switch field { case "session": holder.Session++ holder.Stamp = holder.Session holder.Admissions[len(holder.Admissions)-1].Session = holder.Session case "op": c.Op.ID = "other" case "writer": holder.WriterBox = "other" case "incarnation": holder.Incarnation = "other" case "stamp": holder.Stamp++ case "generation": holder.Generation = "other" case "kind": c.Op.Kind = hostclaim.KindReverting case "phase": c.Op.Phase = "other" case "exec": c.Op.Exec = "other" } _, pending, err := h.pendingRecovery(t.Context(), next) if field == "same" { if err != nil || !pending { t.Fatal(pending, err) } } else if !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("changed %s accepted: %v", field, err) } }) } }