package host import ( "context" "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" "os/exec" "path/filepath" "strings" "testing" "time" ) type plan12FenceAuthority struct { reject bool home string } func (a plan12FenceAuthority) ValidateFence(context.Context, termfence.Record) (hostclaim.FenceBinding, error) { if a.reject { return hostclaim.FenceBinding{}, hostclaim.ErrFence } return hostclaim.FenceBinding{FleetHomeRegion: a.home, Region: "uk", RosterRevision: 1, RosterETag: "roster"}, nil } func plan12Handover(t *testing.T, kind, mode string) (*recoveryFixture, hostclaim.Observed, claimCrossBoxHandover) { t.Helper() f := newRecoveryFixture(t) o := f.read() key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, "m1"), "target") journal := map[string]string{hostclaim.JournalEvidenceTarget: key, hostclaim.JournalTargetSession: "2", hostclaim.JournalTargetGeneration: "target-gen"} if mode == "wrong-session" { journal[hostclaim.JournalTargetSession] = "3" } pub := f.signer.Public().(ed25519.PublicKey) h := claimCrossBoxHandover{Guard: f.guard, Evidence: f.claims, Box: "box-1", PublicKey: pub, FenceAuthority: plan12FenceAuthority{reject: mode == "unverified-fence"}} if mode == "fence" || mode == "unverified-fence" { record := termfence.Record{V: 1, Box: "box-2", Enrollment: strings.Repeat("a", 32), State: "terminated", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)} raw, err := json.Marshal(record) if err != nil { t.Fatal(err) } sum := sha256.Sum256(raw) ref := hostclaim.FenceRef{Enrollment: record.Enrollment, Key: "control/box-fences/" + record.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])} if _, err = f.claims.Put(t.Context(), ref.Key, raw, ""); err != nil { t.Fatal(err) } refRaw, _ := json.Marshal(ref) journal = map[string]string{"fence.box-2": string(refRaw)} } if mode == "loss" || mode == "wrong-loss-op" { journal = map[string]string{"loss.loss_1": `{"kind":"move-target-lost","target":"box-2"}`} } var err error o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op = plan10J2Op(kind, hostclaim.PhaseProbing) c.Op.Journal = journal c.Op.Intent["target_enrollment"] = strings.Repeat("a", 32) c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "box-2" if mode == "loss" || mode == "wrong-loss-op" { c.LossRefs = []string{"loss_1"} } return nil }) if err != nil { t.Fatal(err) } if err = f.guard.Activate(t.Context(), o.Host, "box-1", nil); err != nil { t.Fatal(err) } report := plan12Stopped(o, "target", "box-2") if mode == "wrong-op" { report.OpID = "another" } raw, err := json.Marshal(report) if err != nil { t.Fatal(err) } if _, err = f.claims.Put(t.Context(), key, raw, ""); err != nil { t.Fatal(err) } if mode == "loss" || mode == "wrong-loss-op" { now := time.Now().UTC() r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "m1", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "box-2", Enrollment: "enroll", QuarantineRevision: 1, CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now, ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "box-2", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}} if mode == "wrong-loss-op" { r.OpID = "other" } made, err := hostclaim.CreateLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), r) if err != nil { t.Fatal(err) } r.State, r.ResultClaimGen = "committed", o.Claim.Gen if _, err = hostclaim.ReplaceLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), made, r); err != nil { t.Fatal(err) } } h.Identity = handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "ie"}) h.Inventory, err = openLegacyInventory(t, filepath.Join(t.TempDir(), "inventory.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { h.Inventory.Close() }) h.Registry, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { h.Registry.Close() }) row := registry.Route{Host: o.Host, Kind: "instance", DBPath: provision.NestedDBPath(f.guard.DataDir, o.Host), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: o.Claim.Holder.Incarnation, ClaimStamp: o.Claim.Holder.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-row"}} if err = h.Registry.Upsert(row); err != nil { t.Fatal(err) } if err = h.Inventory.WriteStopInventory(t.Context(), hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: row.ClaimIncarnation, InstallationStamp: row.ClaimStamp, Box: "box-1", Enrollment: "enrollment-1", DirectoryRevision: "7", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: o.Host, RowIdentity: "old-row", Ownership: "old-owner", Materialization: "materialized"}}}); err != nil { t.Fatal(err) } o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Journal[hostclaim.JournalSourceStamp] = fmt.Sprint(row.ClaimStamp) return nil }) if err != nil { t.Fatal(err) } return f, o, h } func TestPlan12RecoveryPredecessorEvidence(t *testing.T) { t.Parallel() for _, kind := range []string{hostclaim.KindRestoring} { for _, mode := range []string{"stop", "loss", "fence", "wrong-session", "wrong-op", "wrong-loss-op", "unverified-fence"} { t.Run(kind+"/"+mode, func(t *testing.T) { _, o, h := plan12Handover(t, kind, mode) if !claimRemotePredecessor(o, "box-1") { t.Fatal("local path selected") } err := h.Before(t.Context(), o) valid := mode == "stop" || mode == "loss" || mode == "fence" if valid && err != nil || !valid && err == nil { t.Fatalf("%s: %v", mode, err) } }) } } } func TestPlan12RecoveryAdmissionAfterTargetSession(t *testing.T) { t.Parallel() for _, kind := range []string{hostclaim.KindRestoring} { t.Run(kind, func(t *testing.T) { f, o, h := plan12Handover(t, kind, "stop") c := f.controller("boot-1", "controller-1", 4101, "start-1") c.wake.Inputs = plan9D2Inputs{recoveryInputs{f}} c.wake.Operations = &plan9D2Attempts{} c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } wireClaimAdmission(c.wake, "box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { t.Fatal("local predecessor handover") return nil }, h, nil, nil) if err := c.wake.stopPredecessor(t.Context(), o); err != nil { t.Fatal(err) } live, cancel := context.WithCancel(t.Context()) defer cancel() done := make(chan struct{}) go func() { <-live.Done(); close(done) }() c.wake.mu.Lock() c.wake.pending[o.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} c.wake.mu.Unlock() got, err := c.runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, activator.StartPolicy{Probe: true}) if err != nil { t.Fatal(err) } if got.Claim.Holder.Session <= 2 || got.Claim.Holder.Stamp != got.Claim.Holder.Session || got.Claim.Holder.WriterBox != "box-1" { t.Fatal(got) } }) } } func plan12Stopped(o hostclaim.Observed, role, box string) moves.StoppedReport { return moves.StoppedReport{V: 1, Role: role, Box: box, Host: o.Host, OpID: o.Claim.Op.ID, Incarnation: o.Claim.Holder.Incarnation, ClaimGen: o.Claim.Gen, CheckedAt: time.Now().UTC(), Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}} } func TestPlan12RecoveryLocalRouting(t *testing.T) { for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} { for _, box := range []string{"", "box-1"} { o := plan10J2Observed(kind, hostclaim.PhaseProbing, "box-1") o.Claim.Holder = &hostclaim.Holder{WriterBox: box} if claimRemotePredecessor(o, "box-1") { t.Fatal(kind, box) } called := false hook := claimAdmissionHandover("box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { called = true; return nil }, func(context.Context, hostclaim.Observed, hostclaim.Observed) error { t.Fatal("remote"); return nil }) if err := hook(t.Context(), o, o); err != nil || !called { t.Fatal(err) } } } } func TestPlan12ConsumedFenceSurvivesRosterChange(t *testing.T) { t.Parallel() f, o, h := plan12Handover(t, hostclaim.KindRestoring, "fence") rb := "rb_abcdefghijklmnopqrstuvwxyz" var ref hostclaim.FenceRef if err := json.Unmarshal([]byte(o.Claim.Op.Journal["fence.box-2"]), &ref); err != nil { t.Fatal(err) } o, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Exec = "box-2" c.Op.Phase = hostclaim.PhaseReleasing c.Op.Intent["rollback"] = rb return nil }) if err != nil { t.Fatal(err) } fence, err := hostclaim.VerifyMoveTargetFence(t.Context(), f.claims, o, ref, h.FenceAuthority) if err != nil { t.Fatal(err) } // Admission consumes a previously grant-released claim. The grant itself // is exercised in hostops.TestPlan12RecoveryFenceGrant. o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { raw, _ := json.Marshal(ref) sum := sha256.Sum256([]byte(c.Op.ID + "\x00box-2\x00" + ref.Enrollment + "\x00" + ref.SHA256)) id := "auto-" + hex.EncodeToString(sum[:16]) loss := hostclaim.AutomaticLoss{ID: id, Box: "box-2", Enrollment: ref.Enrollment, Fence: ref, ObservedAt: fence.ObservedAt().Format(time.RFC3339Nano), Op: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase, Role: "target", Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp, Session: c.Holder.Session, Generation: c.Holder.Generation, Timers: hostclaim.TimersUnknown} if len(c.Holder.Admissions) > 0 { a := c.Holder.Admissions[len(c.Holder.Admissions)-1] loss.LastAdmission = &a } c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseReleased, "none" c.Op.Intent["rollback"] = rb c.Op.Participants = []string{c.Op.Intent["source"]} c.Op.Journal["fence.box-2"], c.Op.Journal["fenced_target_released"] = string(raw), string(raw) c.Lost, c.LossRefs, c.EvidenceRefs = []string{"box-2"}, []string{id}, []hostclaim.AutomaticLoss{loss} return nil }) if err != nil { t.Fatal(err) } o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Exec = "box-1"; c.Op.Phase = hostclaim.PhaseProbing; return nil }) if err != nil { t.Fatal(err) } h.FenceAuthority = plan12FenceAuthority{reject: true} if err := h.Before(t.Context(), o); err != nil { t.Fatal("consumed evidence depended on roster", err) } o.Claim.EvidenceRefs = nil if err := h.Before(t.Context(), o); err == nil { t.Fatal("journal alone accepted as durable loss") } }