package hostops import ( "context" "crypto/ed25519" "encoding/json" "errors" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" "github.com/superfly/ltx" "strings" "testing" ) type plan12BumpFence struct { c7ReplayAuthority bump func() reject bool } func (a *plan12BumpFence) ValidateFence(ctx context.Context, f termfence.Record) (hostclaim.FenceBinding, error) { if a.reject { return hostclaim.FenceBinding{}, hostclaim.ErrFence } if a.bump != nil { fn := a.bump a.bump = nil fn() } return a.c7ReplayAuthority.ValidateFence(ctx, f) } func TestPlan12RecoveryFenceGrant(t *testing.T) { for _, mode := range []string{"ok", "admitted", "installed", "from-landing", "from-restoring", "record-released", "changed-sequence", "dirty", "dirty-unregistered", "escaped", "wrong-ref"} { t.Run(mode, func(t *testing.T) { w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, a *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseProbing, "source" c.Op.Intent["source"], c.Op.Intent["target"], c.Op.Intent["to"], c.Op.Intent["target_enrollment"] = "target", "source", "ie", strings.Repeat("a", 32) c.Op.Participants = []string{"target", "source"} c.Obligations = nil c.Op.Journal = map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"} r.Region = "uk" r.Move.From, r.Move.To, r.Move.State = "uk", "ie", provision.MoveRollingBack at := r.CreatedAt r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", Reason: provision.RollbackTargetFenced, Evidence: "control/box-fences/" + strings.Repeat("a", 32) + ".json", RequestedAt: &at, TargetPhase: provision.RbTargetLost, LossRef: "control/box-fences/" + strings.Repeat("a", 32) + ".json", LostAt: &at, SourcePhase: provision.RbSourceWaiting, Dirty: mode == "dirty" || mode == "dirty-unregistered"} if mode == "dirty-unregistered" { // The registration gone: the grant refuses before it // verifies the fence, so the refusal is not verified-dirty. delete(a.Registrations, c.Op.ID) } if mode == "admitted" || mode == "installed" { c.Holder.Session = 7 c.Holder.Stamp = 7 c.Holder.Generation = "target-gen" c.Holder.WriterBox = "source" c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 7, Box: "source", Generation: "target-gen", Mode: "cold", Region: "ie"}} } if mode == "installed" { c.Inventory = map[string]hostclaim.InventoryEntry{"source": {Layout: "flat", Paths: map[string]uint64{"app.db": 7}}} } if mode == "from-restoring" || mode == "record-released" { c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseReleasing c.Op.Intent["rollback"] = r.Move.Rollback.ID c.Op.Journal[hostclaim.JournalTargetStamp] = "7" r.Move.Rollback.TargetPhase = provision.RbTargetReleasing r.Move.Rollback.LossRef = "" r.Move.Rollback.LostAt = nil r.Move.Rollback.Reason = provision.RollbackLandingDeadline } if mode == "record-released" { r.Move.Rollback.TargetPhase = provision.RbTargetReleased r.Move.Rollback.Evidence = moves.StoppedKey(moves.ReportPrefix("bab", "app", c.Host, c.Op.ID), "target") } if mode == "from-landing" { r.Region = "ie" r.Move.State = provision.MoveLanding r.Move.Rollback = nil } }) if mode == "record-released" { c := w.claim(t).Claim report := moves.StoppedReport{V: 1, Role: "target", Box: "source", Host: w.host, OpID: c.Op.ID, Incarnation: c.Holder.Incarnation, CheckedAt: w.now, Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}} body, _ := json.Marshal(report) if _, err := w.raw.Put(t.Context(), moves.StoppedKey(moves.ReportPrefix("bab", "app", w.host, c.Op.ID), "target"), body, ""); err != nil { t.Fatal(err) } } authority := &plan12BumpFence{c7ReplayAuthority: w.authority} if mode == "changed-sequence" { authority.bump = func() { r, e := ReadRecord(t.Context(), w.raw, w.key) if e != nil { t.Fatal(e) } if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, r.Body, r.Body, r.ETag); e != nil { t.Fatal(e) } } } ref := w.ref if mode == "wrong-ref" { ref.SHA256 = strings.Repeat("e", 64) } var escaped *MoveRecoveryGrant var out FencedTargetOutcome run := func() error { return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { if mode == "escaped" { escaped = g return nil } var err error out, err = g.ReleaseFencedTarget(ctx, ref) return err }, authority) }) } err := run() if mode == "escaped" { out, err = escaped.ReleaseFencedTarget(t.Context(), ref) } // Only a dirty target behind a verified fence is verified-dirty // (Task 5 fix round 1, item 1). if (mode == "dirty") != out.VerifiedDirty || (mode == "dirty" || mode == "dirty-unregistered") && !errors.Is(err, ErrPending) { t.Fatalf("%s: verified-dirty %v, err %v", mode, out.VerifiedDirty, err) } if mode != "ok" && mode != "admitted" && mode != "installed" && mode != "from-landing" && mode != "from-restoring" && mode != "record-released" { if err == nil { t.Fatal("accepted", mode) } if mode == "changed-sequence" && !errors.Is(err, release.ErrCASConflict) { t.Fatal(err) } return } if err != nil { t.Fatal(err) } // The outcome carries the step's own write (Task 5 fix round 1, // item 2): the record's transition, or the claim's rollback. wantEvent := map[string]string{"from-landing": "rolling_back", "from-restoring": "declared_lost"}[mode] if got := ""; out.Transition != nil || wantEvent != "" { if out.Transition != nil { got = out.Transition.Event } if got != wantEvent { t.Fatalf("%s: the record's transition %q, want %q", mode, got, wantEvent) } } if wantRollback := mode == "ok" || mode == "admitted" || mode == "installed"; out.ClaimRolledBack != wantRollback { t.Fatalf("%s: claim rollback %v, want %v", mode, out.ClaimRolledBack, wantRollback) } if mode == "from-landing" || mode == "from-restoring" { if mode == "from-landing" && w.claim(t).Claim.Op.Kind != hostclaim.KindLanding { t.Fatal("record-first pass changed claim") } if err := run(); err != nil { t.Fatal("claim pass", err) } if out.Transition != nil || out.ClaimRolledBack != (mode == "from-landing") { t.Fatalf("%s: the claim pass's outcome %+v", mode, out) } } o := w.claim(t) if o.Claim.Op.Phase != hostclaim.PhaseReleased || len(o.Claim.Op.Participants) != 1 || len(o.Claim.Obligations) != func() int { if mode == "installed" { return 3 } return 2 }() { t.Fatal(o.Claim) } if len(o.Claim.Lost) != 1 || len(o.Claim.LossRefs) != 1 || len(o.Claim.EvidenceRefs) != 1 || o.Claim.EvidenceRefs[0].Reported { t.Fatal("missing pending automatic loss", o.Claim) } if mode == "admitted" || mode == "installed" { l := o.Claim.EvidenceRefs[0] if l.Session != 7 || l.Generation != "target-gen" || l.LastAdmission == nil || l.LastAdmission.Covered { t.Fatal("lost writer provenance", l) } } // A retired target or witness no longer validates against today's roster. authority.reject = true before, _ := json.Marshal(o.Claim.Obligations) if err := run(); err != nil { t.Fatal("replay", err) } after, _ := json.Marshal(w.claim(t).Claim.Obligations) if string(before) != string(after) { t.Fatal("requeued parks") } rec, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host) if err != nil || rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || (mode != "record-released" && rec.Move.Rollback.Evidence != ref.Key) { t.Fatal(rec, err) } if err := run(); err != nil { t.Fatal("completed replay", err) } if mode == "from-restoring" && rec.Move.Rollback.Reason != provision.RollbackLandingDeadline { t.Fatal("rewrote first reason") } act := func(fn func(context.Context, *MoveRecoveryGrant) error) error { return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), fn, authority) }) } if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err == nil { t.Fatal("stamped with parks owed") } current := w.claim(t) for _, ob := range current.Claim.Obligations { var e error current, e = hostclaim.Update(t.Context(), w.raw, current, hostclaim.CompleteObligation(ob.Op, ob.ID)) if e != nil { t.Fatal(e) } } if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err != nil { t.Fatal("stamp", err) } _, stampTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } claimTag := w.claim(t).ETag if err := run(); err != nil { t.Fatal("release replay after parked stamp", err) } _, replayTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host) if err != nil || replayTag != stampTag || w.claim(t).ETag != claimTag { t.Fatal("parked replay wrote state", err) } if mode == "admitted" { plan12GrantLossMaximum(t, w) } // Source opening is owned by the source; seed its completed evidence. current = w.claim(t) if _, err := hostclaim.Update(t.Context(), w.raw, current, func(c *hostclaim.Claim) error { c.Op.Phase = hostclaim.PhaseOpened c.Op.Exec = c.Op.Intent["source"] return nil }); err != nil { t.Fatal(err) } rr, e := ReadRecord(t.Context(), w.raw, w.key) if e != nil { t.Fatal(e) } decoded, e := provision.DecodeRecord(rr.Body) if e != nil { t.Fatal(e) } rec = &decoded rec.Move.Rollback.OpenedAt, rec.Move.Rollback.RecoveredAt = &w.now, &w.now body, _ := json.Marshal(rec) if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, rr.Body, body, rr.ETag); e != nil { t.Fatal(e) } if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, RecoveryMoveRolledBackClose) }); err != nil { t.Fatal("close", err) } if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil { t.Fatal("release", err) } afterClose := w.claim(t).Claim if afterClose.Op.Kind != hostclaim.KindServing || len(afterClose.EvidenceRefs) != 1 || len(afterClose.LossRefs) != 1 { t.Fatal("loss evidence erased", afterClose) } // Reporting works after operation replacement and needs no live roster. pub, priv, e := ed25519.GenerateKey(nil) if e != nil { t.Fatal(e) } w.control.pub = pub if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host) }); err != nil { t.Fatal("report", err) } l := w.claim(t).Claim.EvidenceRefs[0] reported, e := hostclaim.ReadLossRecord(t.Context(), w.raw, pub, l.ID) if e != nil || reported.Record.State != "committed" || !reported.Record.AlertsPending || reported.Record.Automatic == nil || reported.Record.Automatic.Fence != ref || !l.Reported { t.Fatal("durable reporting", reported, e) } if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host) }); err != nil { t.Fatal("report replay", err) } }) } } func plan12GrantLossMaximum(t *testing.T, w *c7World) { t.Helper() o := w.claim(t) source := o.Claim.Op.Intent["source"] o, err := hostclaim.Update(t.Context(), w.raw, o, hostclaim.ReturnToSource(o.Claim.Op.ID, source, false, hostclaim.ReturnFence{})) if err != nil { t.Fatal(err) } o, err = hostclaim.Update(t.Context(), w.raw, o, hostclaim.SetRestoringPhase(o.Claim.Op.ID, hostclaim.PhaseClaimed, hostclaim.PhaseProbing)) if err != nil { t.Fatal(err) } in := hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: source, Mode: "cold", NewGeneration: "recovered", Region: "uk", PredecessorRegion: "ie", RemoteMaxTXID: 3} checked := false writer := hostbox.Writer{Claims: w.raw, Box: source, Region: "uk", AdmissionEvidence: func(_ context.Context, _ hostclaim.Observed, a hostclaim.Admission, p *genselect.Plan) error { checked = true if a.RemoteMaxTXID != 7 || p.MaxTXID != 3 { t.Fatalf("loss=%d restore=%d", a.RemoteMaxTXID, p.MaxTXID) } return nil }, Loss: &hostbox.LossAdmission{Sources: func(_ context.Context, _ hostclaim.Observed, l hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) { if len(l.WriteSources) != 1 { t.Fatalf("sources %+v", l.WriteSources) } s := l.WriteSources[0] return []hostclaim.ValidatedWriteSource{{Reason: s.Reason, Box: s.Box, Session: s.Session, Generation: s.Generation, Region: s.Region, Baseline: s.Baseline, RemoteMaxTXID: 7}}, nil }}} // The restore plan's selected files reach its maximum // (genselect.GenerationMax, moves Task 4). root := genselect.File{Key: strings.TrimSuffix(o.Claim.Holder.Location, "/") + "/g-src/0009/" + ltx.FormatFilename(1, 3), VersionID: "v1", Level: 9, MinTXID: 1, MaxTXID: 3, Size: 100} admitted, err := writer.Admit(t.Context(), o, in, &genselect.Plan{Generation: "src", Files: []genselect.File{root}, Root: &root, MaxTXID: 3}) if err != nil { t.Fatal(err) } records := admitted.Claim.Holder.Admissions if !checked || records[len(records)-1].WritesLost.RemoteMaxTXID != 7 { t.Fatal("fenced maximum lost") } }