package hostops import ( "context" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" ) // Moves into the home region, part 2, Task 4: the loss and the fenced // release of a home target, from clean claim state. The fixtures are the // loss world and Plan 13's fence world, each turned into a move into home: // the record's To is the home's code "", the op's to is @home, written with // home_region set to the target box's roster name, and the console names // the same home. The home's roster name may appear only in what the claim // recorded about the box itself (its admissions, the loss record's write // sources); every region the moves code supplies is @home. // onlyHomeToken asserts a claim that started clean: every chore that is // not the named source's is in @home, owned by region:@home; nothing names // the home's roster name or the unset region in a chore, touched or // quiet_after; and the op's to is still @home. func onlyHomeToken(t *testing.T, c *hostclaim.Claim, source, rosterHome string) { t.Helper() for _, o := range c.Obligations { switch { case o.Kind == "park-local": case o.Region == source: if o.Owner != "region:"+source { t.Fatalf("a source chore %+v", o) } case o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion: t.Fatalf("a target chore not named %s: %+v", park.HomeRegion, o) } } for _, r := range c.Touched.Regions { if r == rosterHome || r == "" { t.Fatalf("touched names %q: %+v", r, c.Touched) } } for k := range c.QuietAfter { if k == rosterHome || k == "" { t.Fatalf("quiet_after names %q: %v", k, c.QuietAfter) } } if to := c.Op.Intent["to"]; to != "" && to != park.HomeRegion { t.Fatalf("the op's to is %q", to) } } // TestMoveIntoHomeDeclareLost: a home target (uk-box, roster region uk, // the fleet's home) stopped mid-landing and declared lost through Plan 4 // Task 10's door. The signed loss intent's TargetRegion is @home, copied // from op.intent.to, and its SourceRegion is ie; ReleaseLostTarget succeeds // with TargetRegion @home against intent.to @home and queues the target's // location and lane parks in @home; the record cites the loss. The source // is then re-admitted with the loss through AdmitWriterWithLoss and the // moves loss owner, its predecessor the lost target's last admission in // the spelling it recorded, uk. func TestMoveIntoHomeDeclareLost(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Intent["to"], c.Op.Intent["home_region"] = park.HomeRegion, "uk" r.Move.To, r.Region = "", "" }) w.control.homeRegion = "uk" before := w.claim(t) if len(before.Claim.Obligations) != 0 || len(before.Claim.QuietAfter) != 0 { t.Fatalf("fixture: the claim is not clean: %+v", before.Claim) } onlyHomeToken(t, before.Claim, "ie", "uk") in := w.inspect(t, "uk-label") if in.Details.Box != "uk-box" || in.Details.Op.Kind != hostclaim.KindLanding { t.Fatalf("inspection = %+v", in.Details) } after, err := w.declare(w.submission(in)) if err != nil { t.Fatal(err) } id := in.Details.ID loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id) if err != nil || loss.Record.State != "committed" { t.Fatalf("loss record = %+v %v", loss.Record, err) } intent, _, err := lossIntentOf(loss.Record) if err != nil { t.Fatal(err) } if intent.TargetRegion != park.HomeRegion || intent.SourceRegion != "ie" || intent.Target != "uk-box" || intent.Source != "ie-box" { t.Fatalf("the signed loss intent: %+v", intent) } // The write sources are what the claim recorded about the box: its own // region, uk. for _, s := range loss.Record.WriteSources { if s.Reason == "target" && s.Region != "uk" { t.Fatalf("a target write source %+v", s) } } c := w.claim(t) if c.ETag != after.ETag || c.Claim.Op.Kind != hostclaim.KindRestoring || c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Exec != "none" || !hostclaim.LostTargetReleased(c.Claim, id) || c.Claim.Op.Intent["to"] != park.HomeRegion { t.Fatalf("the claim after the declaration: %+v", c.Claim.Op) } parks := map[string]string{} for _, o := range c.Claim.Obligations { parks[o.Region+"|"+o.Park.Location] = o.Owner } if want := map[string]string{park.HomeRegion + "|instances/" + lossHost: "region:" + park.HomeRegion, park.HomeRegion + "|moves/" + lossHost + "/" + lossOp: "region:" + park.HomeRegion}; !reflect.DeepEqual(parks, want) { t.Fatalf("parks = %v, want %v", parks, want) } onlyHomeToken(t, c.Claim, "ie", "uk") lossRef, _ := hostclaim.LossRecordKey(id) rec, _ := w.record(t) if rb := rec.Move.Rollback; rec.Region != "ie" || rec.Move.To != "" || rb == nil || rb.LossRef != lossRef || rb.Reason != provision.RollbackDeclaredLost { t.Fatalf("the record: region %q move %+v rollback %+v", rec.Region, rec.Move, rb) } // The source takes execution back on the declaration's authority and // probes; the recovery's admission carries the loss. g := provision.MoveGuard{ID: lossOp, Incarnation: w.inc} for _, step := range [][2]string{{provision.RbSourceWaiting, provision.RbSourceClaimed}, {provision.RbSourceClaimed, provision.RbSourceRestoring}, {provision.RbSourceRestoring, provision.RbSourceVerified}, {provision.RbSourceVerified, provision.RbSourceProbing}} { if _, err := provision.SetRecordMove(ctx, w.raw, "bab", "app", lossHost, provision.MoveRollbackSourcePhase(g, step[0], step[1], nil)); err != nil { t.Fatalf("%s to %s: %v", step[0], step[1], err) } } o := w.claim(t) if o, err = hostclaim.Update(ctx, w.raw, o, hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})); err != nil { t.Fatalf("the source's claimed: %v", err) } if o, err = hostclaim.Update(ctx, w.raw, o, hostclaim.SetMovePhase(lossOp, hostclaim.KindRestoring, hostclaim.PhaseClaimed, hostclaim.PhaseProbing)); err != nil { t.Fatalf("the source's probing: %v", err) } h := o.Claim.Holder last := h.Admissions[len(h.Admissions)-1] if last.Box != "uk-box" || last.Region != "uk" { t.Fatalf("fixture: the target's last admission %+v", last) } validated := make([]hostclaim.ValidatedWriteSource, 0, len(loss.Record.WriteSources)) var more []hostclaim.WritesLost for _, s := range loss.Record.WriteSources { v := hostclaim.ValidatedWriteSource{Reason: s.Reason, Box: s.Box, Generation: s.Generation, Session: s.Session, Region: s.Region, Baseline: s.Baseline} if s.Reason == "source-cut" { v.RemoteMaxTXID = 42 } validated = append(validated, v) if s.Box != last.Box || s.Generation != last.Generation || s.Session != last.Session { more = append(more, hostclaim.WritesLost{Box: s.Box, Generation: s.Generation, Session: s.Session, RemoteMaxTXID: v.RemoteMaxTXID}) } } admission := hostclaim.Admission{ObservedOpID: lossOp, ObservedPhase: hostclaim.PhaseProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "ie-box", Mode: "cold", NewGeneration: "fresh1", Region: "ie", PredecessorRegion: last.Region, AdditionalWritesLost: more} if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, loss.Record, admission); err != nil { t.Fatalf("the moves loss owner on the signed intent: %v (claim %+v holder stamp %d)", err, o.Claim.Op, h.Stamp) } admitted, err := hostclaim.AdmitWriterWithLoss(ctx, w.raw, w.pub, o, admission, id, validated, MoveLossRecoveryOwner()) if err != nil { t.Fatalf("the source's admission with the loss: %v", err) } a := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1] if a.LossID != id || a.Box != "ie-box" || a.Region != "ie" || a.Predecessor != last.Generation || a.PredecessorRegion != "uk" || a.WritesLost == nil || a.WritesLost.Box != "uk-box" { t.Fatalf("the loss admission: %+v", a) } if !hostclaim.OperationLossConsumed(admitted.Claim, id) || admitted.Claim.Holder.WriterBox != "ie-box" { t.Fatalf("the loss was not consumed: %+v", admitted.Claim.Holder) } onlyHomeToken(t, admitted.Claim, "ie", "uk") // The moves loss owner holds the loss intent's @home to op.intent.to // exactly: a loss record whose intent names the roster name is refused. bad := loss named := intent named.TargetRegion = "uk" raw, err := named.encode() if err != nil { t.Fatal(err) } bad.Record.ConfirmedConsequences.RecoveryIntent = raw if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, bad.Record, admission); err == nil { t.Fatal("a loss intent naming uk was accepted against intent.to @home") } if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, loss.Record, admission); err != nil { t.Fatalf("the signed intent: %v", err) } } // TestMoveIntoHomeFencedTargetRelease: a home target terminated after it // released, from clean claim state, on Plan 13's fence world. The recovery // grant's ReleaseFencedTarget builds FencedTargetRelease with TargetRegion // park.PlacementRegion(m.To), @home, and SourceRegion uk (the world's // named source); hostclaim.ReleaseFencedTarget accepts it against // intent.to @home; the location and lane parks it queues are in @home, // owned by region:@home; the record keeps the home's code. func TestMoveIntoHomeFencedTargetRelease(t *testing.T) { w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseProbing, "source" c.Op.Intent["source"], c.Op.Intent["target"], c.Op.Intent["target_enrollment"] = "target", "source", strings.Repeat("a", 32) c.Op.Intent["to"], c.Op.Intent["home_region"] = park.HomeRegion, "ie" c.Op.Participants = []string{"target", "source"} c.Obligations = nil c.Op.Journal = map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"} r.Region = "uk" r.Move.From, r.Move.To, r.Move.State = "uk", "", provision.MoveRollingBack at := r.CreatedAt key := "control/box-fences/" + strings.Repeat("a", 32) + ".json" r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", Reason: provision.RollbackTargetFenced, Evidence: key, RequestedAt: &at, TargetPhase: provision.RbTargetLost, LossRef: key, LostAt: &at, SourcePhase: provision.RbSourceWaiting} }) w.authority.home = "ie" w.control.homeRegion = "ie" before := w.claim(t).Claim if len(before.Obligations) != 0 { t.Fatalf("fixture: the claim owes %+v", before.Obligations) } onlyHomeToken(t, before, "uk", "ie") run := func() error { return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { _, err := g.ReleaseFencedTarget(ctx, w.ref) return err }, w.authority) }) } if err := run(); err != nil { t.Fatal(err) } c := w.claim(t).Claim if c.Op.Phase != hostclaim.PhaseReleased || c.Op.Intent["to"] != park.HomeRegion || c.Op.Journal["fence.source"] == "" { t.Fatalf("the claim after the fenced release: %+v", c.Op) } parks := map[string]string{} for _, o := range c.Obligations { if o.Park == nil { t.Fatalf("a chore without a scope: %+v", o) } parks[o.Region+"|"+o.Park.Location] = o.Owner } wantLocation, wantLane := false, false for k, owner := range parks { region, location, _ := strings.Cut(k, "|") if region != park.HomeRegion || owner != "region:"+park.HomeRegion { t.Fatalf("a park in %q owned by %q: %v", region, owner, parks) } wantLocation = wantLocation || location == strings.TrimSuffix(c.Holder.Location, "/") wantLane = wantLane || strings.HasPrefix(location, "moves/"+w.host+"/") } if !wantLocation || !wantLane { t.Fatalf("the fenced release's parks: %v", parks) } onlyHomeToken(t, c, "uk", "ie") // The record's completion keeps the home's code. if err := run(); err != nil { t.Fatal("record completion", err) } rec, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host) if err != nil || rec.Move.To != "" { t.Fatalf("the record's target placement: %+v %v", rec.Move, err) } }