package hostops import ( "context" "errors" "fmt" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) func TestMoveRecoveryStartDurableFence(t *testing.T) { for _, landed := range []bool{false, true} { t.Run(map[bool]string{false: "absent", true: "applied"}[landed], func(t *testing.T) { r := newResolveFixture(t, landed) ctx := context.Background() if r.effect(t).Inputs["record_base_seq"] != "0" { t.Fatal("no durable base") } // Discard the issuing handle: only the durable evidence survives. r.h = nil run := func() (Outcome, error) { var out Outcome err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error { var err error out, err = g.ResolveStart(ctx) return err }) }) return out, err } out, err := run() if landed { if err != nil || out != OutcomeComplete { t.Fatalf("applied=%s,%v", out, err) } return } if !errors.Is(err, ErrMoveStartFenceUnproven) || out != OutcomeUncertain { t.Fatalf("equal base=%s,%v", out, err) } cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey) if err != nil { t.Fatal(err) } if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil { t.Fatal(err) } out, err = run() if err != nil || out != OutcomeNotApplied { t.Fatalf("advanced=%s,%v", out, err) } if _, err := r.lost.Store.Put(ctx, r.raw.recordKey, r.lost.body, r.lost.etag); !errors.Is(err, release.ErrCASConflict) { t.Fatalf("late put=%v", err) } }) } } func TestMoveRecoveryGrantCannotEscape(t *testing.T) { r := newResolveFixture(t, false) ctx := context.Background() if err := runMoveRecovery(ctx, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) { t.Fatal(err) } var saved *MoveRecoveryGrant if err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = g; return nil }) }); err != nil { t.Fatal(err) } if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) { t.Fatalf("escaped=%v", err) } if _, err := (&MoveRecoveryGrant{}).ResolveStart(ctx); !errors.Is(err, ErrPending) { t.Fatalf("zero grant=%v", err) } if err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, nil) }); !errors.Is(err, ErrContract) { t.Fatal(err) } } func TestMoveRecoveryRejectsChangedSequence(t *testing.T) { r := newResolveFixture(t, false) ctx := context.Background() err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error { cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey) if err != nil { return err } if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil { return err } _, err = g.ResolveStart(ctx) return err }) }) if !errors.Is(err, release.ErrCASConflict) { t.Fatalf("changed sequence=%v", err) } } func TestMoveRecoveryGenericGrantCannotRelease(t *testing.T) { r := newResolveFixture(t, false) ctx := context.Background() err := r.w.RunRecovery(ctx, func(ctx context.Context) error { _, err := hostclaim.Update(ctx, r.data, r.moving, hostclaim.ReleaseMove(moveStartID, "ie-box")) return err }) if err == nil { t.Fatal("generic recovery released moving claim") } } func TestMoveRecoveryConvertUsesFenceAndReplaysDecision(t *testing.T) { w := newC7World(t, nil) w.mh = nil ctx := context.Background() run := func(ref hostclaim.FenceRef) error { return w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Convert(ctx, ref) }, w.authority) }) } bad := w.ref bad.SHA256 = "wrong" if err := run(bad); err == nil { t.Fatal("bad fence accepted") } if err := run(w.ref); err != nil { t.Fatal(err) } if w.claim(t).Claim.Op.Kind != hostclaim.KindMoving { t.Fatal("decision pass converted claim") } rec, err := provision.GetRecord(ctx, w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } if rec.Move.State != provision.MoveRelocating { t.Fatal("decision missing") } if err := run(w.ref); err != nil { t.Fatal(err) } c := w.claim(t).Claim if c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != "pinning" || c.Op.Exec != "none" { t.Fatalf("conversion=%+v", c.Op) } if err := run(w.ref); err != nil { t.Fatal(err) } } func TestMoveRecoveryRecordCloseThenRelease(t *testing.T) { w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseCommitted, "target" c.Obligations = nil r.Region = "uk" r.Move.State = provision.MoveCommitted now := r.CreatedAt r.Move.CommittedAt = &now r.Move.OpenedAt = &now r.Move.SourceParkedAt = &now }) ctx := context.Background() run := func(action func(context.Context, *MoveRecoveryGrant) error) error { return w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), action) }) } if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, RecoveryMoveClose) }); err != nil { t.Fatal(err) } if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil { t.Fatal(err) } if w.claim(t).Claim.Op.Kind != hostclaim.KindServing { t.Fatal("not released") } } func TestMoveRecoveryStartRequiresDurableBaseAndHonorsAppliedJournal(t *testing.T) { for _, marker := range []string{"missing-base", "applied-effect", "issued"} { t.Run(marker, func(t *testing.T) { r := newResolveFixture(t, false) ctx := context.Background() cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey) if err != nil { t.Fatal(err) } if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil { t.Fatal(err) } _, err = r.w.retryAdmission(ctx, "bab", "app", "canonical", AdmissionObserved{}, func(a *AppAdmission) error { e := a.Effects[moveStartID] switch marker { case "missing-base": delete(e.Inputs, "record_base_seq") case "applied-effect": e.ResultReference = moveStartID case "issued": e.State = "issued" } a.Effects[moveStartID] = e return nil }) if err != nil { t.Fatal(err) } var out Outcome err = r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error { var err error out, err = g.ResolveStart(ctx) return err }) }) switch marker { case "missing-base": if !errors.Is(err, ErrMoveStartFenceUnproven) { t.Fatal(err) } case "applied-effect": if err != nil || out != OutcomeComplete { t.Fatalf("%s,%v", out, err) } case "issued": if !errors.Is(err, ErrPending) { t.Fatal(err) } } }) } } func TestMoveRecoveryContextCannotMintAfterRunReturns(t *testing.T) { r := newResolveFixture(t, false) var saved context.Context if err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error { saved = ctx; return nil }); err != nil { t.Fatal(err) } if err := runMoveRecovery(saved, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) { t.Fatalf("escaped recovery context=%v", err) } } func TestMoveRecoveryCloseWaitsForObligations(t *testing.T) { w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseCommitted, "target" r.Region = "uk" r.Move.State = provision.MoveCommitted now := r.CreatedAt r.Move.CommittedAt = &now r.Move.OpenedAt = &now r.Move.SourceParkedAt = &now }) err := w.control.RunRecovery(context.Background(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, RecoveryMoveClose) }) }) if !errors.Is(err, ErrPending) { t.Fatalf("outstanding parks=%v", err) } } func TestMoveRecoveryGrantValueCopyExpires(t *testing.T) { r := newResolveFixture(t, false) ctx := context.Background() var saved MoveRecoveryGrant if err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = *g; return nil }) }); err != nil { t.Fatal(err) } if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) { t.Fatalf("copied grant=%v", err) } } func TestMoveRecoveryHookCannotRemint(t *testing.T) { r := newResolveFixture(t, false) err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error { if err := runMoveRecovery(ctx, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) { t.Fatalf("hook reminted=%v", err) } return nil }) }) if err != nil { t.Fatal(err) } } func TestMoveRecoveryRejectsTargetParkedAction(t *testing.T) { w := newJ17World(t, nil) ctx := context.Background() before, tag, err := w.raw.Get(ctx, w.key) if err != nil { t.Fatal(err) } err = w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, MoveRecoveryRecordAction("target-parked")) }) }) if !errors.Is(err, ErrIdentity) { t.Fatalf("unrecognized action=%v", err) } after, afterTag, err := w.raw.Get(ctx, w.key) if err != nil { t.Fatal(err) } if string(before) != string(after) || tag != afterTag { t.Fatal("unrecognized action changed record") } } func TestMoveRecoveryGrantOverwriteCannotRedirectRevocation(t *testing.T) { r := newResolveFixture(t, false) err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error { var saved MoveRecoveryGrant if err := runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = *g; *g = MoveRecoveryGrant{}; return nil }); err != nil { return err } if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) { t.Fatalf("overwritten callback grant=%v", err) } return nil }) if err != nil { t.Fatal(err) } } type moveRecoveryBumpAuthority struct { c7ReplayAuthority bump func() } func (a *moveRecoveryBumpAuthority) CheckOneBoxRegion(ctx context.Context, source, region string) error { if a.bump != nil { f := a.bump a.bump = nil f() } return a.c7ReplayAuthority.CheckOneBoxRegion(ctx, source, region) } func TestMoveRecoveryConvertRefusesSequenceBumpDuringFenceCheck(t *testing.T) { w := newC7World(t, nil) ctx := context.Background() run := func(a hostclaim.FenceAuthority) error { return w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Convert(ctx, w.ref) }, a) }) } if err := run(w.authority); err != nil { t.Fatal(err) } authority := &moveRecoveryBumpAuthority{c7ReplayAuthority: w.authority, bump: func() { cur, err := ReadRecord(ctx, w.raw, w.key) if err != nil { t.Fatal(err) } if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), w.raw, w.key, cur.Body, cur.Body, cur.ETag); err != nil { t.Fatal(err) } }} if err := run(authority); !errors.Is(err, release.ErrCASConflict) { t.Fatalf("sequence changed during verification=%v", err) } if w.claim(t).Claim.Op.Kind != hostclaim.KindMoving { t.Fatal("converted at stale record sequence") } } func TestMoveRecoveryPassUsesTypedHookOnlyInsideRunRecovery(t *testing.T) { f := newResolveFixture(t, true) noWork := func(context.Context) error { return nil } wrongKind := func(context.Context, hostclaim.Observed) error { t.Fatal("move reached generic recovery"); return nil } calls := 0 r := Recovery{Controller: Controller{Store: f.raw, Control: f.w, Authority: &f.authority, Now: func() time.Time { return f.now }, NewID: func() string { return "recovery" }}, Purge: wrongKind, Placement: wrongKind, SweepRecords: noWork, SweepPurges: noWork, SweepAdmissions: noWork} r.Moves = func(ctx context.Context, g *MoveRecoveryGrant) error { calls++ if g.Observation().OpID != f.moving.Claim.Op.ID { t.Fatal("wrong move grant") } outcome, err := g.ResolveStart(ctx) if err != nil || outcome != OutcomeComplete { t.Fatalf("settle=%s,%v", outcome, err) } return nil } if err := r.Pass(t.Context()); !errors.Is(err, ErrContract) || calls != 0 { t.Fatalf("outside recovery: %v calls=%d", err, calls) } missing := r missing.Moves = nil if err := f.w.RunRecovery(t.Context(), missing.Pass); err != nil { t.Fatalf("missing hook: %v", err) } var saved context.Context if err := f.w.RunRecovery(t.Context(), func(ctx context.Context) error { saved = ctx; return r.Pass(ctx) }); err != nil { t.Fatal(err) } if calls != 1 { t.Fatalf("calls=%d", calls) } if err := r.Pass(saved); !errors.Is(err, ErrContract) || calls != 1 { t.Fatalf("retained pass context: %v calls=%d", err, calls) } } func TestMoveRecoveryReleaseRequiresMatchingOutcomeAndExecutor(t *testing.T) { for _, kind := range []string{hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindMoving} { for _, outcome := range []string{"done", "rolled_back", "aborted"} { for _, foreign := range []bool{false, true} { t.Run(kind+"/"+outcome+fmt.Sprint(foreign), func(t *testing.T) { valid := !foreign && ((kind == hostclaim.KindLanding && outcome == "done") || ((kind == hostclaim.KindRestoring || kind == hostclaim.KindReverting) && outcome == "rolled_back") || (kind == hostclaim.KindMoving && outcome == "aborted")) w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = kind, hostclaim.PhaseOpened, c.Op.Intent["source"] if kind == hostclaim.KindLanding { c.Op.Phase, c.Op.Exec = hostclaim.PhaseCommitted, c.Op.Intent["target"] } if kind == hostclaim.KindMoving { c.Op.Phase = hostclaim.PhaseAborting } if foreign { c.Op.Exec = "foreign" } c.Obligations = nil r.LastMove = &provision.MoveOutcome{ID: r.Move.ID, Outcome: outcome} r.Move = nil }) before := w.claim(t) err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, before, func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }) }) if valid && err != nil { t.Fatal(err) } if !valid && (err == nil || w.claim(t).ETag != before.ETag) { t.Fatalf("invalid release: %v", err) } }) } } } }