package host import ( "context" "errors" "net/http" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/schedule" ) // statusDeliverer is a schedule.Deliverer answering one status to every tick. type statusDeliverer int func (s statusDeliverer) Deliver(context.Context, registry.Route, *http.Request) (int, error) { return int(s), nil } const ( mhHost = "a.go.example.com" mhID = "mv_abcdefghijklmnopqrstuvwxyz" mhRbID = "rb_abcdefghijklmnopqrstuvwxyz" ) // mhRecord is a Phase 1 record mid-move from ie to uk, in state with the // given phases; region is the record's region at that state. func mhRecord(region, state, cutPhase string) provision.Record { return provision.Record{Schema: 1, Host: mhHost, App: "app", Account: "bab", Region: region, Channel: "stable", Move: &provision.MoveState{ID: mhID, From: "ie", To: "uk", State: state, CutPhase: cutPhase, Rollback: &provision.MoveRollback{ID: mhRbID}}} } // mhClaims is a claim reader answering kind with participants ie-box and // uk-box for mhHost; err makes every read fail. func mhClaims(kind string, err error) func(context.Context, release.Store, string) (hostclaim.Observed, error) { return func(_ context.Context, _ release.Store, host string) (hostclaim.Observed, error) { if err != nil { return hostclaim.Observed{}, err } return hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Host: host, Op: hostclaim.Op{Kind: kind, ID: mhID, Participants: []string{"ie-box", "uk-box"}}}}, nil } } func mhView(box, region, kind string, err error, recs ...provision.Record) *movesHoldView { v := newMovesHoldView(box, region, mhClaims(kind, err), nil) v.refresh(context.Background(), nil, recs) return v } // hold's table (moves spec §5.2, §9.6, §19.1): the target holds its row // landing after the flip, the source moving from stopping, restoring in a // revert or a rollback, and the target releasing in a rollback. A box the // claim does not name, and any state with nothing held, answers no hold. func TestMovesHoldViewHold(t *testing.T) { cases := []struct { name string box, region, kind string rec provision.Record wantStatus, wantOp string wantOK bool }{ {"source preparing", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MovePreparing, ""), "", "", false}, {"source cut requested", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutRequested), "", "", false}, {"source cut admitted", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutAdmitted), "", "", false}, {"source stopping", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutStopping), "moving", mhID, true}, {"source archived", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutArchived), "moving", mhID, true}, {"source after the flip", "ie-box", "ie", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "moving", mhID, true}, {"target after the flip", "uk-box", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "landing", mhID, true}, {"target committed", "uk-box", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveCommitted, provision.CutFlipped), "", "", false}, {"source reverting", "ie-box", "ie", hostclaim.KindReverting, mhRecord("ie", provision.MoveReverting, provision.CutStopped), "restoring", mhID, true}, {"source aborting a revert", "ie-box", "ie", hostclaim.KindReverting, mhRecord("ie", provision.MoveAborting, ""), "restoring", mhID, true}, {"source aborting before the cut", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveAborting, ""), "", "", false}, {"source rolling back", "ie-box", "ie", hostclaim.KindRestoring, mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped), "restoring", mhRbID, true}, {"target rolling back", "uk-box", "uk", hostclaim.KindRestoring, mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped), "releasing", mhRbID, true}, {"target stuck", "uk-box", "uk", hostclaim.KindRestoring, mhRecord("ie", provision.MoveStuck, provision.CutFlipped), "releasing", mhRbID, true}, {"not a participant", "uk-box-2", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "", "", false}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { v := mhView(c.box, c.region, c.kind, nil, c.rec) status, op, ok := v.hold(c.rec) if status != c.wantStatus || op != c.wantOp || ok != c.wantOK { t.Fatalf("hold = (%q, %q, %v), want (%q, %q, %v)", status, op, ok, c.wantStatus, c.wantOp, c.wantOK) } }) } // No move, or a legacy one: nothing held. v := mhView("uk-box", "uk", hostclaim.KindLanding, nil) if _, _, ok := v.hold(provision.Record{Host: mhHost}); ok { t.Fatal("a record at rest is held") } if _, _, ok := v.hold(provision.Record{Host: mhHost, Move: &provision.MoveState{To: "uk", State: provision.MoveCutting}}); ok { t.Fatal("a legacy move is held") } } // Without a claim answer for the host (the view was built before the move, // or its claim would not read) hold falls back to the record's regions, and // never answers "no hold" for a state that holds: the row is never minted // ready. A rollback with no rollback id holds under the move id. func TestMovesHoldViewHoldFailsClosedWithoutTheClaim(t *testing.T) { stale := mhView("uk-box", "uk", hostclaim.KindLanding, nil) // built from a list with no move in it rec := mhRecord("uk", provision.MoveLanding, provision.CutFlipped) if status, op, ok := stale.hold(rec); status != "landing" || op != mhID || !ok { t.Fatalf("stale view: hold = (%q, %q, %v), want landing by the move", status, op, ok) } unread := mhView("ie-box", "ie", "", errors.New("store down"), mhRecord("ie", provision.MoveCutting, provision.CutStopping)) if status, _, ok := unread.hold(mhRecord("ie", provision.MoveCutting, provision.CutStopping)); status != "moving" || !ok { t.Fatalf("unreadable claim: hold = (%q, %v), want moving", status, ok) } noRb := mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped) noRb.Move.Rollback = nil if status, op, ok := stale.hold(noRb); status != "releasing" || op != mhID || !ok { t.Fatalf("rollback with no id: hold = (%q, %q, %v), want releasing by the move", status, op, ok) } never := newMovesHoldView("uk-box", "uk", mhClaims(hostclaim.KindLanding, nil), nil) // never built if status, _, ok := never.hold(rec); status != "landing" || !ok { t.Fatalf("unbuilt view: hold = (%q, %v), want landing", status, ok) } } // keep and frozen: a move-kind op freezes the host, and keeps it on a box // the op names as a participant. A claim that is serving does neither. Fail // closed: before the first build, and for a host whose claim would not read, // both answer true. func TestMovesHoldViewKeepAndFrozen(t *testing.T) { rec := mhRecord("uk", provision.MoveLanding, provision.CutFlipped) cases := []struct { name, box, kind string err error keep, frozen bool }{ {"moving, source", "ie-box", hostclaim.KindMoving, nil, true, true}, {"landing, source", "ie-box", hostclaim.KindLanding, nil, true, true}, {"landing, target", "uk-box", hostclaim.KindLanding, nil, true, true}, {"reverting, target", "uk-box", hostclaim.KindReverting, nil, true, true}, {"restoring, target", "uk-box", hostclaim.KindRestoring, nil, true, true}, {"landing, not a participant", "uk-box-2", hostclaim.KindLanding, nil, false, true}, {"serving", "ie-box", hostclaim.KindServing, nil, false, false}, {"relocating is region HA's", "ie-box", hostclaim.KindRelocating, nil, false, false}, {"unreadable claim", "ie-box", "", errors.New("store down"), true, true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { v := mhView(c.box, "", c.kind, c.err, rec) if got := v.keep(mhHost); got != c.keep { t.Errorf("keep = %v, want %v", got, c.keep) } if got := v.frozen(mhHost); got != c.frozen { t.Errorf("frozen = %v, want %v", got, c.frozen) } if v.keep("other.go.example.com") || v.frozen("other.go.example.com") { t.Error("a host with no move is kept or frozen") } }) } never := newMovesHoldView("ie-box", "ie", mhClaims(hostclaim.KindServing, nil), nil) if !never.keep("any.go.example.com") || !never.frozen("any.go.example.com") { t.Fatal("an unbuilt view does not fail closed") } // A later build that cannot list keeps the last answer: refresh is only // called with a list, so a failed list is no refresh at all. built := mhView("ie-box", "ie", hostclaim.KindMoving, nil, mhRecord("ie", provision.MoveCutting, provision.CutStopping)) if !built.keep(mhHost) { t.Fatal("the participant is not kept") } built.refresh(context.Background(), nil, nil) if built.keep(mhHost) || built.frozen(mhHost) { t.Fatal("a host whose move ended is still kept or frozen") } } // The view against a real claim in a store, through hostclaim.Read: the // door's moving claim names ie-box and uk-box. func TestMovesHoldViewReadsTheClaim(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) created := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) inc, err := hostclaim.Incarnation("bab", "app", mhHost, created) if err != nil { t.Fatal(err) } obs, err := hostclaim.Update(ctx, s, hostclaim.Observed{Host: mhHost, Absent: true}, func(c *hostclaim.Claim) error { c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", mhHost), Account: "bab", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/" + mhHost + "/" + inc, Generation: "cur", Stamp: 1} c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}} return nil }) if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(ctx, s, obs, hostclaim.BeginMove(mhID, "op", "ie-box", "uk-box", "uk", created)); err != nil { t.Fatal(err) } rec := mhRecord("ie", provision.MoveCutting, provision.CutStopping) ie := newMovesHoldView("ie-box", "ie", hostclaim.Read, nil) ie.refresh(ctx, s, []provision.Record{rec}) if !ie.keep(mhHost) || !ie.frozen(mhHost) { t.Fatal("the source is not kept and frozen under the door's moving claim") } if status, op, ok := ie.hold(rec); status != "moving" || op != mhID || !ok { t.Fatalf("hold = (%q, %q, %v), want moving by the move", status, op, ok) } other := newMovesHoldView("ie-box-2", "ie", hostclaim.Read, nil) other.refresh(ctx, s, []provision.Record{rec}) if other.keep(mhHost) { t.Fatal("a box the claim does not name is kept") } } // The drain reaches the schedule wiring's gate once armScheduleHolds has // run, and is a no-op before (no scheduled work armed). func TestMovesHoldViewDrainsTheScheduleGate(t *testing.T) { v := newMovesHoldView("ie-box", "ie", mhClaims(hostclaim.KindMoving, nil), nil) if err := v.drainTimers(context.Background(), mhHost); err != nil { t.Fatalf("drain with no schedule armed = %v", err) } sd := &scheduleDeps{runner: &schedule.Runner{}, conv: &schedule.Converger{}} armScheduleHolds(sd, v) if sd.runner.Gate == nil || sd.conv.Gate != sd.runner.Gate || sd.conv.Frozen == nil { t.Fatalf("the schedule halves do not share one gate and the freeze: %+v %+v", sd.runner, sd.conv) } if v.gate.Load() != sd.runner.Gate { t.Fatal("the view does not drain the runner's gate") } if err := v.drainTimers(context.Background(), mhHost); err != nil { t.Fatalf("drain = %v", err) } } // The switch off (moves.Enabled false) is main: no hold view, so reconcile // gets no Hold, KeepParticipant or HoldWritten, and the schedule halves no // gate and no freeze. func TestMovesHoldsAreMainWithTheSwitchOff(t *testing.T) { prev := armedMovesWorker.Load() t.Cleanup(func() { armedMovesWorker.Store(prev) }) off := &movesWorker{cfg: moveCfg{Enabled: false}, holds: newMovesHoldView("ie-box", "ie", hostclaim.Read, nil)} armedMovesWorker.Store(off) if armedHoldView() != nil { t.Fatal("a hold view with the switch off") } cfg := withMovesHolds(provision.ReconcileConfig{}) if cfg.Hold != nil || cfg.KeepParticipant != nil || cfg.HoldWritten != nil { t.Fatal("reconcile hooks with the switch off") } // The schedule halves, wired as the agent wires them with the switch // off, against a row whose status is a hold: the runner delivers its // due timer and records the outcome, and the converger converges its // declaration, exactly as main does (fix round 1: behaviour, not the // absence of hooks). ctx := context.Background() reg, err := registry.Open(filepath.Join(t.TempDir(), "r.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) const host = "held.bab.oncarlos.com" if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: "bab", App: "app", Status: "ready"}); err != nil { t.Fatal(err) } if err := reg.UpsertTimer(registry.Timer{Host: host, Name: "remind", Kind: schedule.KindOnce, Spec: "at 1", Path: "/r", NextAt: 1, DueAt: 1}); err != nil { t.Fatal(err) } if err := reg.SetHold(host, "moving", mhID); err != nil { t.Fatal(err) } now := time.Date(2026, 8, 23, 13, 7, 0, 0, time.UTC) st := release.NewDirStore(t.TempDir()) if _, err := schedule.UpsertSchedule(ctx, st, "bab", "app", schedule.Schedule{Name: "sync", Every: "6h", Path: "/jobs/sync"}, now); err != nil { t.Fatal(err) } runner := &schedule.Runner{Reg: reg, Deliver: statusDeliverer(503), Now: func() time.Time { return now }, Timeout: time.Minute, Box: "ie-box", Build: "test", Token: func(string) (string, error) { return "tok", nil }} conv := schedule.NewConverger(reg, st, "ie-box", nil) conv.Now = func() time.Time { return now } sd := &scheduleDeps{runner: runner, conv: conv} armScheduleHolds(sd, armedHoldView()) if runner.Gate != nil || conv.Gate != nil || conv.Frozen != nil { t.Fatal("schedule hooks with the switch off") } runner.Tick(ctx) // Tick launches the delivery and returns: wait for the record, then // for the host's goroutine to let go. var tm *registry.Timer for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { if tm, _ = reg.GetTimer(host, "remind"); tm != nil && tm.Attempts == 1 && !runner.Busy(host) { break } } if tm == nil || tm.Attempts != 1 || tm.LastStatus != "503" { t.Fatalf("with the switch off a held row's due timer was not delivered and recorded as main does: %+v", tm) } if err := conv.Pass(ctx); err != nil { t.Fatal(err) } if _, err := reg.GetTimer(host, "sync"); err != nil { t.Fatalf("with the switch off the held host's declaration did not converge: %v", err) } armedMovesWorker.Store(nil) if cfg := withMovesHolds(provision.ReconcileConfig{}); cfg.Hold != nil { t.Fatal("reconcile hooks with no worker armed") } on := &movesWorker{cfg: moveCfg{Enabled: true}, holds: newMovesHoldView("ie-box", "ie", hostclaim.Read, func(string) {})} armedMovesWorker.Store(on) cfg = withMovesHolds(provision.ReconcileConfig{}) if cfg.Hold == nil || cfg.KeepParticipant == nil || cfg.HoldWritten == nil { t.Fatal("no reconcile hooks with the switch on") } } // Ruling J3 (fix round 1): the hand-over of a move's hold to its rollback // touches the stage-requested marker when it lands, and only then. func TestHandHoldTouchesTheStageMarkerOnlyWhenItLands(t *testing.T) { const rb = "rb_abcdefghijklmnopqrstuvwxyz" reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() if err := reg.Upsert(registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Status: "ready", Provisioned: true}); err != nil { t.Fatal(err) } if err := reg.SetHold(p1Host, "landing", p1ID); err != nil { t.Fatal(err) } marker := func(dd string) bool { _, err := os.Stat(filepath.Join(dd, stageRequestMarker)) return err == nil } // A refusal: the row is held by p1ID, not p1ID2. Nothing moves, no marker. refused := t.TempDir() if ok, err := handHold(reg, holdStageRequester(refused))(p1Host, "releasing", rb, p1ID2, nil); err != nil || ok || marker(refused) { t.Fatalf("refused hand-over: %v %v, marker %v", ok, err, marker(refused)) } if rt, _ := reg.Get(p1Host); rt.Status != "landing" || rt.HoldOp != p1ID { t.Fatalf("row changed by a refusal: %+v", rt) } // An absent row: no marker. if ok, err := handHold(reg, holdStageRequester(refused))("absent.example.com", "releasing", rb, p1ID, nil); err != nil || ok || marker(refused) { t.Fatalf("absent row: %v %v, marker %v", ok, err, marker(refused)) } // The hand-over: the row is the rollback's, and the marker is written. landed := t.TempDir() if ok, err := handHold(reg, holdStageRequester(landed))(p1Host, "releasing", rb, p1ID, nil); err != nil || !ok || !marker(landed) { t.Fatalf("hand-over: %v %v, marker %v", ok, err, marker(landed)) } if rt, _ := reg.Get(p1Host); rt.Status != "releasing" || rt.HoldOp != rb { t.Fatalf("after the hand-over: %+v", rt) } // A replay finds the row already the rollback's: done, nothing written. var touched []string if ok, err := handHold(reg, func(h string) { touched = append(touched, h) })(p1Host, "releasing", rb, p1ID, nil); err != nil || !ok || len(touched) != 0 { t.Fatalf("replay: %v %v, marker touched %d times", ok, err, len(touched)) } } // Task 4 fix round 1, ruling 1: the hand-over's caller validates its own // record under the hold lock, after the row is re-read and before the hold // moves. A validation that refuses moves nothing and writes no marker; one // that passes is asked once, and the hold moves. The lock is the wrapper's // alone: the callback runs inside it and must not take it again. func TestHandHoldValidatesUnderTheLock(t *testing.T) { const rb = "rb_abcdefghijklmnopqrstuvwxyz" reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() if err := reg.Upsert(registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Status: "ready", Provisioned: true}); err != nil { t.Fatal(err) } if err := reg.SetHold(p1Host, "landing", p1ID); err != nil { t.Fatal(err) } changed := errors.New("the record is no longer this rollback's release") var seen []string refused := t.TempDir() ok, err := handHold(reg, holdStageRequester(refused))(p1Host, "releasing", rb, p1ID, func() error { rt, _ := reg.Get(p1Host) seen = append(seen, rt.Status+"/"+rt.HoldOp) return changed }) if !errors.Is(err, changed) || ok { t.Fatalf("a refused validation: %v %v", ok, err) } if _, err := os.Stat(filepath.Join(refused, stageRequestMarker)); err == nil { t.Fatal("a refused validation wrote the marker") } if rt, _ := reg.Get(p1Host); rt.Status != "landing" || rt.HoldOp != p1ID { t.Fatalf("the hold moved under a refused validation: %+v", rt) } landed := t.TempDir() if ok, err := handHold(reg, holdStageRequester(landed))(p1Host, "releasing", rb, p1ID, func() error { rt, _ := reg.Get(p1Host) seen = append(seen, rt.Status+"/"+rt.HoldOp) return nil }); err != nil || !ok { t.Fatalf("a passing validation: %v %v", ok, err) } if rt, _ := reg.Get(p1Host); rt.Status != "releasing" || rt.HoldOp != rb { t.Fatalf("after the hand-over: %+v", rt) } if strings.Join(seen, " ") != "landing/"+p1ID+" landing/"+p1ID { t.Fatalf("validations saw %v, want the row before the hand-over, once each", seen) } }