//go:build unix package hostbox import ( "context" "errors" "io/fs" "maps" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/park" ) // relocationReconciler is the reconciler over workFixture's guard and // store, with the real fence observer (Guard.AdmissionFenced) and a // recording hold writer. The claim relocates the host off "box". func relocationReconciler(t *testing.T) (Reconciler, RecordSelection, *[]string, hostguard.Guard) { t.Helper() source, store, obs := workFixture(t) source.Policy = Policy{Held: func(context.Context, string) (string, error) { return "", nil }, Relocation: func(context.Context, *hostclaim.Claim, Selection, hostwrite.Identity) error { return nil }}.Check // One cold writer session, so the source has a session a stop can fence. obs, err := hostclaim.Update(context.Background(), store, obs, hostclaim.AdmitWriter(hostclaim.Admission{Box: "box", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) if err != nil { t.Fatal(err) } h := obs.Claim.Holder selected := RecordSelection{Host: obs.Host, Key: h.Key, ETag: "record-etag", Incarnation: h.Incarnation, Family: h.Family, Layout: "flat", Paths: []string{"shop.example.db"}} holds := &[]string{} r := Reconciler{Claims: store, Guard: source.Guard, Local: park.Local{Guard: source.Guard, Claims: store, DataDir: source.Guard.DataDir, Box: "box"}, Writes: hostwrite.Config{Enabled: true, Selector: source}, Box: "box", Region: "ie", Now: func() time.Time { return time.Unix(100, 0) }, CheckRecord: func(context.Context, RecordSelection) error { return nil }, CheckLocal: func(context.Context, hostclaim.Observed, RecordSelection) error { return nil }, CheckHeld: func(context.Context, hostclaim.Observed, RecordSelection) error { return nil }, HoldRow: func(_ context.Context, _ string, op string) error { *holds = append(*holds, op); return nil }} r.FenceClosed = func(_ context.Context, o hostclaim.Observed) (bool, error) { captured, err := hostguard.Capture(source.Guard.DataDir, o.Host) if err != nil { return false, err } return source.Guard.AdmissionFenced(o.Host, o.Claim, captured) } return r, selected, holds, source.Guard } func relocate(t *testing.T, r Reconciler, host, phase string) hostclaim.Observed { t.Helper() ctx := context.Background() obs, err := hostclaim.Read(ctx, r.Claims, host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: phase, Exec: "box", Participants: []string{"box", "dst"}, Intent: map[string]string{"region": "ie", "source_box": "box", "target_box": "dst"}, Journal: map[string]string{"exec_ack.box": "{}", "exec_ack.dst": "{}"}} return nil }) if err != nil { t.Fatal(err) } return obs } // The source in releasing takes the activation path (ready) until root // begins its stop; then the hold-only path, under the flock, with the // observer read inside the existing locked operation: a bounded context // finishes both, so neither reacquires the host lock. // // Claim Plan 11's policy gate admits no "reconcile" action under a // relocating claim (R1: only op.exec's probe), so the activation path is // refused by the first check, before the guard is activated or the // inventory written, dry run or not; the hold-only path, which takes no // policy, still keeps the fenced source's row held. func TestReconcileRelocationSourceReadyThenHeldWithTheRealFenceObserver(t *testing.T) { r, selected, holds, guard := relocationReconciler(t) obs := relocate(t, r, selected.Host, hostclaim.PhaseReleasing) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() writes := 0 before := snapshotDir(t, guard.DataDir) for _, dry := range []bool{true, false} { sel := selected sel.DryRun = dry if err := r.Install(ctx, sel, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) { t.Fatalf("the source before its fence closed (dry run %v): %v (claim Plan 11 refuses a relocating reconcile)", dry, err) } } if writes != 0 || len(*holds) != 0 { t.Fatalf("writes %d holds %v", writes, *holds) } if after := snapshotDir(t, guard.DataDir); !maps.Equal(before, after) { t.Fatal("a refused relocating reconcile changed the guard directory") } if now, err := hostclaim.Read(ctx, r.Claims, selected.Host); err != nil || now.ETag != obs.ETag { t.Fatalf("a refused relocating reconcile wrote the claim: %v", err) } // Root begins the source's stop: the durable admission fence closes. captured, err := hostguard.Capture(guard.DataDir, selected.Host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } target := hostguard.FenceTarget{FenceID: "fence-1", Host: selected.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Session: obs.Claim.Holder.Session, Op: "rel-1", Phase: hostclaim.PhaseReleasing, Box: "box", Enrollment: "enrollment-1", BootID: "boot", ControllerID: "ctl", ClaimETag: obs.ETag, DirectoryRevision: "4"} if _, err := guard.BeginFence(ctx, target); err != nil { t.Fatal(err) } if err := r.Install(ctx, selected, func() error { writes++; return nil }); err != nil { t.Fatalf("the fenced source's hold: %v", err) } if writes != 0 || len(*holds) != 1 || (*holds)[0] != "rel-1" { t.Fatalf("after the fence: writes %d holds %v (the guarded path must not run)", writes, *holds) } // A box with no role mints nothing; an observer error keeps rows and // mints none. r.Box = "zzz" if err := r.Install(ctx, selected, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) || writes != 0 { t.Fatalf("no role: %v writes %d", err, writes) } r.Box = "box" broken := errors.New("record unreadable") r.FenceClosed = func(context.Context, hostclaim.Observed) (bool, error) { return false, broken } if err := r.Install(ctx, selected, func() error { writes++; return nil }); !errors.Is(err, broken) || writes != 0 { t.Fatalf("an observer error: %v writes %d", err, writes) } } // With no observer (region HA off) the reconciler is today's: the serving // executor is ready; an executor of any other operation is refused by the // first check, before the guard is activated or inventory recorded, dry // run or not (round 5 finding 3). func TestReconcileWithoutAFenceObserverIsTodays(t *testing.T) { r, selected, _, guard := relocationReconciler(t) r.FenceClosed, r.HoldRow = nil, nil ctx := context.Background() writes := 0 if err := r.Install(ctx, selected, func() error { writes++; return nil }); err != nil || writes != 1 { t.Fatalf("serving: %v writes %d", err, writes) } moving := func() hostclaim.Observed { obs, err := hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv-1", Phase: "copying", Exec: "box", Participants: []string{"box", "dst"}, Intent: map[string]string{"admission_id": "adm"}} return nil }) if err != nil { t.Fatal(err) } return obs } for _, phase := range []string{hostclaim.PhaseStarting, hostclaim.PhaseReleasing, "moving"} { var obs hostclaim.Observed if phase == "moving" { obs = moving() // a moves executor: main's path with HA on too } else { obs = relocate(t, r, selected.Host, phase) // ready with HA on (RowReady) } before := snapshotDir(t, guard.DataDir) for _, dry := range []bool{true, false} { sel := selected sel.DryRun = dry if err := r.Install(ctx, sel, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) || writes != 1 { t.Fatalf("%s (dry run %v) with HA off: %v writes %d", phase, dry, err, writes) } } after, err := hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } if after.ETag != obs.ETag { t.Fatalf("%s: the claim (inventory) was written with HA off", phase) } if got := snapshotDir(t, guard.DataDir); !maps.Equal(got, before) { t.Fatalf("%s: the guard's local state changed with HA off", phase) } } } // With the observer set (region HA on) a moves claim still takes main's // path: Serving refuses it first. The target mints and holds nothing // here (moves owns its holds, in provision's pass), and the executor is // refused before the guard is activated or inventory recorded, dry run // or not. Only relocating claims take the row decision. func TestReconcileWithAFenceObserverLeavesMovesToMain(t *testing.T) { for _, box := range []string{"dst", "box"} { t.Run(box, func(t *testing.T) { r, selected, holds, guard := relocationReconciler(t) ctx := context.Background() observed := 0 fence := r.FenceClosed r.FenceClosed = func(ctx context.Context, o hostclaim.Observed) (bool, error) { observed++ return fence(ctx, o) } obs, err := hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv-1", Phase: "copying", Exec: "box", Participants: []string{"box", "dst"}, Intent: map[string]string{"admission_id": "adm"}} return nil }) if err != nil { t.Fatal(err) } r.Box = box before := snapshotDir(t, guard.DataDir) writes := 0 for _, dry := range []bool{true, false} { sel := selected sel.DryRun = dry if err := r.Install(ctx, sel, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) { t.Fatalf("%s (dry run %v) under a move with HA on: %v", box, dry, err) } } if writes != 0 || len(*holds) != 0 || observed != 0 { t.Fatalf("%s under a move with HA on: writes %d holds %v observer calls %d", box, writes, *holds, observed) } after, err := hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } if after.ETag != obs.ETag { t.Fatalf("%s: the claim (inventory) was written under a move", box) } if got := snapshotDir(t, guard.DataDir); !maps.Equal(got, before) { t.Fatalf("%s: the guard's local state changed under a move", box) } }) } } // Install asks the fence observer once for its own decision, and its // check and its path both act on that answer: the hold-only path acts on // the answer the decision was made from. The source in releasing is the // one row case that depends on the observer. Before the final review's // fixes it was asked three times here (check, Install, holdOnly); now // Install's one answer serves its check too, so it is asked twice // (Install, holdOnly under the flock). A box whose case does not depend on // it is never asked (TestReconcileRelocationTargetWithNoLocalRecordMintsItsRow). func TestReconcileInstallAsksTheFenceObserverOnce(t *testing.T) { r, selected, holds, _ := relocationReconciler(t) relocate(t, r, selected.Host, hostclaim.PhaseReleasing) ctx := context.Background() // Install's answer is closed, and every later one open: acting on one // answer, Install takes the hold path, whose decision under the flock // then differs, so nothing is written and nothing held. answers := []bool{true} calls := 0 r.FenceClosed = func(context.Context, hostclaim.Observed) (bool, error) { calls++ if calls <= len(answers) { return answers[calls-1], nil } return false, nil } writes := 0 if err := r.Install(ctx, selected, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) { t.Fatalf("a changed answer under the flock: %v", err) } if calls != 2 || writes != 0 || len(*holds) != 0 { t.Fatalf("observer calls %d (Install, holdOnly) writes %d holds %v", calls, writes, *holds) } } // Final review: the hold-only write is not execution, so the region's // quiet window does not gate it. The activation path still waits for it. func TestReconcileQuietWindowGatesExecutionNotTheHold(t *testing.T) { r, selected, holds, guard := relocationReconciler(t) obs := relocate(t, r, selected.Host, hostclaim.PhaseReleasing) ctx := context.Background() obs, err := hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.QuietAfter = map[string]time.Time{"ie": r.Now().Add(time.Minute)} return nil }) if err != nil { t.Fatal(err) } writes := 0 // The source before its fence closes would take the activation path, // which a relocating claim now refuses before the quiet window is // reached (main's policy admits only the probe; checkOn refuses first). if err := r.Install(ctx, selected, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) || writes != 0 || len(*holds) != 0 { t.Fatalf("activation in the quiet window: %v writes %d holds %v", err, writes, *holds) } captured, err := hostguard.Capture(guard.DataDir, selected.Host) if err != nil { t.Fatal(err) } target := hostguard.FenceTarget{FenceID: "fence-1", Host: selected.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Session: obs.Claim.Holder.Session, Op: "rel-1", Phase: hostclaim.PhaseReleasing, Box: "box", Enrollment: "enrollment-1", BootID: "boot", ControllerID: "ctl", ClaimETag: obs.ETag, DirectoryRevision: "4"} if _, err := guard.BeginFence(ctx, target); err != nil { t.Fatal(err) } // The fenced source's row is held at once, inside the quiet window. if err := r.Install(ctx, selected, func() error { writes++; return nil }); err != nil { t.Fatalf("the fenced source's hold in the quiet window: %v", err) } if writes != 0 || len(*holds) != 1 || (*holds)[0] != "rel-1" { t.Fatalf("writes %d holds %v", writes, *holds) } } // targetReconciler is relocationReconciler moved onto the target "dst": // its own guard directory, with no local record for the host, and the // real fence observer over it, which errors on a box with no record. func targetReconciler(t *testing.T, r Reconciler) (Reconciler, hostguard.Guard, *int) { t.Helper() dir := t.TempDir() locks := filepath.Join(dir, "locks") for _, p := range []string{locks, filepath.Join(dir, "hostguard")} { if err := os.Mkdir(p, 0770); err != nil { t.Fatal(err) } } guard := hostguard.Guard{DataDir: dir, LockDir: locks, Claims: hostguard.StoreReader{Store: r.Claims}} target := WorkSource{Guard: guard, Box: "dst", Region: "ie", Now: r.Now, Policy: Policy{Held: func(context.Context, string) (string, error) { return "", nil }, Relocation: func(context.Context, *hostclaim.Claim, Selection, hostwrite.Identity) error { return nil }}.Check, Local: func(context.Context, *hostclaim.Claim, Selection) error { return nil }} r.Box, r.Guard, r.Writes.Selector = "dst", guard, target r.Local = park.Local{Guard: guard, Claims: r.Claims, DataDir: dir, Box: "dst"} calls := new(int) r.FenceClosed = func(_ context.Context, o hostclaim.Observed) (bool, error) { *calls++ captured, err := hostguard.Capture(dir, o.Host) if err != nil { return false, err } return guard.AdmissionFenced(o.Host, o.Claim, captured) } return r, guard, calls } // Final review Important 2: with region HA on, a relocation target that // has no local guard record yet decides its row (held, as op.exec and a // participant) without the fence observer, which errors without a record // and is never asked. Claim Plan 11's policy gate admits no "reconcile" // action under a relocating claim (R1), so the activation path is refused // by the first check, dry run or not: nothing is activated or written, // and no row is minted (stage B settles how one is). func TestReconcileRelocationTargetWithNoLocalRecordIsRefusedBeforeActivation(t *testing.T) { r, selected, holds, _ := relocationReconciler(t) r, guard, calls := targetReconciler(t, r) ctx := context.Background() obs, err := hostclaim.Read(ctx, r.Claims, selected.Host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"box", "dst"}, Intent: map[string]string{"region": "ie", "source_box": "box", "target_box": "dst"}, Journal: map[string]string{"exec_ack.box": "{}", "exec_ack.dst": "{}"}} return nil }) if err != nil { t.Fatal(err) } // The production observer cannot answer here: no record. if _, err := r.FenceClosed(ctx, obs); err == nil { t.Fatal("the observer answered for a box with no local record") } *calls = 0 writes := 0 before := snapshotDir(t, guard.DataDir) for _, dry := range []bool{true, false} { sel := selected sel.DryRun = dry if err := r.Install(ctx, sel, func() error { writes++; return nil }); !errors.Is(err, ErrChanged) { t.Fatalf("the target's first install (dry run %v): %v (claim Plan 11 refuses a relocating reconcile)", dry, err) } } if writes != 0 || len(*holds) != 0 || *calls != 0 { t.Fatalf("writes %d holds %v observer calls %d", writes, *holds, *calls) } if after := snapshotDir(t, guard.DataDir); !maps.Equal(before, after) { t.Fatal("a refused relocating reconcile changed the target's guard directory") } if _, err := hostguard.Capture(guard.DataDir, selected.Host); err == nil { t.Fatal("the target's installation was activated") } if now, err := hostclaim.Read(ctx, r.Claims, selected.Host); err != nil || now.ETag != obs.ETag { t.Fatalf("a refused relocating reconcile wrote the claim: %v", err) } } // snapshotDir is every regular file under dir, by relative path. func snapshotDir(t *testing.T, dir string) map[string]string { t.Helper() out := map[string]string{} err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil || !d.Type().IsRegular() { return err } raw, err := os.ReadFile(path) if err != nil { return err } out[path] = string(raw) return nil }) if err != nil { t.Fatal(err) } return out }