//go:build unix package host import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostrelocate" ) // stopProbeActivator is a real activator.Service's fences, the calls // recorded in order. Replacing svc is an agent restart: the activator's // fences live only in its process. failNext, when set, is the next // FenceAndStopAs's answer instead of the service's. type stopProbeActivator struct { order *[]string svc *activator.Service failNext error } func newStopProbeActivator(order *[]string) *stopProbeActivator { return &stopProbeActivator{order: order, svc: &activator.Service{}} } func (a *stopProbeActivator) ProbeWake(context.Context, activator.ProbeConfig) (activator.ProbeResult, error) { return activator.ProbeResult{}, errors.New("not a probe") } func (a *stopProbeActivator) FenceAndStopAs(ctx context.Context, host string, generation uint64, owner string) (*activator.StopEvidence, error) { *a.order = append(*a.order, "fence-as") if err := a.failNext; err != nil { a.failNext = nil return nil, err } return a.svc.FenceAndStopAs(ctx, host, generation, owner) } func (a *stopProbeActivator) LiftFenceAs(host string, generation uint64, owner string) bool { *a.order = append(*a.order, "lift") return a.svc.LiftFenceAs(host, generation, owner) } // foreignFence installs another operation's plain fence on a.svc. func (a *stopProbeActivator) foreignFence(t *testing.T, host string, generation uint64) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() if _, err := a.svc.FenceAndStop(ctx, host, generation); err != nil && !errors.Is(err, activator.ErrStopUnproven) { t.Fatal(err) } } // stopProbeFixture is a target at relocating/probing whose probe passed // with session 12: the guard activated at the install stamp, main's // Wake.Stop standing for root's writer-session stop (it fences and // completes the stop in the guard journal with a receipt), and the probe // stop's provider bound to them. type stopProbeFixture struct { t *testing.T order []string act *stopProbeActivator a ClaimAssembly cfg RelocationAssemblyConfig obs hostclaim.Observed receipt []byte target hostguard.FenceTarget } func newStopProbeFixture(t *testing.T) *stopProbeFixture { t.Helper() ctx := t.Context() const host, box = "shop.example", "dst" created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) inc, err := hostclaim.Incarnation("acct", "app", host, created) if err != nil { t.Fatal(err) } probe, _ := json.Marshal(hostclaim.Probe{Box: box, Session: 12, At: created.Add(time.Hour)}) f := &stopProbeFixture{t: t} f.obs = hostclaim.Observed{Host: host, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Gen: 30, Host: host, Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, Journal: map[string]string{"probe": string(probe)}}}} dataDir := t.TempDir() if err := os.Mkdir(filepath.Join(dataDir, "hostguard"), 0o770); err != nil { t.Fatal(err) } guard := hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: &guardClaims{obs: f.obs}} if err := guard.Activate(ctx, host, box, nil); err != nil { t.Fatal(err) } f.target = hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: host, Incarnation: inc, Stamp: 9, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: "e1", DirectoryRevision: "1", RosterRevision: 1} at := created.Add(2 * time.Hour) f.receipt = mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: f.target.ObligationID, FenceID: "fence-1", Host: host, Incarnation: inc, InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: testReceiptNumeral(f.target), Binding: "1", EvidenceAt: at}) wake := &ClaimWakeAssembly{Stop: func(ctx context.Context, o hostclaim.Observed) error { f.order = append(f.order, "wake-stop") // Root's writer-session stop: deterministic, so a repeat meets the // same, already completed, fence. if got, err := guard.CompletedWriterStops(ctx, host, hostguard.Captured{Incarnation: inc, Stamp: 9}, "rel-1", box); err == nil && len(got) == 1 { return nil } if _, err := guard.BeginFence(ctx, f.target); err != nil { return err } return guard.CompleteFence(ctx, f.target, f.receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }) }} f.act = newStopProbeActivator(&f.order) f.a = ClaimAssembly{Guard: guard, Wake: wake, stopLocal: func(context.Context) (claimStopLocal, error) { return claimStopLocal{Actor: hostbox.Actor{Box: box, Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil }} f.cfg = RelocationAssemblyConfig{Activator: f.act, DataDir: dataDir} return f } func (f *stopProbeFixture) stop() (hostclaim.ProbeStop, error) { return f.a.relocationStopProbe(f.cfg)(f.t.Context(), f.obs) } // marker is the probe stop's persisted owner for rel-1. func (f *stopProbeFixture) marker() relocationProbeFence { f.t.Helper() path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") if err != nil { f.t.Fatal(err) } rec, ok, err := readRelocationProbeFence(path) if err != nil || !ok { f.t.Fatalf("no probe-fence marker: %v", err) } return rec } // Ruling 49 steps 1 to 3 with ruling 75's owned fence, against the real // activator: the owner is persisted, then the fence goes in before root is // asked; an unproven stop (the probe's entry is gone) still asks root; // root's receipt is the guard journal's writer-session stop of the probe's // session; a repeat meets its own fence and returns the same ProbeStop; // another operation's fence and an uninstalled fence stop everything with // nothing asked of root and the other fence as it was. func TestRelocationStopProbeFencesStopsAndReadsRootsReceipt(t *testing.T) { sum := func(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } const host = "shop.example" t.Run("the probe's session", func(t *testing.T) { f := newStopProbeFixture(t) s, err := f.stop() if err != nil { t.Fatal(err) } if want := (hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: sum(f.receipt)}); s != want { t.Fatalf("stop %+v, want %+v", s, want) } if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { t.Fatalf("order %v, want %v", f.order, want) } m := f.marker() if m.Host != host || m.Session != 12 || len(m.Owner) != 32 || f.act.svc.FenceGeneration(host) != 12 { t.Fatalf("marker %+v, fence %d", m, f.act.svc.FenceGeneration(host)) } // A repeat (the claim CAS lost, or a later pass) meets its own // fence under the same owner. f.order = nil again, err := f.stop() if err != nil || again != s { t.Fatalf("repeat: %+v %v", again, err) } if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) || f.marker() != m { t.Fatalf("repeat order %v, marker %+v", f.order, f.marker()) } }) for _, gen := range []uint64{5, 12, 20} { t.Run(fmt.Sprintf("another operation's fence at %d", gen), func(t *testing.T) { f := newStopProbeFixture(t) f.act.foreignFence(t, host, gen) if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { t.Fatalf("got %v, want ErrFenceHeld", err) } if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen { t.Fatalf("root was asked or the fence changed: %v, fence %d", f.order, f.act.svc.FenceGeneration(host)) } }) } t.Run("no fence installed", func(t *testing.T) { f := newStopProbeFixture(t) f.act.failNext = activator.ErrFenceNotInstalled if _, err := f.stop(); !errors.Is(err, activator.ErrFenceNotInstalled) { t.Fatalf("got %v", err) } if want := []string{"fence-as"}; !slices.Equal(f.order, want) { t.Fatalf("root was asked: %v", f.order) } }) t.Run("not the probe's session", func(t *testing.T) { f := newStopProbeFixture(t) f.obs.Claim.Holder.Session = 13 if _, err := f.stop(); !errors.Is(err, hostbox.ErrChanged) || len(f.order) != 0 { t.Fatalf("%v %v", err, f.order) } }) } // Astra round 6, finding 1: the agent restarts after the probe stop // installed its fence and persisted its owner. The restart empties the // activator's fences; the marker survives. Another operation then fences // the host at a lower or equal generation. The surviving marker must not // let the probe stop take that fence over: ErrFenceHeld, the other fence // exactly as it was, root not asked. Once the other operation lifts its // fence, the next pass installs this probe's own fence and completes. func TestRelocationStopProbeAfterARestartNeverTakesAForeignFence(t *testing.T) { const host = "shop.example" for _, gen := range []uint64{5, 12} { t.Run(fmt.Sprint(gen), func(t *testing.T) { f := newStopProbeFixture(t) first, err := f.stop() if err != nil { t.Fatal(err) } before := f.marker() f.act.svc = &activator.Service{} // the restart f.act.foreignFence(t, host, gen) f.order = nil if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { t.Fatalf("got %v, want ErrFenceHeld", err) } if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen || f.marker() != before { t.Fatalf("order %v, fence %d, marker %+v", f.order, f.act.svc.FenceGeneration(host), f.marker()) } if f.act.svc.LiftFenceAs(host, gen, before.Owner) { t.Fatal("the surviving owner lifted the other operation's fence") } if !f.act.svc.LiftFence(host, gen) { t.Fatal("the other operation's fence was not as it installed it") } f.order = nil again, err := f.stop() if err != nil || again != first || f.act.svc.FenceGeneration(host) != 12 { t.Fatalf("after the other fence went: %+v %v, fence %d", again, err, f.act.svc.FenceGeneration(host)) } }) } } // Astra round 6, finding 2: the owner is persisted before the activator is // asked, so a failed write installs nothing and blocks nothing. The // retry persists it and proceeds, under the owner it persisted. func TestRelocationStopProbeRetriesAFailedMarkerWrite(t *testing.T) { const host = "shop.example" f := newStopProbeFixture(t) path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") if err != nil { t.Fatal(err) } // The op's journal directory cannot be made: a file stands in its way. if err := os.MkdirAll(filepath.Dir(filepath.Dir(path)), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Dir(path), []byte("in the way"), 0o600); err != nil { t.Fatal(err) } if _, err := f.stop(); err == nil { t.Fatal("the probe stop went on without its owner persisted") } if len(f.order) != 0 || f.act.svc.Fenced(host) { t.Fatalf("the activator was asked before the owner was persisted: %v", f.order) } if err := os.Remove(filepath.Dir(path)); err != nil { t.Fatal(err) } s, err := f.stop() if err != nil || s.Session != 12 { t.Fatalf("retry: %+v %v", s, err) } if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { t.Fatalf("retry order %v", f.order) } if m := f.marker(); !f.act.svc.LiftFenceAs(host, 12, m.Owner) { t.Fatal("the fence installed is not the persisted owner's") } } // Absence is never evidence: a probe-fence marker that is present is the // only record of the owner a fence may already be installed under, so one // that cannot be read, or that names another host, session or no owner, // is never replaced by a fresh owner. The stop refuses before the // activator is asked, no fence is installed, and the marker stays as it // was, for an operator. func TestRelocationStopProbeNeverReplacesAPresentMarker(t *testing.T) { const host = "shop.example" owner := strings.Repeat("a", 32) for name, body := range map[string][]byte{ "unreadable": []byte("{not json"), "another session": []byte(`{"host":"shop.example","session":13,"owner":"` + owner + `"}`), "another host": []byte(`{"host":"other.example","session":12,"owner":"` + owner + `"}`), "no owner": []byte(`{"host":"shop.example","session":12}`), } { t.Run(name, func(t *testing.T) { f := newStopProbeFixture(t) path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") if err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(path, body, 0o600); err != nil { t.Fatal(err) } if _, err := f.stop(); !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("the probe stop over a marker it did not write: %v", err) } if len(f.order) != 0 || f.act.svc.Fenced(host) { t.Fatalf("the activator was asked: %v", f.order) } if after, err := os.ReadFile(path); err != nil || !bytes.Equal(after, body) { t.Fatalf("the marker was replaced: %q %v", after, err) } }) } } // Two completed writer-session stops of the probe's session under the same // op on this box (a second fence under another roster revision) leave no // one receipt to record or to hand over from. The stop and N6 refuse with // hostguard.ErrFenceTarget, as main's CompletedColdPredecessor does, // rather than take the first the journal lists. func TestRelocationStopProbeRefusesTwoCompletedStopsOfTheSession(t *testing.T) { ctx := t.Context() f := newStopProbeFixture(t) first, err := f.stop() if err != nil { t.Fatal(err) } // Root's second stop of session 12 under rel-1: a repeat under the next // roster revision, which main's fence target carries, so the guard // journals it beside the first rather than meeting it again. second := f.target second.FenceID, second.DirectoryRevision, second.RosterRevision = "fence-2", "2", 2 if _, err := f.a.Guard.BeginFence(ctx, second); err != nil { t.Fatal(err) } at := time.Date(2026, 10, 7, 4, 0, 0, 0, time.UTC) receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r2", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: second.ObligationID, FenceID: "fence-2", Host: f.obs.Host, Incarnation: second.Incarnation, InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: testReceiptNumeral(second), Binding: "2", EvidenceAt: at}) if err := f.a.Guard.CompleteFence(ctx, second, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { t.Fatal(err) } if _, err := f.stop(); !errors.Is(err, hostguard.ErrFenceTarget) { t.Fatalf("the stop over two receipts: %v", err) } // N6 over the serving record of the first stop refuses the same way. done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: first, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) c := *f.obs.Claim c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} h := relocationCompletedHandover{Guard: f.a.Guard, Box: "dst"} if _, err := h.receipt(ctx, hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c}); !errors.Is(err, hostguard.ErrFenceTarget) { t.Fatalf("N6 over two receipts: %v", err) } }