package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostrelocate" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/superfly/ltx" ) const ( roHost = "shop.example" roOp = "rel-1" roP = "P1" ) var roProbe = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second, Probe: true} // roReleases is the release authority the owners verify a drain's release // with; it records every call. type roReleases struct { calls int err error } func (r *roReleases) VerifyRelease(context.Context, hostclaim.Claim, hostclaim.Release) error { r.calls++ return r.err } // roFiles is moves' C8 provider: it records each Establish under the work // it is given. type roFiles struct{ established []string } func (f *roFiles) Establish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { return work.With(ctx, in.Identity, func() error { f.established = append(f.established, fmt.Sprintf("%s@%d", o.Host, o.Claim.Holder.Session)) return nil }) } // roFixture is a relocation target at relocating/probing, P built and not // opened: the claim as the participant's ToProbing leaves it, P's snapshot, // anchor and install manifest in the versioned bucket, P's prepared files // manifest in the data bucket, and the wake inputs bound as RelocationSource // binds them. type roFixture struct { t *testing.T claims *release.DirStore files *release.DirStore bucket *leasetest.VersionedBucket releases *roReleases c8 *roFiles inputs *RelocationWakeInputs route registry.Route manifest moves.InstallManifest seq int stamp uint64 } type roOptions struct { empty bool // an evidenced empty pin: no snapshot, no install manifest failover bool // no journaled release; the source lost and fenced } func newROFixture(t *testing.T, o roOptions) *roFixture { t.Helper() ctx := t.Context() f := &roFixture{t: t, claims: release.NewDirStore(t.TempDir()), files: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket(), releases: &roReleases{}, c8: &roFiles{}} f.bucket.AddBucket("data") created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC) enrollment := strings.Repeat("a", 32) nh := hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + roHost + ".json", Account: "acct", App: "app", Host: roHost, Region: "ie", RecordRegion: "ie", CreatedAt: created} body := []byte(`{"host":"` + roHost + `","account":"acct","app":"app","region":"ie","created_at":"2026-10-07T01:00:00Z"}`) obs, err := hostclaim.Read(ctx, f.claims, roHost) if err != nil { t.Fatal(err) } for _, tr := range []func(*hostclaim.Claim) error{ hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment}), hostclaim.CompleteCreate("c1", "src"), hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}), } { if obs, err = hostclaim.Update(ctx, f.claims, obs, tr); err != nil { t.Fatal(err) } } h := obs.Claim.Holder prefix := moves.GenerationPrefix(h.Location, roP) auth := hostclaim.PreparedAuthority{Generation: roP, SourceFilesSeq: 5} ref := hostclaim.RecoveryPin{Key: "control/relocations/" + roHost + "/" + roOp + "/pin-1.json", SHA256: strings.Repeat("e", 64), Generation: "g1", FilesSeq: 5, Release: "v1"} if o.empty { ref.Empty = true } else { ref.MaxTXID = 42 snap, anchor := moves.SnapshotKey(prefix, 42), moves.AnchorKey(prefix, 42) f.manifest = moves.InstallManifest{V: 1, Op: roOp, Host: roHost, N: 42, SnapshotKey: snap, SnapshotVersion: f.bucket.Put("data", snap, ltxHeader(t, 1, 42)), SnapshotCRC64: 7, AnchorKey: anchor, AnchorVersion: f.bucket.Put("data", anchor, ltxHeader(t, 42, 42)), IntegrityCheck: "ok", DBSHA256: strings.Repeat("d", 64)} raw, _ := json.Marshal(f.manifest) f.bucket.Put("data", moves.InstallManifestKey(prefix), raw) sum := sha256.Sum256(raw) auth.InstallManifest, auth.SHA256, auth.SnapshotTXID = moves.InstallManifestKey(prefix), hex.EncodeToString(sum[:]), 42 } pm, _ := json.Marshal(hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}}) if _, err := f.files.Put(ctx, hostfiles.ManifestKey(moves.GenerationFilesPrefix(h.Location, roP)), pm, ""); err != nil { t.Fatal(err) } pmSum := sha256.Sum256(pm) auth.FilesManifestSHA256 = hex.EncodeToString(pmSum[:]) if obs, err = hostclaim.Update(ctx, f.claims, obs, func(c *hostclaim.Claim) error { f.stamp = c.Gen + 1 c.Holder.Stamp = f.stamp auth.InstallStamp = f.stamp stamp := strconv.FormatUint(f.stamp, 10) journal := map[string]string{"pin": string(mustJSON(t, ref)), "prepared": roP, "install_stamp": stamp, "install_release": "v1", "prepared_authority": string(mustJSON(t, auth)), "probing_since": created.Add(time.Hour).Format(time.RFC3339Nano), "exec_ack.src": `{"op":"rel-1"}`, "exec_ack.dst": `{"op":"rel-1"}`, "local_install": string(mustJSON(t, hostclaim.LocalInstall{Box: "dst", Enrollment: enrollment, InstallStamp: f.stamp, JournalSHA256: strings.Repeat("c", 64)}))} if !o.failover { journal["release"] = string(mustJSON(t, hostclaim.Release{ReleasedBy: "src", Enrollment: enrollment, Generation: c.Holder.Generation, Session: c.Holder.Session, FinalTXID: ref.MaxTXID, ManifestSeq: 5, Receipt: "r", ReceiptSHA256: strings.Repeat("f", 64)})) } else { c.Lost = []string{"src"} journal["fence.src"] = "control/hosts/" + roHost + "/fences/src.json" } c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: roOp, Phase: hostclaim.PhaseRelocatingProbing, Exec: "dst", Participants: []string{"src", "dst"}, Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst", "target_enrollment": enrollment}, Journal: journal} return nil }); err != nil { t.Fatal(err) } f.route = registry.Route{Host: roHost, Kind: "instance", App: "app", Account: "acct", Status: "relocating", HoldOp: roOp, DBPath: provision.NestedDBPath("/data", roHost), Socket: provision.NestedSocket(roHost)} f.inputs = &RelocationWakeInputs{ Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/opt/carlos/live/" + roHost, ExecutableSHA256: strings.Repeat("b", 64), Version: "v1"}, nil }, FilesManifest: f.c8, Markers: plan9D5Markers{f.bucket}, Claims: f.claims, FilesStore: f.files, Data: f.bucket, Bucket: "data", Region: "ie", Box: "dst", NewID: func() string { f.seq++; return fmt.Sprintf("fresh%d", f.seq) }, Now: func() time.Time { return created.Add(2 * time.Hour) }, Releases: f.releases} return f } func ltxHeader(t *testing.T, min, max uint64) []byte { t.Helper() body, err := (<x.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: ltx.TXID(min), MaxTXID: ltx.TXID(max)}).MarshalBinary() if err != nil { t.Fatal(err) } return body } func (f *roFixture) read() hostclaim.Observed { f.t.Helper() o, err := hostclaim.Read(f.t.Context(), f.claims, roHost) if err != nil { f.t.Fatal(err) } return o } func (f *roFixture) update(tr func(*hostclaim.Claim) error) hostclaim.Observed { f.t.Helper() o, err := hostclaim.Update(f.t.Context(), f.claims, f.read(), tr) if err != nil { f.t.Fatal(err) } return o } // admit counts the attempt and applies the selected admission with the // claim's own CAS, as the participant and the wake do. func (f *roFixture) admit(o hostclaim.Observed, in ClaimLaunchInputs) hostclaim.Observed { f.t.Helper() counted, err := hostclaim.Update(f.t.Context(), f.claims, o, hostclaim.BeginProbeAttempt(roOp, "dst")) if err != nil { f.t.Fatal(err) } in.Admission.ObservedSession, in.Admission.ObservedGeneration = counted.Claim.Holder.Session, counted.Claim.Holder.Generation admitted, err := hostclaim.Update(f.t.Context(), f.claims, counted, hostclaim.AdmitWriter(in.Admission)) if err != nil { f.t.Fatal(err) } return admitted } // C4, ruling 47: P's first opening is cold into P, keeping the install // stamp, prepared from P's snapshot, and certified only by this // operation's verified release; a failover's opening is not certified; an // empty pin's opening is empty, with no baseline. func TestRelocationSelectCertifiesOnlyPsOpeningAfterItsRelease(t *testing.T) { ctx := t.Context() t.Run("a drain's release", func(t *testing.T) { f := newROFixture(t, roOptions{}) o := f.read() in, err := f.inputs.Select(ctx, f.route, o, roProbe) if err != nil { t.Fatal(err) } h := o.Claim.Holder want := hostclaim.Admission{ObservedOpID: roOp, ObservedPhase: hostclaim.PhaseRelocatingProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "dst", Mode: "cold", NewGeneration: roP, KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 42} if !reflect.DeepEqual(in.Admission, want) { t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want) } if !in.Prepared || in.Empty || in.Baseline == nil || in.Baseline.Generation != roP || in.Baseline.MaxTXID != 42 || f.releases.calls != 1 { t.Fatalf("prepared=%v empty=%v baseline=%+v release checks=%d", in.Prepared, in.Empty, in.Baseline, f.releases.calls) } if in.DBPath != f.route.DBPath || in.Version != "v1" || len(in.Roles) != 1 || in.Roles[0] != "main" || in.ReadyPath != "/" { t.Fatalf("launch inputs %+v", in) } f.releases.err = errors.New("the receipt does not verify") if _, err := f.inputs.Select(ctx, f.route, o, roProbe); !errors.Is(err, f.releases.err) { t.Fatalf("an unverified release certified: %v", err) } }) t.Run("a failover", func(t *testing.T) { f := newROFixture(t, roOptions{failover: true}) in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } if in.Admission.PredecessorCertified || !in.Admission.KeepStamp || in.Admission.NewGeneration != roP || f.releases.calls != 0 { t.Fatalf("failover admission %+v (release checks %d)", in.Admission, f.releases.calls) } }) t.Run("an empty pin", func(t *testing.T) { f := newROFixture(t, roOptions{empty: true}) in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } if !in.Empty || in.Prepared || in.Baseline != nil || in.Admission.NewGeneration != roP || !in.Admission.KeepStamp || in.Admission.RemoteMaxTXID != 0 { t.Fatalf("empty opening %+v", in) } if err := f.inputs.EmptyAuthority(ctx, f.read(), in); err != nil { t.Fatalf("the evidenced empty pin: %v", err) } }) } // P is opened once. After its opening every attempt is a fresh generation // from the predecessor, keeping no stamp, never certified, and evidence for // P's opening is refused on the opened claim. func TestRelocationSelectNeverReopensP(t *testing.T) { f := newROFixture(t, roOptions{}) ctx := t.Context() first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } opened := f.admit(f.read(), first) if !hostclaim.PreparedOpened(opened.Claim) || opened.Claim.Holder.Stamp != f.stamp { t.Fatalf("P's opening: %+v", opened.Claim.Holder) } again, err := f.inputs.Select(ctx, f.route, opened, roProbe) if err != nil { t.Fatal(err) } a := again.Admission if a.NewGeneration != "fresh1" || a.KeepStamp || a.PredecessorCertified || again.Prepared || a.ObservedGeneration != roP || a.ObservedSession != opened.Claim.Holder.Session { t.Fatalf("the later attempt %+v prepared=%v", a, again.Prepared) } if err := f.inputs.AdmissionEvidence(ctx, opened, again.Admission, again.Baseline); err != nil { t.Fatalf("the later attempt's evidence: %v", err) } reopen := again.Admission reopen.NewGeneration, reopen.KeepStamp = roP, true if err := f.inputs.AdmissionEvidence(ctx, opened, reopen, first.Baseline); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("P reopened: %v", err) } } // The inputs serve only this box's relocation probe, of the pinned // release, at a nested route. func TestRelocationSelectRefusesEverythingButThisBoxsProbe(t *testing.T) { ctx := t.Context() for name, tc := range map[string]struct { claim func(*hostclaim.Claim) error policy activator.StartPolicy route func(*registry.Route) launch string want error }{ "installed": {claim: func(c *hostclaim.Claim) error { c.Op.Phase = hostclaim.PhaseInstalled; return nil }, policy: roProbe, want: hostrelocate.ErrRefused}, "another executor": {claim: func(c *hostclaim.Claim) error { c.Op.Exec, c.Op.Intent["target_box"] = "src", "src"; return nil }, policy: roProbe, want: hostrelocate.ErrRefused}, "stuck": {claim: func(c *hostclaim.Claim) error { c.Op.Journal["stuck"] = "an operator"; return nil }, policy: roProbe, want: hostrelocate.ErrRefused}, "not a probe": {policy: activator.StartPolicy{MaxStarts: 5, Window: time.Minute}, want: hostrelocate.ErrRefused}, "another release": {policy: roProbe, launch: "v2", want: hostrelocate.ErrRefused}, "a relative database": {policy: roProbe, route: func(rt *registry.Route) { rt.DBPath = "shop.db" }, want: hostbox.ErrUnsupported}, "another host": {policy: roProbe, route: func(rt *registry.Route) { rt.Host = "other.example" }, want: hostbox.ErrUnsupported}, } { t.Run(name, func(t *testing.T) { f := newROFixture(t, roOptions{}) if tc.claim != nil { f.update(tc.claim) } rt := f.route if tc.route != nil { tc.route(&rt) } if tc.launch != "" { launch := f.inputs.Launch f.inputs.Launch = func(ctx context.Context, rt registry.Route) (ClaimServingLaunch, error) { l, err := launch(ctx, rt) l.Version = tc.launch return l, err } } if _, err := f.inputs.Select(ctx, rt, f.read(), tc.policy); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } }) } } // Ruling 47: P's opening is bound to its observation, its certified // baseline (P's snapshot at the pin's N) and P's prepared files manifest by // the digest prepared_authority journals, read again. func TestRelocationAdmissionEvidenceBindsTheBaselineAndThePreparedManifest(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) o := f.read() in, err := f.inputs.Select(ctx, f.route, o, roProbe) if err != nil { t.Fatal(err) } if err := f.inputs.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { t.Fatal(err) } for name, mutate := range map[string]func(*hostclaim.Admission, *genselect.Plan){ "a stale baseline": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = 41, 41 }, "another generation": func(_ *hostclaim.Admission, p *genselect.Plan) { p.Generation = "g1" }, "stamp not kept": func(a *hostclaim.Admission, _ *genselect.Plan) { a.KeepStamp = false }, "not certified": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorCertified = false }, "another box": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Box = "src" }, "sealed coverage": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Coverage = "sealed" }, "another region": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Region = "uk" }, "another session": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedSession++ }, "warm": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Mode = "warm" }, "lost writes claimed": func(a *hostclaim.Admission, _ *genselect.Plan) { a.AdditionalWritesLost = []hostclaim.WritesLost{{}} }, } { a, p := in.Admission, *in.Baseline mutate(&a, &p) if err := f.inputs.AdmissionEvidence(ctx, o, a, &p); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } // P's files manifest replaced after the build: the digest no longer // matches the authority. key := hostfiles.ManifestKey(moves.GenerationFilesPrefix(o.Claim.Holder.Location, roP)) _, etag, err := f.files.Get(ctx, key) if err != nil { t.Fatal(err) } other, _ := json.Marshal(hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 0, Entries: map[string]hostfiles.Entry{"x": {Seq: 1, Deleted: true}}, Platform: map[string]hostfiles.Entry{}}) if _, err := f.files.Put(ctx, key, other, etag); err != nil { t.Fatal(err) } if err := f.inputs.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another files manifest: %v", err) } // An empty pin's opening has no baseline; a non-empty one must. e := newROFixture(t, roOptions{empty: true}) eo := e.read() ein, err := e.inputs.Select(ctx, e.route, eo, roProbe) if err != nil { t.Fatal(err) } if err := e.inputs.AdmissionEvidence(ctx, eo, ein.Admission, nil); err != nil { t.Fatalf("the empty opening: %v", err) } if err := f.inputs.AdmissionEvidence(ctx, o, in.Admission, nil); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a non-empty opening without its baseline: %v", err) } } // Ruling 41: one admitted session is bound to one record attempt k (the // claim's probe_attempts.); the attempt's first app start is kept // across restarts; a start is reserved, settled and re-reserved under the // same session; a session bound to k is refused once the record moves on. // The journal is keyed by op (ruling 44). func TestRelocationProbeAttemptsBindOneSessionPerRecordAttempt(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } admitted := f.admit(f.read(), in) h := admitted.Claim.Holder dataDir := t.TempDir() now := time.Date(2026, 10, 7, 4, 0, 0, 0, time.UTC) p := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: func() time.Time { return now }} owner := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "dst", Enrollment: strings.Repeat("a", 32), BootID: "boot", ControllerID: "ctl"} s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: owner} opened, err := p.OpenProbe(ctx, s, now) if err != nil || !opened.Equal(now) { t.Fatalf("open %v %v", opened, err) } if again, err := p.OpenProbe(ctx, s, now.Add(time.Minute)); err != nil || !again.Equal(now) { t.Fatalf("a restart moved the attempt's first start: %v %v", again, err) } if err := p.Reserve(ctx, s, 0); err != nil { t.Fatal(err) } a := p.CurrentAttempt(s) if a.ID == "" || a.Session != h.Session || a.Box != "dst" || a.ControllerID != "ctl" { t.Fatalf("the reservation %+v", a) } if got, err := (relocationAttemptReader{DataDir: dataDir}).ReadAttempt(ctx, admitted); err != nil || got != a { t.Fatalf("root reads %+v %v", got, err) } if err := p.Reserve(ctx, s, 0); err != nil || p.CurrentAttempt(s) != a { t.Fatalf("a repeated reservation: %v", err) } if err := p.Reserve(ctx, s, 1); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a restart before the first start settled: %v", err) } if err := p.Failed(ctx, s, 0); err != nil { t.Fatal(err) } if err := p.Reserve(ctx, s, 1); err != nil { t.Fatalf("the restart after a failed start: %v", err) } if b := p.CurrentAttempt(s); b.Session != h.Session || b.LaunchInputs != a.LaunchInputs { t.Fatalf("the restart %+v", b) } path, _ := relocationAttemptsPath(dataDir, roOp) if strings.Contains(path, roHost) || filepath.Base(filepath.Dir(path)) != roOp { t.Fatalf("the journal path %s", path) } // The record attempt moves on with the same session admitted: the // session is bound to attempt 1 and refused for attempt 2. moved := f.update(func(c *hostclaim.Claim) error { c.Op.Journal["probe_attempts.dst"] = "2"; return nil }) s.Admitted = moved if err := p.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("one session, two record attempts: %v", err) } // Another session than the admitted writer is refused before any journal. other := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: owner} other.Owner.Session++ if _, err := p.OpenProbe(ctx, other, now); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another session: %v", err) } } // The admission hooks: before P's first admission the source's release is // verified at its receipt (a drain) or its fence is journaled with the // source lost (a failover); after it, this box's install journal is the // receipt's and P keeps the install stamp. func TestRelocationAdmissionHooksVerifyReleaseAndInstall(t *testing.T) { ctx := t.Context() releases := &roReleases{} before := relocationBeforeAdmission("dst", releases) f := newROFixture(t, roOptions{}) if err := before(ctx, f.read()); err != nil || releases.calls != 1 { t.Fatalf("a drain's release: %v (%d checks)", err, releases.calls) } releases.err = errors.New("no receipt") if err := before(ctx, f.read()); !errors.Is(err, releases.err) { t.Fatalf("an unverified release: %v", err) } g := newROFixture(t, roOptions{failover: true}) if err := before(ctx, g.read()); err != nil { t.Fatalf("a fenced failover: %v", err) } unfenced := g.update(func(c *hostclaim.Claim) error { delete(c.Op.Journal, "fence.src"); return nil }) if err := before(ctx, unfenced); !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("neither release nor fence: %v", err) } if err := relocationBeforeAdmission("other", releases)(ctx, f.read()); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another box: %v", err) } // After: the install journal at the receipt's digest. dataDir := t.TempDir() after := relocationAfterAdmission(dataDir, "dst") in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } prior := f.read() path, _ := relocationInstallPath(dataDir, roOp) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } journal := []byte(`{"v":1,"host":"shop.example","op":"rel-1"}`) if err := os.WriteFile(path, journal, 0o600); err != nil { t.Fatal(err) } sum := sha256.Sum256(journal) f.update(func(c *hostclaim.Claim) error { var l hostclaim.LocalInstall _ = json.Unmarshal([]byte(c.Op.Journal["local_install"]), &l) l.JournalSHA256 = hex.EncodeToString(sum[:]) c.Op.Journal["local_install"] = string(mustJSON(t, l)) return nil }) admitted := f.admit(f.read(), in) if err := after(ctx, prior, admitted); err != nil { t.Fatalf("after P's opening: %v", err) } if err := os.WriteFile(path, []byte(`{"v":1}`), 0o600); err != nil { t.Fatal(err) } if err := after(ctx, prior, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another install journal: %v", err) } if err := relocationAfterAdmission(dataDir, "src")(ctx, prior, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another box's admission: %v", err) } } // relocationProbeActivator is the activator's probe and fence, recorded. type relocationProbeActivator struct { probes []activator.ProbeConfig result activator.ProbeResult err error } func (a *relocationProbeActivator) ProbeWake(_ context.Context, pc activator.ProbeConfig) (activator.ProbeResult, error) { a.probes = append(a.probes, pc) return a.result, a.err } // The probe attempt never fences: a fence asked of this double fails the // test's call with an error naming it, and lifts nothing. func (a *relocationProbeActivator) FenceAndStopAs(context.Context, string, uint64, string) (*activator.StopEvidence, error) { return nil, errors.New("the probe attempt asked for a fence") } func (a *relocationProbeActivator) LiftFenceAs(string, uint64, string) bool { return false } // One probe attempt is one ProbeWake, bounded by the probe deadline, // carrying P's install manifest on P's opening; a failed or overdue // attempt is not passed, and any other error is returned. func TestRelocationAdmitRunsOneProbeAttempt(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) act := &relocationProbeActivator{result: activator.ProbeResult{Session: 99}} admit := ClaimAssembly{}.relocationAdmit(RelocationAssemblyConfig{Activator: act, Versions: f.bucket, Bucket: "data"}) o := f.read() session, passed, err := admit(ctx, o, true) if err != nil || !passed || session != 99 || len(act.probes) != 1 { t.Fatalf("passed: %d %v %v", session, passed, err) } pc := act.probes[0] since, _ := time.Parse(time.RFC3339Nano, o.Claim.Op.Journal["probing_since"]) if pc.Host != roHost || pc.Op != roOp || pc.HoldOp != roOp || !pc.Deadline.Equal(since.Add(hostrelocate.ProbeDeadline)) || pc.Prepared == nil || *pc.Prepared != f.manifest { t.Fatalf("probe config %+v", pc) } for _, failed := range []error{activator.ErrProbeFailed, activator.ErrAttemptOver} { act.err = failed if _, passed, err := admit(ctx, o, true); passed || err != nil { t.Fatalf("%v: passed=%v %v", failed, passed, err) } } act.err = errors.New("the activator is closing") if _, _, err := admit(ctx, o, true); !errors.Is(err, act.err) { t.Fatalf("another error: %v", err) } if _, _, err := admit(ctx, o, false); !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("certification disagreed: %v", err) } if n := len(act.probes); n != 4 { t.Fatalf("%d probes, want 4", n) } } // Rulings 37 and 39: with moves' C8 provider bound, Open returns the // complete owner set and the two refusals; the relocating dispatch binds. // Without the region's bucket or a store constructor it is unavailable, // and a nil source is refused before it is locked. func TestRelocationSourceOpensACompleteOwnerSet(t *testing.T) { ctx := t.Context() var opened []string src := &RelocationSource{DataDir: t.TempDir(), FilesStore: func(_ context.Context, bucket string) (release.Store, error) { opened = append(opened, bucket) return release.NewDirStore(t.TempDir()), nil }} identityErr := errors.New("the roster entry has moved") var identity error sources := ClaimServingSources{ Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil }, Markers: plan9D5Markers{leasetest.NewVersionedBucket()}, Files: &roFiles{}, Identity: func(context.Context) error { return identity }, } src.SetServingSources(sources) actor := hostbox.Actor{Box: "dst", Bucket: "data-ie", Region: "ie"} owners, refusals, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor) if err != nil { t.Fatal(err) } if owners.zero() || !owners.complete() || owners.Operations == nil || owners.Policy == nil || owners.BeforeAdmission == nil || owners.AfterAdmission == nil || len(refusals) != 2 { t.Fatalf("owners %+v refusals %d", owners, len(refusals)) } if len(opened) != 1 || opened[0] != "data-ie" { t.Fatalf("the files store was opened for %v", opened) } if bound, err := claimRelocationC1(plan11Env("1", "1"), true, owners); !bound || err != nil { t.Fatalf("the relocating dispatch did not bind: %v %v", bound, err) } // M2: every data-bucket request the inputs make runs the box's own // identity check first, as main's serving owners do. identity = identityErr if _, err := owners.Inputs.(*RelocationWakeInputs).Data.ListVersions(ctx, "data-ie", "instances/"); !errors.Is(err, identityErr) { t.Fatalf("a read without the identity check: %v", err) } identity = nil if _, _, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, hostbox.Actor{Box: "dst", Region: "ie"}); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("no bucket: %v", err) } for name, without := range map[string]func(*ClaimServingSources){ "no launch": func(s *ClaimServingSources) { s.Launch = nil }, "no markers": func(s *ClaimServingSources) { s.Markers = nil }, "no identity": func(s *ClaimServingSources) { s.Identity = nil }, } { partial := sources without(&partial) src.SetServingSources(partial) if _, _, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("%s: %v", name, err) } } src.SetServingSources(sources) var none *RelocationSource if _, _, err := none.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("a nil source: %v", err) } } // Finding 11 (C8): without moves' files-manifest provider the source opens // no owners, and the dispatcher keeps relocating refused. func TestRelocationStaysUnboundWithoutTheC8Provider(t *testing.T) { src := &RelocationSource{DataDir: t.TempDir(), FilesStore: func(context.Context, string) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil }} src.SetServingSources(ClaimServingSources{Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil }, Markers: nil}) owners, refusals, err := src.Open(context.Background(), release.NewDirStore(t.TempDir()), &fakeVersioned{}, hostbox.Actor{Box: "dst", Bucket: "data-ie", Region: "ie"}) if err != nil || !owners.zero() || refusals != nil { t.Fatalf("opened without C8: %+v %v %v", owners, refusals, err) } bound, err := claimRelocationC1(plan11Env("1", "1"), true, owners) if bound || err != nil { t.Fatalf("relocating bound without C8: %v %v", bound, err) } } // authority rewrites the journaled prepared authority in place. func (f *roFixture) authority(mutate func(*hostclaim.PreparedAuthority)) hostclaim.Observed { f.t.Helper() return f.update(func(c *hostclaim.Claim) error { var a hostclaim.PreparedAuthority if err := json.Unmarshal([]byte(c.Op.Journal["prepared_authority"]), &a); err != nil { return err } mutate(&a) c.Op.Journal["prepared_authority"] = string(mustJSON(f.t, a)) return nil }) } // Absence is never evidence: a journaled release that is present but // unreadable, or that names another writer than the predecessor, is not // "no release" (fix round 1, I3). Before the fix a damaged release on a // claim whose source is lost and fenced passed the admission hook as a // failover, and a mismatched one opened P uncertified, declaring the // source's writes lost with no failover evidence. func TestRelocationAnUnreadableReleaseIsNeverAFailover(t *testing.T) { ctx := t.Context() for name, damage := range map[string]func(*hostclaim.Claim){ "unparseable": func(c *hostclaim.Claim) { c.Op.Journal["release"] = `{"released_by":` }, "no releasing box": func(c *hostclaim.Claim) { c.Op.Journal["release"] = `{}` }, "another session": func(c *hostclaim.Claim) { var rel hostclaim.Release _ = json.Unmarshal([]byte(c.Op.Journal["release"]), &rel) rel.Session++ c.Op.Journal["release"] = string(mustJSON(t, rel)) }, "another box": func(c *hostclaim.Claim) { var rel hostclaim.Release _ = json.Unmarshal([]byte(c.Op.Journal["release"]), &rel) rel.ReleasedBy = "other" c.Op.Journal["release"] = string(mustJSON(t, rel)) }, } { t.Run(name, func(t *testing.T) { // A failover shape (the source lost and fenced) for the two // unreadable cases, a drain for the two mismatched ones. f := newROFixture(t, roOptions{failover: strings.HasPrefix(name, "unparseable") || strings.HasPrefix(name, "no ")}) good, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } damaged := f.update(func(c *hostclaim.Claim) error { damage(c); return nil }) releases := &roReleases{} if err := relocationBeforeAdmission("dst", releases)(ctx, damaged); !errors.Is(err, hostrelocate.ErrRelease) || releases.calls != 0 { t.Fatalf("the hook took a damaged release: %v (%d checks)", err, releases.calls) } checked := f.releases.calls if _, err := f.inputs.Select(ctx, f.route, damaged, roProbe); !errors.Is(err, hostrelocate.ErrRelease) || f.releases.calls != checked { t.Fatalf("Select opened P over a damaged release: %v (%d checks)", err, f.releases.calls-checked) } if err := f.inputs.AdmissionEvidence(ctx, damaged, good.Admission, good.Baseline); !errors.Is(err, hostrelocate.ErrRelease) { t.Fatalf("AdmissionEvidence passed over a damaged release: %v", err) } act := &relocationProbeActivator{} probe := ClaimAssembly{}.relocationAdmit(RelocationAssemblyConfig{Activator: act, Versions: f.bucket, Bucket: "data"}) if _, _, err := probe(ctx, damaged, false); !errors.Is(err, hostrelocate.ErrRelease) || len(act.probes) != 0 { t.Fatalf("the probe attempt ran over a damaged release: %v (%d probes)", err, len(act.probes)) } // M1: the preflight finds it before any attempt is counted, // before root is asked. if err := (ClaimAssembly{Wake: &ClaimWakeAssembly{Roles: &ClaimRoleAssembly{}}}).relocationPreflight(RelocationAssemblyConfig{})(ctx, damaged); !errors.Is(err, hostrelocate.ErrRelease) { t.Fatalf("the preflight passed a damaged release: %v", err) } }) } } // A drain's P opening is certified only by a release naming the exact // predecessor; the hook's PreparedOpened short-circuit stands on its own: // once P is open no release or fence is asked for. func TestRelocationBeforeAdmissionIgnoresTheReleaseOncePIsOpen(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } f.admit(f.read(), first) opened := f.update(func(c *hostclaim.Claim) error { delete(c.Op.Journal, "release"); return nil }) releases := &roReleases{err: errors.New("never asked")} if err := relocationBeforeAdmission("dst", releases)(ctx, opened); err != nil || releases.calls != 0 { t.Fatalf("after P's opening: %v (%d checks)", err, releases.calls) } } // Absence is never evidence: a later attempt follows P's admitted // session, which ran, so a listing that finds nothing is nothing // replicated or not yet visible, never an empty predecessor. Main's // serving rule admits an empty cold admission only for a holder that // never admitted a writer, which a later attempt never is (fix round 1, // I2). Over an empty P as over a pinned snapshot, Select refuses, no // EmptyAuthority proves it, and no evidence passes a nil baseline. func TestRelocationLaterAttemptIsNeverEmpty(t *testing.T) { ctx := t.Context() for name, o := range map[string]roOptions{"a pinned snapshot": {}, "an empty P": {empty: true}} { t.Run(name, func(t *testing.T) { f := newROFixture(t, o) first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } opened := f.admit(f.read(), first) // The same inputs over a bucket that lists nothing for the location. blank := leasetest.NewVersionedBucket() blank.AddBucket("data") elsewhere := &RelocationWakeInputs{Launch: f.inputs.Launch, FilesManifest: f.c8, Markers: plan9D5Markers{blank}, Claims: f.claims, FilesStore: f.files, Data: blank, Bucket: "data", Region: "ie", Box: "dst", NewID: f.inputs.NewID, Now: f.inputs.Now, Releases: f.releases} for _, inputs := range []*RelocationWakeInputs{f.inputs, elsewhere} { // Over the pinned snapshot the real bucket lists P's root, a // baseline; the blank bucket, and an empty P's own bucket, list // nothing, which is never an empty admission. if inputs == elsewhere || o.empty { if in, err := inputs.Select(ctx, f.route, opened, roProbe); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("an empty listing after P's session selected empty=%v: %v", in.Empty, err) } } h := opened.Claim.Holder fresh := hostclaim.Admission{ObservedOpID: roOp, ObservedPhase: hostclaim.PhaseRelocatingProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "dst", Mode: "cold", NewGeneration: "fresh9", Region: "ie", PredecessorRegion: "ie"} if err := inputs.EmptyAuthority(ctx, opened, ClaimLaunchInputs{Empty: true, Admission: fresh}); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("EmptyAuthority proved an empty later attempt: %v", err) } if err := inputs.AdmissionEvidence(ctx, opened, fresh, nil); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("AdmissionEvidence passed an empty later attempt: %v", err) } } }) } } // Absence is never evidence: P opens empty only on the pin's word // (RecoveryPin.Empty), which the prepared authority must agree with. A // missing install manifest alone, an authority with a manifest under an // empty pin, or an unreadable pin, refuses at Select and at the probe // attempt, before any admission. func TestRelocationEmptyPNeedsThePinAndTheAuthorityToAgree(t *testing.T) { ctx := t.Context() admit := func(f *roFixture) (func(context.Context, hostclaim.Observed, bool) (uint64, bool, error), *relocationProbeActivator) { act := &relocationProbeActivator{} return ClaimAssembly{}.relocationAdmit(RelocationAssemblyConfig{Activator: act, Versions: f.bucket, Bucket: "data"}), act } t.Run("no manifest under a non-empty pin", func(t *testing.T) { f := newROFixture(t, roOptions{}) o := f.authority(func(a *hostclaim.PreparedAuthority) { a.InstallManifest, a.SHA256, a.SnapshotTXID = "", "", 0 }) if in, err := f.inputs.Select(ctx, f.route, o, roProbe); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("selected empty=%v: %v", in.Empty, err) } probe, act := admit(f) if _, _, err := probe(ctx, o, true); !errors.Is(err, hostbox.ErrChanged) || len(act.probes) != 0 { t.Fatalf("the probe ran without P's manifest: %v (%d probes)", err, len(act.probes)) } }) t.Run("a manifest under an empty pin", func(t *testing.T) { f := newROFixture(t, roOptions{empty: true}) o := f.authority(func(a *hostclaim.PreparedAuthority) { a.InstallManifest = "somewhere/install.json" }) if _, err := f.inputs.Select(ctx, f.route, o, roProbe); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("selected: %v", err) } probe, act := admit(f) if _, _, err := probe(ctx, o, true); !errors.Is(err, hostbox.ErrChanged) || len(act.probes) != 0 { t.Fatalf("the probe ran: %v (%d probes)", err, len(act.probes)) } }) t.Run("an unreadable pin", func(t *testing.T) { f := newROFixture(t, roOptions{}) o := f.update(func(c *hostclaim.Claim) error { c.Op.Journal["pin"] = `{"empty":`; return nil }) if _, err := f.inputs.Select(ctx, f.route, o, roProbe); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("selected: %v", err) } probe, act := admit(f) if _, _, err := probe(ctx, o, true); !errors.Is(err, hostbox.ErrChanged) || len(act.probes) != 0 { t.Fatalf("the probe ran: %v (%d probes)", err, len(act.probes)) } }) } // roWork is a spawn selection whose fence always holds. type roWork struct{} func (roWork) With(_ context.Context, _ hostwrite.Identity, fn func() error) error { return fn() } // putManifest replaces P's install.json in the bucket and the authority's // digest of it, so only the mutated field disagrees. func (f *roFixture) putManifest(mutate func(*moves.InstallManifest)) hostclaim.Observed { f.t.Helper() im := f.manifest mutate(&im) raw, _ := json.Marshal(im) f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix(f.read().Claim.Holder.Location, roP)), raw) sum := sha256.Sum256(raw) return f.authority(func(a *hostclaim.PreparedAuthority) { a.SHA256 = hex.EncodeToString(sum[:]) }) } // atP is the observation as the admission that opens P leaves the holder, // without writing the claim. func atP(o hostclaim.Observed) hostclaim.Observed { c := *o.Claim h := *c.Holder h.Generation = roP c.Holder = &h o.Claim = &c return o } // Rulings 37 and 39, by behaviour: the two policy refusals refuse a // relocating selection at another installation than the holder's, and // any selection by a box the claim lists as lost; they pass everything // else and never grant. func TestRelocationPolicyRefusalsRefuseByBehaviour(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) c := f.read().Claim h := c.Holder at := hostbox.Selection{Host: roHost, Action: "probe", Box: "dst", Region: "ie", Captured: hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}} if err := refuseStaleStamp(ctx, c, at, hostwrite.Identity{}); err != nil { t.Fatalf("the holder's installation: %v", err) } stale := at stale.Captured.Stamp++ if err := refuseStaleStamp(ctx, c, stale, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a retired stamp: %v", err) } other := at other.Captured.Incarnation = strings.Repeat("9", 64) if err := refuseStaleStamp(ctx, c, other, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another incarnation: %v", err) } serving := *c serving.Op = hostclaim.Op{Kind: hostclaim.KindServing} if err := refuseStaleStamp(ctx, &serving, stale, hostwrite.Identity{}); err != nil { t.Fatalf("the stamp refusal reaches beyond relocating: %v", err) } if err := refuseLostBox(ctx, c, at, hostwrite.Identity{}); err != nil { t.Fatalf("a box not lost: %v", err) } lost := *c lost.Lost = []string{"dst"} if err := refuseLostBox(ctx, &lost, at, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a lost box: %v", err) } if err := refuseLostBox(ctx, &lost, hostbox.Selection{Box: "src"}, hostwrite.Identity{}); err != nil { t.Fatalf("another box while dst is lost: %v", err) } } // preparedMarker binds install.json to the prepared authority (its host, // its snapshot position, the holder at P) and takes an existing marker // for P only when it is this box's, this incarnation's, this host's, with // the authority's digest; another op's marker is superseded by its ETag. func TestRelocationPreparedMarkerBindsTheManifestAndTheMarker(t *testing.T) { ctx := t.Context() fence := func(context.Context) error { return nil } t.Run("install.json", func(t *testing.T) { for name, mutate := range map[string]func(*moves.InstallManifest){ "another host": func(im *moves.InstallManifest) { im.Host = "other.example" }, "another position": func(im *moves.InstallManifest) { im.N = 41 }, } { f := newROFixture(t, roOptions{}) o := f.putManifest(mutate) if _, err := f.inputs.preparedMarker(ctx, atP(o), fence); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } f := newROFixture(t, roOptions{}) if _, err := f.inputs.preparedMarker(ctx, f.read(), fence); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("the holder not at P: %v", err) } if _, err := f.inputs.preparedMarker(ctx, atP(f.read()), fence); err != nil { t.Fatalf("the authority's own manifest: %v", err) } }) t.Run("an existing marker", func(t *testing.T) { f := newROFixture(t, roOptions{}) o := atP(f.read()) h := o.Claim.Holder auth, _ := preparedAuthority(o.Claim) mine := genselect.Marker{Op: roP, Incarnation: h.Incarnation, Host: roHost, Box: "dst", ManifestSHA256: auth.SHA256} for name, mutate := range map[string]func(*genselect.Marker){ "another box": func(m *genselect.Marker) { m.Box = "src" }, "another incarnation": func(m *genselect.Marker) { m.Incarnation = strings.Repeat("9", 64) }, "another host": func(m *genselect.Marker) { m.Host = "other.example" }, "another manifest": func(m *genselect.Marker) { m.ManifestSHA256 = strings.Repeat("0", 64) }, } { m := mine mutate(&m) f.bucket.Put("data", markerKey(h), mustJSON(t, m)) if _, err := f.inputs.preparedMarker(ctx, o, fence); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } written := f.bucket.Put("data", markerKey(h), mustJSON(t, mine)) if r, err := f.inputs.preparedMarker(ctx, o, fence); err != nil || r.MarkerVersion != written { t.Fatalf("P's own marker: %+v %v", r, err) } earlier := f.bucket.Put("data", markerKey(h), mustJSON(t, genselect.Marker{Op: "g1", Incarnation: h.Incarnation, Host: roHost, Box: "src"})) r, err := f.inputs.preparedMarker(ctx, o, fence) if err != nil || r.MarkerVersion == earlier || r.MarkerVersion != f.bucket.LatestID("data", markerKey(h)) { t.Fatalf("an earlier generation's marker superseded: %+v %v", r, err) } }) } // PublicationEvidence for P's prepared opening requires the holder still // at P, with the marker exactly the recorded version. func TestRelocationPublicationEvidenceNeedsTheHolderAtP(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } admitted := f.admit(f.read(), in) if err := f.inputs.Materialize(ctx, admitted, in, roWork{}); err != nil { t.Fatal(err) } if err := f.inputs.PublicationEvidence(ctx, admitted); err != nil { t.Fatalf("P's receipt: %v", err) } moved := admitted c := *moved.Claim h := *c.Holder h.Generation = "g9" c.Holder = &h moved.Claim = &c if err := f.inputs.PublicationEvidence(ctx, moved); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("the holder moved off P: %v", err) } f.bucket.Put("data", markerKey(admitted.Claim.Holder), mustJSON(t, genselect.Marker{Op: "g9", Box: "dst"})) if err := f.inputs.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another marker version: %v", err) } } // A later attempt's evidence re-selects the predecessor's baseline and // binds the admission to it: its position, the declared remote position, // and the exact files. func TestRelocationLaterAttemptEvidenceReselectsTheBaseline(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } opened := f.admit(f.read(), first) again, err := f.inputs.Select(ctx, f.route, opened, roProbe) if err != nil || again.Baseline == nil { t.Fatalf("the later attempt: %+v %v", again, err) } if err := f.inputs.AdmissionEvidence(ctx, opened, again.Admission, again.Baseline); err != nil { t.Fatal(err) } for name, mutate := range map[string]func(*hostclaim.Admission, *genselect.Plan){ "a stale position": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = 41, 41 }, "another declared loss": func(a *hostclaim.Admission, _ *genselect.Plan) { a.RemoteMaxTXID++ }, "other files": func(_ *hostclaim.Admission, p *genselect.Plan) { p.Files = append([]genselect.File{{Key: "x"}}, p.Files...) }, "the predecessor's name": func(_ *hostclaim.Admission, p *genselect.Plan) { p.Generation = "g1" }, } { a, p := again.Admission, *again.Baseline mutate(&a, &p) if err := f.inputs.AdmissionEvidence(ctx, opened, a, &p); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } } // After the admission, P's opening keeps the install stamp: an admitted // holder at P with another stamp is refused even with the receipt whole. func TestRelocationAfterAdmissionNeedsTheInstallStampAtP(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } dataDir := t.TempDir() path, _ := relocationInstallPath(dataDir, roOp) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } journal := []byte(`{"v":1,"host":"shop.example","op":"rel-1"}`) if err := os.WriteFile(path, journal, 0o600); err != nil { t.Fatal(err) } sum := sha256.Sum256(journal) f.update(func(c *hostclaim.Claim) error { var l hostclaim.LocalInstall _ = json.Unmarshal([]byte(c.Op.Journal["local_install"]), &l) l.JournalSHA256 = hex.EncodeToString(sum[:]) c.Op.Journal["local_install"] = string(mustJSON(t, l)) return nil }) prior := f.read() admitted := f.admit(prior, in) after := relocationAfterAdmission(dataDir, "dst") if err := after(ctx, prior, admitted); err != nil { t.Fatal(err) } moved := admitted c := *moved.Claim h := *c.Holder h.Stamp++ c.Holder = &h moved.Claim = &c if err := after(ctx, prior, moved); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("P at another stamp: %v", err) } } // The journal refuses once the claim no longer names this box the // executor at probing, unstuck; a reserved start is bound to the // admission's generation and stamp; a settle binds the exact reservation // this process made. func TestRelocationProbeAttemptsRefuseAMovedClaimOrJournal(t *testing.T) { ctx := t.Context() now := time.Date(2026, 10, 7, 4, 0, 0, 0, time.UTC) start := func(t *testing.T, f *roFixture, dataDir string) (*relocationProbeAttempts, OperationStart) { t.Helper() in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } admitted := f.admit(f.read(), in) h := admitted.Claim.Holder p := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: func() time.Time { return now }} owner := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "dst", Enrollment: strings.Repeat("a", 32), BootID: "boot", ControllerID: "ctl"} s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: owner} if err := p.Reserve(ctx, s, 0); err != nil { t.Fatal(err) } return p, s } t.Run("stuck", func(t *testing.T) { f := newROFixture(t, roOptions{}) p, s := start(t, f, t.TempDir()) f.update(func(c *hostclaim.Claim) error { c.Op.Journal["stuck"] = "an operator"; return nil }) if err := p.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a stuck operation: %v", err) } }) t.Run("another executor", func(t *testing.T) { // A journal bound for src, with src's record attempt counted, over // a claim whose executor is dst: the admitted writer still matches, // only the executor does not. f := newROFixture(t, roOptions{}) _, s := start(t, f, t.TempDir()) s.Admitted = f.update(func(c *hostclaim.Claim) error { c.Op.Journal["probe_attempts.src"] = "1"; return nil }) s.Owner.Box = "src" src := &relocationProbeAttempts{Claims: f.claims, DataDir: t.TempDir(), Box: "src", NewID: f.inputs.NewID, Now: func() time.Time { return now }} if err := src.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another executor: %v", err) } }) for name, rewrite := range map[string]func(*relocationStart){ "another generation": func(st *relocationStart) { st.Generation = "g9" }, "another stamp": func(st *relocationStart) { st.Stamp++ }, } { t.Run("a journaled start under "+name, func(t *testing.T) { f := newROFixture(t, roOptions{}) dd := t.TempDir() p, s := start(t, f, dd) j, err := readRelocationAttempts(dd, roHost, roOp) if err != nil { t.Fatal(err) } st := j.Starts["1"] rewrite(&st) j.Starts["1"] = st if err := j.write(dd); err != nil { t.Fatal(err) } if err := p.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } }) } t.Run("a settle of another reservation", func(t *testing.T) { f := newROFixture(t, roOptions{}) dd := t.TempDir() p, s := start(t, f, dd) j, _ := readRelocationAttempts(dd, roHost, roOp) st := j.Starts["1"] st.Attempt.ID = "someone-else" j.Starts["1"] = st if err := j.write(dd); err != nil { t.Fatal(err) } if err := p.Failed(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("settled a reservation this process did not make: %v", err) } }) } // M1: the preflight refuses a claim whose pin, authority or probing start // is damaged before root is asked and before any attempt is counted; a // sound claim goes on to root. func TestRelocationPreflightRefusesADamagedClaimBeforeCounting(t *testing.T) { ctx := t.Context() preflight := (ClaimAssembly{Wake: &ClaimWakeAssembly{Roles: &ClaimRoleAssembly{}}}).relocationPreflight(RelocationAssemblyConfig{}) for name, damage := range map[string]func(*hostclaim.Claim){ "an unreadable pin": func(c *hostclaim.Claim) { c.Op.Journal["pin"] = `{"empty":` }, "an unreadable authority": func(c *hostclaim.Claim) { c.Op.Journal["prepared_authority"] = `{` }, "no probing start": func(c *hostclaim.Claim) { delete(c.Op.Journal, "probing_since") }, } { f := newROFixture(t, roOptions{}) damaged := f.update(func(c *hostclaim.Claim) error { damage(c); return nil }) if err := preflight(ctx, damaged); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } f := newROFixture(t, roOptions{}) if err := preflight(ctx, f.read()); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { t.Fatalf("a sound claim went to root: %v", err) } }