package host import ( "context" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/moves/archive" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/superfly/ltx" ) const ( poHost = "shop.bab.oncarlos.com" poOp = "mv_ownersprobe" ) // poWork is a selected work that records the identities it runs under. type poWork struct { action string f *poFixture shut bool } func (w poWork) With(_ context.Context, id hostwrite.Identity, fn func() error) error { w.f.order = append(w.f.order, w.action+":"+id.Config+"|"+id.Release) if w.shut { return hostwrite.ErrUnavailable } return fn() } type poFixture struct { t *testing.T claims release.Store bucket *leasetest.VersionedBucket rec *provision.Record dataDir, files string route registry.Route manifest moves.InstallManifest launch ClaimServingLaunch admitted int renderETags map[string]string renderOverlay string order []string owners *movesProbeOwners seq int launches, reads int seal moves.Seal inc string snapshotVersion string sourceGeneration string // The move's sides (poSides): the source box and its engine region, the // target box, its engine region (its roster name) and its placement // code, the claim's intent "to" and the fleet's home designation. poSides } // poSides names the fixture's two boxes. The default is ie-box in ie to // uk-box in uk; poIntoHome is a move from ca-box in ca into the home // region, whose box home-box has the roster name ie, the record code "" and // the claim's to @home (home part 2). type poSides struct { srcBox, srcRegion string dstBox, dstEngine, dstCode, dstTo string home string } var poNamed = poSides{srcBox: "ie-box", srcRegion: "ie", dstBox: "uk-box", dstEngine: "uk", dstCode: "uk", dstTo: "uk"} // poOption adjusts the fixture's sides before it is built. type poOption func(*poSides) // poIntoHome is a move from ca-box into the home region: the target is // home-box, its roster name ie, the record's code "" and the claim's to the // placement-derived @home, with the fleet's home designation ie. func poIntoHome(s *poSides) { *s = poSides{srcBox: "ca-box", srcRegion: "ca", dstBox: "home-box", dstEngine: "ie", dstCode: "", dstTo: park.PlacementRegion(""), home: "ie"} } // poFlipClaim is the claim as the flip leaves it (moves §7.5): landing at // probing with uk-box executing; the holder is still the source's session 15 // in generation "src" on ie-box, at the flip's stamp 18. func (f *poFixture) poFlipClaim() *hostclaim.Claim { f.t.Helper() journal := map[string]string{hostclaim.JournalSeal: f.seal.Encode(), hostclaim.JournalCutSession: "15", hostclaim.JournalEvidenceSource: "moves/bab/jam/" + poHost + "/" + poOp + "/stopped-source.json", hostclaim.JournalSourceStamp: "15", hostclaim.JournalSrcGen: "src"} intent := map[string]string{"to": f.dstTo, "source": f.srcBox, "target": f.dstBox} if f.home != "" { intent["home_region"] = f.home } c := &hostclaim.Claim{V: 2, Host: poHost, Gen: 19, Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + poHost + ".json", Account: "bab", App: "jam", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), Incarnation: f.inc, Location: "instances/" + poHost, Generation: "src", Stamp: 18, Session: 15, WriterBox: f.srcBox, Admissions: []hostclaim.AdmissionRecord{{Session: 15, Box: f.srcBox, Generation: "src", Predecessor: "", Region: f.srcRegion, Mode: "cold"}}}, Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: poOp, Phase: hostclaim.PhaseLandingProbing, Exec: f.dstBox, Participants: []string{f.srcBox, f.dstBox}, Intent: intent, Journal: journal}} if err := c.Validate(); err != nil { f.t.Fatal(err) } return c } func newPOFixture(t *testing.T, opts ...poOption) *poFixture { t.Helper() // Claim Plan 15: admissions through the real CAS record their origin // under both switches; the real C8's proof reads that record. t.Setenv("CARLOS_HOST_CLAIM", "1") t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1") at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) inc, err := hostclaim.Incarnation("bab", "jam", poHost, at) if err != nil { t.Fatal(err) } f := &poFixture{t: t, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket(), dataDir: t.TempDir(), files: t.TempDir(), inc: inc, poSides: poNamed} for _, o := range opts { o(&f.poSides) } f.bucket.AddBucket("data") f.seal = moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true, SealedAt: at, LeaseUntil: at.Add(time.Minute), Replica: []moves.SealedObject{{Key: "moves/" + poHost + "/" + poOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}}, Files: []moves.SealedObject{{Key: "moves/" + poHost + "/" + poOp + "/files/archive.sealed", VersionID: "v2", Size: 10}, {Key: "moves/" + poHost + "/" + poOp + "/files/archive.json", VersionID: "v3", Size: 10}}} f.putClaim(f.poFlipClaim()) // The prepared generation the build wrote, and its manifest in the op // journal (Plan 2 Task 8). key := "instances/" + poHost + "/g-" + poOp + "/0009/" + ltx.FormatFilename(1, 42) f.snapshotVersion = f.bucket.Put("data", key, f.ltxHeader(1, 42)) f.manifest = moves.InstallManifest{V: 1, Op: poOp, Host: poHost, N: 42, SnapshotKey: key, SnapshotVersion: f.snapshotVersion, SnapshotCRC64: 7, DBSHA256: strings.Repeat("d", 64)} f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix("instances/"+poHost, poOp)), mustJSON(t, f.manifest)) j, err := moves.ReadJournal(f.dataDir, poHost, poOp) if err != nil { t.Fatal(err) } body, _ := json.Marshal(f.manifest) j.Mark(landStepInstall, string(body)) // The install receipt the landing journals: the files directory and // .env, placed (moveslandinstall.go). // The files the landing extracted (fix round 3 changes them in place). if err := os.WriteFile(filepath.Join(f.files, "a.txt"), []byte("the landing's file"), 0o600); err != nil { t.Fatal(err) } envPath := filepath.Join(f.dataDir, poHost+".env") if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600); err != nil { t.Fatal(err) } filesID, err := pathIdentity(f.files) if err != nil { t.Fatal(err) } envID, err := pathIdentity(envPath) if err != nil { t.Fatal(err) } placed, _ := json.Marshal(stagedFiles{Files: filesID, Env: envID}) j.Mark(landStepFilesPlaced, string(placed)) if err := j.Commit(f.dataDir); err != nil { t.Fatal(err) } // The prepared files manifest local wrote at the flip's stamp, before // .files-gen, and its binding in the op journal (C8, Task 6). f.preparedManifestJournalledAt(poOp, poOp, 18, f.files, envPath) f.route = registry.Route{Host: poHost, Kind: "instance", App: "jam", Account: "bab", Status: "landing", HoldOp: poOp, Socket: filepath.Join(f.dataDir, poHost+".sock"), DBPath: filepath.Join(f.dataDir, poHost+".db")} f.writeLocal(poOp, poOp, inc) pins := mpPins() pins.EnvReport = "sha256:env" deadline := time.Date(2026, 10, 9, 12, 10, 0, 0, time.UTC) f.rec = &provision.Record{Host: poHost, Account: "bab", App: "jam", Region: f.dstCode, Move: &provision.MoveState{ID: poOp, Incarnation: inc, From: f.srcRegion, To: f.dstCode, State: provision.MoveLanding, LandingPhase: provision.LandProbing, Inputs: pins, CutDeadline: &deadline, Landing: &provision.MoveLandingJournal{Attempts: 1, InstalledDBSHA256: "sha256:" + f.manifest.DBSHA256}}} f.launch = ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:snapshot", Release: "sha256:bytes"}, Executable: "/opt/carlos/live/" + poHost, ExecutableSHA256: strings.Repeat("b", 64), Args: []string{"--socket", f.route.Socket, "--db", f.route.DBPath}, Env: []string{"A=1", "CARLOS_MOVE_PROBE=0"}, Version: pins.Release} pins.EnvReport = envDigest(poEnvKey, f.launch.Env) f.admitted = 3 f.renderETags, f.renderOverlay = pins.BundleETags, pins.OverlayETag f.owners = &movesProbeOwners{Claims: f.claims, Data: f.bucket, Bucket: "data", Region: f.dstEngine, Box: f.dstBox, DataDir: f.dataDir, Records: func(context.Context, string, string, string) (*provision.Record, error) { f.reads++; return f.rec, nil }, ReadyPath: func(context.Context, string, string) (string, error) { return "/healthz", nil }, FilesDir: func(string) (string, string) { return f.files, filepath.Join(f.dataDir, poHost+".env") }, Markers: plan9D5Markers{f.bucket}, Route: func(string) (*registry.Route, error) { rt := f.route; return &rt, nil }, // The launch and its env report, made from the same env as the // production launch source makes it (claimMoveLaunchSource). Launch: func(context.Context, registry.Route) (ClaimServingLaunch, moves.EnvReport, error) { f.launches++ return f.launch, moves.ReportLaunch(poEnvKey, launchVars(f.launch.Env), nil), nil }, EnvKey: poEnvKey, Admitted: func(context.Context, *provision.Record) (int, error) { return f.admitted, nil }, Works: func(_ context.Context, host, action string, captured hostguard.Captured) (hostwrite.Work, error) { f.order = append(f.order, "select:"+action) return poWork{action: action, f: f}, nil }, Render: func(ctx context.Context, rt registry.Route, op string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) { f.order = append(f.order, "render") w, err := work(appconfig.HeldRender{BundleETags: f.renderETags, OverlayETag: f.renderOverlay}) if err != nil { return appconfig.HeldRender{}, err } if err := w.With(ctx, hostwrite.Identity{Config: "rendered-content-identity"}, func() error { return nil }); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{Path: "p", SHA256: "s", BundleETags: f.renderETags, OverlayETag: f.renderOverlay}, nil }, Release: func(ctx context.Context, rt registry.Route, version string, work hostwrite.Work) error { f.order = append(f.order, "release:"+version) return work.With(ctx, hostwrite.Identity{Release: "ignored"}, func() error { return nil }) }, NewID: func() string { f.seq++; return fmt.Sprintf("fresh%d", f.seq) }, Now: func() time.Time { return time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) }} if f.home != "" { // The target is the home box: the roster grants it the alias. f.owners.HomeRegion, f.owners.FleetHomeRegion = f.home, f.home } // The C8 provider is the real one over the fixture's bucket, as the // claim assembly binds it (whole-branch fixes, item 1). f.owners.Files = f.realC8() return f } func (f *poFixture) ltxHeader(min, max uint64) []byte { h := ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: ltx.TXID(min), MaxTXID: ltx.TXID(max)} body, err := h.MarshalBinary() if err != nil { f.t.Fatal(err) } return body } // writeLocal writes the two local markers the landing left: .files-gen // naming files/inc and .gen naming db ("" removes it). func (f *poFixture) writeLocal(files, db, inc string) { f.t.Helper() body, _ := json.Marshal(moves.FilesGen{Op: files, Incarnation: inc}) if err := os.WriteFile(filepath.Join(f.files, archive.MarkerName), body, 0o600); err != nil { f.t.Fatal(err) } if db == "" { os.Remove(f.route.DBPath + ".gen") return } if err := hostbox.WriteGenerationMarker(f.route.DBPath+".gen", db); err != nil { f.t.Fatal(err) } } func (f *poFixture) putClaim(c *hostclaim.Claim) { f.t.Helper() body, err := json.Marshal(c) if err != nil { f.t.Fatal(err) } key, err := hostclaim.Key(poHost) if err != nil { f.t.Fatal(err) } etag := "" if _, current, err := f.claims.Get(f.t.Context(), key); err == nil { etag = current } if _, err := f.claims.Put(f.t.Context(), key, body, etag); err != nil { f.t.Fatal(err) } } func (f *poFixture) read() hostclaim.Observed { f.t.Helper() o, err := hostclaim.Read(f.t.Context(), f.claims, poHost) if err != nil { f.t.Fatal(err) } return o } var poEnvKey = []byte("0123456789abcdef0123456789abcdef") var poProbe = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second, Probe: true} // admit applies the selected admission with the claim's own CAS. func (f *poFixture) admit(o hostclaim.Observed, in ClaimLaunchInputs) hostclaim.Observed { f.t.Helper() admitted, err := hostclaim.Update(f.t.Context(), f.claims, o, hostclaim.AdmitWriter(in.Admission)) if err != nil { f.t.Fatal(err) } return admitted } // TestProbeIsAColdAdmissionIntoThePreparedGeneration (owner half; the // runtime half is TestMovesProbeOwnersThroughTheClaimRuntime): the first // probe's admission is cold, into the move id, keeping the flip's stamp, // sealed, from the prepared pin; the restarted attempt's is a fresh // generation that keeps no stamp and claims no coverage. func TestProbeIsAColdAdmissionIntoThePreparedGeneration(t *testing.T) { f := newPOFixture(t) ctx := t.Context() o := f.read() in, err := f.owners.Select(ctx, f.route, o, poProbe) if err != nil { t.Fatal(err) } want := hostclaim.Admission{ObservedOpID: poOp, ObservedPhase: hostclaim.PhaseLandingProbing, ObservedSession: 15, ObservedGeneration: "src", Box: "uk-box", Mode: "cold", NewGeneration: poOp, KeepStamp: true, Coverage: "sealed", Region: "uk", PredecessorRegion: "ie", RemoteMaxTXID: 42} if !reflect.DeepEqual(in.Admission, want) { t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want) } if !in.Prepared || in.Empty || in.Baseline == nil || in.Baseline.Generation != poOp || in.Baseline.MaxTXID != 42 || in.Baseline.Pin == nil || in.Baseline.Pin.SnapshotVersion != f.snapshotVersion { t.Fatalf("baseline: prepared=%v %+v", in.Prepared, in.Baseline) } if in.Identity != mpLaunchIdentity(f.rec.Move.Inputs) || in.ReadyPath != "/healthz" || in.DBPath != f.route.DBPath || in.Version != f.launch.Version || in.Executable != f.launch.Executable || len(in.Roles) != 1 || in.Roles[0] != "main" { t.Fatalf("launch inputs: %+v", in) } probe := 0 for _, kv := range in.Env { if strings.HasPrefix(kv, "CARLOS_MOVE_PROBE=") { probe++ if kv != "CARLOS_MOVE_PROBE=1" { t.Fatalf("env %q", kv) } } } if probe != 1 || !contains(in.Env, "A=1") { t.Fatalf("env: %v", in.Env) } if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { t.Fatal(err) } for name, mutate := range map[string]func(*hostclaim.Admission, *genselect.Plan){ "no coverage": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Coverage = "" }, "stamp not kept": func(a *hostclaim.Admission, _ *genselect.Plan) { a.KeepStamp = false }, "another region": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Region = "ie" }, "another box": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Box = "ie-box" }, "a stale baseline": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = 41, 41 }, "another session": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedSession = 14 }, "warm": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Mode = "warm" }, "another new gen": func(a *hostclaim.Admission, _ *genselect.Plan) { a.NewGeneration = "other" }, "a loss": func(a *hostclaim.Admission, _ *genselect.Plan) { a.LossID = "loss" }, "certified": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorCertified = true }, "predecessor where": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorRegion = "uk" }, } { a, p := in.Admission, *in.Baseline mutate(&a, &p) if err := f.owners.AdmissionEvidence(ctx, o, a, &p); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } // The first attempt is admitted; the next attempt over it is a fresh // generation with no kept stamp and no coverage, restored from the // prepared pin while the move id published nothing. admitted := f.admit(o, in) h := admitted.Claim.Holder if h.Generation != poOp || h.Stamp != 18 || h.WriterBox != "uk-box" || h.Admissions[len(h.Admissions)-1].Coverage != "sealed" { t.Fatalf("first admission: %+v", h) } again, err := f.owners.Select(ctx, f.route, admitted, poProbe) if err != nil { t.Fatal(err) } if a := again.Admission; a.NewGeneration != "fresh1" || a.KeepStamp || a.Coverage != "" || a.ObservedGeneration != poOp || a.ObservedSession != h.Session || a.Mode != "cold" || a.PredecessorRegion != "uk" || again.Prepared { t.Fatalf("restarted admission: %+v prepared=%v", a, again.Prepared) } if again.Baseline == nil || again.Baseline.Generation != poOp || again.Baseline.MaxTXID != 42 { t.Fatalf("restarted baseline: %+v", again.Baseline) } if err := f.owners.AdmissionEvidence(ctx, admitted, again.Admission, again.Baseline); err != nil { t.Fatal(err) } reused := again.Admission reused.NewGeneration = poOp if err := f.owners.AdmissionEvidence(ctx, admitted, reused, again.Baseline); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("the move id twice: %v", err) } } // realC8 is the production C8 provider over the fixture's bucket, data // directory and files paths, as the claim assembly binds it: the probe // owners' Files is this, not a fake (whole-branch fixes, item 1). func (f *poFixture) realC8() *movesFiles { return &movesFiles{Files: casBucketStore{b: f.bucket, bucket: "data"}, Data: f.bucket, Bucket: "data", HomeRegion: "ie", Region: f.owners.Region, Box: f.owners.Box, DataDir: f.dataDir, EnvFile: envFileOf(f.owners.FilesDir), // Plan 15's proof over the fixture's claim store: the admissions the // fixture makes through the real CAS are recorded (both switches // on), and recordOrigins records what a hand-built claim or the // #250 backfill would. Proof: ClaimOriginProof{Claims: f.claims, HomeRegion: "ie", Region: f.owners.Region}} } // preparedManifestJournalled builds the prepared manifest local writes for // generation at the installation's stamp and journals its binding, as // local does (movesinstall.go). func (f *poFixture) preparedManifestJournalled(generation, op string, stamp uint64) preparedManifest { f.t.Helper() _, envPath := f.owners.FilesDir(poHost) return f.preparedManifestJournalledAt(generation, op, stamp, f.files, envPath) } func (f *poFixture) preparedManifestJournalledAt(generation, op string, stamp uint64, dir, envPath string) preparedManifest { f.t.Helper() pm, err := buildPreparedManifest(f.t.Context(), f.bucket, "data", "instances/"+poHost, generation, op, stamp, dir, envPath, true, hostfiles.WakeState{V: 1, Host: poHost, Timers: []hostfiles.WakeTimer{}}) if err != nil { f.t.Fatal(err) } j, err := moves.ReadJournal(f.dataDir, poHost, generation) if err != nil { f.t.Fatal(err) } j.Mark(landStepFilesManifest, string(mustJSON(f.t, pm))) if err := j.Commit(f.dataDir); err != nil { f.t.Fatal(err) } return pm } // The landing's first probe and its committed wake establish the admitted // generation's files manifest through the registered C8 provider before // the start check (whole-branch fixes, item 1): the probe's establishment // takes P's prepared manifest at the journalled version, stages the env // file on disk and drops the adoption record; the committed wake's copies // it into the fresh generation. A refusal from C8 stops the start, and // without a provider the wake is refused by name. func TestLandingWakesEstablishThroughC8(t *testing.T) { f := newPOFixture(t) ctx := t.Context() j, err := moves.ReadJournal(f.dataDir, poHost, poOp) if err != nil { t.Fatal(err) } var pm preparedManifest if err := json.Unmarshal([]byte(j.Steps[landStepFilesManifest]), &pm); err != nil { t.Fatal(err) } // The launch source minted the instance key into .env before the // establishment (O1'): the env file differs from the prepared item. env := []byte("CARLOS_ADMIN_TOKEN=t\nCARLOS_INSTANCE_KEY=abc\n") if err := os.WriteFile(filepath.Join(f.dataDir, poHost+".env"), env, 0o600); err != nil { t.Fatal(err) } adoption := filepath.Join(f.dataDir, poHost+instancekey.AdoptionSuffix) if err := os.WriteFile(adoption, []byte(`{"v":2}`), 0o600); err != nil { t.Fatal(err) } o := f.read() in, err := f.owners.Select(ctx, f.route, o, poProbe) if err != nil { t.Fatal(err) } admitted := f.admit(o, in) if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil { t.Fatalf("the probe's materialization: %v", err) } prefix := movesFilesPrefix("instances/"+poHost, poOp) m, found, err := hostfiles.Load(ctx, casBucketStore{b: f.bucket, bucket: "data"}, prefix) if err != nil || !found { t.Fatal(err, found) } if m.Seq != 1 || m.Platform[hostfiles.EnvItem].SHA256 != sha(env) || m.Entries["a.txt"].Seq != 0 { t.Fatalf("P's manifest after the probe: %+v", m) } if _, has := m.Platform[instancekey.AdoptionItem]; has { t.Fatal("the adoption item was carried") } if _, err := os.Stat(adoption); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the adoption record remains: %v", err) } if raw, err := f.bucket.GetVersion(ctx, "data", pm.Key, pm.Version); err != nil || sha(raw) != pm.SHA256 { t.Fatalf("the prepared version: %v", err) } if err := f.owners.Publish(ctx, admitted, in); err != nil { t.Fatal(err) } // The committed wake: the claim at landing/committed, the record // committed, a cold admission into a fresh generation whose files // manifest is copied from P's. c := *f.read().Claim c.Op.Phase = hostclaim.PhaseLandingCommitted f.putClaim(&c) f.rec.Move.State, f.rec.Move.LandingPhase = provision.MoveCommitted, "" f.route.Status, f.route.HoldOp = "ready", "" o = f.read() in, err = f.owners.Select(ctx, f.route, o, activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second}) if err != nil { t.Fatalf("the committed select: %v", err) } if in.Prepared || in.Admission.NewGeneration == poOp { t.Fatalf("the committed admission: %+v", in.Admission) } admitted = f.admit(o, in) f.writeLocal(poOp, admitted.Claim.Holder.Generation, f.inc) if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil { t.Fatalf("the committed materialization: %v", err) } fresh, found, err := hostfiles.Load(ctx, casBucketStore{b: f.bucket, bucket: "data"}, movesFilesPrefix("instances/"+poHost, in.Admission.NewGeneration)) if err != nil || !found { t.Fatalf("the committed wake's manifest: %v %v", err, found) } if fresh.Entries["a.txt"].Object != m.Entries["a.txt"].Object || fresh.Platform[hostfiles.EnvItem].SHA256 != sha(env) { t.Fatalf("the committed wake's manifest: %+v", fresh) } // C8's refusal stops the start: no binding in the op journal. g := newPOFixture(t) gj, err := moves.ReadJournal(g.dataDir, poHost, poOp) if err != nil { t.Fatal(err) } gj.Unmark(landStepFilesManifest) if err := gj.Commit(g.dataDir); err != nil { t.Fatal(err) } o = g.read() in, err = g.owners.Select(ctx, g.route, o, poProbe) if err != nil { t.Fatal(err) } admitted = g.admit(o, in) if err := g.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: g}); err == nil || !errors.Is(err, errLandingChanged) { t.Fatalf("a probe without a prepared manifest binding: %v", err) } if m, found, _ := hostfiles.Load(ctx, casBucketStore{b: g.bucket, bucket: "data"}, movesFilesPrefix("instances/"+poHost, poOp)); !found || m.Seq != 0 { t.Fatalf("a refused establishment changed the prepared manifest: %+v %v", m, found) } // Without a provider the wake is refused by name, before anything. g.owners.Files = nil var dep *ClaimDependencyError if err := g.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: g}); !errors.As(err, &dep) { t.Fatalf("no provider: %v", err) } } // noopFiles is a C8 provider that establishes nothing: for tests of the // runtime's wiring, where the real provider's inputs are not laid out. type noopFiles struct{} func (noopFiles) Establish(ctx context.Context, _ hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { return work.With(ctx, in.Identity, func() error { return nil }) } func mpLaunchIdentity(p *provision.MoveInputs) hostwrite.Identity { return hostwrite.Identity{Config: HeldConfigIdentity(p.BundleETags, p.OverlayETag).Config, Release: p.Release} } func contains(xs []string, x string) bool { for _, v := range xs { if v == x { return true } } return false } // TestProbeRefusesUnheldRowOrForeignExec: the row must be held by the // move, and the claim must name this box the executor of the move's // landing at probing; anything else is refused before any launch is read. func TestProbeRefusesUnheldRowOrForeignExec(t *testing.T) { for name, tc := range map[string]struct { route func(*registry.Route) claim func(*hostclaim.Claim) rec func(*provision.Record) policy func(*activator.StartPolicy) }{ "a ready row": {route: func(r *registry.Route) { r.Status, r.HoldOp = "ready", "" }}, "held by another op": {route: func(r *registry.Route) { r.HoldOp = "mv_other" }}, "another host's row": {route: func(r *registry.Route) { r.Host = "other.bab.oncarlos.com" }}, "a foreign executor": {claim: func(c *hostclaim.Claim) { c.Op.Exec = "ie-box" }}, "not the target": {claim: func(c *hostclaim.Claim) { c.Op.Intent["target"] = "ie-box" }}, "another region": {claim: func(c *hostclaim.Claim) { c.Op.Intent["to"] = "ie" }}, "not probing": {claim: func(c *hostclaim.Claim) { c.Op.Phase = provision.LandRestored }}, "not a landing": {claim: func(c *hostclaim.Claim) { c.Op.Kind = hostclaim.KindReverting }}, "declared lost": {claim: func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }}, "a marked record": {rec: func(r *provision.Record) { r.Move.Reason = provision.RollbackProbeAttempts }}, "a record past probing": {rec: func(r *provision.Record) { r.Move.LandingPhase = provision.LandProbed }}, "no pinned inputs": {rec: func(r *provision.Record) { r.Move.Inputs = nil }}, "not a probe": {policy: func(p *activator.StartPolicy) { p.Probe = false }}, "other pins": {policy: func(p *activator.StartPolicy) { p.Inputs = &provision.MoveInputs{Release: "v0"} }}, "another manifest": {policy: func(p *activator.StartPolicy) { p.Prepared = &moves.InstallManifest{Op: poOp, N: 41} }}, } { t.Run(name, func(t *testing.T) { f := newPOFixture(t) if tc.claim != nil { c := f.poFlipClaim() tc.claim(c) f.putClaim(c) } if tc.route != nil { tc.route(&f.route) } if tc.rec != nil { tc.rec(f.rec) } p := poProbe if tc.policy != nil { tc.policy(&p) } if _, err := f.owners.Select(t.Context(), f.route, f.read(), p); err == nil { t.Fatal("selected") } if f.launches != 0 { t.Fatal("the launch was read") } }) } } // The prepared inputs are validated before any admission: the local markers // name the move id and the holder's incarnation, and the journalled // manifest is the seal's. func TestMovesProbeOwnersValidateThePreparedInputs(t *testing.T) { for name, tc := range map[string]struct { mutate func(*poFixture) want error }{ "files of another op": {func(f *poFixture) { f.writeLocal("mv_other", poOp, f.inc) }, moves.ErrStartFiles}, "files of another incarnation": {func(f *poFixture) { f.writeLocal(poOp, poOp, "other") }, moves.ErrStartFiles}, "no files marker": {func(f *poFixture) { os.Remove(filepath.Join(f.files, archive.MarkerName)) }, moves.ErrStartFiles}, "a database of another op": {func(f *poFixture) { f.writeLocal(poOp, "src", f.inc) }, moves.ErrStartDB}, "no database marker": {func(f *poFixture) { f.writeLocal(poOp, "", f.inc) }, moves.ErrStartDB}, "a manifest at another N": {func(f *poFixture) { f.manifest.N = 41 j, _ := moves.ReadJournal(f.dataDir, poHost, poOp) body, _ := json.Marshal(f.manifest) j.Mark(landStepInstall, string(body)) j.Commit(f.dataDir) }, hostbox.ErrChanged}, "no manifest": {func(f *poFixture) { j, _ := moves.ReadJournal(f.dataDir, poHost, poOp) j.Unmark(landStepInstall) j.Commit(f.dataDir) }, hostbox.ErrChanged}, } { t.Run(name, func(t *testing.T) { f := newPOFixture(t) tc.mutate(f) if _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } }) } } // The launch-input pin at spawn (§7.7, round-2 finding 14): the live // release and the launch environment's report are the ones pinned at // readiness, else the probe fails with the recorded rollback sentence // before any admission. func TestMovesProbeOwnersPinTheLaunchInputs(t *testing.T) { for name, mutate := range map[string]func(*poFixture){ "another release": func(f *poFixture) { f.launch.Version = "v9" }, "another environment": func(f *poFixture) { f.rec.Move.Inputs.EnvReport = "sha256:other" }, } { t.Run(name, func(t *testing.T) { f := newPOFixture(t) mutate(f) _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe) var changed *ProbeInputsChanged if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged { t.Fatalf("got %v", err) } }) } } // TestFirstProbeFromTheSourceGenerationIsAdmitted: from the claim as the // flip left it, the prepared inputs validate, the admission names the move // id, and Materialize, before every start, renders and materializes the held // inputs under their own selections and passes the §5.1 start check after // the CAS. A hand-built claim already in the move id whose files name // another op is refused there. func TestFirstProbeFromTheSourceGenerationIsAdmitted(t *testing.T) { f := newPOFixture(t) ctx := t.Context() o := f.read() in, err := f.owners.Select(ctx, f.route, o, poProbe) if err != nil { t.Fatal(err) } admitted := f.admit(o, in) f.order = nil spawn := poWork{action: "spawn", f: f} if err := f.owners.Materialize(ctx, admitted, in, spawn); err != nil { t.Fatal(err) } pins := f.rec.Move.Inputs launch := mpLaunchIdentity(pins) want := []string{ "spawn:" + launch.Config + "|" + launch.Release, "render", "select:held-config", "held-config:" + HeldConfigIdentity(pins.BundleETags, pins.OverlayETag).Config + "|", "select:held-release", "release:" + pins.Release, "held-release:|" + pins.Release, // The prepared marker under the writer's fence: before the request // and in the store before its attempt. "spawn:" + launch.Config + "|" + launch.Release, "spawn:" + launch.Config + "|" + launch.Release, // C8's establishment (item 1): the env file is still the prepared // item's and there is no adoption item, so nothing is published and // only the local effect runs under the work. "spawn:" + launch.Config + "|" + launch.Release, } if !reflect.DeepEqual(f.order, want) { t.Fatalf("order:\n got %q\nwant %q", f.order, want) } // A render that is not the pinned configuration fails the probe and // materializes nothing. f.order = nil f.renderOverlay = "e-changed" err = f.owners.Materialize(ctx, admitted, in, spawn) var changed *ProbeInputsChanged if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged || contains(f.order, "select:held-release") { t.Fatalf("a changed render: %v %v", err, f.order) } f.renderOverlay = pins.OverlayETag // The start check against the claim after the CAS. f.writeLocal("mv_other", poOp, f.inc) if err := f.owners.Materialize(ctx, admitted, in, spawn); !errors.Is(err, moves.ErrStartFiles) { t.Fatalf("files of another op: %v", err) } f.writeLocal(poOp, "src", f.inc) if err := f.owners.Materialize(ctx, admitted, in, spawn); !errors.Is(err, moves.ErrStartDB) { t.Fatalf("a database the restore did not write: %v", err) } f.writeLocal(poOp, poOp, f.inc) // The spawn selection itself refused: nothing is rendered. f.order = nil if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f, shut: true}); !errors.Is(err, hostwrite.ErrUnavailable) || contains(f.order, "render") { t.Fatalf("a refused spawn selection: %v %v", err, f.order) } } func mustJSON(t *testing.T, v any) []byte { t.Helper() b, err := json.Marshal(v) if err != nil { t.Fatal(err) } return b } // The prepared marker (moves §7.14 marker row, R10-3): the first probe's // admission publishes generation.json naming the move id with the build's // pin, superseding an earlier stay's marker by its exact ETag, before // replication; the evidence is this process's receipt. func TestMovesProbeOwnersPublishThePreparedMarker(t *testing.T) { f := newPOFixture(t) ctx := t.Context() key := "instances/" + poHost + "/generation.json" f.bucket.Put("data", key, mustJSON(t, genselect.Marker{Op: "earlier", Incarnation: f.inc, Host: poHost, Box: "uk-box", InstalledAt: time.Unix(1, 0).UTC(), Pin: genselect.Pin{SnapshotTXID: 3}})) o := f.read() in, err := f.owners.Select(ctx, f.route, o, poProbe) if err != nil { t.Fatal(err) } admitted := f.admit(o, in) if err := f.owners.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("evidence before publication: %v", err) } // Materialize publishes under the writer's fence; Group D's Publish // records it. if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil { t.Fatal(err) } if err := f.owners.Publish(ctx, admitted, in); err != nil { t.Fatal(err) } body, ok := f.bucket.Current("data", key) var m genselect.Marker if !ok || json.Unmarshal(body, &m) != nil || m.Op != poOp || m.Box != "uk-box" || m.Incarnation != f.inc || m.Host != poHost || m.Pin != (genselect.Pin{SnapshotKey: f.manifest.SnapshotKey, SnapshotVersion: f.manifest.SnapshotVersion, SnapshotTXID: 42, SnapshotCRC64: 7}) || m.ManifestSHA256 == "" { t.Fatalf("marker: %s", body) } if err := f.owners.PublicationEvidence(ctx, admitted); err != nil { t.Fatal(err) } written := f.bucket.LatestID("data", key) if err := f.owners.Publish(ctx, admitted, in); err != nil || f.bucket.LatestID("data", key) != written { t.Fatalf("replay: %v", err) } // A restarted attempt's baseline is now the published move id. again, err := f.owners.Select(ctx, f.route, admitted, poProbe) if err != nil || again.Baseline.Generation != poOp || again.Baseline.Marker == nil || again.Baseline.Marker.Op != poOp { t.Fatalf("restarted baseline after publication: %+v %v", again.Baseline, err) } f.bucket.Put("data", key, mustJSON(t, genselect.Marker{Op: poOp, Incarnation: f.inc, Host: poHost, Box: "ie-box", InstalledAt: time.Unix(2, 0).UTC(), Pin: m.Pin, ManifestSHA256: m.ManifestSHA256})) if err := f.owners.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("evidence for a marker this session did not write: %v", err) } if err := f.owners.EmptyAuthority(ctx, admitted, in); err == nil { t.Fatal("a move admission claimed the empty bootstrap") } } // A restarted attempt's fresh generation is published fenced, after its own // first root, as serving publishes. func TestMovesProbeOwnersPublishARestartedAttemptFenced(t *testing.T) { f := newPOFixture(t) ctx := t.Context() o := f.read() in, err := f.owners.Select(ctx, f.route, o, poProbe) if err != nil { t.Fatal(err) } first := f.admit(o, in) again, err := f.owners.Select(ctx, f.route, first, poProbe) if err != nil { t.Fatal(err) } admitted := f.admit(first, again) f.restore(admitted.Claim.Holder.Generation) if err := f.owners.Materialize(ctx, admitted, again, newFenceWork()); err != nil { t.Fatal(err) } synced := 0 pub := ClaimPublication{Sync: func(context.Context) error { synced++; return nil }, Check: func(context.Context) error { return nil }} if err := f.owners.Publish(ctx, admitted, again); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("an unfenced marker for a fresh generation: %v", err) } if _, err := f.owners.PublishFenced(ctx, admitted, again, pub); !errors.Is(err, hostbox.ErrChanged) || synced != 1 { t.Fatalf("a marker before the root: %v synced=%d", err, synced) } root := "instances/" + poHost + "/g-" + again.Admission.NewGeneration + "/0009/" + ltx.FormatFilename(1, 44) f.bucket.Put("data", root, f.ltxHeader(1, 44)) receipt, err := f.owners.PublishFenced(ctx, admitted, again, pub) if err != nil || receipt.Root.Key != root || receipt.MarkerVersion == "" { t.Fatalf("publish: %+v %v", receipt, err) } if err := f.owners.PublicationEvidence(ctx, admitted); err != nil { t.Fatal(err) } } // TestMovesProbeOwnersThroughTheClaimRuntime drives the claim's own runtime // (claim Plan 9 Group D's ClaimWakeAssembly, the recovery fixture's // controller) with this owner set and the moves probe attempts bound, as // production binds them: the admission is the prepared cold admission, // journalled before any start; the probe start carries the moves attempt // journal; and Materialize renders the held inputs before any launch is // prepared. func TestMovesProbeOwnersThroughTheClaimRuntime(t *testing.T) { f := newRecoveryFixture(t) ctx := t.Context() journalDir, filesDir := t.TempDir(), t.TempDir() seal := moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true, SealedAt: f.now, LeaseUntil: f.now.Add(time.Minute), Replica: []moves.SealedObject{{Key: "moves/" + recoveryHost + "/" + poOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}}, Files: []moves.SealedObject{{Key: "moves/" + recoveryHost + "/" + poOp + "/files/archive.sealed", VersionID: "v2", Size: 10}}} if _, err := hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: poOp, Phase: hostclaim.PhaseLandingProbing, Exec: "box-1", Participants: []string{"box-0", "box-1"}, Intent: map[string]string{"to": "ie", "source": "box-0", "target": "box-1"}, Journal: map[string]string{hostclaim.JournalSeal: seal.Encode()}} return nil }); err != nil { t.Fatal(err) } o := f.read() h := o.Claim.Holder bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") key := strings.TrimSuffix(h.Location, "/") + "/g-" + poOp + "/0009/" + ltx.FormatFilename(1, 42) hdr, _ := (<x.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: 1, MaxTXID: 42}).MarshalBinary() manifest := moves.InstallManifest{V: 1, Op: poOp, Host: recoveryHost, N: 42, SnapshotKey: key, SnapshotVersion: bucket.Put("data", key, hdr), DBSHA256: strings.Repeat("d", 64)} bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix(h.Location, poOp)), mustJSON(t, manifest)) j, _ := moves.ReadJournal(journalDir, recoveryHost, poOp) body, _ := json.Marshal(manifest) j.Mark(landStepInstall, string(body)) if err := j.Commit(journalDir); err != nil { t.Fatal(err) } route := registry.Route{Host: recoveryHost, Kind: "instance", App: "jam", Account: "bab", Status: "landing", HoldOp: poOp, Socket: filepath.Join(f.guard.DataDir, "app.sock"), DBPath: filepath.Join(f.guard.DataDir, "state", "data.db")} if err := os.MkdirAll(filepath.Dir(route.DBPath), 0o700); err != nil { t.Fatal(err) } if err := hostbox.WriteGenerationMarker(route.DBPath+".gen", poOp); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(filesDir, archive.MarkerName), mustJSON(t, moves.FilesGen{Op: poOp, Incarnation: h.Incarnation}), 0o600); err != nil { t.Fatal(err) } pins := mpPins() pins.EnvReport = envDigest(poEnvKey, []string{"A=1"}) deadline := f.now.Add(time.Hour) rec := &provision.Record{Host: recoveryHost, Account: "bab", App: "jam", Region: "ie", Move: &provision.MoveState{ID: poOp, Incarnation: h.Incarnation, From: "uk", To: "ie", State: provision.MoveLanding, LandingPhase: provision.LandProbing, Inputs: pins, CutDeadline: &deadline, CutAttempts: 3, Landing: &provision.MoveLandingJournal{Attempts: 1, InstalledDBSHA256: "sha256:" + manifest.DBSHA256}}} records := func(context.Context, string, string, string) (*provision.Record, error) { return rec, nil } var order []string launches := 0 owners := &movesProbeOwners{Claims: f.claims, Records: records, Data: bucket, Bucket: "data", Region: "ie", Box: "box-1", DataDir: journalDir, Markers: plan9D5Markers{bucket}, Files: noopFiles{}, ReadyPath: func(context.Context, string, string) (string, error) { return "/ready", nil }, FilesDir: func(string) (string, string) { return filesDir, "" }, Route: func(string) (*registry.Route, error) { rt := route; return &rt, nil }, Launch: func(context.Context, registry.Route) (ClaimServingLaunch, moves.EnvReport, error) { return ClaimServingLaunch{Executable: "/usr/bin/true", ExecutableSHA256: strings.Repeat("a", 64), Env: []string{"A=1"}, Version: pins.Release}, moves.ReportLaunch(poEnvKey, map[string]string{"A": "1"}, nil), nil }, EnvKey: poEnvKey, Admitted: MovesRecordAdmittedAttempts, Works: func(_ context.Context, _, action string, _ hostguard.Captured) (hostwrite.Work, error) { order = append(order, "select:"+action) return plan9D5Work{}, nil }, Render: func(ctx context.Context, _ registry.Route, _ string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) { order = append(order, "render") if _, err := work(appconfig.HeldRender{BundleETags: pins.BundleETags, OverlayETag: pins.OverlayETag}); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{BundleETags: pins.BundleETags, OverlayETag: pins.OverlayETag}, nil }, Release: func(context.Context, registry.Route, string, hostwrite.Work) error { order = append(order, "release") return nil }, NewID: f.newID, Now: func() time.Time { return f.now }} attempts := &movesProbeAttempts{Claims: f.claims, Records: records, DataDir: journalDir, NewID: f.newID, Now: func() time.Time { return f.now }, Admitted: MovesRecordAdmittedAttempts} c := f.controller("boot-1", "controller-1", 4101, "start-1") bound := owners.MovesOwners(attempts) c.wake.Inputs = ClaimOwnerDispatch{Moves: &bound.ClaimKindOwners} c.wake.Continue = hostclaim.SessionContinues c.wake.Operations = bound.Operations c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } c.wake.Launch = func(context.Context, hostclaim.Observed, hostwrite.Work, ClaimLaunchInputs, *activator.RoleLaunch) (*activator.RoleLaunch, error) { launches++ return nil, hostwrite.ErrUnavailable } live, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) done := make(chan struct{}) go func() { <-live.Done(); close(done) }() c.wake.mu.Lock() c.wake.pending[recoveryHost] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} c.wake.mu.Unlock() admitted, err := c.runtime.Admit(ctx, route, o, poProbe) if err != nil { t.Fatal(err) } ah := admitted.Claim.Holder last := ah.Admissions[len(ah.Admissions)-1] if ah.Generation != poOp || ah.Stamp != h.Stamp || ah.WriterBox != "box-1" || last.Mode != "cold" || last.Coverage != "sealed" || last.Generation != poOp || last.Region != "ie" { t.Fatalf("the admission: %+v %+v", ah, last) } s, err := c.runtime.Prepare(ctx, route, admitted, poProbe) if err != nil { t.Fatal(err) } if s.OpenProbe == nil || s.Identity != mpLaunchIdentity(pins) { t.Fatalf("the probe start: %+v", s.Identity) } if opened, err := s.OpenProbe(ctx, f.now); err != nil || !opened.Equal(f.now) { t.Fatalf("open: %v %v", opened, err) } if err := s.Reserve(ctx, 0); err != nil { t.Fatal(err) } if a := s.CurrentAttempt(); a.ID == "" || a.Session != ah.Session || a.ControllerID != "controller-1" { t.Fatalf("the reservation: %+v", a) } if err := s.Materialize(ctx); err != nil { t.Fatal(err) } if want := []string{"render", "select:held-config", "select:held-release", "release"}; !reflect.DeepEqual(order, want) || launches != 0 { t.Fatalf("before the launch: %v launches=%d", order, launches) } if _, err := s.PrepareLaunch(ctx); err == nil || launches != 1 { t.Fatalf("the launch after Materialize: %v %d", err, launches) } }