package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "os/exec" "path/filepath" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // relocationProbeRig is P's first probe on box-1 through the claim's own // runtime (the recovery fixture's controller), the relocation owners bound // as claimGroupDWiring binds them, the probe attempt counted and pending. type relocationProbeRig struct { f *recoveryFixture bucket *leasetest.VersionedBucket files *release.DirStore c8 *roFiles inputs *RelocationWakeInputs c *recoveryController h *hostclaim.Holder o hostclaim.Observed im moves.InstallManifest auth hostclaim.PreparedAuthority route registry.Route } // roProbeLaunch is the serving launch source's result for the rig: the // live link hashed as build v1, staged under v1. func roProbeLaunch() ClaimServingLaunch { return ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/usr/bin/true", ExecutableSHA256: strings.Repeat("a", 64), Version: "v1", Staged: release.StagedIdentity{Version: "v1", StagedPath: "/opt/carlos/versions/v1/bab/jam", Digest: strings.Repeat("a", 64)}} } func newRelocationProbeRig(t *testing.T, launch ClaimServingLaunch) *relocationProbeRig { t.Helper() ctx := t.Context() f := newRecoveryFixture(t) bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") files := release.NewDirStore(t.TempDir()) o := f.read() h := o.Claim.Holder prefix := moves.GenerationPrefix(h.Location, roP) snap, anchor := moves.SnapshotKey(prefix, 42), moves.AnchorKey(prefix, 42) im := moves.InstallManifest{V: 1, Op: roOp, Host: recoveryHost, N: 42, SnapshotKey: snap, SnapshotVersion: bucket.Put("data", snap, ltxHeader(t, 1, 42)), SnapshotCRC64: 7, AnchorKey: anchor, AnchorVersion: bucket.Put("data", anchor, ltxHeader(t, 42, 42)), IntegrityCheck: "ok", DBSHA256: strings.Repeat("d", 64)} imRaw := mustJSON(t, im) bucket.Put("data", moves.InstallManifestKey(prefix), imRaw) pm := mustJSON(t, hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}}) if _, err := files.Put(ctx, hostfiles.ManifestKey(moves.GenerationFilesPrefix(h.Location, roP)), pm, ""); err != nil { t.Fatal(err) } imSum, pmSum := sha256.Sum256(imRaw), sha256.Sum256(pm) auth := hostclaim.PreparedAuthority{Generation: roP, InstallManifest: moves.InstallManifestKey(prefix), SHA256: hex.EncodeToString(imSum[:]), SnapshotTXID: 42, SourceFilesSeq: 5, FilesManifestSHA256: hex.EncodeToString(pmSum[:])} ref := hostclaim.RecoveryPin{Key: "control/relocations/" + recoveryHost + "/" + roOp + "/pin-1.json", SHA256: strings.Repeat("e", 64), Generation: h.Generation, MaxTXID: 42, FilesSeq: 5, Release: "v1"} o, err := hostclaim.Update(ctx, f.claims, o, func(c *hostclaim.Claim) error { c.Holder.Stamp = c.Gen + 1 auth.InstallStamp = c.Holder.Stamp c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: roOp, Phase: hostclaim.PhaseRelocatingProbing, Exec: "box-1", Participants: []string{"box-0", "box-1"}, Intent: map[string]string{"region": "ie", "source_box": "box-0", "target_box": "box-1", "target_enrollment": "enrollment-1"}, Journal: map[string]string{"pin": string(mustJSON(t, ref)), "prepared": roP, "install_stamp": strconv.FormatUint(c.Holder.Stamp, 10), "install_release": "v1", "prepared_authority": string(mustJSON(t, auth)), "probing_since": f.now.Format(time.RFC3339Nano), "exec_ack.box-0": `{"op":"rel-1"}`, "exec_ack.box-1": `{"op":"rel-1"}`, "local_install": string(mustJSON(t, hostclaim.LocalInstall{Box: "box-1", Enrollment: "enrollment-1", InstallStamp: c.Holder.Stamp, JournalSHA256: strings.Repeat("c", 64)}))}} return nil }) if err != nil { t.Fatal(err) } if o, err = hostclaim.Update(ctx, f.claims, o, hostclaim.BeginProbeAttempt(roOp, "box-1")); err != nil { t.Fatal(err) } c8 := &roFiles{} inputs := &RelocationWakeInputs{ Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return launch, nil }, FilesManifest: c8, Markers: plan9D5Markers{bucket}, Claims: f.claims, FilesStore: files, Data: bucket, Bucket: "data", Region: "ie", Box: "box-1", NewID: f.newID, Now: func() time.Time { return f.now }, Releases: &roReleases{}} attempts := &relocationProbeAttempts{Claims: f.claims, DataDir: t.TempDir(), Box: "box-1", NewID: f.newID, Now: func() time.Time { return f.now }} c := f.controller("boot-1", "controller-1", 4101, "start-1") c.wake.Inputs = ClaimOwnerDispatch{Relocation: &ClaimKindOwners{Inputs: inputs, AdmissionEvidence: inputs.AdmissionEvidence, PublicationEvidence: inputs.PublicationEvidence}} c.wake.Continue = hostclaim.SessionContinues // As claimGroupDWiring routes them once the relocation owners bind. ready := func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } c.wake.Operations = claimOperationRouter{hostclaim.KindRelocating: {Attempts: attempts, ProbeReady: ready}} live, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) done := make(chan struct{}) go func() { <-live.Done(); close(done) }() c.wake.mu.Lock() c.wake.pending[recoveryHost] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} c.wake.mu.Unlock() route := registry.Route{Host: recoveryHost, Kind: "instance", App: "jam", Account: "bab", Status: "relocating", HoldOp: roOp, Socket: filepath.Join(f.guard.DataDir, "app.sock"), DBPath: filepath.Join(f.guard.DataDir, "state", "data.db")} return &relocationProbeRig{f: f, bucket: bucket, files: files, c8: c8, inputs: inputs, c: c, h: h, o: o, im: im, auth: auth, route: route} } // Finding 6, ruling 47: P's first probe publishes through the shared // assembly. The claim's own runtime (the recovery fixture's controller) // admits P's opening through the relocation dispatch, binds region HA's // attempt journal to the probe start, and Materialize, under the admitted // spawn selection, runs moves' C8 Establish and writes P's marker: P as // the marker's op, the install manifest's pin, prepared_authority's // digest. Publish (the pre-replication call) finds that marker and // records the receipt; PublicationEvidence then passes. Publish without // Materialize's fence refuses. func TestRelocationFirstProbePublishesThroughTheAssembly(t *testing.T) { r := newRelocationProbeRig(t, roProbeLaunch()) ctx := t.Context() f, bucket, files, c8, inputs, c, h, o, im, auth, route := r.f, r.bucket, r.files, r.c8, r.inputs, r.c, r.h, r.o, r.im, r.auth, r.route admitted, err := c.runtime.Admit(ctx, route, o, roProbe) if err != nil { t.Fatal(err) } ah := admitted.Claim.Holder if ah.Generation != roP || ah.Stamp != o.Claim.Holder.Stamp || ah.WriterBox != "box-1" || !hostclaim.PreparedOpened(admitted.Claim) { t.Fatalf("P's opening: %+v", ah) } s, err := c.runtime.Prepare(ctx, route, admitted, roProbe) if err != nil { t.Fatal(err) } if s.OpenProbe == nil { t.Fatal("the probe start carries no attempt journal") } if _, err := s.OpenProbe(ctx, f.now); err != nil { t.Fatal(err) } if err := s.Reserve(ctx, 0); err != nil { t.Fatal(err) } if a := s.CurrentAttempt(); a.ID == "" || a.Session != ah.Session || a.ControllerID != "controller-1" { t.Fatalf("the reservation: %+v", a) } if err := s.Materialize(ctx); err != nil { t.Fatal(err) } if len(c8.established) != 1 { t.Fatalf("C8 established %v", c8.established) } markerKey := strings.TrimSuffix(h.Location, "/") + "/generation.json" body, ok := bucket.Current("data", markerKey) var m genselect.Marker if !ok || json.Unmarshal(body, &m) != nil || m.Op != roP || m.Box != "box-1" || m.Host != recoveryHost || m.ManifestSHA256 != auth.SHA256 || m.Pin != manifestPin(im) { t.Fatalf("P's marker: %s", body) } written := bucket.LatestID("data", markerKey) // The pre-replication publish (ClaimWakeAssembly s.Replicate's first // call for a prepared admission) finds the marker and records it. in, err := inputs.Select(ctx, route, o, roProbe) if err != nil { t.Fatal(err) } if err := inputs.Publish(ctx, admitted, in); err != nil { t.Fatal(err) } if bucket.LatestID("data", markerKey) != written { t.Fatal("Publish rewrote P's marker") } if err := inputs.PublicationEvidence(ctx, admitted); err != nil { t.Fatalf("publication evidence: %v", err) } // A fresh wake inputs (another process) has no fence for the session. other := &RelocationWakeInputs{Launch: inputs.Launch, FilesManifest: c8, Markers: inputs.Markers, Claims: f.claims, FilesStore: files, Data: bucket, Bucket: "data", Region: "ie", Box: "box-1", NewID: f.newID, Now: inputs.Now, Releases: inputs.Releases} if err := other.Publish(ctx, admitted, in); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("Publish without Materialize's fence: %v", err) } } // L18 joint round, final 4, item 1 (ruling R-EXEC-CLAIMVERSION): the // relocation probe's start names the file its launch source hashed. With // the production BeforeExec hook set, the activator refuses any claimed // start whose Staged.Version is not its Version (Service.claimedBuild, // run on the start the runtime's Prepare returns), so a probe selection // that drops Staged never starts. The start Prepare hands the activator // for P's prepared opening carries the launch's Staged, and so does every // selection: a later cold attempt's and an empty P's. func TestTheRelocationProbeStartNamesTheBuildItHashed(t *testing.T) { ctx := t.Context() launch := roProbeLaunch() r := newRelocationProbeRig(t, launch) admitted, err := r.c.runtime.Admit(ctx, r.route, r.o, roProbe) if err != nil { t.Fatal(err) } s, err := r.c.runtime.Prepare(ctx, r.route, admitted, roProbe) if err != nil { t.Fatal(err) } // claimedBuild's refusal, as the activator applies it with BeforeExec set. if s.Staged != launch.Staged || s.Version != launch.Version || s.Staged.Version != s.Version { t.Fatalf("the probe start names build %q but staged %+v; the launch hashed %+v", s.Version, s.Staged, launch.Staged) } for _, o := range []roOptions{{}, {empty: true}} { f := newROFixture(t, o) source := f.inputs.Launch staged := release.StagedIdentity{Version: "v1", StagedPath: "/opt/carlos/versions/v1/acct/app", Digest: strings.Repeat("b", 64)} f.inputs.Launch = func(ctx context.Context, rt registry.Route) (ClaimServingLaunch, error) { l, err := source(ctx, rt) l.Staged = staged return l, err } first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } if first.Staged != staged || first.Staged.Version != first.Version { t.Fatalf("empty=%v: the first attempt names build %q but staged %+v", o.empty, first.Version, first.Staged) } if o.empty { continue } again, err := f.inputs.Select(ctx, f.route, f.admit(f.read(), first), roProbe) if err != nil { t.Fatal(err) } if again.Prepared || again.Staged != staged || again.Staged.Version != again.Version { t.Fatalf("a later attempt (prepared %v) names build %q but staged %+v", again.Prepared, again.Version, again.Staged) } } } // Ruling 47: a later attempt's fresh generation, and an empty P's opening, // publish through main's fenced writer only after their first root. func TestRelocationLaterAttemptsPublishFenced(t *testing.T) { ctx := t.Context() f := newROFixture(t, roOptions{}) first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } opened := f.admit(f.read(), first) again, err := f.inputs.Select(ctx, f.route, opened, roProbe) if err != nil { t.Fatal(err) } later := f.admit(opened, again) gen := later.Claim.Holder.Generation synced := 0 pub := ClaimPublication{Sync: func(context.Context) error { synced++; return nil }, Check: func(context.Context) error { return nil }} if err := f.inputs.Publish(ctx, later, again); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("an unfenced marker for a fresh generation: %v", err) } if _, err := f.inputs.PublishFenced(ctx, later, again, pub); err == nil || synced != 1 { t.Fatalf("a marker before the root: %v synced=%d", err, synced) } root := moves.SnapshotKey(moves.GenerationPrefix(later.Claim.Holder.Location, gen), 44) f.bucket.Put("data", root, ltxHeader(t, 1, 44)) receipt, err := f.inputs.PublishFenced(ctx, later, again, pub) if err != nil || receipt.Root.Key != root || receipt.MarkerVersion == "" { t.Fatalf("publish: %+v %v", receipt, err) } if err := f.inputs.PublicationEvidence(ctx, later); err != nil { t.Fatalf("fresh evidence: %v", err) } prepared := again prepared.Prepared = true if _, err := f.inputs.PublishFenced(ctx, later, prepared, pub); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a prepared admission through the fenced writer: %v", err) } // An empty P: no snapshot, no root until the first transaction. e := newROFixture(t, roOptions{empty: true}) ein, err := e.inputs.Select(ctx, e.route, e.read(), roProbe) if err != nil { t.Fatal(err) } eopened := e.admit(e.read(), ein) if _, err := e.inputs.PublishFenced(ctx, eopened, ein, pub); err == nil { t.Fatal("an empty P published a marker with no root") } }