package host import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http/httptest" "os" "path/filepath" "strconv" "strings" "testing" "time" "github.com/benbjohnson/litestream" lsfile "github.com/benbjohnson/litestream/file" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostrelocate" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/release" "github.com/superfly/ltx" ) const buildHost = "shop.example" // buildFixture is a relocation target at installing on an empty box: the // claim journals a pin of a real litestream replica sealed into a // versioned bucket (served over a loopback S3 endpoint, so the replica // lease is lease.Acquire's own), a retained files manifest with one entry, // an empty guard directory, and main's Local recorded. type buildFixture struct { t *testing.T claims *release.DirStore files *hookedStore bucket *leasetest.VersionedBucket versions *hookedVersions a ClaimAssembly cfg RelocationAssemblyConfig obs hostclaim.Observed locals []hostbox.Selection localErr error // localHook, when set, is asked on every Local check that localErr // passes: a test loses the lease at one write group. localHook func(hostbox.Selection) error verifier PreparedFromInstallManifest } // hookedStore and hookedVersions run after, once a write has landed, so a // test can move the build's authority between two durable writes. type hookedStore struct { release.Store afterPut func(key string) } func (s *hookedStore) Put(ctx context.Context, key string, body []byte, etag string) (string, error) { out, err := s.Store.Put(ctx, key, body, etag) if err == nil && s.afterPut != nil { s.afterPut(key) } return out, err } type hookedVersions struct { lease.VersionedStore afterPut func(key string) } func (s *hookedVersions) PutIfAbsent(ctx context.Context, bucket, key string, body []byte) (string, error) { out, err := s.VersionedStore.PutIfAbsent(ctx, bucket, key, body) if err == nil && s.afterPut != nil { s.afterPut(key) } return out, err } // buildOptions shape a buildFixture: a pin with or without a database, the // data root (a fresh one with a hostguard directory when empty), and the // pinned manifest's entries and platform items, each stored as an object // under the source generation's prefix. NoFiles pins a host that never // published files instead (no digest, no retained copy: the pinner's // evidenced empty files set), and Files and Platform are ignored. type buildOptions struct { DB bool Data string Files map[string]string Platform map[string]string NoFiles bool } func newBuildFixture(t *testing.T, withDB bool) *buildFixture { return newBuildFixtureWith(t, buildOptions{DB: withDB, Files: map[string]string{"public/hello.txt": "hello"}}) } func newBuildFixtureWith(t *testing.T, o buildOptions) *buildFixture { t.Helper() ctx := t.Context() withDB := o.DB f := &buildFixture{t: t, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()} f.bucket.AddBucket("data") // P's files manifests and the pinned objects live in the versioned data // bucket, as production's files store and versioned client are one bucket. f.files = &hookedStore{Store: bucketStore{b: f.bucket, bucket: "data"}} f.versions = &hookedVersions{VersionedStore: f.bucket} handler, _ := leasetest.NewS3Server(f.bucket, "data") srv := httptest.NewServer(handler) // 127.0.0.1 only t.Cleanup(srv.Close) t.Setenv("CARLOS_S3_ENDPOINT", srv.URL) t.Setenv("AWS_ACCESS_KEY_ID", "test-key") t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") t.Setenv("AWS_SESSION_TOKEN", "") t.Setenv("AWS_PROFILE", "") t.Setenv("AWS_REGION", "") created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC) enrollment := strings.Repeat("a", 32) nh := hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + buildHost + ".json", Account: "acct", App: "app", Host: buildHost, Region: "ie", RecordRegion: "ie", CreatedAt: created} body := []byte(`{"host":"` + buildHost + `","account":"acct","app":"app","region":"ie","created_at":"2026-10-07T01:00:00Z"}`) obs, err := hostclaim.Read(ctx, f.claims, buildHost) if err != nil { t.Fatal(err) } for _, tr := range []func(*hostclaim.Claim) error{ hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment}), hostclaim.CompleteCreate("c1", "src"), hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}), } { if obs, err = hostclaim.Update(ctx, f.claims, obs, tr); err != nil { t.Fatal(err) } } location := obs.Claim.Holder.Location const op = "rel-1" pin := hostrelocate.PinObject{V: 1, Host: buildHost, Op: op, ClaimGen: obs.Claim.Gen, PinnedAt: created, Release: "v1"} if withDB { files, n := sealedReplica(t, f.bucket, moves.GenerationPrefix(location, "g1")) pin.DB = &genselect.Plan{Generation: "g1", Files: files, MaxTXID: n, MaxLevel0TXID: n} } else { pin.Empty = "session-0" } srcPrefix := location + "/files/g-g1/" pinned := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 5, Platform: map[string]hostfiles.Entry{}, Entries: map[string]hostfiles.Entry{}} object := func(body string, mode uint32) hostfiles.Entry { sum := sha256.Sum256([]byte(body)) sha := hex.EncodeToString(sum[:]) key := srcPrefix + "objects/" + fmt.Sprintf("%016d-%s", 5, sha) if _, err := f.files.Store.Put(ctx, key, []byte(body), ""); err != nil { t.Fatal(err) } return hostfiles.Entry{Object: key, SHA256: sha, Size: int64(len(body)), Mode: mode, Seq: 5} } for rel, body := range o.Files { pinned.Entries[rel] = object(body, 0o644) } for name, body := range o.Platform { pinned.Platform[name] = object(body, 0o600) } pinnedRaw, err := json.Marshal(pinned) if err != nil { t.Fatal(err) } pinnedSum := sha256.Sum256(pinnedRaw) retained := hostrelocate.RetainedManifestKey(buildHost, op, hex.EncodeToString(pinnedSum[:])) if _, err := f.claims.Put(ctx, retained, pinnedRaw, ""); err != nil { t.Fatal(err) } pin.Files = hostrelocate.FilesPin{Prefix: srcPrefix, Seq: 5, SHA256: hex.EncodeToString(pinnedSum[:]), Origins: []string{srcPrefix}, Retained: retained} filesManifest, filesSeq := retained, int64(5) if o.NoFiles { pin.Files = hostrelocate.FilesPin{Prefix: srcPrefix, Origins: []string{srcPrefix}} filesManifest, filesSeq = hostfiles.ManifestKey(srcPrefix), 0 } pinRaw, err := json.Marshal(pin) if err != nil { t.Fatal(err) } pinKey := hostrelocate.PinKey(buildHost, op, obs.Claim.Gen) if _, err := f.claims.Put(ctx, pinKey, pinRaw, ""); err != nil { t.Fatal(err) } pinSum := sha256.Sum256(pinRaw) ref := hostclaim.RecoveryPin{Key: pinKey, SHA256: hex.EncodeToString(pinSum[:]), Generation: "g1", FilesManifest: filesManifest, FilesSeq: filesSeq, PinnedAt: created, Release: "v1", Empty: pin.DB == nil} if pin.DB != nil { ref.MaxTXID = pin.DB.MaxTXID } refRaw, err := json.Marshal(ref) if err != nil { t.Fatal(err) } // The installing CAS: this box executes the op, the holder's stamp is // the install stamp, P is named (hostclaim's install()). if obs, err = hostclaim.Update(ctx, f.claims, obs, func(c *hostclaim.Claim) error { stamp := c.Gen + 1 c.Holder.Stamp = stamp c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: op, Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}, Journal: map[string]string{"pin": string(refRaw), "prepared": "p1", "install_stamp": strconv.FormatUint(stamp, 10), "install_release": "v1"}} return nil }); err != nil { t.Fatal(err) } f.obs = obs data := o.Data if data == "" { data = t.TempDir() } if err := os.MkdirAll(filepath.Join(data, "hostguard"), 0o770); err != nil { t.Fatal(err) } f.a = ClaimAssembly{Claims: f.claims, Guard: hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.claims}}} f.cfg = RelocationAssemblyConfig{Data: f.files, Versions: f.versions, Bucket: "data", Region: "ie", LeaseRegion: "eu-west-1", DataDir: data, WorkDir: filepath.Join(data, "relocate", "build"), Now: func() time.Time { return created.Add(time.Hour) }, Source: hostbox.WorkSource{Box: "dst", Region: "ie", Local: func(_ context.Context, c *hostclaim.Claim, s hostbox.Selection) error { f.locals = append(f.locals, s) if f.localErr == nil && f.localHook != nil { return f.localHook(s) } return f.localErr }}} f.verifier = PreparedFromInstallManifest{Claims: f.claims, Files: f.files, Data: f.bucket, Bucket: "data"} return f } // sealedReplica writes a real SQLite database through litestream's file // replica and seals every LTX file into the bucket under prefix, keyed as // genselect lists a generation (/-.ltx, prefix // moves.GenerationPrefix). func sealedReplica(t *testing.T, b *leasetest.VersionedBucket, prefix string) ([]genselect.File, uint64) { t.Helper() ctx := t.Context() dir := t.TempDir() path := filepath.Join(dir, "src.db") write := func(n int) { d, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } defer d.Close() if _, err := d.Exec(`CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, body TEXT)`); err != nil { t.Fatal(err) } for range n { if _, err := d.Exec(`INSERT INTO t (body) VALUES (?)`, strings.Repeat("x", 100)); err != nil { t.Fatal(err) } } } write(1) client := lsfile.NewReplicaClient(filepath.Join(dir, "replica")) db := litestream.NewDB(path) db.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) db.SetMetaPath(filepath.Join(dir, "meta")) db.MonitorInterval = time.Hour db.Replica = litestream.NewReplicaWithClient(db, client) db.Replica.SyncInterval = time.Hour if err := db.Open(); err != nil { t.Fatal(err) } for range 2 { if err := db.SyncAndWait(ctx); err != nil { t.Fatal(err) } write(2) } if err := db.SyncAndWait(ctx); err != nil { t.Fatal(err) } if err := db.Close(ctx); err != nil { t.Fatal(err) } var out []genselect.File var n uint64 for level := 0; level <= litestream.SnapshotLevel; level++ { itr, err := client.LTXFiles(ctx, level, 0, false) if err != nil { t.Fatal(err) } infos, err := ltx.SliceFileIterator(itr) if err != nil { t.Fatal(err) } for _, info := range infos { rc, err := client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0) if err != nil { t.Fatal(err) } body, err := io.ReadAll(rc) rc.Close() if err != nil { t.Fatal(err) } key := prefix + fmt.Sprintf("%04x/", level) + ltx.FormatFilename(info.MinTXID, info.MaxTXID) v := b.Put("data", key, body) out = append(out, genselect.File{Key: key, VersionID: v, Level: level, MinTXID: uint64(info.MinTXID), MaxTXID: uint64(info.MaxTXID), Size: int64(len(body))}) if level == 0 && uint64(info.MaxTXID) > n { n = uint64(info.MaxTXID) } } } if n == 0 { t.Fatal("the source replica has no level-0 files") } return out, n } func (f *buildFixture) build() error { return f.a.relocationBuild(f.cfg)(f.t.Context(), f.obs) } func (f *buildFixture) prefix() string { return moves.GenerationPrefix(f.obs.Claim.Holder.Location, "p1") } func (f *buildFixture) installManifestExists() bool { _, ok := f.bucket.Current("data", moves.InstallManifestKey(f.prefix())) return ok } // moveExec moves the op to another executor, as a replaced target sees it. func (f *buildFixture) moveExec() { f.t.Helper() o, err := hostclaim.Read(context.Background(), f.claims, buildHost) if err != nil { f.t.Fatal(err) } if _, err := hostclaim.Update(context.Background(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Exec, c.Op.Intent["target_box"] = "other", "other" c.Op.Participants = []string{"src", "other"} return nil }); err != nil { f.t.Fatal(err) } } func (f *buildFixture) retireStamp() { f.t.Helper() if err := f.a.Guard.RetireInstallation(context.Background(), buildHost, func(*hostclaim.Claim) error { return nil }); err != nil { f.t.Fatal(err) } } // Prepared is done only when P's files manifest and, for a non-empty pin, // its install manifest both verify; an unconfigured verifier reports // nothing built. Done carries what ToInstalled requires: P's manifest at // sequence 0 built from the pinned sequence, and the install manifest at // the pin's N. func TestPreparedFromInstallManifestNeedsBothManifests(t *testing.T) { if _, done, err := (PreparedFromInstallManifest{}).Verify(t.Context(), hostclaim.Observed{Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{}}}); done || err == nil { t.Fatalf("an unconfigured verifier reported P built: %v", err) } f := newBuildFixture(t, true) if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || err != nil { t.Fatalf("before the build: done=%v %v", done, err) } if err := f.build(); err != nil { t.Fatal(err) } a, done, err := f.verifier.Verify(t.Context(), f.obs) if err != nil || !done { t.Fatalf("after the build: done=%v %v", done, err) } var ref hostclaim.RecoveryPin _ = json.Unmarshal([]byte(f.obs.Claim.Op.Journal["pin"]), &ref) if a.Generation != "p1" || a.FilesSeq != 0 || a.SourceFilesSeq != 5 || len(a.FilesManifestSHA256) != 64 || a.InstallManifest != moves.InstallManifestKey(f.prefix()) || len(a.SHA256) != 64 || a.SnapshotTXID != ref.MaxTXID { t.Fatalf("authority: %+v", a) } // An empty bootstrap needs only P's files manifest. empty := newBuildFixture(t, false) if err := empty.build(); err != nil { t.Fatal(err) } if a, done, err := empty.verifier.Verify(t.Context(), empty.obs); err != nil || !done || a.InstallManifest != "" || a.SnapshotTXID != 0 { t.Fatalf("empty bootstrap: %+v done=%v %v", a, done, err) } if empty.installManifestExists() { t.Fatal("an empty pin built a snapshot") } } // A crash after the files manifest and before the install manifest, and a // store holding the install manifest without P's files manifest, each // leave P unverified, never an error that would stop the worker. func TestPreparedRefusesAPartialBuild(t *testing.T) { t.Run("files manifest only", func(t *testing.T) { f := newBuildFixture(t, true) // The process dies after P's files manifest: the next authority // check refuses, as a crash would stop it. f.files.afterPut = func(string) { f.localErr = errors.New("crashed") } if err := f.build(); err == nil { t.Fatal("the stopped build succeeded") } if _, _, err := f.files.Get(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/")); err != nil { t.Fatalf("P's files manifest was not written first: %v", err) } if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || err != nil { t.Fatalf("done=%v err=%v", done, err) } }) t.Run("install manifest only", func(t *testing.T) { f := newBuildFixture(t, true) if err := f.build(); err != nil { t.Fatal(err) } v := f.verifier v.Files = release.NewDirStore(t.TempDir()) // P's files manifest is not there if _, done, err := v.Verify(t.Context(), f.obs); done || err != nil { t.Fatalf("done=%v err=%v", done, err) } }) t.Run("a files manifest that is not the pinned entries", func(t *testing.T) { f := newBuildFixture(t, true) bad := hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} raw, _ := json.Marshal(bad) if _, err := f.files.Store.Put(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/"), raw, ""); err != nil { t.Fatal(err) } if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("done=%v err=%v", done, err) } }) } // A build whose authority moved under it stops before the next durable // write: after P's files manifest the executor moved; after the snapshot // PUT the executor moved, or the guard retired the stamp. No install // manifest is written for a box that no longer builds. func TestRelocationBuildStopsWhenAuthorityChanges(t *testing.T) { for _, tc := range []struct { name string hook func(f *buildFixture) after string }{ {"after the files manifest", func(f *buildFixture) { f.moveExec() }, "files"}, {"after the snapshot PUT", func(f *buildFixture) { f.moveExec() }, "snapshot"}, {"after the stamp is retired", func(f *buildFixture) { f.retireStamp() }, "snapshot"}, } { t.Run(tc.name, func(t *testing.T) { f := newBuildFixture(t, true) fired := false fire := func(string) { if !fired { fired = true tc.hook(f) } } if tc.after == "files" { f.files.afterPut = fire } else { f.versions.afterPut = func(key string) { if strings.HasPrefix(key, f.prefix()) && strings.HasSuffix(key, ".ltx") { fire(key) } } } if err := f.build(); !errors.Is(err, hostrelocate.ErrRefused) || !fired { t.Fatalf("the build went on after its authority changed: %v (fired %v)", err, fired) } if f.installManifestExists() { t.Fatal("the install manifest was written for a box that no longer builds") } }) } } // Ruling 59: the replica lease, the claim and the guard all stay valid; // only main's Local refuses (the execution lease lost), and the next write // is refused with its error. func TestRelocationBuildStopsWhenOnlyTheExecutionLeaseIsLost(t *testing.T) { f := newBuildFixture(t, true) f.versions.afterPut = func(key string) { if strings.HasSuffix(key, ".ltx") { f.localErr = execlease.ErrLost } } if err := f.build(); !errors.Is(err, execlease.ErrLost) { t.Fatalf("the build went on without its execution lease: %v", err) } if f.installManifestExists() { t.Fatal("the install manifest was written after the execution lease was lost") } if c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost); err != nil || c.Stamp != f.obs.Claim.Holder.Stamp { t.Fatalf("the guard moved: %+v %v", c, err) } } // A replay from installing (a crash after the build, before ToInstalled) // is a success that changes nothing: the same files manifest bytes, the // same install manifest version. func TestRelocationBuildReplaysToTheSameBytes(t *testing.T) { f := newBuildFixture(t, true) if err := f.build(); err != nil { t.Fatal(err) } key := moves.InstallManifestKey(f.prefix()) first := f.bucket.LatestID("data", key) filesKey := hostfiles.ManifestKey(f.obs.Claim.Holder.Location + "/files/g-p1/") before, _, err := f.files.Get(t.Context(), filesKey) if err != nil { t.Fatal(err) } if err := f.build(); err != nil { t.Fatalf("replay: %v", err) } after, _, err := f.files.Get(t.Context(), filesKey) if err != nil || string(after) != string(before) || f.bucket.LatestID("data", key) != first { t.Fatalf("the replay rewrote P: %v", err) } if _, done, err := f.verifier.Verify(t.Context(), f.obs); !done || err != nil { t.Fatalf("after the replay: done=%v %v", done, err) } } // Finding 2, ruling 43: the build activates the target's guard at // install_stamp from an empty box, under main's Local, before Capture; // never for another executor, never past a refusing Local, never at // another phase, never a retired stamp. The build's host-named scratch is // gone once P is built. func TestRelocationBuildActivatesTheGuardFromAnEmptyTarget(t *testing.T) { t.Run("from an empty box", func(t *testing.T) { f := newBuildFixture(t, true) if err := f.build(); err != nil { t.Fatal(err) } c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost) if err != nil || c.Incarnation != f.obs.Claim.Holder.Incarnation || c.Stamp != f.obs.Claim.Holder.Stamp { t.Fatalf("capture %+v %v", c, err) } if len(f.locals) == 0 || f.locals[0].Action != relocationActionInstall || f.locals[0].Captured.Stamp != f.obs.Claim.Holder.Stamp || f.locals[0].Box != "dst" { t.Fatalf("main's Local was not asked for the install: %+v", f.locals) } if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the build's scratch survived: %v", err) } }) for _, tc := range []struct { name string change func(f *buildFixture) want error }{ {"another executor", func(f *buildFixture) { f.moveExec() f.obs, _ = hostclaim.Read(context.Background(), f.claims, buildHost) }, hostrelocate.ErrRefused}, {"main's Local refuses", func(f *buildFixture) { f.localErr = execlease.ErrLost }, execlease.ErrLost}, {"installed, not installing", func(f *buildFixture) { o, _ := hostclaim.Read(context.Background(), f.claims, buildHost) f.obs, _ = hostclaim.Update(context.Background(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Phase = hostclaim.PhaseInstalled; return nil }) }, hostrelocate.ErrRefused}, } { t.Run(tc.name, func(t *testing.T) { f := newBuildFixture(t, true) tc.change(f) before := f.bucket.Snapshot() if err := f.build(); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } if entries, _ := os.ReadDir(filepath.Join(f.a.Guard.DataDir, "hostguard")); len(entries) != 0 { t.Fatalf("the guard was written: %v", entries) } if f.bucket.Snapshot() != before { t.Fatal("the bucket was written") } }) } t.Run("a retired stamp", func(t *testing.T) { f := newBuildFixture(t, true) if _, err := f.a.relocationActivate(f.cfg)(t.Context(), f.obs); err != nil { t.Fatal(err) } f.retireStamp() before := f.bucket.Snapshot() if err := f.build(); !errors.Is(err, hostguard.ErrRetiredStamp) { t.Fatalf("a retired stamp was reactivated: %v", err) } if f.bucket.Snapshot() != before { t.Fatal("the bucket was written") } }) } // A pin whose files digest names no retained copy is refused, never read // as a host that published no files (absence is never evidence). The // pinned manifest here is itself empty, so only that check tells the two // apart. func TestPreparedRefusesAFilesDigestWithoutItsRetainedCopy(t *testing.T) { ctx := t.Context() f := newBuildFixtureWith(t, buildOptions{}) if err := f.build(); err != nil { t.Fatal(err) } if _, done, err := f.verifier.Verify(ctx, f.obs); !done || err != nil { t.Fatalf("the built empty pin: done=%v %v", done, err) } var ref hostclaim.RecoveryPin if err := json.Unmarshal([]byte(f.obs.Claim.Op.Journal["pin"]), &ref); err != nil { t.Fatal(err) } pin, err := hostrelocate.ReadPin(ctx, f.claims, ref) if err != nil { t.Fatal(err) } if pin.Files.SHA256 == "" || pin.Files.Retained == "" { t.Fatalf("the fixture's pin has no retained manifest: %+v", pin.Files) } pin.Files.Retained = "" raw, err := json.Marshal(pin) if err != nil { t.Fatal(err) } ref.Key += ".forged" if _, err := f.claims.Put(ctx, ref.Key, raw, ""); err != nil { t.Fatal(err) } sum := sha256.Sum256(raw) ref.SHA256 = hex.EncodeToString(sum[:]) refRaw, err := json.Marshal(ref) if err != nil { t.Fatal(err) } c := *f.obs.Claim c.Op.Journal = map[string]string{} for k, v := range f.obs.Claim.Op.Journal { c.Op.Journal[k] = v } c.Op.Journal["pin"] = string(refRaw) obs := f.obs obs.Claim = &c if _, done, err := f.verifier.Verify(ctx, obs); done || !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("done=%v err=%v", done, err) } } // staleObs is the build's observation with change applied to a copy of its // claim, as a worker holding an older read sees it; the stored claim is // unchanged. func (f *buildFixture) staleObs(change func(*hostclaim.Claim)) hostclaim.Observed { c := *f.obs.Claim h := *c.Holder c.Holder = &h c.Op.Journal = map[string]string{} for k, v := range f.obs.Claim.Op.Journal { c.Op.Journal[k] = v } change(&c) o := f.obs o.Claim = &c return o } // Rulings 43 and 52: the guard is activated, and P built, only for the // installation the installing CAS issued. A stored claim whose holder stamp // is not the install stamp, that lost this box, that is stuck or whose P // was opened writes no guard record and nothing in the bucket. An // observation older than the claim, at another install stamp, of another // incarnation, or naming another P or pin, writes nothing in the bucket. func TestRelocationBuildRefusesAnInstallationTheClaimDidNotIssue(t *testing.T) { for _, tc := range []struct { name string stored func(*hostclaim.Claim) // a change to the stored claim stale func(*buildFixture, *hostclaim.Claim) // a change to the observation only want error }{ {name: "a holder stamp that is not the install stamp", stored: func(c *hostclaim.Claim) { c.Holder.Stamp++ }, want: hostrelocate.ErrRefused}, {name: "this box declared lost", stored: func(c *hostclaim.Claim) { c.Lost = append(c.Lost, "dst") }, want: hostrelocate.ErrRefused}, {name: "the op stuck", stored: func(c *hostclaim.Claim) { c.Op.Journal["stuck"] = "an operator's" }, want: hostrelocate.ErrRefused}, {name: "P opened", stored: func(c *hostclaim.Claim) { c.Op.Journal["prepared_opened"] = c.Op.Journal["prepared"] }, want: hostrelocate.ErrRefused}, {name: "an observation at another install stamp", stale: func(_ *buildFixture, c *hostclaim.Claim) { c.Op.Journal["install_stamp"] = strconv.FormatUint(c.Holder.Stamp+1, 10) }, want: hostguard.ErrStampMismatch}, {name: "an observation of another incarnation", stale: func(_ *buildFixture, c *hostclaim.Claim) { c.Holder.Incarnation = "another" }, want: hostguard.ErrStampMismatch}, {name: "an observation naming another P", stale: func(_ *buildFixture, c *hostclaim.Claim) { c.Op.Journal["prepared"] = "p2" }, want: hostrelocate.ErrRefused}, {name: "an observation naming another pin", stale: func(f *buildFixture, c *hostclaim.Claim) { // The same pin bytes at another key: ReadPin accepts them, the // journal's text differs. var ref hostclaim.RecoveryPin if err := json.Unmarshal([]byte(c.Op.Journal["pin"]), &ref); err != nil { f.t.Fatal(err) } raw, _, err := f.claims.Get(context.Background(), ref.Key) if err != nil { f.t.Fatal(err) } ref.Key += ".copy" if _, err := f.claims.Put(context.Background(), ref.Key, raw, ""); err != nil { f.t.Fatal(err) } refRaw, _ := json.Marshal(ref) c.Op.Journal["pin"] = string(refRaw) }, want: hostrelocate.ErrRefused}, } { t.Run(tc.name, func(t *testing.T) { f := newBuildFixture(t, true) if tc.stored != nil { o, err := hostclaim.Read(t.Context(), f.claims, buildHost) if err != nil { t.Fatal(err) } if f.obs, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { tc.stored(c); return nil }); err != nil { t.Fatal(err) } } if tc.stale != nil { f.obs = f.staleObs(func(c *hostclaim.Claim) { tc.stale(f, c) }) } before := f.bucket.Snapshot() if err := f.build(); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } if f.bucket.Snapshot() != before { t.Fatal("the bucket was written") } if tc.stored != nil { if entries, _ := os.ReadDir(filepath.Join(f.a.Guard.DataDir, "hostguard")); len(entries) != 0 { t.Fatalf("the guard was written: %v", entries) } } }) } } // relocationBuildP checks its authority before it takes the replica lease // (a stale build never writes the lease object on holder.location), again // once the lease is held, before P's files manifest, and before the // database build, so a refusal at each stops what follows it. func TestRelocationBuildPChecksItsAuthorityBeforeEachStep(t *testing.T) { refused := errors.New("sentinel: the authority refused") for _, tc := range []struct { name string at int // the auth call that refuses }{ {"before the replica lease", 1}, {"before P's files manifest", 2}, {"before the database build", 3}, } { t.Run(tc.name, func(t *testing.T) { f := newBuildFixture(t, true) calls := 0 auth := func(context.Context) error { if calls++; calls == tc.at { return refused } return nil } before := f.bucket.Snapshot() if err := f.a.relocationBuildP(f.cfg)(t.Context(), f.obs, auth); !errors.Is(err, refused) || calls != tc.at { t.Fatalf("got %v after %d checks", err, calls) } if tc.at == 1 && f.bucket.Snapshot() != before { t.Fatal("the replica lease was taken without authority") } _, _, err := f.files.Get(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/")) if wrote := err == nil; wrote != (tc.at > 2) { t.Fatalf("P's files manifest written=%v (%v)", wrote, err) } if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the database build started: %v", err) } if f.installManifestExists() { t.Fatal("the install manifest was written") } }) } } // lossyLease is a real replica lease whose loss the test signals. type lossyLease struct { *lease.Held lost chan struct{} } func (l lossyLease) Lost() <-chan struct{} { return l.lost } // The replica lease lost while the claim, the guard and main's Local all // still admit the build stops it before the database build. func TestRelocationBuildPStopsWhenTheReplicaLeaseIsLost(t *testing.T) { f := newBuildFixture(t, true) lost := make(chan struct{}) f.a.acquireReplicaLease = func(ctx context.Context, bucket, prefix, region string) (replicaLease, error) { h, err := lease.Acquire(ctx, bucket, prefix, region) if err != nil { return nil, err } return lossyLease{Held: h, lost: lost}, nil } calls := 0 auth := func(ctx context.Context) error { if calls++; calls == 3 { // after P's files manifest close(lost) <-ctx.Done() // the build has seen the loss } return nil } if err := f.a.relocationBuildP(f.cfg)(t.Context(), f.obs, auth); !errors.Is(err, context.Canceled) { t.Fatalf("the build went on without its replica lease: %v", err) } if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the database build started: %v", err) } if f.installManifestExists() { t.Fatal("the install manifest was written after the replica lease was lost") } } // Verify accepts only this pin's build of P: P's files manifest at // sequence 0, and an install manifest of this op and host at the pin's N, // whose integrity check passed and whose snapshot and anchor versions // exist. Anything else built at P is refused, never reported done. func TestPreparedRefusesAnythingButThisPinsBuild(t *testing.T) { for _, tc := range []struct { name string files func(*hostfiles.Manifest) im func(*buildFixture, *moves.InstallManifest) }{ {name: "P's files manifest not at sequence 0", files: func(m *hostfiles.Manifest) { m.Seq = 5 }}, {name: "another op", im: func(_ *buildFixture, m *moves.InstallManifest) { m.Op = "rel-2" }}, {name: "another host", im: func(_ *buildFixture, m *moves.InstallManifest) { m.Host = "other.example" }}, {name: "another N", im: func(f *buildFixture, m *moves.InstallManifest) { // A complete build at N-1: its snapshot and anchor exist, at // their keys for N-1. Only the pin's N tells it apart. copyAt := func(from, to string) string { body, ok := f.bucket.Current("data", from) if !ok { f.t.Fatalf("%s is not there", from) } return f.bucket.Put("data", to, body) } n := m.N - 1 m.SnapshotVersion = copyAt(m.SnapshotKey, moves.SnapshotKey(f.prefix(), n)) m.AnchorVersion = copyAt(m.AnchorKey, moves.AnchorKey(f.prefix(), n)) m.N, m.SnapshotKey, m.AnchorKey = n, moves.SnapshotKey(f.prefix(), n), moves.AnchorKey(f.prefix(), n) }}, {name: "a failed integrity check", im: func(_ *buildFixture, m *moves.InstallManifest) { m.IntegrityCheck = "corrupt" }}, {name: "a snapshot version that is not there", im: func(_ *buildFixture, m *moves.InstallManifest) { m.SnapshotVersion = "missing" }}, {name: "an anchor version that is not there", im: func(_ *buildFixture, m *moves.InstallManifest) { m.AnchorVersion = "missing" }}, } { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() f := newBuildFixture(t, true) if err := f.build(); err != nil { t.Fatal(err) } if _, done, err := f.verifier.Verify(ctx, f.obs); !done || err != nil { t.Fatalf("the untouched build: done=%v %v", done, err) } if tc.files != nil { key := hostfiles.ManifestKey(f.obs.Claim.Holder.Location + "/files/g-p1/") raw, etag, err := f.files.Get(ctx, key) if err != nil { t.Fatal(err) } var m hostfiles.Manifest if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } tc.files(&m) if raw, err = json.Marshal(m); err != nil { t.Fatal(err) } if _, err := f.files.Store.Put(ctx, key, raw, etag); err != nil { t.Fatal(err) } } if tc.im != nil { im, _, err := readInstallManifest(ctx, f.bucket, "data", f.prefix()) if err != nil { t.Fatal(err) } tc.im(f, &im) raw, err := json.Marshal(im) if err != nil { t.Fatal(err) } f.bucket.Put("data", moves.InstallManifestKey(f.prefix()), raw) } if _, done, err := f.verifier.Verify(ctx, f.obs); done || !errors.Is(err, hostrelocate.ErrRefused) { t.Fatalf("done=%v err=%v", done, err) } }) } }