//go:build linux package host import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/superfly/ltx" "golang.org/x/sys/unix" ) // This is the explicit Task 0 root seam, not a production parking witness. // It retains immutable replies across controller restarts so the agent's // enrollment/completion order can be tested independently of Task 0R. type servingParkFixture struct { continuePending bool t *testing.T f *recoveryFixture db string records map[string]servingParkAuthorization calls []string inside bool failComplete bool afterPark func() } func (p *servingParkFixture) park(current, prior servingArtifactIdentity, legacy bool) (servingParkAuthorization, error) { if p.inside { p.t.Fatal("root called while Work held") } p.calls = append(p.calls, "park") for _, a := range p.records { if a.Successor == current { return a, nil } } if p.continuePending { for id, a := range p.records { a.Successor = current db, _ := filepath.Rel(p.f.guard.DataDir, p.db) a.TagsFrom = p.f.read().Claim.Inventory[current.Box].Paths[db] p.records[id] = a return a, nil } } if !legacy && prior == (servingArtifactIdentity{}) { return servingParkAuthorization{}, hostbox.ErrChanged } a := servingParkAuthorization{TagsFrom: prior.Stamp, Predecessor: prior, Successor: current, OriginalSuccessor: current, Legacy: legacy} a.ID = servingRetirementID(prior, current, legacy) var stat unix.Statx_t if err := unix.Statx(unix.AT_FDCWD, filepath.Dir(p.db), unix.AT_SYMLINK_NOFOLLOW, unix.STATX_BASIC_STATS|unix.STATX_MNT_ID, &stat); err != nil { return a, err } a.Directory = servingArtifactDirectory{Device: unix.Mkdev(stat.Dev_major, stat.Dev_minor), Inode: stat.Ino, Mount: stat.Mnt_id} dest := filepath.Join(p.f.guard.DataDir, "test-parks", a.ID) if err := os.MkdirAll(dest, 0700); err != nil { return a, err } for _, suffix := range []string{"", ".gen", "-wal", "-shm"} { if err := os.Rename(p.db+suffix, filepath.Join(dest, filepath.Base(p.db)+suffix)); err != nil && !errors.Is(err, os.ErrNotExist) { return a, err } } p.records[a.ID] = a if p.afterPark != nil { p.afterPark() } return a, nil } func (p *servingParkFixture) Park(_ context.Context, current, prior servingArtifactIdentity) (servingParkAuthorization, error) { return p.park(current, prior, false) } func (p *servingParkFixture) ParkLegacy(_ context.Context, current servingArtifactIdentity, _ servingLegacyPredecessor) (servingParkAuthorization, error) { return p.park(current, servingArtifactIdentity{}, true) } func (p *servingParkFixture) Complete(ctx context.Context, id string, current servingArtifactIdentity) error { if p.inside { p.t.Fatal("completion under Work") } p.calls = append(p.calls, "complete") a, ok := p.records[id] if !ok || a.Successor != current { return hostbox.ErrChanged } o, err := hostclaim.Read(ctx, p.f.claims, current.Host) if err != nil { return err } db, _ := filepath.Rel(p.f.guard.DataDir, p.db) if o.Claim.Inventory[current.Box].Paths[db] != current.Stamp || o.Claim.Inventory[current.Box].Paths[db+".gen"] != current.Stamp { p.t.Fatal("root completion before paired enrollment") } if p.failComplete { return hostbox.ErrChanged } return nil } type servingRetirementWork struct { p *servingParkFixture r *claimServingArtifactRetirement admitted hostclaim.Observed } func (w servingRetirementWork) With(ctx context.Context, _ hostwrite.Identity, fn func() error) error { if w.p.inside { w.p.t.Fatal("recursive Work") } h := w.admitted.Claim.Holder return w.r.Guard.With(ctx, w.admitted.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, func(c *hostclaim.Claim) error { if !claimSameWriter(w.admitted, hostclaim.Observed{Host: w.admitted.Host, Claim: c}, nil) { return hostbox.ErrChanged } return nil }, func() error { w.p.inside = true; defer func() { w.p.inside = false }(); return fn() }) } func servingRetirementFixture(t *testing.T, hosts ...string) (*recoveryFixture, *claimServingArtifactRetirement, *servingParkFixture, ClaimLaunchInputs) { t.Helper() f := newRecoveryFixture(t, hosts...) db := provision.NestedDBPath(f.guard.DataDir, f.hostName()) if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil { t.Fatal(err) } p := &servingParkFixture{t: t, f: f, db: db, records: make(map[string]servingParkAuthorization)} r := &claimServingArtifactRetirement{Claims: f.claims, Guard: f.guard, DataDir: f.guard.DataDir, Box: "box-1", Region: "ie", Root: p} return f, r, p, ClaimLaunchInputs{DBPath: db} } func servingAdmit(t *testing.T, f *recoveryFixture, in *ClaimLaunchInputs, generation string) hostclaim.Observed { t.Helper() o := f.read() in.Admission = hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: generation} admitted, err := hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(in.Admission)) if err != nil { t.Fatal(err) } if err := f.guard.Activate(t.Context(), recoveryHost, "box-1", nil); err != nil { t.Fatal(err) } return admitted } func servingStop(t *testing.T, f *recoveryFixture, o hostclaim.Observed, boots ...string) hostguard.FenceReceipt { t.Helper() return servingStopFor(t, f, o, hostbox.Actor{Box: "box-1", Region: "ie", Enrollment: "enrollment", Revision: 7, Binding: "7"}, boots...) } func servingStopFor(t *testing.T, f *recoveryFixture, o hostclaim.Observed, actor hostbox.Actor, boots ...string) hostguard.FenceReceipt { t.Helper() boot := "boot" if len(boots) != 0 { boot = boots[0] } target := recoveryTuple(o, claimStopLocal{Actor: actor, BootID: boot, ControllerID: "controller"}) target.FenceID, target.ClaimGen = "stop-"+o.Claim.Holder.Generation, o.Claim.Gen if _, err := f.guard.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } s := StopReceipt{V: 1, ID: target.FenceID, Kind: StopReceiptStopped, Region: target.Region, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, FenceID: target.FenceID, Host: target.Host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, Box: target.Box, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, EvidenceAt: f.now} body := mustMarshalStopReceipt(s) receipt := hostguard.FenceReceipt{Target: target, Bytes: body, EvidenceAt: f.now} if err := f.guard.CompleteFence(t.Context(), target, body, f.now, func(ctx context.Context, _ hostguard.FenceTarget, _ []byte, _ time.Time) error { return verifyRetirementReceipt(ctx, receipt) }); err != nil { t.Fatal(err) } return receipt } func TestServingRetirementEnrollmentAndProductionRestore(t *testing.T) { f, r, p, in := servingRetirementFixture(t) plan := plan12RestoreBaseline(t, f) if err := os.WriteFile(in.DBPath, []byte("legacy source"), 0600); err != nil { t.Fatal(err) } files := filepath.Join(filepath.Dir(in.DBPath), "files") if err := os.Mkdir(files, 0700); err != nil { t.Fatal(err) } env := filepath.Join(filepath.Dir(in.DBPath), "env") if err := os.WriteFile(env, []byte("KEEP=1"), 0600); err != nil { t.Fatal(err) } for _, generation := range []string{"G", "G2"} { admitted := servingAdmit(t, f, &in, generation) work := servingRetirementWork{p: p, r: r, admitted: admitted} restore := hostbox.ColdRestore{Reader: f.data, Bucket: f.bucket, Generation: generation, Work: work, BeforeLegacyInstall: func(ctx context.Context, _, _ string) error { return r.BeforeLegacyInstall(ctx, admitted, in, work) }, BeforeInstallLocked: func(ctx context.Context, _, _ string) error { return r.EnrollLocked(ctx, admitted, in) }, RetireArtifacts: func(ctx context.Context, generation string) error { return r.Retire(ctx, admitted, in, generation) }, } if err := r.Recover(t.Context(), admitted, in); err != nil { t.Fatal(err) } if err := restore.Install(t.Context(), in.DBPath, plan); err != nil { t.Fatal(err) } if got := mwReadDB(t, in.DBPath); got != "selected recovery baseline" { t.Fatal(got) } marker, err := hostbox.ReadGenerationMarker(in.DBPath + ".gen") if err != nil || marker != generation { t.Fatalf("marker %q: %v", marker, err) } servingStop(t, f, f.read()) } if body, err := os.ReadFile(env); err != nil || string(body) != "KEEP=1" { t.Fatalf("env: %s %v", body, err) } if _, err := os.Stat(files); err != nil { t.Fatal(err) } if len(p.records) != 2 { t.Fatalf("park records: %v", p.records) } } func TestServingRetirementRefusesUnprovedEnrollment(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) { t.Fatal(err) } if len(p.calls) != 0 { t.Fatal(p.calls) } for _, a := range []string{"", ".gen", "-wal", "-shm"} { t.Run("replacement"+a, func(t *testing.T) { if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil { t.Fatal(err) } if err := os.WriteFile(in.DBPath+a, []byte("unexpected"), 0600); err != nil { t.Fatal(err) } if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) { t.Fatal(err) } if err := os.Remove(in.DBPath + a); err != nil { t.Fatal(err) } }) } } func TestServingRetirementCrashBetweenTagsAndCompletion(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") p.failComplete = true if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); !errors.Is(err, hostbox.ErrChanged) { t.Fatal(err) } if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) { t.Fatal(err) } p.failComplete = false restarted := &claimServingArtifactRetirement{Claims: r.Claims, Guard: r.Guard, DataDir: r.DataDir, Box: r.Box, Region: r.Region, Root: p} before := f.read().ETag if err := restarted.Recover(t.Context(), admitted, in); err != nil { t.Fatal(err) } if f.read().ETag != before { t.Fatal("recovery rewrote current tags") } if err := restarted.EnrollLocked(t.Context(), admitted, in); err != nil { t.Fatal(err) } if len(p.records) != 1 { t.Fatal("recovery invented another retirement") } } func TestServingRetirementAuthenticatesEveryReceiptField(t *testing.T) { t.Parallel() fields := []string{"Region", "ClaimETag", "ClaimGen", "ObligationID", "FenceID", "Host", "Incarnation", "InstallationStamp", "WriterSession", "OperationID", "Phase", "Box", "Enrollment", "BootID", "ControllerID", "DirectoryRevision", "EvidenceAt"} for _, field := range fields { t.Run(field, func(t *testing.T) { f, r, p, in := servingRetirementFixture(t) old := servingAdmit(t, f, &in, "G") if err := r.BeforeLegacyInstall(t.Context(), old, in, servingRetirementWork{p, r, old}); err != nil { t.Fatal(err) } receipt := servingStop(t, f, f.read()) admitted := servingAdmit(t, f, &in, "G2") var body StopReceipt if err := json.Unmarshal(receipt.Bytes, &body); err != nil { t.Fatal(err) } v := reflect.ValueOf(&body).Elem().FieldByName(field) switch v.Kind() { case reflect.String: v.SetString(v.String() + "bad") case reflect.Uint64: v.SetUint(v.Uint() + 1) default: v.Set(reflect.ValueOf(f.now.Add(time.Second))) } receipt.Bytes = mustMarshalStopReceipt(body) r.Guard.FenceView = func(_ context.Context, host string, c hostguard.Captured) (hostguard.FenceView, error) { return hostguard.FenceView{Schema: 1, Host: host, Captured: c, Installation: hostguard.RootInstallation{Host: host, Migration: 1}, Receipts: []hostguard.FenceReceipt{receipt}}, nil } p.calls = nil before := f.read().ETag if err := r.Retire(t.Context(), admitted, in, "G"); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("forged %s: %v", field, err) } if len(p.calls) != 0 || f.read().ETag != before { t.Fatal("forged receipt caused effects") } }) } } func TestServingRetirementRefusesWrongTagsAndWriter(t *testing.T) { t.Parallel() for _, name := range []string{"missing-one", "zero", "mixed", "future", "foreign-box", "foreign-generation", "stale-location", "flat", "ancestor", "outside-path", "noncold", "duplicate-admission", "missing-receipt", "wrong-session"} { t.Run(name, func(t *testing.T) { f, r, p, in := servingRetirementFixture(t) old := servingAdmit(t, f, &in, "G") if err := r.BeforeLegacyInstall(t.Context(), old, in, servingRetirementWork{p, r, old}); err != nil { t.Fatal(err) } if name != "missing-receipt" { servingStop(t, f, f.read()) } admitted := servingAdmit(t, f, &in, "G2") db, _ := filepath.Rel(r.DataDir, in.DBPath) if name == "outside-path" { in.DBPath = filepath.Join(r.DataDir, "foreign.db") } else if name == "stale-location" { admitted.Claim.Holder.Location += "/stale" } else { o := f.read() _, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { e := c.Inventory[r.Box] switch name { case "missing-one": delete(e.Paths, db+".gen") case "zero": e.Paths[db], e.Paths[db+".gen"] = 0, 0 case "mixed": e.Paths[db]++ case "future": e.Paths[db], e.Paths[db+".gen"] = c.Gen+50, c.Gen+50 case "foreign-box": c.Holder.Admissions[0].Box = "foreign" case "foreign-generation": c.Holder.Admissions[0].Generation = "foreign" case "flat": e.Layout = "flat" case "ancestor": e.Paths[filepath.Dir(db)] = old.Claim.Holder.Stamp case "noncold": c.Holder.Admissions[0].Mode = "warm" case "duplicate-admission": c.Holder.Admissions = append(c.Holder.Admissions, c.Holder.Admissions[0]) case "wrong-session": c.Holder.Admissions[0].Session++ } c.Inventory[r.Box] = e return nil }) // Some invalid claims are refused by the writer itself. That also // prevents retirement; never persist malformed bytes to bypass it. if err != nil { return } } before := f.read().ETag p.calls = nil if err := r.Retire(t.Context(), admitted, in, "G"); err == nil { t.Fatal("invalid authority accepted") } if len(p.calls) != 0 { t.Fatal(p.calls) } if f.read().ETag != before { t.Fatal("invalid authority retagged inventory") } }) } } type servingCASFailure struct { release.Store afterCommit bool calls int } func (s *servingCASFailure) Put(ctx context.Context, key string, b []byte, etag string) (string, error) { s.calls++ if s.afterCommit { if _, err := s.Store.Put(ctx, key, b, etag); err != nil { return "", err } } return "", fmt.Errorf("injected lost CAS reply") } func TestServingEnrollmentResolvesOnlyExactCommittedCAS(t *testing.T) { for _, committed := range []bool{false, true} { t.Run(fmt.Sprint(committed), func(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") r.Claims = &servingCASFailure{Store: f.claims, afterCommit: committed} err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}) if (err == nil) != committed { t.Fatalf("committed %v: %v", committed, err) } if !committed { if err := r.EnrollLocked(t.Context(), admitted, in); err == nil { t.Fatal("uncommitted enrollment admitted") } r.Claims = f.claims if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil { t.Fatal(err) } } if len(p.records) != 1 { t.Fatal("CAS retry reparks legacy source") } }) } } func TestServingRetirementPreservesOperationDispatch(t *testing.T) { _, r, _, _ := servingRetirementFixture(t) var calls []string w := &ClaimWakeAssembly{ RetireArtifacts: func(_ context.Context, o hostclaim.Observed, _ ClaimLaunchInputs, _ string) error { calls = append(calls, "retire:"+o.Claim.Op.Kind) return nil }, RecoverArtifacts: func(_ context.Context, o hostclaim.Observed, _ ClaimLaunchInputs) error { calls = append(calls, "recover:"+o.Claim.Op.Kind) return nil }, } if err := bindServingArtifactRetirement(w, r); err != nil { t.Fatal(err) } for _, kind := range []string{hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindRelocating} { o := hostclaim.Observed{Claim: &hostclaim.Claim{Op: hostclaim.Op{Kind: kind}}} if err := w.RetireArtifacts(t.Context(), o, ClaimLaunchInputs{}, "G"); err != nil { t.Fatal(err) } if err := w.RecoverArtifacts(t.Context(), o, ClaimLaunchInputs{}); err != nil { t.Fatal(err) } } if len(calls) != 8 { t.Fatal(calls) } o := hostclaim.Observed{Claim: &hostclaim.Claim{Op: hostclaim.Op{Kind: hostclaim.KindServing}}} if err := w.RetireArtifacts(t.Context(), o, ClaimLaunchInputs{}, "G"); err == nil { t.Fatal("invalid serving authority accepted") } if len(calls) != 8 { t.Fatal("serving fell through to operations") } if !w.RequireServingEnrollment || w.BeforeInstallLocked == nil || w.BeforeLegacyInstall == nil { t.Fatal("missing serving hooks") } } func TestServingArtifactDirectoryRefusesReplacement(t *testing.T) { for _, kind := range []string{"missing", "replacement", "symlink", "mount", "device", "inode", "leaf-symlink"} { t.Run(kind, func(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil { t.Fatal(err) } a := r.completed[admitted.Host] switch kind { case "missing", "replacement", "symlink": dir := filepath.Dir(in.DBPath) if err := os.Rename(dir, dir+".original"); err != nil { t.Fatal(err) } if kind == "replacement" { if err := os.Mkdir(dir, 0700); err != nil { t.Fatal(err) } } if kind == "symlink" { if err := os.Symlink(dir+".original", dir); err != nil { t.Fatal(err) } } case "mount": a.Directory.Mount++ case "device": a.Directory.Device++ case "inode": a.Directory.Inode++ case "leaf-symlink": if err := os.Symlink(filepath.Join(t.TempDir(), "absent"), in.DBPath); err != nil { t.Fatal(err) } } if err := checkServingArtifactAbsence(in.DBPath, a.Directory); err == nil { t.Fatal("unsafe directory accepted") } }) } } func TestServingInventoryPairValidationAndPreservation(t *testing.T) { f, _, _, in := servingRetirementFixture(t) o := servingAdmit(t, f, &in, "G") current, err := servingWriterIdentity(o, "box-1", "ie") if err != nil { t.Fatal(err) } db, _ := filepath.Rel(f.guard.DataDir, in.DBPath) for _, name := range []string{"absent", "current", "older", "missing-one", "mixed", "zero", "future", "ancestor", "descendant", "unrelated-overlap", "unnormalized", "flat"} { t.Run(name, func(t *testing.T) { c := *o.Claim paths := map[string]uint64{"unrelated": 1} entry := hostclaim.InventoryEntry{Layout: "nested", Paths: paths} expected := false prior := uint64(0) switch name { case "absent": expected = true case "current": paths[db], paths[db+".gen"] = current.Stamp, current.Stamp expected = true case "older": paths[db], paths[db+".gen"] = 1, 1 prior = 1 expected = true case "missing-one": paths[db] = 1 case "mixed": paths[db], paths[db+".gen"] = 1, 2 case "zero": paths[db], paths[db+".gen"] = 0, 0 case "future": paths[db], paths[db+".gen"] = current.Stamp+1, current.Stamp+1 case "ancestor": paths[filepath.Dir(db)] = 1 case "descendant": paths[db+"/child"] = 1 case "unrelated-overlap": paths["unrelated/child"] = 1 case "unnormalized": paths["a/../b"] = 1 case "flat": entry.Layout = "flat" } c.Inventory = map[string]hostclaim.InventoryEntry{"box-1": entry, "other": {Layout: "flat", Paths: map[string]uint64{"other-db": 99}}} before, _ := json.Marshal(c) err := servingInventoryTransition(current, db, prior)(&c) if (err == nil) != expected { t.Fatalf("accepted=%v: %v", expected, err) } if !expected { after, _ := json.Marshal(c) if string(after) != string(before) { t.Fatal("refused transition mutated claim") } return } if c.Inventory["box-1"].Paths[db] != current.Stamp || c.Inventory["box-1"].Paths[db+".gen"] != current.Stamp || c.Inventory["box-1"].Paths["unrelated"] != 1 || c.Inventory["other"].Paths["other-db"] != 99 { t.Fatal(c.Inventory) } }) } } func TestServingRetirementParkThenClaimRaceRefusesRetag(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") p.afterPark = func() { p.afterPark = nil newer := in servingAdmit(t, f, &newer, "racer") } if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err == nil { t.Fatal("stale writer enrolled") } if len(f.read().Claim.Inventory) != 0 { t.Fatal("stale writer tagged new installation") } if err := r.EnrollLocked(t.Context(), admitted, in); err == nil { t.Fatal("stale writer permitted restore") } } type servingBaselineInputs struct{ recoveryInputs } func (s servingBaselineInputs) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) (ClaimLaunchInputs, error) { in, err := s.recoveryInputs.Select(ctx, rt, o, policy) if err != nil { return in, err } in.DBPath = provision.NestedDBPath(s.f.guard.DataDir, o.Host) in.Empty = false in.Baseline, err = genselect.Select(ctx, s.f.data, genselect.Request{Bucket: s.f.bucket, Region: "ie", Host: o.Host, Holder: *o.Claim.Holder}) if err == nil { in.Admission.RemoteMaxTXID = in.Baseline.MaxTXID } return in, err } func TestServingRetirementOrdinaryPreparationThenSuccessor(t *testing.T) { f, r, p, in := servingRetirementFixture(t) bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") key := f.read().Claim.Holder.Location + "/0009/" + ltx.FormatFilename(1, 1) payload := mwSnapshot(t, 1, "ordinary serving baseline") version := bucket.Put("data", key, payload) f.data, f.bucket = bucket, "data" if err := os.WriteFile(in.DBPath, []byte("unmarked legacy"), 0600); err != nil { t.Fatal(err) } controller := f.controller("boot-1", "controller-1", 4101, "start-1") controller.wake.Inputs = servingBaselineInputs{recoveryInputs{f}} if err := bindServingArtifactRetirement(controller.wake, r); err != nil { t.Fatal(err) } var generations []string for i := 0; i < 2; i++ { admitted := controller.admit(t, f) generations = append(generations, admitted.Claim.Holder.Generation) start, err := controller.runtime.Prepare(t.Context(), registry.Route{Host: recoveryHost}, admitted, recoveryPolicy) if err != nil { t.Fatal(err) } if err = start.Restore(t.Context()); err != nil { t.Fatal(err) } if got := mwReadDB(t, in.DBPath); got != "ordinary serving baseline" { t.Fatal(got) } // Stop before Materialize: an unseeded ordinary C8 correctly refuses. // FailedAdmission drives the real serving stop and retry settlement. if err = controller.runtime.FailedAdmission(t.Context(), admitted, hostbox.ErrChanged); err != nil { t.Fatal(err) } f.now = f.now.Add(time.Minute) } if generations[0] == generations[1] { t.Fatal("successor reused generation") } if len(f.receipts()) != 2 { t.Fatal("missing exact serving stops") } if f.retry().AttemptsReserved != 2 { t.Fatal("retirement changed retry accounting") } remote, err := bucket.GetVersion(t.Context(), "data", key, version) if err != nil || !bytes.Equal(remote, payload) { t.Fatal("baseline changed") } if len(p.records) != 2 { t.Fatal("retirement did not park both sources") } } func TestServingRetirementContinuationKeepsOriginalIdentity(t *testing.T) { f, r, p, in := servingRetirementFixture(t) original := servingAdmit(t, f, &in, "G") p.failComplete = true if err := r.BeforeLegacyInstall(t.Context(), original, in, servingRetirementWork{p, r, original}); err == nil { t.Fatal("incomplete park accepted") } var old servingParkAuthorization for _, a := range p.records { old = a } servingStop(t, f, f.read()) successor := servingAdmit(t, f, &in, "G2") p.failComplete = false p.continuePending = true if err := r.Recover(t.Context(), successor, in); err != nil { t.Fatal(err) } a := r.completed[successor.Host] if a.ID != old.ID || a.OriginalSuccessor != old.OriginalSuccessor || a.Successor.Generation != "G2" || len(p.records) != 1 { t.Fatal("continuation relabeled immutable retirement") } if err := r.EnrollLocked(t.Context(), successor, in); err != nil { t.Fatal(err) } if err := r.EnrollLocked(t.Context(), original, in); err == nil { t.Fatal("old admission reused continuation") } } func TestServingRetirementAuthorizationBinding(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil { t.Fatal(err) } a := r.completed[admitted.Host] fresh := f.read() current, err := servingWriterIdentity(fresh, r.Box, r.Region) if err != nil { t.Fatal(err) } for _, field := range []string{"Host", "Box", "Region", "Incarnation", "Location", "Generation", "Stamp", "Session"} { for _, target := range []string{"Successor", "OriginalSuccessor"} { t.Run(target+"/"+field, func(t *testing.T) { changed := a v := reflect.ValueOf(&changed).Elem().FieldByName(target).FieldByName(field) if v.Kind() == reflect.String { v.SetString(v.String() + "wrong") } else { v.SetUint(v.Uint() + 1) } // Even a recomputed journal ID cannot bind a foreign original tuple. changed.ID = servingRetirementID(changed.Predecessor, changed.OriginalSuccessor, changed.Legacy) if err := r.validateAuthorization(t.Context(), fresh, current, changed); err == nil { t.Fatal("foreign authorization accepted") } }) } } for _, name := range []string{"id", "mount", "inode", "device", "future-tags"} { t.Run(name, func(t *testing.T) { changed := a switch name { case "id": changed.ID = "wrong" case "mount": changed.Directory.Mount = 0 case "inode": changed.Directory.Inode = 0 case "device": changed.Directory.Device = 0 case "future-tags": changed.TagsFrom = current.Stamp + 1 } if err := r.validateAuthorization(t.Context(), fresh, current, changed); err == nil { t.Fatal("invalid authorization accepted") } }) } } func TestServingRetirementFailedReplayInvalidatesPermission(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil { t.Fatal(err) } if err := r.EnrollLocked(t.Context(), admitted, in); err != nil { t.Fatal(err) } p.failComplete = true if err := r.Recover(t.Context(), admitted, in); err == nil { t.Fatal("refused completion accepted") } if err := r.EnrollLocked(t.Context(), admitted, in); err == nil { t.Fatal("old completion survived failed replay") } } // Root and the hardened edge see the same inode through different mounts. type servingNamespaceParker struct { servingDatabaseParker change func(*servingParkAuthorization) } func (p servingNamespaceParker) Park(ctx context.Context, current, prior servingArtifactIdentity) (servingParkAuthorization, error) { a, err := p.servingDatabaseParker.Park(ctx, current, prior) p.change(&a) return a, err } func (p servingNamespaceParker) ParkLegacy(ctx context.Context, current servingArtifactIdentity, prior servingLegacyPredecessor) (servingParkAuthorization, error) { a, err := p.servingDatabaseParker.ParkLegacy(ctx, current, prior) p.change(&a) return a, err } func TestServingEnrollmentMountNamespaces(t *testing.T) { for _, kind := range []string{"namespace", "device", "inode", "replacement"} { t.Run(kind, func(t *testing.T) { f, r, p, in := servingRetirementFixture(t) admitted := servingAdmit(t, f, &in, "G") r.Root = servingNamespaceParker{p, func(a *servingParkAuthorization) { a.Directory.Mount++ switch kind { case "device": a.Directory.Device++ case "inode": a.Directory.Inode++ case "replacement": dir := filepath.Dir(in.DBPath) if err := os.Rename(dir, dir+".old"); err != nil { t.Fatal(err) } if err := os.Mkdir(dir, 0700); err != nil { t.Fatal(err) } } }} err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}) if kind == "namespace" { if err != nil { t.Fatal(err) } if err := r.EnrollLocked(t.Context(), admitted, in); err != nil { t.Fatal(err) } if err := r.Recover(t.Context(), admitted, in); err != nil { t.Fatal("namespace recovery", err) } } else if err == nil { t.Fatal("changed source accepted") } }) } }