//go:build linux package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "net" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "golang.org/x/sys/unix" ) // Disk persistence and lock exclusion are real. Root ownership, mount policy, // roster and authenticated process observations remain explicit test seams. type servingAuthorityDisk struct { *rollbackDisk mu sync.Mutex locks map[string]chan struct{} trace *[]string prefix string } func (s *servingAuthorityDisk) Lock(ctx context.Context, name string) (func(), error) { s.mu.Lock() if s.locks == nil { s.locks = make(map[string]chan struct{}) } lock := s.locks[name] if lock == nil { lock = make(chan struct{}, 1) s.locks[name] = lock } s.mu.Unlock() select { case lock <- struct{}{}: if s.trace != nil { *s.trace = append(*s.trace, s.prefix+name) } return func() { <-lock }, nil case <-ctx.Done(): return nil, ctx.Err() } } type servingRootFixture struct { t *testing.T f *recoveryFixture p *servingParkRoot disk *servingAuthorityDisk rootDisk *servingAuthorityDisk conn net.Conn actor hostbox.Actor current, prior servingArtifactIdentity admitted hostclaim.Observed predecessor hostclaim.Observed receipt hostguard.FenceReceipt db string trace []string } // servingFixtureActor is the serving fixtures' root actor. Its binding is the // fixture shorthand "7"; newServingRootFixtureFor takes a production actor. var servingFixtureActor = hostbox.Actor{Box: "box-1", Region: "ie", Enrollment: "enrollment", Revision: 7, Binding: "7", Eligible: true} func newServingRootFixture(t *testing.T, launched bool) *servingRootFixture { t.Helper() return newServingRootFixtureFor(t, launched, servingFixtureActor) } // newServingRootFixtureFor stamps the installation, its stop receipt and the // root's inventory actor with actor (box-1 in ie). func newServingRootFixtureFor(t *testing.T, launched bool, actor hostbox.Actor) *servingRootFixture { t.Helper() f, _, _, in := servingRetirementFixture(t) first := servingAdmit(t, f, &in, "G") disk := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}, prefix: "park:"} roots := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}, prefix: "host:"} f.guard.Root = hostguard.NewRootAuthority(roots) journals, err := hostbox.NewJournalService(f.journal) if err != nil { t.Fatal(err) } fx := &servingRootFixture{t: t, f: f, disk: disk, rootDisk: roots, db: in.DBPath, actor: actor} fx.prior, err = servingWriterIdentity(first, "box-1", "ie") if err != nil { t.Fatal(err) } fs := newServingParkFS(f.guard.DataDir, disk) fs.owner = uint32(os.Geteuid()) fs.mount = func(string) (bool, error) { return true, nil } fx.p = &servingParkRoot{ledger: &servingParkLedger{state: disk, fs: fs, journal: f.journal, now: func() time.Time { return f.now }}} fx.p.root = ClaimRootHTTP{Roots: map[string]string{"data": f.guard.DataDir}, Journals: journals, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: f.guard, Evidence: f.claims}, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{UID: 1001, PID: 77, ProcessStart: "edge-start", BootID: "boot", ControllerID: "controller"}, nil }}, InventoryActor: func(context.Context) (hostbox.Actor, error) { return fx.actor, nil }, RequireNewWork: func(context.Context) error { return nil }} fx.install(first) fx.tags(fx.prior.Stamp) boot, err := fs.currentBoot() if err != nil { t.Fatal(err) } fx.receipt = servingStopFor(t, f, f.read(), fx.actor, boot) target := fx.receipt.Target if err := f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}); err != nil { t.Fatal(err) } if launched { res := hostbox.LaunchReservation{ID: "attempt:main", Role: "main", InvocationID: "attempt", Inputs: "digest", Target: target, Group: hostbox.LaunchGroup{RelativePath: "system.slice/carlos-app@shop.service"}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}} if err := f.journal.IntendUnit(t.Context(), res); err != nil { t.Fatal(err) } if err := f.journal.StartedUnit(t.Context(), res, hostbox.LaunchGroup{RelativePath: res.Group.RelativePath, Device: 1, Inode: 4, InvocationID: "unit"}, 99, "tenant-start"); err != nil { t.Fatal(err) } if err := f.journal.SettleUnit(t.Context(), target, res.ID); err != nil { t.Fatal(err) } } o := f.read() fx.predecessor = o fx.admitted, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) if err != nil { t.Fatal(err) } fx.current, err = servingWriterIdentity(fx.admitted, "box-1", "ie") if err != nil { t.Fatal(err) } for suffix, body := range map[string]string{"": "served bytes", ".gen": "G", "-wal": "WAL", "-shm": "SHM"} { if err := os.WriteFile(fx.db+suffix, []byte(body), 0600); err != nil { t.Fatal(err) } } left, right := net.Pipe() fx.conn = left t.Cleanup(func() { left.Close(); right.Close() }) disk.trace, roots.trace = &fx.trace, &fx.trace return fx } func (f *servingRootFixture) install(o hostclaim.Observed) { f.t.Helper() h := o.Claim.Holder d, err := os.Open(filepath.Dir(f.db)) if err != nil { f.t.Fatal(err) } id, err := servingParkDirID(d) d.Close() if err != nil { f.t.Fatal(err) } owner := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%d\x00%s", o.Host, h.Incarnation, h.Stamp, f.actor.Box))) in := hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: f.actor.Box, Enrollment: f.actor.Enrollment, DirectoryRevision: f.actor.Binding, Entries: []hostbox.StopInventoryEntry{{Kind: "path", ManagedRoot: "data", RelativePath: "instances/" + provision.InstanceID(o.Host), ArtifactID: "state", Ownership: hex.EncodeToString(owner[:]), Materialization: "materialized", Device: id.Device, Inode: id.Inode, Type: unix.S_IFDIR}}} inventory := f.p.root.Journals.Inventory if err := inventory.WriteStopInventory(f.t.Context(), in); err != nil { f.t.Fatal(err) } _, revision, err := inventory.Snapshot(f.t.Context(), o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) if err != nil { f.t.Fatal(err) } root := hostguard.RootInstallation{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Migration: 1, InventoryRevision: revision, Account: h.Account, App: h.App, Record: h.Key, Box: f.actor.Box, Enrollment: f.actor.Enrollment} if err := f.f.guard.EstablishRoot(f.t.Context(), root, func(*hostclaim.Claim) error { return nil }); err != nil { f.t.Fatal(err) } } func (f *servingRootFixture) tags(stamp uint64) { f.t.Helper() _, err := hostclaim.Update(f.t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { rel, _ := filepath.Rel(f.f.guard.DataDir, f.db) c.Inventory = map[string]hostclaim.InventoryEntry{f.actor.Box: {Layout: "nested", Paths: map[string]uint64{rel: stamp, rel + ".gen": stamp}}} return nil }) if err != nil { f.t.Fatal(err) } } func (f *servingRootFixture) park() (servingParkAuthorization, error) { return f.p.parkServingDatabase(f.t.Context(), f.conn, f.current, f.prior) } func (f *servingRootFixture) mutateRoot(edit func(map[string]any)) { f.t.Helper() name := f.current.Host + ".json" b, err := f.rootDisk.Read(name) if err != nil { f.t.Fatal(err) } var m map[string]any if err = json.Unmarshal(b, &m); err != nil { f.t.Fatal(err) } edit(m) b, err = json.Marshal(m) if err != nil { f.t.Fatal(err) } if err = f.rootDisk.Write(name, b); err != nil { f.t.Fatal(err) } } func TestServingParkRootGenerationAuthority(t *testing.T) { for _, launched := range []bool{false, true} { t.Run(fmt.Sprint(launched), func(t *testing.T) { f := newServingRootFixture(t, launched) a, err := f.park() if err != nil { t.Fatal(err) } want := []string{"park:namespace.lock", "host:" + f.current.Host + ".lock", "park:slot-" + servingParkKey(f.current.Box, f.current.Host) + ".lock", "park:retirement-" + a.ID + ".lock"} if !reflect.DeepEqual(f.trace, want) { t.Fatalf("lock order: %v", f.trace) } if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err == nil { t.Fatal("completed before retag") } f.tags(f.current.Stamp) if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { t.Fatal(err) } r, err := f.p.ledger.read(a.ID) if err != nil || !r.Complete { t.Fatalf("record: %+v %v", r, err) } if (r.Disposition == servingParkRetain) != launched { t.Fatalf("wrong disposition: %v", r.Disposition) } again, err := f.p.parkServingDatabase(t.Context(), f.conn, f.current, servingArtifactIdentity{}) if err != nil || again != a { t.Fatalf("lost completion reply: %+v %v", again, err) } if err = os.WriteFile(f.db, []byte("replacement"), 0600); err != nil { t.Fatal(err) } if _, err = f.p.parkServingDatabase(t.Context(), f.conn, f.current, servingArtifactIdentity{}); err == nil { t.Fatal("accepted replacement source") } }) } } func TestServingParkRootReplaysOriginalSelectorAfterRetag(t *testing.T) { for _, complete := range []bool{false, true} { t.Run(fmt.Sprintf("complete=%v", complete), func(t *testing.T) { f := newServingRootFixture(t, true) a, err := f.park() if err != nil { t.Fatal(err) } f.tags(f.current.Stamp) if complete { if err := f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { t.Fatal(err) } } before, err := f.p.ledger.read(a.ID) if err != nil { t.Fatal(err) } f.f.now = f.f.now.Add(time.Hour) again, err := f.park() if err != nil || again != a { t.Fatalf("original selector after retag: %+v %v", again, err) } for _, field := range []string{"generation", "stamp", "session", "box", "incarnation"} { bad := f.prior switch field { case "generation": bad.Generation = "foreign" case "stamp": bad.Stamp = f.current.Stamp case "session": bad.Session = f.current.Session case "box": bad.Box = "foreign" case "incarnation": bad.Incarnation = "foreign" } if _, err := f.p.parkServingDatabase(t.Context(), f.conn, f.current, bad); err == nil { t.Fatalf("accepted changed %s", field) } } after, err := f.p.ledger.read(a.ID) if err != nil || !reflect.DeepEqual(before, after) { t.Fatalf("replay changed retirement: %+v %v", after, err) } if err := f.p.ledger.fs.CheckParked(&after); err != nil { t.Fatal(err) } }) } } func TestServingParkRootRefusesBeforeEffects(t *testing.T) { for _, attack := range []string{"peer", "peer-root", "census", "roster", "quarantine", "enrollment", "binding", "claim", "prior", "tags", "half-tags", "source", "inventory-revision", "receipt", "pending", "root-quarantine", "root-pending", "barrier", "canceled"} { t.Run(attack, func(t *testing.T) { f := newServingRootFixture(t, true) ctx := t.Context() switch attack { case "peer": f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, ErrClaimStopIdentity } case "peer-root": f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{PID: 77, ProcessStart: "start", BootID: "boot", ControllerID: "controller"}, nil } case "census": f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } case "roster": f.actor.Eligible = false case "quarantine": f.actor.Quarantined = true case "enrollment": f.actor.Enrollment = "other" case "binding": f.actor.Binding = "8" case "claim": f.current.Generation = "foreign" case "prior": f.prior.Generation = "foreign" case "tags": f.tags(f.current.Stamp) case "half-tags": _, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { rel, _ := filepath.Rel(f.f.guard.DataDir, f.db) delete(c.Inventory[f.actor.Box].Paths, rel+".gen") return nil }) if err != nil { t.Fatal(err) } case "source": if err := os.Rename(filepath.Dir(f.db), filepath.Dir(f.db)+".saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Dir(f.db), 0700); err != nil { t.Fatal(err) } case "inventory-revision": in, _, err := f.p.root.Journals.Inventory.Snapshot(t.Context(), f.prior.Host, hostguard.Captured{Incarnation: f.prior.Incarnation, Stamp: f.prior.Stamp}) if err != nil { t.Fatal(err) } if err = f.p.root.Journals.Inventory.WriteStopInventory(t.Context(), in); err != nil { t.Fatal(err) } case "receipt": f.mutateRoot(func(m map[string]any) { s := m["State"].(map[string]any) s["fence_journal"].([]any)[0].(map[string]any)["bytes"] = "Zm9yZ2Vk" }) case "pending": f.mutateRoot(func(m map[string]any) { m["State"].(map[string]any)["fence_journal"] = []any{} }) case "root-quarantine": f.mutateRoot(func(m map[string]any) { m["Quarantined"] = true }) case "root-pending": f.mutateRoot(func(m map[string]any) { m["PendingInstallation"] = m["Installation"] m["PendingInventory"] = "cGVuZGluZw==" }) case "barrier": f.mutateRoot(func(m map[string]any) { m["Barrier"] = m["State"].(map[string]any)["active_fence"] }) case "canceled": var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) cancel() } if _, err := f.p.parkServingDatabase(ctx, f.conn, f.current, f.prior); err == nil { t.Fatal("unproved authority accepted") } names, err := f.disk.Names() if err != nil { t.Fatal(err) } if len(names) != 0 { t.Fatalf("refusal wrote private state: %v", names) } db := f.db if attack == "source" { db = filepath.Join(filepath.Dir(db)+".saved", filepath.Base(db)) } if b, err := os.ReadFile(db); err != nil || string(b) != "served bytes" { t.Fatalf("source changed: %q %v", b, err) } }) } } type servingRootClientFixture struct{ f *servingRootFixture } func (c servingRootClientFixture) Park(ctx context.Context, current, prior servingArtifactIdentity) (servingParkAuthorization, error) { return c.f.p.parkServingDatabase(ctx, c.f.conn, current, prior) } func (c servingRootClientFixture) ParkLegacy(ctx context.Context, current servingArtifactIdentity, prior servingLegacyPredecessor) (servingParkAuthorization, error) { return c.f.p.parkLegacyServingDatabase(ctx, c.f.conn, current, prior) } func (c servingRootClientFixture) Complete(ctx context.Context, id string, current servingArtifactIdentity) error { return c.f.p.completeServingDatabasePark(ctx, c.f.conn, id, current) } type servingRootWork struct { g hostguard.Guard o hostclaim.Observed } func (w servingRootWork) With(ctx context.Context, _ hostwrite.Identity, fn func() error) error { return w.g.With(ctx, w.o.Host, hostguard.Captured{Incarnation: w.o.Claim.Holder.Incarnation, Stamp: w.o.Claim.Holder.Stamp}, func(c *hostclaim.Claim) error { if !claimSameWriter(w.o, hostclaim.Observed{Host: w.o.Host, Claim: c}, nil) { return hostbox.ErrChanged } return nil }, fn) } func TestServingParkRootProductionRestoreSuccessor(t *testing.T) { f := newServingRootFixture(t, true) plan := plan12RestoreBaseline(t, f.f) f.install(f.admitted) r := &claimServingArtifactRetirement{Claims: f.f.claims, Guard: f.f.guard, DataDir: f.f.guard.DataDir, Box: f.actor.Box, Region: f.actor.Region, Root: servingRootClientFixture{f}} in := ClaimLaunchInputs{DBPath: f.db} work := servingRootWork{f.f.guard, f.admitted} restore := hostbox.ColdRestore{Reader: f.f.data, Bucket: f.f.bucket, Generation: f.current.Generation, Work: work, BeforeLegacyInstall: func(ctx context.Context, _, _ string) error { return r.BeforeLegacyInstall(ctx, f.admitted, in, work) }, BeforeInstallLocked: func(ctx context.Context, _, _ string) error { return r.EnrollLocked(ctx, f.admitted, in) }, RetireArtifacts: func(ctx context.Context, generation string) error { return r.Retire(ctx, f.admitted, in, generation) }} if err := r.Recover(t.Context(), f.admitted, in); err != nil { t.Fatal(err) } if err := restore.Install(t.Context(), f.db, plan); err != nil { t.Fatal(err) } if got := mwReadDB(t, f.db); got != "selected recovery baseline" { t.Fatal(got) } marker, err := hostbox.ReadGenerationMarker(f.db + ".gen") if err != nil || marker != "G2" { t.Fatalf("marker %q %v", marker, err) } names, err := f.disk.Names() if err != nil { t.Fatal(err) } for _, name := range names { if strings.HasPrefix(name, "retirement-") && strings.HasSuffix(name, ".json") { rec, err := f.p.ledger.read(strings.TrimSuffix(strings.TrimPrefix(name, "retirement-"), ".json")) if err != nil || !rec.Complete { t.Fatalf("incomplete: %v", err) } b, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", rec.Authorization.ID, filepath.Base(f.db))) if err != nil || string(b) != "served bytes" { t.Fatalf("parked predecessor: %q %v", b, err) } } } } func TestServingParkRootWaitReauthenticates(t *testing.T) { f := newServingRootFixture(t, true) unlock, err := f.disk.Lock(t.Context(), "namespace.lock") if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) defer cancel() if _, err = f.p.parkServingDatabase(ctx, f.conn, f.current, f.prior); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("lock bypass: %v", err) } unlock() f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, ErrClaimStopIdentity } if _, err = f.park(); !errors.Is(err, ErrClaimStopIdentity) { t.Fatalf("stale authentication: %v", err) } } func TestServingParkRootContinuesOriginalRetirement(t *testing.T) { for _, point := range []string{"rename", "parked", "retag", "lost-continuation"} { t.Run(point, func(t *testing.T) { f := newServingRootFixture(t, true) original := f.current fs := f.p.ledger.fs.(*servingParkFS) rename := fs.rename if point == "rename" { once := false fs.rename = func(a int, b string, c int, d string, e uint) error { err := rename(a, b, c, d, e) if err == nil && !once { once = true return errors.New("lost rename reply") } return err } } a, err := f.park() if point == "rename" { if err == nil { t.Fatal("fault missed") } fs.rename = rename a = servingParkAuthorization{ID: servingRetirementID(f.prior, original, false)} } else if err != nil { t.Fatal(err) } before, err := f.p.ledger.read(a.ID) if err != nil { t.Fatal(err) } if point == "retag" || point == "lost-continuation" { f.tags(original.Stamp) } f.install(f.f.read()) servingStop(t, f.f, f.f.read()) o := f.f.read() later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G3"})) if err != nil { t.Fatal(err) } f.current, err = servingWriterIdentity(later, "box-1", "ie") if err != nil { t.Fatal(err) } f.f.now = f.f.now.Add(2 * time.Hour) requested := f.prior if point == "retag" || point == "lost-continuation" { requested = original } if point == "lost-continuation" { disk := &servingParkFaultStore{Store: f.disk, failAt: 1} f.p.ledger.state = disk if _, err = f.p.parkServingDatabase(t.Context(), f.conn, f.current, requested); err == nil { t.Fatal("lost continuation reply missed") } f.p.ledger.state = f.disk } continued, err := f.p.parkServingDatabase(t.Context(), f.conn, f.current, requested) if err != nil { t.Fatal(err) } if continued.ID != a.ID || continued.OriginalSuccessor != original || continued.Successor != f.current || continued.Predecessor != f.prior { t.Fatalf("relabelled retirement: %+v", continued) } f.tags(f.current.Stamp) if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { t.Fatal(err) } after, err := f.p.ledger.read(a.ID) if err != nil { t.Fatal(err) } if after.Authorization != before.Authorization || len(after.Continuations) != 1 || !after.Complete { t.Fatalf("history: %+v", after) } if !before.ParkedAt.IsZero() && (!before.ParkedAt.Equal(after.ParkedAt) || !before.ReapAfter.Equal(after.ReapAfter)) { t.Fatal("continuation reset retention") } if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, original); err == nil { t.Fatal("old successor completed continuation") } again, err := f.p.parkServingDatabase(t.Context(), f.conn, f.current, servingArtifactIdentity{}) if err != nil || again != continued { t.Fatalf("replay: %+v %v", again, err) } again, err = f.p.parkServingDatabase(t.Context(), f.conn, f.current, requested) if err != nil || again != continued { t.Fatalf("original continuation selector: %+v %v", again, err) } }) } } func TestServingParkRootRechecksClaimAfterInputs(t *testing.T) { f := newServingRootFixture(t, true) original := f.p.root.Stop.Authenticate f.p.root.Stop.Authenticate = func(ctx context.Context, conn net.Conn) (StopPeer, error) { _, err := hostclaim.Update(ctx, f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Inventory = nil; return nil }) if err != nil { return StopPeer{}, err } return original(ctx, conn) } if _, err := f.park(); err == nil { t.Fatal("used claim captured before peer and inventory checks") } names, err := f.disk.Names() if err != nil || len(names) != 0 { t.Fatalf("effects: %v %v", names, err) } } func TestServingParkRootContinuationNeedsStop(t *testing.T) { f := newServingRootFixture(t, true) a, err := f.park() if err != nil { t.Fatal(err) } f.install(f.admitted) original := f.current o := f.f.read() next, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G3"})) if err != nil { t.Fatal(err) } f.current, err = servingWriterIdentity(next, "box-1", "ie") if err != nil { t.Fatal(err) } if _, err = f.park(); err == nil { t.Fatal("continued without original successor stop") } rec, err := f.p.ledger.read(a.ID) if err != nil || rec.activeAuthorization().Successor != original || len(rec.Continuations) != 0 { t.Fatalf("changed record: %+v %v", rec, err) } }