//go:build linux package host import ( "errors" "fmt" "os" "reflect" "testing" "github.com/carlosframework/platform/internal/hostbox" ) // Put the historical records on a different boot-local device number without // changing any persistent proof or anything in the actual filesystem. func servingRenumberHistory(t *testing.T, f *servingParkDiskFixture, r *servingParkRecord) { t.Helper() old := r.Source.Parent.Device + 123 for _, d := range []*servingArtifactDirectory{&r.Source.Parent, &r.Source.Directory, &r.Authorization.Directory, &r.Namespace, &r.Event} { if d.Inode != 0 { d.Device = old } } for i := range r.Source.Files { if r.Source.Files[i].Present { r.Source.Files[i].Device = old } } if err := f.ledger.save(*r); err != nil { t.Fatal(err) } p, err := f.fs.historicalNamespaceParent() if err != nil { t.Fatal(err) } p.Parent.Device = old if err := f.ledger.write("namespace-parent.json", p); err != nil { t.Fatal(err) } var n servingParkNamespace body, err := f.disk.Read("namespace.json") if err != nil || servingParkDecode(body, &n) != nil { t.Fatal(err) } n.Parent.Device, n.Root.Device = old, old if err := f.ledger.write("namespace.json", n); err != nil { t.Fatal(err) } if r.Authorization.Legacy { c, err := f.ledger.legacyConsumption(r.Authorization.Successor) if err != nil { t.Fatal(err) } if c == nil { return } c.Source, c.Authorization = r.Source, r.Authorization if err := f.ledger.write(servingLegacyKey("legacy", r.Authorization.Successor), c); err != nil { t.Fatal(err) } } } func TestServingParkRebootDeviceRenumber(t *testing.T) { for _, phase := range []string{"captured", "prepared", "rename-reply", "parked", "complete", "unlink-reply", "partial-unlink", "rmdir-reply", "closed"} { t.Run(phase, func(t *testing.T) { f, r := servingRebootRecordFixture(t, phase) servingRenumberHistory(t, f, &r) before := servingBootSnapshot(t, f.disk.dir) got, err := f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID) if err != nil || got.Topology.CurrentParent.Device == r.Source.Parent.Device || !reflect.DeepEqual(got.Record, r) { t.Fatal("renumbered-device inspection", got, err) } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("inspection rewrote history") } if err := f.fs.CheckParked(&r); err == nil { t.Fatal("inspection granted old-boot authority") } if phase == "captured" || phase == "prepared" || phase == "rename-reply" { return } var active *servingParkRebootAuthority if !r.Complete { a := r.activeAuthorization() active = &servingParkRebootAuthority{ClaimETag: "fixture", Actor: hostbox.Actor{Box: a.Successor.Box, Region: a.Successor.Region, Enrollment: "fixture", Revision: 1, Binding: "1", Eligible: true}, Directory: got.Source, Tags: a.TagsFrom} } if err := f.ledger.prepareRebootLocked(t.Context(), r, active); err != nil { t.Fatal("prepare", err) } if err := f.ledger.commitRebootLocked(t.Context(), r, active); err != nil { t.Fatal("commit", err) } current, err := f.ledger.read(r.Authorization.ID) if err != nil || current.Source.Parent.Device != got.Topology.CurrentParent.Device { t.Fatal("current device", current, err) } if err := f.ledger.save(current); err != nil { t.Fatal("reverse binding", err) } raw, err := f.ledger.readHistorical(r.Authorization.ID) if err != nil || !reflect.DeepEqual(raw, r) { t.Fatal("round trip changed history", raw, err) } if r.Parked && !r.Deleting { if err := f.fs.CheckParked(¤t); !r.Complete && err != nil { t.Fatal("bound park", err) } if r.Complete { f.now = current.ReapAfter if err := f.ledger.reapLocked(t.Context(), r.Authorization.ID); err != nil { t.Fatal("reap", err) } if _, err := os.Stat(f.fs.data + "/instances/.serving-parked/" + r.Authorization.ID); !os.IsNotExist(err) { t.Fatal("event survived reap", err) } } } }) } } func TestServingParkRebootDeviceRenumberRefusesReplacement(t *testing.T) { for _, fault := range []string{"filesystem", "root-handle", "source-handle", "event-handle", "file-handle", "inode", "same-boot"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootRecordFixture(t, "parked") servingRenumberHistory(t, f, &r) var n servingParkNamespace body, err := f.disk.Read("namespace.json") if err != nil || servingParkDecode(body, &n) != nil { t.Fatal(err) } switch fault { case "filesystem": n.Topology.Parent.Filesystem[0]++ n.Topology.Root.Filesystem = n.Topology.Parent.Filesystem case "root-handle": n.Topology.Root.Handle = "aa" case "source-handle": r.Source.Persistent.Handle = "aa" case "event-handle": r.EventPersistent.Handle = "aa" case "file-handle": r.Source.Files[0].Persistent.Handle = "aa" case "inode": r.Source.Directory.Inode++ r.Authorization.Directory = r.Source.Directory case "same-boot": f.fs.boot = func() (string, error) { return r.Source.BootID, nil } } if err := f.ledger.save(r); err != nil { t.Fatal(err) } if err := f.ledger.write("namespace.json", n); err != nil { t.Fatal(err) } before := servingBootSnapshot(t, f.disk.dir) if _, err := f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID); err == nil { t.Fatal("renumbering accepted replaced identity") } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("refusal changed authority") } }) } } func TestServingParkRebootDeviceRenumberLostWrite(t *testing.T) { for _, before := range []bool{true, false} { for at := 1; at <= 3; at++ { f, r := servingRebootRecordFixture(t, "complete") servingRenumberHistory(t, f, &r) fault := &servingParkFaultStore{Store: f.disk, failAt: at, before: before} f.ledger.state, f.fs.state = fault, fault root := &servingParkRoot{ledger: f.ledger} err := root.commitServingReboot(t.Context(), r.Authorization.ID) if !errors.Is(err, os.ErrPermission) { t.Fatalf("%v/%d: missing write fault: %v", before, at, err) } f.ledger.state, f.fs.state = f.disk, f.disk if err := root.commitServingReboot(t.Context(), r.Authorization.ID); err != nil { t.Fatal("commit replay", err) } if err := f.fs.recoverRecordedNamespace(t.Context()); err != nil { t.Fatal("namespace recovery", err) } current, err := f.ledger.read(r.Authorization.ID) if err != nil { t.Fatal(err) } f.now = current.ReapAfter if err := f.ledger.reapLocked(t.Context(), r.Authorization.ID); err != nil { t.Fatal("reap after lost reply", err) } raw, err := f.ledger.readHistorical(r.Authorization.ID) if err != nil || raw.Source.BootID != r.Source.BootID || raw.Source.Parent != r.Source.Parent || !raw.Closed { t.Fatal("history after recovery", raw, err) } } } } func TestServingParkRetirementAfterDeviceRenumber(t *testing.T) { for _, renumberAgain := range []bool{false, true} { for _, replacement := range []bool{false, true} { t.Run(fmt.Sprint(renumberAgain, "/replacement-", replacement), func(t *testing.T) { f, old := servingRebootRecordFixture(t, "complete") servingRenumberHistory(t, f, &old) // A/D1 -> B/D2. root := &servingParkRoot{ledger: f.ledger} if err := root.commitServingReboot(t.Context(), old.Authorization.ID); err != nil { t.Fatal(err) } f.put("", "boot B served database") f.put(".gen", "G2") a, stops := f.evidence(2, false, "started") park, err := f.park(a, stops) if err != nil { t.Fatal("B retirement and replacement", err) } f.complete(a) park, err = f.ledger.readHistorical(a.ID) if err != nil { t.Fatal(err) } if renumberAgain { // C's real device is D3; B's immutable record used D2. b := servingParkBootBinding{Version: 1, ID: "retirement-" + a.ID, BeforeBoot: park.Source.BootID, BootID: park.Source.BootID, BeforeParent: park.Source.Parent, Parent: park.Source.Parent} b.Parent.Device += 456 if err := b.record(&park, false); err != nil { t.Fatal(err) } if err := f.ledger.save(park); err != nil { t.Fatal(err) } } f.fs.boot = func() (string, error) { return "33333333-3333-4333-8333-333333333333", nil } if err := root.commitServingReboot(t.Context(), a.ID); err != nil { t.Fatal("C recovery", err) } if replacement { f.put("", "boot C served database") f.put(".gen", "G3") next, stops := f.evidence(3, false, "started") if _, err := f.park(next, stops); err != nil { t.Fatal("C replacement", err) } f.complete(next) nextPark, err := f.ledger.read(next.ID) if err != nil { t.Fatal(err) } f.now = nextPark.ReapAfter if err := f.ledger.reapLocked(t.Context(), next.ID); err != nil { t.Fatal("C replacement reaping", err) } } f.now = park.ReapAfter if err := f.ledger.reapLocked(t.Context(), a.ID); err != nil { t.Fatal("C reap", err) } if _, err := os.Stat(f.path(a, "")); !os.IsNotExist(err) { t.Fatal("park survived", err) } }) } } } func TestServingParkLaterRetirementRequiresParentProof(t *testing.T) { for _, fault := range []string{"missing", "handle", "filesystem"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootRecordFixture(t, "complete") servingRenumberHistory(t, f, &r) root := &servingParkRoot{ledger: f.ledger} if err := root.commitServingReboot(t.Context(), r.Authorization.ID); err != nil { t.Fatal(err) } f.put("", "later served database") f.put(".gen", "G2") a, stops := f.evidence(2, false, "started") if _, err := f.park(a, stops); err != nil { t.Fatal(err) } f.complete(a) later, err := f.ledger.readHistorical(a.ID) if err != nil { t.Fatal(err) } switch fault { case "missing": later.Source.ParentPersistent = servingParkPersistentDirectory{} case "handle": later.Source.ParentPersistent.Handle = "aa" case "filesystem": later.Source.ParentPersistent.Filesystem[0]++ } if err := f.ledger.write("retirement-"+a.ID+".json", later); err != nil { t.Fatal(err) } f.fs.boot = func() (string, error) { return "33333333-3333-4333-8333-333333333333", nil } before := servingBootSnapshot(t, f.disk.dir) if err := root.commitServingReboot(t.Context(), a.ID); err == nil { t.Fatal("unbound historical parent accepted") } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("refusal rewrote authority") } f.retained(a, "later served database", later.ReapAfter) }) } }