//go:build linux package host import ( "context" "errors" "net" "os" "path/filepath" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "golang.org/x/sys/unix" ) func servingRebootPreparationFixture(t *testing.T, phase string) (*servingRootFixture, servingParkRecord) { t.Helper() return servingRebootPreparationFixtureFor(t, phase, servingFixtureActor) } func servingRebootPreparationFixtureFor(t *testing.T, phase string, actor hostbox.Actor) (*servingRootFixture, servingParkRecord) { t.Helper() f := newServingRootFixtureFor(t, true, actor) fs := f.p.ledger.fs.(*servingParkFS) fs.volume = func(uint64) string { return "vol0123456789abcdef0" } if phase == "rename-reply" { fs.rename = func(a int, b string, c int, d string, flags uint) error { if err := unix.Renameat2(a, b, c, d, flags); err != nil { return err } return unix.EIO } } a, err := f.park() if phase == "rename-reply" { if !errors.Is(err, unix.EIO) { t.Fatal(err) } a.ID = servingRetirementID(f.prior, f.current, false) } else if err != nil { t.Fatal(err) } fs.rename = unix.Renameat2 if phase == "retagged" || phase == "complete" || phase == "source-gone" { f.tags(f.current.Stamp) } if phase == "complete" || phase == "source-gone" { if err := f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { t.Fatal(err) } } if phase == "source-gone" { if err := os.Rename(filepath.Dir(f.db), filepath.Dir(f.db)+"-gone"); err != nil { t.Fatal(err) } } r, err := f.p.ledger.read(a.ID) if err != nil { t.Fatal(err) } fs.boot = func() (string, error) { return servingOtherBoot, nil } f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { panic("reboot preparation authenticated a peer") } return f, r } func TestServingParkRebootPreparationPreservesHistory(t *testing.T) { for _, phase := range []string{"rename-reply", "parked", "retagged", "complete", "source-gone"} { t.Run(phase, func(t *testing.T) { f, r := servingRebootPreparationFixture(t, phase) before := servingBootSnapshot(t, f.disk.dir) claim := f.f.read() root := servingBootSnapshot(t, f.rootDisk.dir) name := servingRebootPreparationName(r.Authorization.ID, servingOtherBoot) f.trace = nil if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { t.Fatal(err) } body, err := f.disk.Read(name) var prep servingParkRebootPreparation if err != nil || servingParkDecode(body, &prep) != nil || prep.Version != 1 || !reflect.DeepEqual(prep.Observation.Record, r) || (prep.Active == nil) != r.Complete { t.Fatal("preparation", err) } after := servingBootSnapshot(t, f.disk.dir) delete(after, name) if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(claim, f.f.read()) || !reflect.DeepEqual(root, servingBootSnapshot(t, f.rootDisk.dir)) { t.Fatal("preparation changed authority or history") } if prep.Active != nil && (prep.Active.ClaimETag != claim.ETag || prep.Active.Directory != prep.Observation.Source) { t.Fatal("active installation not bound") } trace := strings.Join(f.trace, "\n") namespace := strings.Index(trace, "park:namespace.lock") slot := strings.Index(trace, "park:slot-") record := strings.Index(trace, "park:retirement-") if namespace < 0 || slot < namespace || record < slot { t.Fatal("lock order", trace) } if !r.Complete { host := strings.Index(trace, "host:") if host < namespace || host > slot { t.Fatal("host lock order", trace) } } f.disk.beforeWrite = func(string, []byte) { t.Fatal("unchanged preparation replay wrote") } restarted := *f.p if err := restarted.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { t.Fatal("replay", err) } f.disk.beforeWrite = nil fs := f.p.ledger.fs.(*servingParkFS) if err := fs.recoverRecordedNamespace(t.Context()); err == nil { t.Fatal("preparation granted namespace authority") } if err := fs.CheckParked(&r); err == nil { t.Fatal("preparation granted artifact authority") } if err := fs.Delete(&r, func() error { t.Fatal("old authority wrote deletion progress"); return nil }); err == nil { t.Fatal("preparation granted deletion") } }) } } func TestServingParkRebootPreparationRefusesAuthority(t *testing.T) { for _, fault := range []string{"enrollment", "binding", "claim", "tags", "receipt", "launch-intent", "census", "cancel", "slot-id", "slot-pending", "slot-last", "slot-foreign", "slot-missing", "record-missing"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootPreparationFixture(t, "parked") ctx := t.Context() var err error switch fault { case "enrollment": f.actor.Enrollment = "changed" case "binding": f.actor.Binding = "8" case "claim": _, err = hostclaim.Update(ctx, f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Holder.WriterBox = "other"; return nil }) case "tags": f.tags(f.current.Stamp + 1) case "receipt": f.mutateRoot(func(m map[string]any) { m["State"].(map[string]any)["fence_journal"] = []any{} }) case "census": f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } case "cancel": var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) cancel() case "launch-intent": target := f.receipt.Target target.Stamp, target.Session = f.current.Stamp, f.current.Session err = f.f.journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}) if err == nil { err = f.f.journal.IntendUnit(ctx, hostbox.LaunchReservation{ID: "new:main", Role: "main", InvocationID: "new", Inputs: "digest", Target: target, Group: hostbox.LaunchGroup{RelativePath: "system.slice/carlos-app@shop.service"}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}}) } case "record-missing": err = f.disk.Remove("retirement-" + r.Authorization.ID + ".json") default: slot, e := f.p.ledger.slot(r.Authorization) if e != nil { t.Fatal(e) } name := "slot-" + servingParkKey(slot.Box, slot.Host) + ".json" switch fault { case "slot-id": slot.ID = "" case "slot-pending": slot.Pending = "" case "slot-last": slot.Last = "" case "slot-foreign": slot.Host = "other" case "slot-missing": err = f.disk.Remove(name) } if fault != "slot-missing" { err = f.p.ledger.write(name, slot) } } if err != nil { t.Fatal(err) } before := servingBootSnapshot(t, f.disk.dir) if err := f.p.prepareServingReboot(ctx, r.Authorization.ID); err == nil { t.Fatal("accepted", fault) } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("refusal wrote preparation") } }) } } func TestServingParkRebootPreparationLegacyConsumption(t *testing.T) { for _, fault := range []string{"", "missing", "collected", "handle", "source-boot", "receipt"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootRecordFixture(t, "complete") key := servingLegacyKey("legacy", r.Authorization.Successor) c, err := f.ledger.legacyConsumption(r.Authorization.Successor) if err != nil || c == nil { t.Fatal(err) } switch fault { case "missing": err = f.disk.Remove(key) case "collected": c.Collected = true case "handle": c.Source.Persistent.Handle = "aa" case "source-boot": c.Source.BootID = servingOtherBoot case "receipt": c.Receipts = nil } if fault != "missing" { err = f.ledger.write(key, c) } if err != nil { t.Fatal(err) } before := servingBootSnapshot(t, f.disk.dir) p := servingParkRoot{ledger: f.ledger} err = p.prepareServingReboot(t.Context(), r.Authorization.ID) if (err != nil) != (fault != "") { t.Fatal(fault, err) } after := servingBootSnapshot(t, f.disk.dir) if fault == "" { delete(after, servingRebootPreparationName(r.Authorization.ID, servingOtherBoot)) } if !reflect.DeepEqual(before, after) { t.Fatal("changed permanent history") } }) } } func TestServingParkRebootPreparationLostWrite(t *testing.T) { for _, before := range []bool{true, false} { t.Run(map[bool]string{true: "before", false: "after"}[before], func(t *testing.T) { f, r := servingRebootPreparationFixture(t, "parked") original := servingBootSnapshot(t, f.disk.dir) store := &servingParkFaultStore{Store: f.disk, failAt: 1, before: before} f.p.ledger.state = store if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); !errors.Is(err, os.ErrPermission) { t.Fatal("write failure", err) } f.p.ledger.state = f.disk if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { t.Fatal("retry", err) } after := servingBootSnapshot(t, f.disk.dir) delete(after, servingRebootPreparationName(r.Authorization.ID, servingOtherBoot)) if !reflect.DeepEqual(original, after) { t.Fatal("write retry changed history") } }) } } func TestServingParkRebootPreparationNeverRefreshesStaleEvidence(t *testing.T) { for _, fault := range []string{"slot", "record", "claim", "boot", "body", "unknown-field"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootPreparationFixture(t, "parked") id := r.Authorization.ID name := servingRebootPreparationName(id, servingOtherBoot) if err := f.p.prepareServingReboot(t.Context(), id); err != nil { t.Fatal(err) } switch fault { case "slot": s, err := f.p.ledger.slot(r.Authorization) if err != nil { t.Fatal(err) } s.ID = "" if err := f.p.ledger.write("slot-"+servingParkKey(s.Box, s.Host)+".json", s); err != nil { t.Fatal(err) } case "record": r.Cause = "fixture-change" if err := f.p.ledger.save(r); err != nil { t.Fatal(err) } case "claim": f.tags(f.current.Stamp) case "boot": body, e := f.disk.Read(name) if e != nil { t.Fatal(e) } var prep servingParkRebootPreparation if servingParkDecode(body, &prep) != nil { t.Fatal("decode") } prep.Observation.Topology.BootID = "11234567-89ab-4cde-8fab-0123456789ab" if e = f.p.ledger.write(name, prep); e != nil { t.Fatal(e) } case "body": if err := f.disk.Write(name, []byte("{}")); err != nil { t.Fatal(err) } case "unknown-field": body, err := f.disk.Read(name) if err != nil { t.Fatal(err) } body = append([]byte(`{"Unknown":true,`), body[1:]...) if err := f.disk.Write(name, body); err != nil { t.Fatal(err) } } before := servingBootSnapshot(t, f.disk.dir) if err := f.p.prepareServingReboot(t.Context(), id); err == nil { t.Fatal("stale preparation refreshed") } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("stale refusal wrote") } }) } } func TestServingParkRebootPreparationDetectsWriteRace(t *testing.T) { for _, fault := range []string{"file", "slot", "cancel", "claim", "actor", "census"} { t.Run(fault, func(t *testing.T) { f, r := servingRebootPreparationFixture(t, "parked") ctx, cancel := context.WithCancel(t.Context()) defer cancel() f.disk.afterWrite = func(name string, _ []byte) { if !strings.HasPrefix(name, "reboot-") { return } switch fault { case "cancel": cancel() case "claim": f.tags(f.current.Stamp) case "actor": f.actor.Binding = "8" case "census": f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } case "slot": s, err := f.p.ledger.slot(r.Authorization) if err != nil { t.Fatal(err) } s.ID = "" if err := f.p.ledger.write("slot-"+servingParkKey(s.Box, s.Host)+".json", s); err != nil { t.Fatal(err) } case "file": path := filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db") if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { t.Fatal(err) } } } if err := f.p.prepareServingReboot(ctx, r.Authorization.ID); err == nil { t.Fatal("accepted write race") } f.disk.afterWrite = nil before := servingBootSnapshot(t, f.disk.dir) if err := f.p.ledger.fs.(*servingParkFS).recoverRecordedNamespace(t.Context()); err == nil { t.Fatal("failed preparation granted recovery") } if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { t.Fatal("failed preparation changed authority") } }) } } func TestServingParkRebootPreparationLegacyContinuation(t *testing.T) { for _, continued := range []bool{false, true} { t.Run(map[bool]string{false: "original", true: "continued"}[continued], func(t *testing.T) { f, _ := newServingLegacyRootFixture(t, false) original := f.current a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) if err != nil { t.Fatal(err) } if continued { servingStop(t, f.f, f.f.read()) o := f.f.read() later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) if err != nil { t.Fatal(err) } f.current, err = servingWriterIdentity(later, "box-1", "ie") if err != nil { t.Fatal(err) } f.admitted = later f.install(later) if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err != nil { t.Fatal(err) } } fs := f.p.ledger.fs.(*servingParkFS) fs.boot = func() (string, error) { return servingOtherBoot, nil } before := servingBootSnapshot(t, f.disk.dir) if err := f.p.prepareServingReboot(t.Context(), a.ID); err != nil { t.Fatal(err) } name := servingRebootPreparationName(a.ID, servingOtherBoot) body, err := f.disk.Read(name) var prep servingParkRebootPreparation if err != nil || servingParkDecode(body, &prep) != nil || prep.Active == nil || prep.Legacy == nil || prep.Observation.Record.Authorization.OriginalSuccessor != original || prep.Observation.Record.activeAuthorization().Successor != f.current { t.Fatal("lost original or current authority", err) } after := servingBootSnapshot(t, f.disk.dir) delete(after, name) if !reflect.DeepEqual(before, after) { t.Fatal("reboot preparation rewrote legacy or continuation") } }) } }