//go:build linux package host import ( "context" "errors" "os" "path/filepath" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func servingCollectionFixture(t *testing.T, legacy bool) (*servingParkDiskFixture, servingParkAuthorization, servingParkAuthorization) { t.Helper() f := newServingParkDiskFixture(t) f.put("", "old served bytes") if !legacy { f.put(".gen", "G10") } old, receipts := f.evidence(10, legacy, "started") if _, err := f.park(old, receipts); err != nil { t.Fatal(err) } f.complete(old) f.put("", "new served bytes") f.put(".gen", "G11") next, receipts := f.evidence(11, false, "started") if _, err := f.park(next, receipts); err != nil { t.Fatal(err) } f.complete(next) return f, old, next } func TestServingParkCollectionWindowAndReplay(t *testing.T) { f, old, next := servingCollectionFixture(t, false) if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, err := f.ledger.read(old.ID) if err != nil || !r.CollectAfter.Equal(f.now.Add(hostbox.JournalReceiptRetention)) { t.Fatalf("missing retention deadline: %+v %v", r, err) } before, _ := f.disk.Read("retirement-" + old.ID + ".json") for _, now := range []time.Time{f.now.Add(-time.Hour), r.CollectAfter.Add(-time.Nanosecond)} { f.now = now if err = f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } after, err := f.disk.Read("retirement-" + old.ID + ".json") if err != nil || string(before) != string(after) { t.Fatal("deadline refreshed or record collected early", err) } } // A restarted ledger uses only the persisted deadline. restarted := *f.ledger f.now = r.CollectAfter if err = f.disk.Write("retirement-"+old.ID+".lock", nil); err != nil { t.Fatal(err) } if err = restarted.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } for _, suffix := range []string{".json", ".lock"} { if _, err = f.disk.Read("retirement-" + old.ID + suffix); !errors.Is(err, os.ErrNotExist) { t.Fatal("collected file survived", suffix, err) } } f.retained(next, "new served bytes", r.ReapAfter) // Even a fresh transport nonce must not rebuild an old collected intent // from the current source. Root additionally checks the live claim. f.put("", "current database") f.put(".gen", "G10") if _, err = f.park(old, r.Receipts); !errors.Is(err, hostbox.ErrChanged) { t.Fatal("collected replay regained authority", err) } if _, err = f.ledger.read(old.ID); !errors.Is(err, os.ErrNotExist) { t.Fatal("replay recreated retirement", err) } b, err := os.ReadFile(provision.NestedDBPath(f.data, f.base.Host)) if err != nil || string(b) != "current database" { t.Fatal("refused replay changed live source", err) } } func TestServingParkCollectionPreservesSlotReferences(t *testing.T) { for _, reference := range []string{"ID", "Pending", "Last"} { t.Run(reference, func(t *testing.T) { f, old, _ := servingCollectionFixture(t, false) if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, _ := f.ledger.read(old.ID) f.now = r.CollectAfter.Add(time.Hour) slot, err := f.ledger.slot(old) if err != nil { t.Fatal(err) } switch reference { case "ID": slot.ID = old.ID case "Pending": slot.Pending = old.ID case "Last": slot.Last = old.ID } if err = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.collectClosedLocked(t.Context(), &r, slot) }); err != nil { t.Fatal(err) } if _, err = f.ledger.read(old.ID); err != nil { t.Fatal("collected a referenced retirement", err) } }) } } func TestServingParkCollectionKeepsLegacyConsumption(t *testing.T) { f, old, next := servingCollectionFixture(t, true) consumed, err := f.ledger.legacyConsumption(old.Successor) if err != nil || consumed == nil { t.Fatal(err) } if err = f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, _ := f.ledger.read(old.ID) f.now = r.CollectAfter if err = f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } after, err := f.ledger.legacyConsumption(old.Successor) if err != nil || after == nil || !after.Collected { t.Fatal("collection lost the permanent refusal", err) } after.Collected = false if !reflect.DeepEqual(after, consumed) { t.Fatal("collection changed legacy evidence") } f.retained(next, "new served bytes", r.ReapAfter) } func TestServingParkCollectionLegacyRootRefusesRecapture(t *testing.T) { f, _ := newServingLegacyRootFixture(t, false) a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) if err != nil { t.Fatal(err) } c, err := f.p.ledger.legacyConsumption(f.current) if err != nil || c == nil { t.Fatal(err) } c.Collected = true if err = f.p.ledger.write(servingLegacyKey("legacy", f.current), c); err != nil { t.Fatal(err) } if err = f.disk.Remove("retirement-" + a.ID + ".json"); err != nil { t.Fatal(err) } if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatal("root accepted collected legacy source", err) } if _, err = f.p.ledger.read(a.ID); !errors.Is(err, os.ErrNotExist) { t.Fatal("root recreated collected record", err) } } func TestServingParkCollectionLegacyConsumptionBeforeIntentStillResumes(t *testing.T) { f, _ := newServingLegacyRootFixture(t, false) store := &servingCollectionFaultStore{Store: f.disk} f.p.ledger.state = store store.write = func(name string, body []byte) error { if strings.HasPrefix(name, "retirement-") { return unix.EIO } return f.disk.Write(name, body) } if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); !errors.Is(err, unix.EIO) { t.Fatal("intent fault was not reached", err) } c, err := f.p.ledger.legacyConsumption(f.current) if err != nil || c == nil || c.Collected { t.Fatal("pending consumption unavailable", err) } if _, err := f.p.ledger.read(c.Authorization.ID); !errors.Is(err, os.ErrNotExist) { t.Fatal("intent was saved", err) } f.p.ledger.state = f.disk if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err != nil { t.Fatal("interrupted first capture did not resume", err) } } func TestServingParkCollectionRejectsInvalidDeadline(t *testing.T) { f, old, _ := servingCollectionFixture(t, false) r, err := f.ledger.read(old.ID) if err != nil { t.Fatal(err) } r.CollectAfter = f.now.Add(hostbox.JournalReceiptRetention) for _, change := range []string{"incomplete", "open", "event-present", "namespace-unknown", "early"} { t.Run(change, func(t *testing.T) { bad := r switch change { case "incomplete": bad.Complete = false case "open": bad.Closed = false case "event-present": bad.EventRemoved = false case "namespace-unknown": bad.Namespace = servingArtifactDirectory{} case "early": bad.CollectAfter = bad.ParkedAt.Add(hostbox.JournalReceiptRetention - time.Nanosecond) } if err := f.ledger.save(bad); !errors.Is(err, hostbox.ErrChanged) { t.Fatal("invalid collection authority accepted", err) } }) } } type servingCollectionFaultStore struct { rootstate.Store remove func(string) error write func(string, []byte) error } func (s *servingCollectionFaultStore) Remove(name string) error { if s.remove != nil { return s.remove(name) } return s.Store.Remove(name) } func (s *servingCollectionFaultStore) Write(name string, body []byte) error { if s.write != nil { return s.write(name, body) } return s.Store.Write(name, body) } func TestServingParkCollectionInterruptedWrites(t *testing.T) { for _, boundary := range []string{"deadline-before", "deadline-after", "consumption-before", "consumption-after", "lock-before", "lock-after", "record-before", "record-after"} { t.Run(boundary, func(t *testing.T) { f, old, _ := servingCollectionFixture(t, true) deadline := strings.HasPrefix(boundary, "deadline") if !deadline { if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, _ := f.ledger.read(old.ID) f.now = r.CollectAfter } store := &servingCollectionFaultStore{Store: f.disk} f.ledger.state = store fired := false fault := func(name string, apply func() error) error { matches := deadline && strings.HasPrefix(name, "retirement-") && strings.HasSuffix(name, ".json") || strings.HasPrefix(boundary, "consumption") && strings.HasPrefix(name, "legacy-") || strings.HasPrefix(boundary, "lock") && strings.HasSuffix(name, ".lock") || strings.HasPrefix(boundary, "record") && strings.HasPrefix(name, "retirement-") && strings.HasSuffix(name, ".json") if !matches || fired { return apply() } fired = true if strings.HasSuffix(boundary, "after") { if err := apply(); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } return unix.EIO } store.write = func(name string, body []byte) error { return fault(name, func() error { return f.disk.Write(name, body) }) } store.remove = func(name string) error { return fault(name, func() error { return f.disk.Remove(name) }) } if err := f.ledger.recoverCompleted(t.Context(), old.ID); !errors.Is(err, unix.EIO) || !fired { t.Fatalf("fault not reached: fired=%t err=%v", fired, err) } f.ledger.state = f.disk if boundary != "record-after" { if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal("restart did not resume", err) } } if deadline { r, err := f.ledger.read(old.ID) if err != nil || !r.CollectAfter.Equal(f.now.Add(hostbox.JournalReceiptRetention)) { t.Fatal("deadline lost after restart", err) } } else { if _, err := f.ledger.read(old.ID); !errors.Is(err, os.ErrNotExist) { t.Fatal("collection incomplete", err) } consumed, err := f.ledger.legacyConsumption(old.Successor) if err != nil || consumed == nil || !consumed.Collected { t.Fatal("legacy evidence lost", err) } } }) } } func TestServingParkCollectionRefusesChangedFilesystem(t *testing.T) { for _, change := range []string{"event", "namespace", "cancel"} { t.Run(change, func(t *testing.T) { f, old, _ := servingCollectionFixture(t, false) if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, _ := f.ledger.read(old.ID) f.now = r.CollectAfter ctx := t.Context() var err error switch change { case "event": err = os.Mkdir(filepath.Dir(f.path(old, "")), 0700) case "namespace": path := filepath.Join(f.data, "instances", ".serving-parked") err = os.Rename(path, path+"-moved") if err == nil { err = os.Mkdir(path, 0700) } case "cancel": var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) cancel() } if err != nil { t.Fatal(err) } before, _ := f.disk.Read("retirement-" + old.ID + ".json") if err = f.ledger.recoverCompleted(ctx, old.ID); err == nil { t.Fatal("collected through changed evidence") } after, err := f.disk.Read("retirement-" + old.ID + ".json") if err != nil || string(before) != string(after) { t.Fatal("refusal changed retirement", err) } }) } } func TestServingParkCollectionRebootAdjuncts(t *testing.T) { for _, failure := range []string{"none", "preparation-before", "preparation-after", "binding-before", "binding-after"} { t.Run(failure, func(t *testing.T) { f, old, next := servingCollectionFixture(t, false) if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } r, err := f.ledger.read(old.ID) if err != nil { t.Fatal(err) } f.now = r.CollectAfter // Collection treats preparation contents as inert private metadata. prep := "reboot-" + old.ID + "-" + servingOtherBoot + ".json" other := "reboot-" + next.ID + "-" + servingOtherBoot + ".json" binding := "current-retirement-" + old.ID + ".json" for _, name := range []string{prep, other} { if err := f.disk.Write(name, []byte("{}")); err != nil { t.Fatal(err) } } fired := false fault := &servingCollectionFaultStore{Store: f.disk} fault.remove = func(name string) error { match := strings.HasPrefix(failure, "preparation") && name == prep || strings.HasPrefix(failure, "binding") && name == binding if match && !fired { fired = true if strings.HasSuffix(failure, "after") { _ = f.disk.Remove(name) } return unix.EIO } return f.disk.Remove(name) } f.ledger.state = fault err = f.ledger.recoverCompleted(t.Context(), old.ID) if failure == "none" { if err != nil { t.Fatal(err) } } else if !fired || !errors.Is(err, unix.EIO) { t.Fatalf("fault not reached: %v", err) } f.ledger.state = f.disk if _, err := f.disk.Read("retirement-" + old.ID + ".json"); err == nil { if err := f.ledger.recoverCompleted(t.Context(), old.ID); err != nil { t.Fatal(err) } } // A lost record-removal reply leaves only a binding; maintenance resumes it. if err := f.disk.Write(binding, []byte("{}")); err != nil { t.Fatal(err) } if err := f.ledger.maintainParks(t.Context(), func() error { return nil }); err != nil { t.Fatal(err) } for _, name := range []string{prep, binding, "retirement-" + old.ID + ".json"} { if _, err := f.disk.Read(name); !errors.Is(err, os.ErrNotExist) { t.Fatalf("adjunct survived: %s %v", name, err) } } if _, err := f.disk.Read(other); err != nil { t.Fatal("collected another retirement's preparation", err) } }) } }