//go:build linux package host import ( "bytes" "errors" "fmt" "os" "path/filepath" "testing" "github.com/carlosframework/platform/internal/provision" "golang.org/x/sys/unix" ) func displacedParkFixture(t *testing.T, oldRecord bool) (*servingParkDiskFixture, servingParkRecord, servingParkQuarantine, string) { t.Helper() f := newServingParkDiskFixture(t) f.put("", "last tenant writes") f.put("-wal", "last WAL") f.put("-shm", "last SHM") a, stops := f.evidence(10, true, "missing") if _, err := f.park(a, stops); err != nil { t.Fatal(err) } f.complete(a) r, err := f.ledger.read(a.ID) if err != nil { t.Fatal(err) } if oldRecord { r.Namespace = servingArtifactDirectory{} if err := f.ledger.save(r); err != nil { t.Fatal(err) } } path := filepath.Join(f.data, "instances", ".serving-parked") saved := path + "-saved" if err := os.Rename(path, saved); err != nil { t.Fatal(err) } if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil { t.Fatal(err) } q, err := quarantineRecover(t, f, r.Source.Parent) if err != nil || q.Previous == nil { t.Fatal("quarantine", err) } return f, r, q, saved } func TestServingDisplacedParkReplacementAndExpiry(t *testing.T) { for _, old := range []bool{false, true} { for _, replacement := range []bool{false, true} { t.Run(fmt.Sprintf("old=%v/replacement=%v", old, replacement), func(t *testing.T) { f, r, q, saved := displacedParkFixture(t, old) before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json") if err != nil { t.Fatal(err) } if err := f.fs.CheckParked(&r); err != nil { t.Fatal("known displaced park inaccessible", err) } after, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json") if err != nil || !bytes.Equal(before, after) { t.Fatal("inspection wrote authority", err) } f.put("", "current tenant writes") if replacement { next, stops := f.evidence(11, false, "started") if _, err := f.park(next, stops); err != nil { t.Fatal("replacement", err) } f.complete(next) body, err := os.ReadFile(f.path(next, "")) if err != nil || string(body) != "current tenant writes" { t.Fatal("replacement database", err) } } else { f.now = r.ReapAfter if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err != nil { t.Fatal("expiry", err) } body, err := os.ReadFile(provision.NestedDBPath(f.data, f.base.Host)) if err != nil || string(body) != "current tenant writes" { t.Fatal("expiry touched live database", err) } } got, err := f.ledger.read(r.Authorization.ID) if err != nil || !got.Closed || !got.EventRemoved || got.Namespace != q.Previous.Root || !got.ReapAfter.Equal(r.ReapAfter) { t.Fatal("closed record lost identity or deadline", err) } if _, err := os.Stat(filepath.Join(saved, r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) { t.Fatal("old event remains", err) } if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err != nil { t.Fatal("closed replay", err) } quarantinePreserved(t, f, q) }) } } } func TestServingDisplacedParkRetainsUnprovenIdentities(t *testing.T) { for _, fault := range []string{"parent", "parent-mode", "root", "root-mode", "symlink", "nested", "event", "extra-child", "hardlink", "file", "quarantine", "limit"} { t.Run(fault, func(t *testing.T) { f, r, q, saved := displacedParkFixture(t, false) event := filepath.Join(saved, r.Authorization.ID) db := filepath.Join(event, r.Authorization.Successor.Host+".db") keep := db switch fault { case "parent": parent := filepath.Dir(saved) if err := os.Rename(parent, parent+"-saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(parent, 0700); err != nil { t.Fatal(err) } keep = filepath.Join(parent+"-saved", filepath.Base(saved), r.Authorization.ID, filepath.Base(db)) case "parent-mode": if err := os.Chmod(filepath.Dir(saved), 0777); err != nil { t.Fatal(err) } case "root", "symlink", "nested": outside := filepath.Join(f.data, "retained") if fault == "nested" { outside = filepath.Join(filepath.Dir(saved), "container", "retained") if err := os.Mkdir(filepath.Dir(outside), 0700); err != nil { t.Fatal(err) } } if err := os.Rename(saved, outside); err != nil { t.Fatal(err) } keep = filepath.Join(outside, r.Authorization.ID, filepath.Base(db)) if fault == "symlink" { if err := os.Symlink(outside, saved); err != nil { t.Fatal(err) } } else if err := os.Mkdir(saved, 0700); err != nil { t.Fatal(err) } case "root-mode": if err := os.Chmod(saved, 0755); err != nil { t.Fatal(err) } case "event": if err := os.Rename(event, event+"-saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(event, 0700); err != nil { t.Fatal(err) } keep = filepath.Join(event+"-saved", filepath.Base(db)) case "extra-child": if err := os.WriteFile(filepath.Join(event, "unknown"), []byte("unrecorded"), 0600); err != nil { t.Fatal(err) } case "hardlink": if err := os.Link(db, filepath.Join(f.data, "other-link")); err != nil { t.Fatal(err) } case "file": if err := os.Rename(db, db+"-saved"); err != nil { t.Fatal(err) } if err := os.WriteFile(db, []byte("victim"), 0600); err != nil { t.Fatal(err) } keep = db + "-saved" case "quarantine": if err := f.disk.Write("namespace-quarantine.json", []byte("{}")); err != nil { t.Fatal(err) } case "limit": entries, err := os.ReadDir(filepath.Dir(saved)) if err != nil { t.Fatal(err) } // Exactly one over the bound: the limited read includes the // genuine root regardless of the filesystem's enumeration order. for i := len(entries); i < servingParkRootSearchLimit+1; i++ { if err := os.Mkdir(filepath.Join(filepath.Dir(saved), fmt.Sprintf("unknown-%d", i)), 0000); err != nil { t.Fatal(err) } } } before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json") if err != nil { t.Fatal(err) } f.now = r.ReapAfter if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err == nil { t.Fatal("unproven identity deleted") } got, err := os.ReadFile(keep) if err != nil || string(got) != "last tenant writes" { t.Fatal("retained bytes changed", err) } // Deletion intent may be recorded, but no file progress is permitted. after, err := f.ledger.read(r.Authorization.ID) if err != nil { t.Fatal(err) } for _, file := range after.Source.Files { if file.DeleteIntent || file.Deleted { t.Fatal("unsafe deletion progress") } } if after.Closed || after.EventRemoveIntent || len(before) == 0 { t.Fatal("unsafe closure") } if fault != "parent" { quarantinePreserved(t, f, q) } }) } } func TestServingDisplacedParkRecoveryWriteBoundaries(t *testing.T) { for _, fault := range []string{"root-before", "root-after", "unlink", "rmdir"} { t.Run(fault, func(t *testing.T) { f, r, q, saved := displacedParkFixture(t, true) f.now = r.ReapAfter fired := false check := func(name string, body []byte) error { var next servingParkRecord if !fired && name == "retirement-"+r.Authorization.ID+".json" && servingParkDecode(body, &next) == nil && next.Namespace.Inode != 0 { fired = true return unix.ENOSPC } return nil } if fault == "root-before" { f.ledger.state = quarantineWriteFault{Store: f.disk, before: check} } if fault == "root-after" { f.ledger.state = quarantineWriteFault{Store: f.disk, after: check} } if fault == "unlink" || fault == "rmdir" { f.fs.unlink = func(fd int, name string, flags int) error { if err := unix.Unlinkat(fd, name, flags); err != nil { return err } if !fired && (fault == "unlink" && flags == 0 || fault == "rmdir" && flags == unix.AT_REMOVEDIR) { fired = true return unix.ENOSPC } return nil } } reap := func() error { return f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }) } if err := reap(); !errors.Is(err, unix.ENOSPC) || !fired { t.Fatal("missing boundary", err) } if fault == "root-before" || fault == "root-after" { body, err := os.ReadFile(filepath.Join(saved, r.Authorization.ID, r.Authorization.Successor.Host+".db")) if err != nil || string(body) != "last tenant writes" { t.Fatal("deleted before durable root", err) } } f.ledger.state = f.disk fs := newServingParkFS(f.data, f.disk) fs.owner, fs.mount = f.fs.owner, f.fs.mount f.fs, f.ledger.fs = fs, fs if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err != nil { t.Fatal("recovery", err) } got, err := f.ledger.read(r.Authorization.ID) if err != nil || !got.Closed || !got.EventRemoved || got.Namespace != q.Previous.Root { t.Fatal("incomplete recovered deletion", err) } quarantinePreserved(t, f, q) }) } } func TestServingDisplacedParkCannotAdoptEventUnderUnknownRoot(t *testing.T) { for _, old := range []bool{false, true} { t.Run(fmt.Sprint(old), func(t *testing.T) { f, r, q, saved := displacedParkFixture(t, old) outside := filepath.Join(f.data, "known-root") if err := os.Rename(saved, outside); err != nil { t.Fatal(err) } if err := os.Mkdir(saved, 0700); err != nil { t.Fatal(err) } if err := os.Rename(filepath.Join(outside, r.Authorization.ID), filepath.Join(saved, r.Authorization.ID)); err != nil { t.Fatal(err) } f.now = r.ReapAfter if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err == nil { t.Fatal("event identity authorized an unknown root") } body, err := os.ReadFile(filepath.Join(saved, r.Authorization.ID, r.Authorization.Successor.Host+".db")) if err != nil || string(body) != "last tenant writes" { t.Fatal("unknown root lost known event", err) } quarantinePreserved(t, f, q) }) } } func TestServingDisplacedParkPendingRenameResumesOriginalSlot(t *testing.T) { f := newServingParkDiskFixture(t) f.put("", "interrupted database") f.put("-wal", "interrupted WAL") a, stops := f.evidence(10, true, "missing") f.fs.rename = func(a int, b string, c int, d string, flags uint) error { if err := unix.Renameat2(a, b, c, d, flags); err != nil { return err } return unix.EIO } if _, err := f.park(a, stops); !errors.Is(err, unix.EIO) { t.Fatal("missing interrupted rename", err) } r, err := f.ledger.read(a.ID) if err != nil || r.Complete || r.Parked { t.Fatal("pending record", err) } path := filepath.Join(f.data, "instances", ".serving-parked") if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil { t.Fatal(err) } f.fs.rename = unix.Renameat2 q, err := quarantineRecover(t, f, r.Source.Parent) if err != nil { t.Fatal(err) } if _, err := f.park(a, stops); err != nil { t.Fatal("pending replay", err) } f.complete(a) slot, err := f.ledger.slot(a) if err != nil || slot.ID != a.ID || slot.Pending != "" { t.Fatal("recovery changed slot", err) } for suffix, want := range map[string]string{"": "interrupted database", "-wal": "interrupted WAL"} { body, err := os.ReadFile(filepath.Join(path+"-saved", a.ID, a.Successor.Host+".db"+suffix)) if err != nil || string(body) != want { t.Fatal("pending recovery lost bytes", err) } } quarantinePreserved(t, f, q) } func TestServingDisplacedParkOldAbsentEventRemainsAmbiguous(t *testing.T) { f, r, q, saved := displacedParkFixture(t, true) // Model an older deletion whose rmdir reply was lost before the root was // pinned. Neither namespace's absence supplies the missing identity. r.Deleting, r.Cause, r.EventRemoveIntent = true, "expiry", true for i := range r.Source.Files { item := &r.Source.Files[i] if item.Present { item.DeleteIntent, item.Deleted = true, true if err := os.Remove(filepath.Join(saved, r.Authorization.ID, item.Name)); err != nil { t.Fatal(err) } } } if err := os.Remove(filepath.Join(saved, r.Authorization.ID)); err != nil { t.Fatal(err) } if err := f.ledger.save(r); err != nil { t.Fatal(err) } before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json") if err != nil { t.Fatal(err) } if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err == nil { t.Fatal("old absence accepted as root identity") } after, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json") if err != nil || !bytes.Equal(before, after) { t.Fatal("ambiguous recovery wrote progress", err) } if _, err := os.Stat(filepath.Join(saved, r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) { t.Fatal("ambiguous event recreated", err) } quarantinePreserved(t, f, q) }