//go:build linux package host import ( "context" "errors" "os" "path/filepath" "reflect" "testing" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func quarantineFixture(t *testing.T) (*servingParkDiskFixture, servingArtifactDirectory, string) { t.Helper() f := newServingParkDiskFixture(t) parent, id, err := f.fs.parent() if err != nil { t.Fatal(err) } parent.Close() path := filepath.Join(f.data, "instances", ".serving-parked") if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil { t.Fatal(err) } return f, id, path } func quarantineRecover(t *testing.T, f *servingParkDiskFixture, expected servingArtifactDirectory) (servingParkQuarantine, error) { t.Helper() var q servingParkQuarantine err := f.ledger.withNamespace(t.Context(), func() error { var err error q, err = f.fs.recoverNamespace(t.Context(), expected) return err }) return q, err } func quarantinePreserved(t *testing.T, f *servingParkDiskFixture, q servingParkQuarantine) { t.Helper() path := filepath.Join(f.data, "instances", q.name()) body, err := os.ReadFile(filepath.Join(path, "unknown")) if err != nil || string(body) != "keep these bytes" { t.Fatal("quarantine lost data", err) } d, err := os.Open(path) if err != nil { t.Fatal(err) } id, err := servingParkDirID(d) d.Close() if err != nil || id != q.Obstruction { t.Fatal("quarantine changed identity", err) } } func TestServingParkQuarantineUnknownRoot(t *testing.T) { f, parent, path := quarantineFixture(t) // Contents include a symlink to a victim and an unreadable subtree. The // recovery only renames the outer directory and never inspects either. victim := filepath.Join(t.TempDir(), "victim") if err := os.WriteFile(victim, []byte("victim"), 0600); err != nil { t.Fatal(err) } if err := os.Symlink(victim, filepath.Join(path, "link")); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(path, "unreadable"), 0000); err != nil { t.Fatal(err) } q, err := quarantineRecover(t, f, parent) if err != nil || !q.Complete || q.Previous != nil { t.Fatalf("quarantine: %+v %v", q, err) } quarantinePreserved(t, f, q) if body, err := os.ReadFile(victim); err != nil || string(body) != "victim" { t.Fatal("quarantine followed victim", err) } before, err := f.disk.Read("namespace-quarantine.json") if err != nil { t.Fatal(err) } f.fs = newServingParkFS(f.data, f.disk) f.fs.owner, f.fs.mount = uint32(os.Geteuid()), func(string) (bool, error) { return true, nil } f.ledger.fs = f.fs for i := 0; i < 3; i++ { again, err := quarantineRecover(t, f, parent) if err != nil || !reflect.DeepEqual(again, q) { t.Fatal("quarantine replay", err) } } after, err := f.disk.Read("namespace-quarantine.json") if err != nil || string(after) != string(before) { t.Fatal("replay changed quarantine record", err) } // A fresh host can use the empty namespace. Unknown bytes do not become // retirement records and survive an ordinary expiry pass. f.put("", "current database") a, stops := f.evidence(10, true, "missing") r, err := f.park(a, stops) if err != nil { t.Fatal("park after quarantine", err) } f.complete(a) f.now = r.ReapAfter if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil { t.Fatal(err) } quarantinePreserved(t, f, q) // Only one quarantine is allowed. A later replacement is retained in // place for investigation, never adopted or moved to another quarantine. if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if _, err := quarantineRecover(t, f, parent); err == nil { t.Fatal("replaced root accepted") } names, err := os.ReadDir(filepath.Dir(path)) if err != nil || len(names) != 4 { // instance, fresh obstruction, saved root, quarantine t.Fatalf("unexpected recovery directories: %v %v", names, err) } } type quarantineWriteFault struct { rootstate.Store before, after func(string, []byte) error } func (s quarantineWriteFault) Write(name string, body []byte) error { if s.before != nil { if err := s.before(name, body); err != nil { return err } } if err := s.Store.Write(name, body); err != nil { return err } if s.after != nil { return s.after(name, body) } return nil } func TestServingParkQuarantineResumesDurableBoundaries(t *testing.T) { for _, boundary := range []string{"intent", "rename", "sync", "moved", "fresh-sync", "fresh", "namespace", "complete"} { t.Run(boundary, func(t *testing.T) { f, parent, _ := quarantineFixture(t) crash := errors.New("interrupted quarantine") f.fs.state = quarantineWriteFault{Store: f.disk, after: func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" { if err := servingParkDecode(body, &q); err != nil { return err } if boundary == "intent" && !q.Moved || boundary == "moved" && q.Moved && q.Fresh.Inode == 0 || boundary == "fresh" && q.Fresh.Inode != 0 && !q.Complete || boundary == "complete" && q.Complete { return crash } } if boundary == "namespace" && name == "namespace.json" { return crash } return nil }} if boundary == "rename" { f.fs.rename = func(a int, b string, c int, d string, flags uint) error { if err := unix.Renameat2(a, b, c, d, flags); err != nil { return err } return crash } } if boundary == "sync" { f.fs.sync = func(int) error { return crash } } if boundary == "fresh-sync" { f.fs.sync = func(fd int) error { if _, err := os.Stat(filepath.Join(f.data, "instances", ".serving-parked")); err == nil { return crash } return unix.Fsync(fd) } f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 { t.Fatal("fresh identity saved before successful parent sync") } return nil }} } if _, err := quarantineRecover(t, f, parent); !errors.Is(err, crash) { t.Fatal("missing interruption", err) } if boundary != "complete" { if root, _, err := f.fs.namespace(); err == nil { root.Close() t.Fatal("unfinished quarantine allowed ordinary parking") } } f.fs.state, f.fs.rename, f.fs.sync = f.disk, unix.Renameat2, unix.Fsync q, err := quarantineRecover(t, f, parent) if err != nil || !q.Complete { t.Fatalf("recovery: %+v %v", q, err) } quarantinePreserved(t, f, q) }) } } func TestServingParkQuarantineRefusesUnrecordedFreshRoot(t *testing.T) { f, parent, _ := quarantineFixture(t) f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 { return unix.ENOSPC } return nil }} q, err := quarantineRecover(t, f, parent) if !errors.Is(err, unix.ENOSPC) { t.Fatal("expected full-disk fault", err) } // One unknown replacement can be retained. A second lost identity write // exhausts that bounded recovery; neither directory becomes authority. if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.ENOSPC) { t.Fatal("expected second full-disk fault", err) } f.fs.state = f.disk if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.EEXIST) { t.Fatal("adopted fresh root without durable identity", err) } if _, err := f.disk.Read("namespace.json"); !errors.Is(err, os.ErrNotExist) { t.Fatal("published an unrecorded root", err) } quarantinePreserved(t, f, q) } func quarantineUnrecordedFixture(t *testing.T) (*servingParkDiskFixture, servingArtifactDirectory, servingArtifactDirectory) { t.Helper() f, parent, path := quarantineFixture(t) f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 { return unix.ENOSPC } return nil }} if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.ENOSPC) { t.Fatal("missing creation interruption", err) } f.fs.state = f.disk d, err := os.Open(path) if err != nil { t.Fatal(err) } id, err := servingParkDirID(d) d.Close() if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "uncertain"), []byte("never adopt or delete"), 0600); err != nil { t.Fatal(err) } if err := os.Symlink("/unavailable/victim", filepath.Join(path, "link")); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(path, "unreadable"), 0000); err != nil { t.Fatal(err) } return f, parent, id } func TestServingParkQuarantineRetainsUnrecordedReplacement(t *testing.T) { for _, boundary := range []string{"", "intent-before", "intent-after", "rename", "sync", "moved-before", "moved-after", "fresh-after", "namespace", "complete"} { t.Run(boundary, func(t *testing.T) { f, parent, unrecorded := quarantineUnrecordedFixture(t) crash := errors.New("interrupted replacement retention") writeFault := func(after bool) func(string, []byte) error { return func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Unrecorded.Inode != 0 { if boundary == "intent-before" && !after && !q.UnrecordedMoved || boundary == "intent-after" && after && !q.UnrecordedMoved || boundary == "moved-before" && !after && q.UnrecordedMoved || boundary == "moved-after" && after && q.UnrecordedMoved || boundary == "fresh-after" && after && q.Fresh.Inode != 0 || boundary == "complete" && after && q.Complete { return crash } } if boundary == "namespace" && after && name == "namespace.json" { return crash } return nil } } f.fs.state = quarantineWriteFault{Store: f.disk, before: writeFault(false), after: writeFault(true)} if boundary == "rename" { f.fs.rename = func(a int, b string, c int, d string, flags uint) error { if err := unix.Renameat2(a, b, c, d, flags); err != nil { return err } return crash } } if boundary == "sync" { f.fs.sync = func(int) error { return crash } } q, err := quarantineRecover(t, f, parent) if boundary != "" && !errors.Is(err, crash) || boundary == "" && err != nil { t.Fatal("unexpected interruption", err) } f.fs = newServingParkFS(f.data, f.disk) f.fs.owner, f.fs.mount = uint32(os.Geteuid()), func(string) (bool, error) { return true, nil } f.ledger.fs = f.fs q, err = quarantineRecover(t, f, parent) if err != nil || !q.Complete || !q.UnrecordedMoved || q.Unrecorded != unrecorded || q.Fresh == unrecorded { t.Fatalf("replacement recovery: %+v %v", q, err) } quarantinePreserved(t, f, q) body, err := os.ReadFile(filepath.Join(f.data, "instances", q.unrecordedName(), "uncertain")) if err != nil || string(body) != "never adopt or delete" { t.Fatal("lost unrecorded contents", err) } before, err := f.disk.Read("namespace-quarantine.json") if err != nil { t.Fatal(err) } for i := 0; i < 3; i++ { if replay, err := quarantineRecover(t, f, parent); err != nil || !reflect.DeepEqual(replay, q) { t.Fatal("completed replacement replay", err) } } after, err := f.disk.Read("namespace-quarantine.json") if err != nil || string(before) != string(after) { t.Fatal("completed replay wrote authority", err) } names, err := os.ReadDir(filepath.Join(f.data, "instances")) if err != nil || len(names) != 4 { t.Fatalf("unbounded recovery directories: %v %v", names, err) } if boundary == "" { f.put("", "new database") a, stops := f.evidence(10, true, "missing") r, err := f.park(a, stops) if err != nil { t.Fatal(err) } f.complete(a) f.now = r.ReapAfter if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil { t.Fatal(err) } quarantinePreserved(t, f, q) body, err := os.ReadFile(filepath.Join(f.data, "instances", q.unrecordedName(), "uncertain")) if err != nil || string(body) != "never adopt or delete" { t.Fatal("reaper touched retained replacement", err) } } }) } } func TestServingParkQuarantineValidatesUnrecordedPhases(t *testing.T) { f, parent, _ := quarantineUnrecordedFixture(t) q, err := quarantineRecover(t, f, parent) if err != nil { t.Fatal(err) } for _, fault := range []string{"unmoved", "missing", "device", "mount", "inode", "obstruction", "fresh", "previous"} { t.Run(fault, func(t *testing.T) { broken := q switch fault { case "unmoved": broken.UnrecordedMoved = false case "missing": broken.Unrecorded = servingArtifactDirectory{} case "device": broken.Unrecorded.Device++ case "mount": broken.Unrecorded.Mount++ case "inode": broken.Unrecorded.Inode = 0 case "obstruction": broken.Unrecorded = q.Obstruction case "fresh": broken.Unrecorded = q.Fresh case "previous": broken.Previous = &servingParkNamespace{Version: 1, Parent: parent, Root: q.Unrecorded, BootID: q.BootID} } if !errors.Is(broken.validate(), hostbox.ErrChanged) { t.Fatal("accepted invalid retained replacement", fault) } }) } } func TestServingParkQuarantineUnrecordedIdentityRefusals(t *testing.T) { for _, fault := range []string{"parent", "symlink", "destination", "recorded-replacement", "cancel", "retained-replacement"} { t.Run(fault, func(t *testing.T) { f, parent, _ := quarantineUnrecordedFixture(t) path := filepath.Join(f.data, "instances", ".serving-parked") f.fs.state = quarantineWriteFault{Store: f.disk, after: func(name string, body []byte) error { var q servingParkQuarantine if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Unrecorded.Inode != 0 && !q.UnrecordedMoved { return unix.EIO } return nil }} q, err := quarantineRecover(t, f, parent) if !errors.Is(err, unix.EIO) { t.Fatal("missing retention intent", err) } f.fs.state = f.disk switch fault { case "parent": parent.Inode++ case "symlink", "recorded-replacement": if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } if fault == "symlink" { err = os.Symlink(path+"-saved", path) } else { err = os.Mkdir(path, 0700) } if err != nil { t.Fatal(err) } case "destination": if err := os.Mkdir(filepath.Join(filepath.Dir(path), q.unrecordedName()), 0700); err != nil { t.Fatal(err) } case "retained-replacement": q, err = quarantineRecover(t, f, parent) if err != nil { t.Fatal(err) } retained := filepath.Join(filepath.Dir(path), q.unrecordedName()) if err := os.Rename(retained, retained+"-saved"); err != nil { t.Fatal(err) } if err := os.Mkdir(retained, 0700); err != nil { t.Fatal(err) } } beforeNames, err := os.ReadDir(filepath.Dir(path)) if err != nil { t.Fatal(err) } beforeRecord, err := f.disk.Read("namespace-quarantine.json") if err != nil { t.Fatal(err) } if fault == "cancel" { ctx, cancel := context.WithCancel(t.Context()) cancel() _, err = f.fs.recoverNamespace(ctx, parent) } else { _, err = quarantineRecover(t, f, parent) } if err == nil { t.Fatal("unsafe replacement accepted") } afterNames, err := os.ReadDir(filepath.Dir(path)) if err != nil || len(afterNames) != len(beforeNames) { t.Fatal("refusal changed retained directory names", err) } for i := range beforeNames { if beforeNames[i].Name() != afterNames[i].Name() { t.Fatal("refusal moved a replacement directory") } } afterRecord, err := f.disk.Read("namespace-quarantine.json") if err != nil || string(beforeRecord) != string(afterRecord) { t.Fatal("refusal changed retention authority", err) } quarantinePreserved(t, f, q) }) } } func TestServingParkQuarantineRefusesUnsafeAuthority(t *testing.T) { for _, fault := range []string{"parent", "mode", "symlink", "mount", "cancel", "corrupt", "known-root"} { t.Run(fault, func(t *testing.T) { f, parent, path := quarantineFixture(t) switch fault { case "parent": parent.Inode++ case "mode": if err := os.Chmod(filepath.Dir(path), 0777); err != nil { t.Fatal(err) } case "symlink": if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } if err := os.Symlink(path+"-saved", path); err != nil { t.Fatal(err) } case "mount": f.fs.mount = func(string) (bool, error) { return false, nil } case "corrupt": if err := f.disk.Write("namespace.json", []byte("{}")); err != nil { t.Fatal(err) } case "known-root": d, err := os.Open(path) if err != nil { t.Fatal(err) } id, err := servingParkDirID(d) d.Close() if err != nil { t.Fatal(err) } boot, err := f.fs.currentBoot() if err != nil { t.Fatal(err) } if err := f.ledger.write("namespace.json", servingParkNamespace{Version: 1, Parent: parent, Root: id, BootID: boot}); err != nil { t.Fatal(err) } } var err error if fault == "cancel" { ctx, cancel := context.WithCancel(t.Context()) cancel() _, err = f.fs.recoverNamespace(ctx, parent) } else { _, err = quarantineRecover(t, f, parent) } if err == nil { t.Fatal("unsafe quarantine accepted") } if _, err := f.disk.Read("namespace-quarantine.json"); !errors.Is(err, os.ErrNotExist) { t.Fatal("unsafe authority persisted intent", err) } }) } } func TestServingParkQuarantineRetainedParkAllowsRetirement(t *testing.T) { f := newServingParkDiskFixture(t) f.put("", "old served database") a, stops := f.evidence(10, true, "missing") r, err := f.park(a, stops) if err != nil { t.Fatal(err) } f.complete(a) path := filepath.Join(f.data, "instances", ".serving-parked") if err := os.Rename(path, path+"-saved"); err != nil { t.Fatal(err) } // Finding the genuine root does not authorize an unknown event child. if err := os.WriteFile(filepath.Join(path+"-saved", a.ID, "unknown"), []byte("retain"), 0600); err != nil { t.Fatal(err) } if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil { t.Fatal(err) } q, err := quarantineRecover(t, f, r.Source.Parent) if err != nil || q.Previous == nil { t.Fatal("recorded-root recovery", err) } quarantinePreserved(t, f, q) f.put("", "new served database") next, receipts := f.evidence(11, false, "started") parked, err := f.park(next, receipts) if err != nil { t.Fatal("retained quarantine blocked retirement", err) } f.complete(next) f.retained(next, "new served database", parked.ReapAfter) old, err := f.ledger.readHistorical(a.ID) if err != nil || old.Closed || !old.ReapAfter.Equal(r.ReapAfter) { t.Fatal("lost retained quarantine record", old, err) } body, err := os.ReadFile(filepath.Join(path+"-saved", a.ID, a.Successor.Host+".db")) if err != nil || string(body) != "old served database" { t.Fatal("old park changed", err) } body, err = os.ReadFile(filepath.Join(path+"-saved", a.ID, "unknown")) if err != nil || string(body) != "retain" { t.Fatal("unknown retained child changed", err) } if _, err = os.Stat(provision.NestedDBPath(f.data, f.base.Host)); !errors.Is(err, os.ErrNotExist) { t.Fatal("new retirement left its source", err) } f.base.Host, f.target.Host = "other.example", "other.example" if err := os.MkdirAll(filepath.Dir(provision.NestedDBPath(f.data, f.base.Host)), 0700); err != nil { t.Fatal(err) } f.put("", "other host database") other, otherStops := f.evidence(20, true, "missing") if _, err := f.park(other, otherStops); err != nil { t.Fatal("blocked host prevented unrelated park", err) } f.complete(other) quarantinePreserved(t, f, q) if _, err := f.disk.Read("retirement-" + q.ID + ".json"); !errors.Is(err, os.ErrNotExist) { t.Fatal("quarantine became deletion authority", err) } if err := q.validate(); err != nil { t.Fatal(err) } q.Fresh = q.Obstruction if !errors.Is(q.validate(), hostbox.ErrChanged) { t.Fatal("quarantine reused as fresh root") } }