//go:build unix package host import ( "bufio" "bytes" "context" "database/sql" "errors" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) func TestClaimProductionCensusInstallsThroughReconciler(t *testing.T) { for _, flag := range []string{"", "1", "0"} { for _, mode := range []string{"empty", "fenced"} { name := "flag-off/" + mode if flag != "" { name = "flag-on-" + flag + "/" + mode } t.Run(name, func(t *testing.T) { t.Setenv("CARLOS_HOST_CLAIM", flag) testClaimProductionCensusInstallation(t, mode == "fenced") }) } } } // claimCensusInstallLinks is the test's own reading of a unit's [Install] // section: the symlinks `systemctl enable` makes, relative to the unit // directory. It fails on a key it does not model rather than under-enable. func claimCensusInstallLinks(t *testing.T, unit string, body []byte) []string { t.Helper() var links []string install := false scan := bufio.NewScanner(bytes.NewReader(body)) for scan.Scan() { line := strings.TrimSpace(scan.Text()) switch { case line == "" || line[0] == '#' || line[0] == ';': continue case line[0] == '[': install = line == "[Install]" continue case !install: continue } key, value, _ := strings.Cut(line, "=") for _, v := range strings.Fields(value) { switch strings.TrimSpace(key) { case "WantedBy": links = append(links, v+".wants/"+unit) case "RequiredBy": links = append(links, v+".requires/"+unit) case "Alias": links = append(links, v) default: t.Fatalf("%s: [Install] %s is not modelled by the census fixture", unit, key) } } } return links } // claimCensusSystemctl stands in for systemctl over a staging root: enable // writes exactly the links the unit's [Install] metadata names, pointing at // the installed unit file, as `systemctl enable` does. Nothing reaches the // live system manager. func claimCensusSystemctl(t *testing.T, unitsDir string) func(args ...string) (string, error) { return func(args ...string) (string, error) { switch args[0] { case "is-active": return "inactive\n", nil case "is-enabled": return "disabled\n", nil case "enable": default: return "", nil } for _, unit := range args[1:] { target := filepath.Join(unitsDir, unit) body, err := os.ReadFile(target) if err != nil { return "", err } for _, rel := range claimCensusInstallLinks(t, unit, body) { link := filepath.Join(unitsDir, rel) if old, err := os.Readlink(link); err == nil && old == target { continue } if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { return "", err } if err := os.Symlink(target, link); err != nil { return "", err } } } return "", nil } } // claimCensusBox provisions a box under a staging root with the production // installers: bootstrap, the self-update applier's binary rename, unit // install and arming, ensureSystemUnits' second belt and the release store. func claimCensusBox(t *testing.T) ClaimRuntimeResources { t.Helper() ctx := t.Context() root := t.TempDir() stageEnv := func(stage string) { t.Helper() if err := os.MkdirAll(filepath.Join(stage, "etc", "carlos"), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(stage, "etc", "carlos", "host.env"), []byte("CARLOS_HOST_CLAIM="+os.Getenv("CARLOS_HOST_CLAIM")+"\n"), 0600); err != nil { t.Fatal(err) } } stageEnv(root) if err := bootstrap(root, nil); err != nil { t.Fatal(err) } etc := filepath.Join(root, "etc") r := ClaimRuntimeResources{DataDir: filepath.Join(root, "data"), OptDir: filepath.Join(root, "opt", "carlos"), UnitsDir: filepath.Join(etc, "systemd", "system")} systemctl := claimCensusSystemctl(t, r.UnitsDir) a := &systemApplier{optDir: r.OptDir, dataDir: r.DataDir, etcDir: etc, ops: sysOps{ systemctl: systemctl, run: func(context.Context, time.Duration, string, ...string) (string, error) { return "", nil }, logf: t.Logf, }} // Bootstrap never stages the binary; the applier renames a staged one // to its production path. if err := os.MkdirAll(a.stagingDir(), 0o700); err != nil { t.Fatal(err) } staged := filepath.Join(a.stagingDir(), "carlos") if err := os.WriteFile(staged, []byte("#!/bin/sh\necho carlos\n"), 0o755); err != nil { t.Fatal(err) } if err := os.Rename(staged, systemBinaryPath(r.OptDir)); err != nil { t.Fatal(err) } if err := copyFileDurable(systemBinaryPath(r.OptDir), systemPrevPath(r.OptDir), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(prevVersionPath(r.OptDir), []byte("v0.1.0\n"), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(prevAllowedPath(r.OptDir), []byte("v0.1.0\n"), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(seedsPrevAllowedPath(r.OptDir), []byte("v0.1.0\n"), 0o644); err != nil { t.Fatal(err) } newDir := filepath.Join(a.stagingDir(), "new") stageEnv(newDir) if err := bootstrap(newDir, nil); err != nil { t.Fatal(err) } if err := writeJSONDurable(filepath.Join(newDir, "plan.json"), currentBootstrapPlan(), 0o600); err != nil { t.Fatal(err) } if err := a.installUnits(ctx, newDir, nil); err != nil { t.Fatal(err) } plan, err := stagedPlan(newDir) if err != nil { t.Fatal(err) } if err := a.arm(plan); err != nil { t.Fatal(err) } ensureSystemUnits(etc, a.lockPath(), systemctl, systemctl, t.Logf) for _, unit := range plan.Enabled { body, err := os.ReadFile(filepath.Join(r.UnitsDir, unit)) if err != nil { t.Fatal(err) } for _, rel := range claimCensusInstallLinks(t, unit, body) { if _, err := os.Readlink(filepath.Join(r.UnitsDir, rel)); err != nil { t.Fatalf("fixture did not enable %s: %v", unit, err) } } } if hostwrite.Enabled(os.Getenv) { // Rollout uses add-wants for this static service, which has no // [Install] section. Model its exact boot link independently. link := filepath.Join(r.UnitsDir, "multi-user.target.wants/carlos-instance-control.service") if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { t.Fatal(err) } if err := os.Symlink(filepath.Join(r.UnitsDir, "carlos-instance-control.service"), link); err != nil { t.Fatal(err) } } for _, app := range []struct{ acct, ver, app string }{{"acct", "v1", "app"}, {"acct", "v2", "app"}, {"", "v1", "ops"}} { if err := release.StageArtifact(r.OptDir, app.acct, app.ver, app.app, []byte("build "+app.ver)); err != nil { t.Fatal(err) } } for _, name := range []string{"hostguard", ".hostclaims"} { if err := os.MkdirAll(filepath.Join(r.DataDir, name), 0o700); err != nil { t.Fatal(err) } } for unit, mode := range map[string]string{"carlos-edge.service": "edge", "carlos-system-update.service": "apply", "carlos-system-recover.service": "recover"} { path := filepath.Join(r.UnitsDir, unit+".d", "90-host-floor.conf") if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("[Service]\nExecStart=\nExecStart=/opt/carlos/system/guard/carlos system guard-exec "+mode+"\n"), 0644); err != nil { t.Fatal(err) } } return r } // claimCensusMutation changes the box and returns how to put it back. type claimCensusMutation struct { name string mutate func(t *testing.T) func() } func claimCensusReplace(t *testing.T, path string, put func() error) func() { t.Helper() info, err := os.Lstat(path) if err != nil { t.Fatal(err) } var target string var body []byte if info.Mode()&os.ModeSymlink != 0 { target, err = os.Readlink(path) } else { body, err = os.ReadFile(path) } if err != nil { t.Fatal(err) } if err := os.Remove(path); err != nil { t.Fatal(err) } if err := put(); err != nil { t.Fatal(err) } return func() { if err := os.RemoveAll(path); err != nil { t.Fatal(err) } if target != "" { err = os.Symlink(target, path) } else { err = os.WriteFile(path, body, info.Mode().Perm()) } if err != nil { t.Fatal(err) } } } func claimCensusCreate(t *testing.T, path string, put func() error) func() { t.Helper() // Restore removes the outermost directory this created, so a host-named // drop-in directory does not outlive its case. created := path for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { if _, err := os.Lstat(dir); err == nil { break } created = dir } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := put(); err != nil { t.Fatal(err) } return func() { if err := os.RemoveAll(created); err != nil { t.Fatal(err) } } } func claimCensusMutations(r ClaimRuntimeResources, host string) []claimCensusMutation { unit := func(name string) string { return filepath.Join(r.UnitsDir, name) } edgeLink := unit("multi-user.target.wants/carlos-edge.service") socketLink := unit("sockets.target.wants/carlos-instance-control.socket") relink := func(link, target string) func(t *testing.T) func() { return func(t *testing.T) func() { return claimCensusReplace(t, link, func() error { return os.Symlink(target, link) }) } } newLink := func(link, target string) func(t *testing.T) func() { return func(t *testing.T) func() { return claimCensusCreate(t, link, func() error { return os.Symlink(target, link) }) } } newFile := func(path, body string) func(t *testing.T) func() { return func(t *testing.T) func() { return claimCensusCreate(t, path, func() error { return os.WriteFile(path, []byte(body), 0o644) }) } } live := func(h string) func(t *testing.T) func() { return func(t *testing.T) func() { link := release.LiveLink(r.OptDir, h) return claimCensusCreate(t, link, func() error { return release.PointLive(r.OptDir, h, "acct", "v1", "app") }) } } mutations := []claimCensusMutation{ {"unknown guarded-edge drop-in", newFile(unit("carlos-edge.service.d/91-unknown.conf"), "[Service]\nExecStart=/bin/true\n")}, {"guard drop-in symlink", relink(unit("carlos-edge.service.d/90-host-floor.conf"), "/missing/guard")}, {"unknown data file", newFile(filepath.Join(r.DataDir, "unclassified.db"), "old")}, {"unknown opt file", newFile(filepath.Join(r.OptDir, "unexplained"), "old")}, {"unknown vendor link", newLink(unit("multi-user.target.wants/ssh.service"), "/usr/lib/systemd/system/ssh.service")}, {"dangling unknown link", newLink(unit("multi-user.target.wants/unknown.service"), "/missing/unit.service")}, {"unknown unit", newFile(unit("backup.service"), "[Service]\nExecStart=/bin/true\n")}, {"unknown platform unit", newFile(unit("carlos-extra.service"), "[Service]\nExecStart=/bin/true\n")}, {"selected host live link", live(host)}, {"another host live link", live("other.example")}, {"enable link to the host live link", relink(edgeLink, release.LiveLink(r.OptDir, host))}, {"enable link to another platform unit", relink(edgeLink, unit("carlos-app@.service"))}, {"enable link to a host unit instance", relink(socketLink, unit("carlos-app@"+host+".socket"))}, {"relative enable link", relink(edgeLink, "../carlos-edge.service")}, {"enable link under the wrong target", newLink(unit("timers.target.wants/carlos-edge.service"), unit("carlos-edge.service"))}, {"masked platform unit", relink(unit("carlos-edge.service"), "/dev/null")}, {"platform unit symlinked to a release", relink(unit("carlos-edge.service"), release.VersionPath(r.OptDir, "acct", "v1", "app"))}, {"enabled host unit instance", newLink(unit("multi-user.target.wants/carlos-app@other.example.service"), unit("carlos-app@.service"))}, {"host unit drop-in", newFile(unit("carlos-app@"+host+".service.d/10-caps.conf"), "[Service]\n")}, {"platform unit drop-in", newFile(unit("carlos-edge.service.d/override.conf"), "[Service]\n")}, {"binary symlinked to a release", relink(systemBinaryPath(r.OptDir), release.VersionPath(r.OptDir, "acct", "v1", "app"))}, {"symlink in the release store", newLink(release.VersionPath(r.OptDir, "acct", "v1", "shell"), "/bin/sh")}, {"host-named release", newFile(release.VersionPath(r.OptDir, "acct", "v1", host), "build")}, {"foreign link into a host live link", newLink(unit("multi-user.target.wants/backup2.service"), release.LiveLink(r.OptDir, "other.example"))}, {"foreign link into the data root", newLink(unit("multi-user.target.wants/backup3.service"), filepath.Join(r.DataDir, "unit.service"))}, {"foreign unit naming the platform", newFile(unit("evil.service"), "[Service]\nExecStart="+release.LiveLink(r.OptDir, "other.example")+"\n")}, {"foreign unit naming a managed root", newFile(unit("export.service"), "[Service]\nExecStart=/usr/bin/cp "+filepath.Join(r.DataDir, "registry.db")+" /srv\n")}, } rootBoot := unit("multi-user.target.wants/carlos-instance-control.service") rootUnit := unit("carlos-instance-control.service") mutations = append(mutations, claimCensusMutation{"root service under wrong target", newLink(unit("timers.target.wants/carlos-instance-control.service"), rootUnit)}, claimCensusMutation{"root service masked", relink(rootUnit, "/dev/null")}, ) if hostwrite.Enabled(os.Getenv) { mutations = append(mutations, claimCensusMutation{"root boot link to another unit", relink(rootBoot, unit("carlos-edge.service"))}, claimCensusMutation{"root boot link to host instance", relink(rootBoot, unit("carlos-app@"+host+".service"))}, claimCensusMutation{"root boot relative link", relink(rootBoot, "../carlos-instance-control.service")}, claimCensusMutation{"root boot regular file", func(t *testing.T) func() { return claimCensusReplace(t, rootBoot, func() error { return os.WriteFile(rootBoot, []byte("unit"), 0o644) }) }}, ) } else { mutations = append(mutations, claimCensusMutation{"root boot link while flag off", newLink(rootBoot, rootUnit)}) } return mutations } func testClaimProductionCensusInstallation(t *testing.T, fenced bool) { ctx := t.Context() r := claimCensusBox(t) reg, err := registry.Open(filepath.Join(r.DataDir, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() r.Registry = reg inventory, err := openLegacyInventory(t, filepath.Join(r.DataDir, ".hostclaims", "inventory.db")) if err != nil { t.Fatal(err) } defer inventory.Close() launches, err := hostbox.OpenLaunchJournal(filepath.Join(r.DataDir, ".hostclaims", "launches.db")) if err != nil { t.Fatal(err) } defer launches.Close() store := release.NewDirStore(t.TempDir()) created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} if err := provision.CreateRecord(ctx, store, record); err != nil { t.Fatal(err) } key := provision.RecordKey(record.Account, record.App, record.Host) obs, err := hostclaim.Read(ctx, store, record.Host) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: key, Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "box", nil, nil, "adopt", func() string { return "id" })) if err != nil { t.Fatal(err) } if fenced { obs, err = hostclaim.Update(ctx, store, obs, hostclaim.AdmitWriter(hostclaim.Admission{Box: "box", Region: "ie", Mode: "cold", NewGeneration: "generation"})) if err != nil { t.Fatal(err) } } guard := hostguard.Guard{DataDir: r.DataDir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: store}} source := hostbox.WorkSource{Guard: guard, Box: "box", Region: "ie", Now: time.Now, Policy: func(_ context.Context, c *hostclaim.Claim, s hostbox.Selection, _ hostwrite.Identity) error { return hostbox.Serving(c, s) }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }, } census := &hostbox.EmptyInstallationCensus{Inventory: inventory, Registry: reg, Launches: launches, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Roots: map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}, SharedPaths: claimGlobalSharedPaths(r), } adapter := hostbox.ReconcileAdapter{Registry: reg, Inventory: inventory, Census: census, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Reconciler: hostbox.Reconciler{Claims: store, Guard: guard, Local: park.Local{Guard: guard, Claims: store, DataDir: r.DataDir, Box: "box"}, Writes: hostwrite.Config{Enabled: true, Selector: source}, Box: "box", Region: "ie", Now: time.Now, CheckLocal: census.CheckLocal}, } selected := provision.ClaimRecord{Host: record.Host, Key: key, Family: "instance", Account: record.Account, App: record.App, CreatedAt: created, Value: record, Route: registry.Route{Host: record.Host, App: record.App, Account: record.Account, Kind: "instance", Channel: "stable"}} guardPath := filepath.Join(r.DataDir, "hostguard", record.Host+".json") if _, err := os.Stat(guardPath); !errors.Is(err, os.ErrNotExist) { t.Fatalf("guard was active before installation: %v", err) } if fenced { if err := guard.Activate(ctx, record.Host, "box", nil); err != nil { t.Fatal(err) } h := obs.Claim.Holder target := hostguard.FenceTarget{FenceID: "fence", Host: record.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box", Region: "ie", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: obs.ETag, DirectoryRevision: "1"} if _, err := guard.BeginFence(ctx, target); err != nil { t.Fatal(err) } before, err := os.ReadFile(guardPath) if err != nil { t.Fatal(err) } if err := adapter.Install(ctx, selected, func(registry.ClaimOwnership) error { t.Fatal("fenced installation wrote a route"); return nil }); !errors.Is(err, hostguard.ErrFenced) { t.Fatalf("fenced installation: %v", err) } after, err := os.ReadFile(guardPath) if err != nil || !bytes.Equal(before, after) { t.Fatalf("installation changed the durable fence: %v", err) } if row, err := reg.Get(record.Host); err != nil || row != nil { t.Fatalf("fenced installation wrote a row: %+v %v", row, err) } if _, err := inventory.ReadInstallation(ctx, record.Host, h.Incarnation, h.Stamp); !errors.Is(err, sql.ErrNoRows) { t.Fatalf("fenced installation wrote inventory: %v", err) } return } // Unknown files and substituted links still refuse before activation; // the installers' shared artifacts do not make any of them eligible for // a new ownership stamp. for _, m := range claimCensusMutations(r, record.Host) { restore := m.mutate(t) if err := adapter.Install(ctx, selected, func(registry.ClaimOwnership) error { t.Fatalf("%s: admitted", m.name); return nil }); !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("%s: refusal: %v", m.name, err) } if _, err := os.Stat(guardPath); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s: refused installation activated guard: %v", m.name, err) } if row, err := reg.Get(record.Host); err != nil || row != nil { t.Fatalf("%s: refused installation wrote a row: %+v %v", m.name, row, err) } if _, err := inventory.ReadInstallation(ctx, record.Host, obs.Claim.Holder.Incarnation, obs.Claim.Holder.Stamp); !errors.Is(err, sql.ErrNoRows) { t.Fatalf("%s: refused installation wrote inventory: %v", m.name, err) } restore() } writes := 0 install := func(owner registry.ClaimOwnership) error { writes++ if !owner.Valid() { t.Fatal("installation has no ownership") } if _, err := os.Stat(guardPath); err != nil { t.Fatalf("installation did not activate the real guard record: %v", err) } route := selected.Route route.ClaimOwnership = owner return reg.Upsert(route) } if err := adapter.Install(ctx, selected, install); err != nil { t.Fatalf("first installation on a provisioned box: %v", err) } row, err := reg.Get(record.Host) if err != nil || row == nil || !row.ClaimOwnership.Valid() || writes != 1 { t.Fatalf("installed row=%+v writes=%d err=%v", row, writes, err) } in, err := inventory.ReadInstallation(ctx, record.Host, obs.Claim.Holder.Incarnation, obs.Claim.Holder.Stamp) if err != nil || !in.Complete || len(in.Entries) != 1 || in.Entries[0].Kind != "route" || in.Entries[0].Materialization != "materialized" || in.Entries[0].Ownership != row.ClaimOwner { t.Fatalf("installation inventory=%+v err=%v", in, err) } if err := adapter.Install(ctx, selected, install); err != nil { t.Fatalf("repeat installation: %v", err) } again, err := reg.Get(record.Host) if err != nil || again == nil || again.ClaimOwnership != row.ClaimOwnership || writes != 2 { t.Fatalf("repeat changed ownership: row=%+v writes=%d err=%v", again, writes, err) } }