//go:build linux && rootrunner package host import ( "bytes" "context" "crypto/ed25519" "crypto/rand" "crypto/sha256" "database/sql" "encoding/base64" "encoding/hex" "encoding/json" "errors" "io" "net" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/controlsock" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rollout" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) const lifecycleHost = "lifecycle.bab.example" const lifecycleSibling = "sibling.bab.example" const lifecycleHome = "/var/lib/plan14-lifecycle" func init() { if lifecyclePayloadArgs(os.Args) { db, err := sql.Open("sqlite", os.Args[4]) if err != nil { panic(err) } if _, err = db.Exec("PRAGMA journal_mode=WAL; CREATE TABLE IF NOT EXISTS lifecycle (value TEXT)"); err != nil { panic(err) } listener, err := net.Listen("unix", os.Args[2]) if err != nil { panic(err) } if err = http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "lifecycle tenant") })); err != nil { panic(err) } os.Exit(0) } } func lifecyclePayloadArgs(args []string) bool { if len(args) != 5 { return false } host := lifecycleHost if args[2] == filepath.Join(provision.InstanceRunDir(lifecycleSibling), "app.sock") { host = lifecycleSibling } return args[1] == "--socket" && args[2] == filepath.Join(provision.InstanceRunDir(host), "app.sock") && args[3] == "--db" && args[4] == provision.NestedDBPath("/data", host) } func TestPlan14LifecyclePayloadDispatch(t *testing.T) { ours := []string{"tenant", "--socket", filepath.Join(provision.InstanceRunDir(lifecycleHost), "app.sock"), "--db", provision.NestedDBPath("/data", lifecycleHost)} if !lifecyclePayloadArgs(ours) { t.Fatal("lifecycle payload refused") } for _, index := range []int{1, 2, 3, 4} { other := append([]string(nil), ours...) other[index] = "another-test" if lifecyclePayloadArgs(other) { t.Fatalf("another payload intercepted at argument %d", index) } } if lifecyclePayloadArgs(ours[:4]) { t.Fatal("incomplete payload intercepted") } } func lifecycleCheck(t *testing.T, step string, err error) { t.Helper() if err != nil { t.Fatalf("%s: %v", step, err) } } // Only the external box enrollment and object service are supplied by this // harness. Local authority, execution, settings, files and launch wiring use // the same constructors as instance-control and the edge. type lifecycleOwners struct{ t *testing.T } func (p lifecycleOwners) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) { key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) identity := handoverRecoveryIdentity(p.t, hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Revision: 1, Binding: "1", Region: "ie", FleetHomeRegion: "ie", ControlBucket: "deployment"}) identity.HomeRegion = "ie" actor, err := identity.Verify(ctx) if err != nil { return ClaimOwnerBindings{}, err } directory := hostactors.Directory{Store: store, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket} data, err := lease.NewVersionedS3(ctx, map[string]string{actor.Bucket: actor.Cloud.Region}) if err != nil { return ClaimOwnerBindings{}, err } markers, err := newProductionClaimMarkers(ctx, actor.Bucket, actor.Cloud.Region) if err != nil { return ClaimOwnerBindings{}, err } local := &productionClaimLocal{Resources: r, Identity: identity, Directory: directory, Store: store, slots: map[string]*claimParticipation{}} backend := claimExecBackend(r, productionInstanceKeyReaders()) tokens := claimAdminTokens{Guard: hostguard.Guard{DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}}, Claims: store, Verify: identity.Verify, Local: identity.Local(local), Policy: hostbox.Policy{Held: hostbox.RegistryHoldPolicy(r.Registry)}.Check, Now: time.Now} input := claimUnitInputReader(rootClaimUnitInputs) if r.Mode == "agent" { input = NewClaimRoleRootClient(controlsock.Path).UnitInputs } regional := productionClaimRegional{Identity: identity, Local: local} out := ClaimOwnerBindings{Identity: identity, RetirementAuthority: &hostops.ProductionAuthority{Store: store, Directory: directory, Now: time.Now}, EvidenceKey: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize)), Controllers: productionStopControllers{}, Data: data, Local: local, Serving: ClaimServingSources{Launch: claimServingLaunchSource(backend, tokens, input), Markers: markers}, InputPolicy: claimInputPolicy(r.Registry.Get, store, key.Public().(ed25519.PublicKey), backend, input), MovesPolicy: MovesClaimPolicy(MovesRecordsFrom(store)), Evidence: regional, LeaseSet: regional.LeaseSet, View: regional.View, Authorize: regional.Authorize, Close: local.Close} source := &movesFilesSource{DataDir: r.DataDir, Routes: r.Registry, newFiles: newProductionFilesStore} _, err = openClaimFilesManifest(ctx, os.Getenv, source, store, data, actor, &out, claimBoundIdentity(identity.Verify, actor)) if r.Mode != "instance-control" { out.EvidenceKey = nil } bindClaimLegacyOwners(&out) return out, err } func lifecycleEnvironment(t *testing.T, endpoint string) { t.Helper() for k, v := range map[string]string{"CARLOS_HOST_CLAIM": "1", "CARLOS_HOST_CLAIM_ORIGINS": "1", "CARLOS_MOVES_PHASE1": "", "CARLOS_REGION_HA": "", "CARLOS_DATA_DIR": "/data", "CARLOS_OPT_DIR": "/opt/carlos", "CARLOS_DEPLOYMENT_BUCKET": "", "CARLOS_DEPLOYMENT_DIR": lifecycleHome + "/deployment", "CARLOS_REGION": "", "AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_CONFIG_FILE": "/nonexistent", "AWS_SHARED_CREDENTIALS_FILE": "/nonexistent", "CARLOS_S3_ENDPOINT": endpoint, "AWS_ENDPOINT_URL_S3": endpoint, "CARLOS_RELEASE_PUBKEY": base64.StdEncoding.EncodeToString(ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)).Public().(ed25519.PublicKey))} { t.Setenv(k, v) } } func TestPlan14LifecycleColdWakeRootRunner(t *testing.T) { t.Setenv("PLAN14_LIFECYCLE_PHASE", "probe") lifecycleRun(t) } func TestRootRebootPlan14LifecycleEndToEndPhase1(t *testing.T) { t.Setenv("PLAN14_LIFECYCLE_PHASE", "1") TestPlan14LifecycleEndToEndRootRunnerPayload(t) } func TestRootRebootPlan14LifecycleEndToEndPhase2(t *testing.T) { t.Setenv("PLAN14_LIFECYCLE_PHASE", "2") TestPlan14LifecycleEndToEndRootRunnerPayload(t) } func TestPlan14LifecycleEndToEndRootRunnerPayload(t *testing.T) { if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "" { t.Fatal("lifecycle E2E requires root-runner --reboot Plan14LifecycleEndToEnd") } t.Run("ordinary", lifecycleRun) if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" && os.Getenv("PLAN14_LIFECYCLE_AGENT") == "" { for _, point := range []string{"reservation", "handover", "retirement"} { t.Run("crash-"+point, func(t *testing.T) { t.Setenv("PLAN14_LIFECYCLE_PHASE", "crash") t.Setenv("PLAN14_LIFECYCLE_CRASH", point) lifecycleRun(t) }) } } } func lifecycleRun(t *testing.T) { if os.Getenv("PLAN14_LIFECYCLE_AGENT") == "1" { lifecycleAgent(t) return } TestPlan14RootJournalNamespaceRootRunner(t) phase := os.Getenv("PLAN14_LIFECYCLE_PHASE") resume := phase == "2" uid, gid := generationRunnerRegistryUser(t) for _, path := range []string{lifecycleHome, "/data", "/opt/carlos", "/run/carlos", "/run/carlos-control", "/etc/carlos", "/var/lib/carlos-claims", "/var/lib/carlos-release", rootRetryProjection} { if !resume || strings.HasPrefix(path, "/run/") { lifecycleCheck(t, "exclusive fixture "+path, os.Mkdir(path, 0755)) } t.Cleanup(func() { if phase == "1" && !t.Failed() { return } lifecycleCheck(t, "cleanup "+path, os.RemoveAll(path)) }) } lifecycleCheck(t, "private authority", os.Chmod("/var/lib/carlos-claims", 0700)) lifecycleCheck(t, "input service owner", os.Chown(lifecycleHome, uid, gid)) lifecycleCheck(t, "runtime service owner", os.Chown("/run/carlos", uid, gid)) lifecycleCheck(t, "data owner", os.Chown("/data", uid, gid)) for _, path := range []string{lifecycleHome + "/deployment", "/data/hostguard", "/data/.parked", "/data/layout", "/data/instances", "/data/.instances", "/run/carlos/.instances", "/run/carlos/.logs", "/data/.hostexec", "/run/carlos/instances", "/run/carlos/logs", "/run/carlos/hostguard", "/run/carlos/hostexec"} { lifecycleCheck(t, "directory", os.MkdirAll(path, 0755)) lifecycleCheck(t, "directory owner", os.Chown(path, uid, gid)) } for _, path := range []string{"/data/.instances", "/data/instances", "/data/layout", "/run/carlos/.logs", "/run/carlos/logs"} { lifecycleCheck(t, "private tree", os.Chmod(path, 0700)) } model := leasetest.NewVersionedBucket() model.AddBucket("data") if resume { lifecycleRestoreBucket(t, model) } handler, _ := leasetest.NewS3Server(model) server := httptest.NewServer(lifecycleS3(handler, model)) t.Cleanup(server.Close) lifecycleEnvironment(t, server.URL) store := release.NewDirStore(lifecycleHome + "/deployment") key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Revision: 1, Binding: "1", Region: "ie", FleetHomeRegion: "ie", ControlBucket: "deployment"}) identity.HomeRegion = "ie" actor, err := identity.Verify(t.Context()) lifecycleCheck(t, "identity", err) if !resume { roster := hostactors.Roster{V: 1, HomeRegion: "ie", DeploymentBucket: actor.ControlBucket, Revision: 1, RequestLimitNS: int64(hostbox.RequestLimit), Boxes: map[string]hostactors.Box{actor.Box: {BoxID: actor.Box, Enrollment: actor.Enrollment, Namespace: actor.Namespace, Label: actor.Box, Region: actor.Region, Pool: "default", Cloud: hostactors.Cloud{Provider: actor.Cloud.Provider, Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), CensusPublicKey: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize)).Public().(ed25519.PublicKey), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} raw, err := release.SignRootEnvelope(release.RootSigner(key), roster) lifecycleCheck(t, "sign roster", err) _, err = store.Put(t.Context(), hostactors.Key, raw, "") lifecycleCheck(t, "roster", err) filesSeedConstructionMigration(t, "/opt/carlos", actor.ControlBucket, actor.Box, actor.Enrollment) } binary, err := os.Executable() lifecycleCheck(t, "test binary", err) image, err := os.ReadFile(binary) lifecycleCheck(t, "test image", err) if !resume { lifecycleCheck(t, "initial version namespace", os.MkdirAll("/opt/carlos/versions", 0755)) lifecycleCheck(t, "executor", os.WriteFile("/opt/carlos/carlos", image, 0755)) lifecycleCheck(t, "previous executor", os.WriteFile("/opt/carlos/carlos.prev", image, 0755)) } sum := sha256.Sum256(image) platform := runtime.GOOS + "-" + runtime.GOARCH pointer := release.SystemPointer{Schema: 1, Component: "router", Channel: "stable", Spread: "0s", PromotedAt: time.Now(), Current: release.SystemRelease{Version: "v2.0.0", ReleaseTag: "v2.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{platform: {Key: release.SystemArtifactKey("v2.0.0", platform), SHA256: hex.EncodeToString(sum[:])}}}} if !resume { signed, err := release.SignRootEnvelope(release.RootSigner(key), pointer) lifecycleCheck(t, "sign executor", err) identities, err := rootstate.Open("/opt/carlos/system/guard/identities", false, true) lifecycleCheck(t, "executor identities", err) lifecycleCheck(t, "executor identity", identities.Write("v2.0.0.json", signed)) identities.Close() } lifecycleArmGuard(t, store, key, actor, pointer, resume) if !resume { state, err := rootstate.Open(rootparticipation.Namespace, true, false) lifecycleCheck(t, "census state", err) census := rootparticipation.Census{Schema: 1, Box: actor.Box, Enrollment: actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true} raw, err := json.Marshal(census) lifecycleCheck(t, "census encode", err) lifecycleCheck(t, "census", state.Write("census.json", raw)) state.Close() failures, err := rootstate.Open(rollout.RootFailuresDir, true, false) lifecycleCheck(t, "failure state", err) _, err = rollout.InitializeRootFailures(failures, rootUpdatesMigration(census), nil) lifecycleCheck(t, "initialize migration failures", err) failures.Close() pins, err := rootstate.Open(release.RootGenerationsDir, true, false) lifecycleCheck(t, "generation store", err) pins.Close() } now := time.Now().UTC() rec := provision.Record{Schema: 1, Host: lifecycleHost, Account: "bab", App: "lifecycle", Channel: "stable", Region: "", CreatedAt: now} completeCreate := func() error { return nil } if !resume { completeCreate = lifecycleCreate(t, store, key, actor, pointer, rec) manifest := release.Manifest{App: rec.App, Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} lifecycleCheck(t, "ship", release.Ship(t.Context(), store, rec.Account, manifest, map[string][]byte{release.AdoptPlatform(): image})) lifecycleCheck(t, "promote", release.Promote(t.Context(), store, key, rec.Account, rec.App, "stable", "v1")) } else { found, _, e := provision.FindRecordByHost(t.Context(), store, lifecycleHost) lifecycleCheck(t, "retained record", e) rec = *found } withSibling := phase == "1" || phase == "2" || phase == "crash" var completeSibling func() error createSibling := func() { if !withSibling || resume { return } if completeSibling == nil { sibling := provision.Record{Schema: 1, Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable", CreatedAt: time.Now().UTC()} completeSibling = lifecycleCreate(t, store, key, actor, pointer, sibling) manifest := release.Manifest{App: "sibling", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} lifecycleCheck(t, "ship active sibling", release.Ship(t.Context(), store, "bab", manifest, map[string][]byte{release.AdoptPlatform(): image})) lifecycleCheck(t, "promote active sibling", release.Promote(t.Context(), store, key, "bab", "sibling", "stable", "v1")) _, _, e := release.StageSignedHead(t.Context(), store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable"}) lifecycleCheck(t, "signed sibling stage", e) lifecycleCheck(t, "new deployment owner", filepath.Walk(lifecycleHome+"/deployment", func(path string, info os.FileInfo, e error) error { if e != nil { return e } return os.Chown(path, uid, gid) })) } if err := completeSibling(); errors.Is(err, hostops.ErrPending) { return } else { lifecycleCheck(t, "complete populated-box create", err) } lifecycleCheck(t, "sibling create ack", os.WriteFile("/run/carlos/lifecycle-sibling-created", []byte("ready"), 0644)) } // Only deployment inputs are service-owned. Root authority remains private. lifecycleCheck(t, "deployment owner", filepath.Walk(lifecycleHome+"/deployment", func(path string, info os.FileInfo, err error) error { if err != nil { return err } return os.Chown(path, uid, gid) })) ctx := t.Context() lifecycleInstallUnits(t) out, mountErr := exec.Command("systemctl", "daemon-reload").CombinedOutput() lifecycleCheck(t, string(out), mountErr) out, mountErr = exec.Command("systemctl", "start", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount").CombinedOutput() lifecycleCheck(t, string(out), mountErr) recoverControl := lifecycleStartControl(t, server.URL) lifecycleStartAgent(t, server.URL, completeCreate, createSibling, func() { _, _, err = release.StageSignedHead(ctx, store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: rec.Host, Account: rec.Account, App: rec.App, Channel: "stable"}) lifecycleCheck(t, "signed stage", err) lifecycleCheck(t, "point initial version", release.PointLive("/opt/carlos", rec.Host, rec.Account, "v1", rec.App)) if withSibling && resume { _, _, err := release.StageSignedHead(ctx, store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable"}) lifecycleCheck(t, "signed sibling stage", err) if resume { lifecycleCheck(t, "sibling live", release.PointLive("/opt/carlos", lifecycleSibling, "bab", "v1", "sibling")) } } }, lifecycleDeleteDriver(t, phase, store, key, actor, pointer), recoverControl) if phase == "1" { lifecycleSaveBucket(t, model) } if phase == "2" || phase == "crash" { lifecycleVerifyRetiredPins(t, store, key) lifecycleReinstall(t, store, key, actor, pointer) lifecycleCheck(t, "release agent", os.WriteFile("/run/carlos/lifecycle-finish", []byte("ready"), 0644)) deadline := time.Now().Add(time.Minute) for { state, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output() if strings.TrimSpace(string(state)) == "inactive" { break } if strings.TrimSpace(string(state)) == "failed" || time.Now().After(deadline) { log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "80").CombinedOutput() t.Fatalf("agent completion: %s", log) } time.Sleep(20 * time.Millisecond) } if phase == "crash" { if _, err := os.Stat(lifecycleCrashPath); err != nil { t.Fatal("crash point was not reached", err) } } } } func lifecycleStartAgent(t *testing.T, endpoint string, completeCreate func() error, createSibling func(), stage func(), deletion func(), recoverControl func()) { t.Helper() unit, err := unitsFS.ReadFile("units/carlos-edge.service") lifecycleCheck(t, "edge template", err) var lines []string for _, line := range strings.Split(string(unit), "\n") { key, _, _ := strings.Cut(line, "=") switch key { case "ExecReload", "Restart": continue } lines = append(lines, line) } lines = append(lines, "[Service]", "Restart=no", "Environment=PLAN14_LIFECYCLE_AGENT=1", "Environment=PLAN14_LIFECYCLE_PHASE="+os.Getenv("PLAN14_LIFECYCLE_PHASE"), "Environment=PLAN14_LIFECYCLE_DATA_ENDPOINT="+endpoint, "Environment=TMPDIR=/run/carlos", "ReadWritePaths="+lifecycleHome) path := "/run/systemd/system/carlos-edge.service" f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) lifecycleCheck(t, "edge exclusive", err) _, err = f.WriteString(strings.Join(lines, "\n")) f.Close() lifecycleCheck(t, "edge write", err) t.Cleanup(func() { exec.Command("systemctl", "stop", "carlos-edge.service").Run() os.Remove(path) exec.Command("systemctl", "daemon-reload").Run() }) out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput() lifecycleCheck(t, string(out), err) out, err = exec.Command("systemctl", "start", "carlos-edge.service").CombinedOutput() lifecycleCheck(t, string(out), err) deadline := time.Now().Add(3 * time.Minute) staged, created, recovered := false, false, false for time.Now().Before(deadline) { if _, err := os.Stat(lifecycleCrashPath); err == nil && !recovered { recoverControl() until := time.Now().Add(110 * time.Second) for { state, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output() if strings.TrimSpace(string(state)) == "failed" || strings.TrimSpace(string(state)) == "inactive" { break } if time.Now().After(until) { lifecycleCrashDiagnostic(t) t.Fatal("edge did not finish cleanup after controller crash") } time.Sleep(100 * time.Millisecond) } os.Remove("/run/carlos/lifecycle-result") if os.Getenv("PLAN14_LIFECYCLE_CRASH") == "retirement" { lifecycleCheck(t, "resume retirement checkpoint", os.WriteFile("/run/carlos/lifecycle-resume-retirement", []byte("ready"), 0644)) } recoverControl() out, err := exec.Command("systemctl", "start", "carlos-edge.service").CombinedOutput() lifecycleCheck(t, string(out), err) recovered = true t.Log("controller and edge restarted after", os.Getenv("PLAN14_LIFECYCLE_CRASH")) } if _, err := os.Stat("/run/carlos/lifecycle-create-sibling"); err == nil { createSibling() } if _, err := os.Stat("/run/carlos/lifecycle-sibling-installed"); err == nil { lifecycleCheck(t, "sibling live", release.PointLive("/opt/carlos", lifecycleSibling, "bab", "v1", "sibling")) } if !created { err := completeCreate() if err == nil { created = true lifecycleCheck(t, "create ack", os.WriteFile("/run/carlos/lifecycle-created", []byte("ready"), 0644)) } else if !errors.Is(err, hostops.ErrPending) { lifecycleCheck(t, "complete create", err) } } if !staged { if _, err := os.Stat("/run/carlos/lifecycle-stage"); err == nil { stage() lifecycleCheck(t, "stage ack", os.WriteFile("/run/carlos/lifecycle-staged", []byte("ready"), 0644)) staged = true } } if _, err := os.Stat("/run/carlos/lifecycle-delete"); err == nil { deletion() } raw, err := os.ReadFile("/run/carlos/lifecycle-result") if err == nil { if string(raw) != "pass" { log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager").CombinedOutput() t.Fatalf("%s: %s", raw, log) } log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "100").CombinedOutput() t.Logf("completed agent: %s", log) return } status, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output() if strings.TrimSpace(string(status)) == "failed" { break } time.Sleep(100 * time.Millisecond) } log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "100").CombinedOutput() t.Fatalf("agent did not complete: %s", log) } func lifecycleAgent(t *testing.T) { lifecycleStackTrace(t) deadlineReady := time.Now().Add(3 * time.Minute) for { if _, err := os.Stat("/run/carlos-control/lifecycle-ready"); err == nil { break } if time.Now().After(deadlineReady) { t.Fatal("controller activation timeout") } time.Sleep(20 * time.Millisecond) } lifecycleEnvironment(t, os.Getenv("CARLOS_S3_ENDPOINT")) completed := false defer func() { result := "agent failed; inspect journal" if completed && !t.Failed() { result = "pass" } os.WriteFile("/run/carlos/lifecycle-result", []byte(result), 0600) }() store := release.NewDirStore(lifecycleHome + "/deployment") reg, err := registry.Open("/data/registry.db") lifecycleCheck(t, "agent registry", err) defer reg.Close() manager := cgroup.SetupRetaining(t.Logf) resources := ClaimRuntimeResources{Mode: "agent", Registry: reg, Manager: manager, DataDir: "/data", OptDir: "/opt/carlos", UnitsDir: "/etc/systemd/system"} ctx := context.WithValue(t.Context(), claimEntryKey{}, claimEntry{Enabled: true, FlagRead: true, Store: store, Resources: resources}) assembly, err := OpenProductionClaimAssembly(ctx, store, resources, lifecycleOwners{t}) lifecycleCheck(t, "agent assembly", err) defer func() { cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() for { t.Log("agent assembly close enter") err := assembly.Close(cleanup) t.Log("agent assembly close returned", err) if err == nil { return } if cleanup.Err() != nil { t.Error("agent cleanup", err) return } time.Sleep(100 * time.Millisecond) } }() providers, err := assembly.Build(ctx) lifecycleCheck(t, "agent build", err) lifecycleTraceRegional(t, &providers) oldPrerequisites := providers.Wake.Prerequisites providers.Wake.Prerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) error { err := oldPrerequisites(ctx, rt, o, policy) if err != nil && strings.Contains(err.Error(), "no restorable data") { h := o.Claim.Holder if assembly.Guard.FenceView != nil { view, e := assembly.Guard.FenceView(ctx, o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) t.Logf("empty recovery root view %+v error=%v", view, e) for _, r := range view.Receipts { t.Logf("empty recovery receipt target=%+v body=%s", r.Target, r.Bytes) } } launches, e := assembly.Launches.HostLaunches(ctx, o.Host) t.Logf("empty recovery launches %+v error=%v", launches, e) } return err } lifecycleTraceWake(t, assembly.Wake) deadline := time.Now().Add(time.Minute) for { if _, err := os.Stat("/run/carlos/lifecycle-created"); err == nil { break } if time.Now().After(deadline) { t.Fatal("create acknowledgement timeout") } time.Sleep(20 * time.Millisecond) } cfg := provision.ReconcileConfig{Region: "", DataDir: "/data", Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }} providers.Provision(&cfg) resumingRetirement := false if _, e := os.Stat("/run/carlos/lifecycle-resume-retirement"); e == nil { obs, e := hostclaim.Read(ctx, store, lifecycleHost) lifecycleCheck(t, "retirement recovery claim", e) if obs.Claim == nil || (!obs.Claim.Free() && obs.Claim.Op.Kind != hostclaim.KindParking) { t.Fatal("retirement recovery lost parking claim") } _, _, e = store.Get(ctx, provision.RecordKey("bab", "lifecycle", lifecycleHost)) if !errors.Is(e, os.ErrNotExist) { t.Fatal("retirement recovery record still present", e) } resumingRetirement = true } lifecycleCheck(t, "provision", provision.Reconcile(ctx, store, reg, cfg)) if !resumingRetirement { route, err := reg.Get(lifecycleHost) lifecycleCheck(t, "route", err) if route == nil { t.Fatal("create did not install route") } lifecycleCheck(t, "request staging", os.WriteFile("/run/carlos/lifecycle-stage", []byte("ready"), 0600)) stageDeadline := time.Now().Add(time.Minute) for { if _, err := os.Stat("/run/carlos/lifecycle-staged"); err == nil { break } if time.Now().After(stageDeadline) { t.Fatal("stage timeout") } time.Sleep(20 * time.Millisecond) } wakes := 2 if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" { wakes = 1 } for wake := 0; wake < wakes; wake++ { route, err = reg.Get(lifecycleHost) lifecycleCheck(t, "fresh route for wake", err) if route == nil { t.Fatal("route disappeared before wake") } finish := lifecycleLaunch(t, ctx, providers.Wake, assembly.Wake, *route) if wake == 0 && (os.Getenv("PLAN14_LIFECYCLE_PHASE") == "1" || os.Getenv("PLAN14_LIFECYCLE_PHASE") == "crash") { lifecycleCheck(t, "request populated-box create", os.WriteFile("/run/carlos/lifecycle-create-sibling", []byte("ready"), 0644)) until := time.Now().Add(time.Minute) for { if _, e := os.Stat("/run/carlos/lifecycle-sibling-created"); e == nil { break } if time.Now().After(until) { t.Fatal("populated-box creation timeout") } lifecycleCheck(t, "populated-box reconcile", provision.Reconcile(ctx, store, reg, cfg)) time.Sleep(20 * time.Millisecond) } lifecycleCheck(t, "provision sibling while main active", provision.Reconcile(ctx, store, reg, cfg)) sibling, e := reg.Get(lifecycleSibling) lifecycleCheck(t, "new sibling row", e) if sibling == nil { t.Fatal("populated-box creation has no route") } lifecycleCheck(t, "sibling installed", os.WriteFile("/run/carlos/lifecycle-sibling-installed", []byte("ready"), 0644)) } finish() t.Logf("completed cold wake %d", wake+1) } } if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" || os.Getenv("PLAN14_LIFECYCLE_PHASE") == "crash" { if !resumingRetirement { parkDeadline := time.Now().Add(time.Minute) for { obs, err := hostclaim.Read(ctx, store, lifecycleHost) lifecycleCheck(t, "pre-delete observation", err) if len(obs.Claim.Obligations) == 0 { break } if time.Now().After(parkDeadline) { lifecycleTraceParkState(t, providers, obs) t.Fatal("pre-delete obligations did not drain", obs.Claim.Obligations) } if err := providers.Regional.(hostbox.RegionalWorker).Run(ctx); err != nil { t.Log("pre-delete parks", err) } time.Sleep(100 * time.Millisecond) } } var finishSibling func() { sibling, err := reg.Get(lifecycleSibling) lifecycleCheck(t, "sibling route", err) if sibling == nil { t.Fatal("missing sibling installation") } finishSibling = lifecycleLaunch(t, ctx, providers.Wake, assembly.Wake, *sibling) lifecycleCheck(t, "active sibling ready", os.WriteFile("/run/carlos/lifecycle-sibling", []byte("active"), 0644)) } lifecycleCheck(t, "request deletion", os.WriteFile("/run/carlos/lifecycle-delete", []byte("ready"), 0600)) deadline := time.Now().Add(time.Minute) for time.Now().Before(deadline) { if e := provision.Reconcile(ctx, store, reg, cfg); e != nil { t.Log("retirement reconcile", e) } if e := providers.Regional.(hostbox.RegionalWorker).Run(ctx); e != nil { t.Log("regional retirement", e) } obs, e := hostclaim.Read(ctx, store, lifecycleHost) lifecycleCheck(t, "retirement observation", e) if obs.Claim.Free() { lifecycleCheck(t, "agent checkpoint", os.WriteFile("/run/carlos/lifecycle-result", []byte("pass"), 0600)) lifecycleAgentReinstall(t, ctx, store) if finishSibling != nil { finishSibling() } completed = true return } time.Sleep(100 * time.Millisecond) } t.Fatal("retirement did not complete") } completed = true } func lifecycleWake(t *testing.T, ctx context.Context, runtime *activator.ClaimRuntime, wake *ClaimWakeAssembly, route registry.Route) { lifecycleLaunch(t, ctx, runtime, wake, route)() } func lifecycleLaunch(t *testing.T, ctx context.Context, runtime *activator.ClaimRuntime, wake *ClaimWakeAssembly, route registry.Route) func() { t.Helper() observed, err := runtime.Read(ctx, route.Host) lifecycleCheck(t, "read for wake", err) policy := activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second} prerequisiteDeadline := time.Now().Add(2 * time.Minute) for { e := runtime.Prerequisites(ctx, route, observed, policy) if e == nil { break } fresh, readErr := runtime.Read(ctx, route.Host) retry := errors.Is(e, hostbox.ErrChanged) && readErr == nil && fresh.ETag != observed.ETag if errors.Is(e, hostclaim.ErrWrongState) && readErr == nil && fresh.Claim.Holder.WakeRetry != nil && fresh.Claim.Holder.WakeRetry.State == "open" { retry = true } if !retry || time.Now().After(prerequisiteDeadline) { t.Logf("prerequisite state: holder=%+v route=%+v", observed.Claim.Holder, route) lifecycleCheck(t, "wake prerequisites", e) } observed = fresh time.Sleep(100 * time.Millisecond) } admitted, err := runtime.Admit(ctx, route, observed, policy) lifecycleCheck(t, "cold admission", err) start, err := runtime.Prepare(ctx, route, admitted, policy) lifecycleCheck(t, "wake prepare", err) lifecycleCheck(t, "restore", start.Restore(ctx)) lifecycleCheck(t, "reserve", start.Reserve(ctx, 0)) lifecycleCheck(t, "materialize", start.Materialize(ctx)) launch, err := start.PrepareLaunch(ctx) lifecycleCheck(t, "launch preparation", err) cmd, _, settle, producer, err := launch.Start(ctx, "main", start.CurrentAttempt(), start.Build, func() error { return ctx.Err() }) if err != nil && settle != nil { cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) cleanupErr := lifecycleRetry(cleanup, settle) cancel() t.Log("failed launch settlement", cleanupErr) } lifecycleCheck(t, "root launch", err) var stopMu sync.Mutex stopped := make(chan struct{}) stop := func(call context.Context) error { stopMu.Lock() defer stopMu.Unlock() select { case <-stopped: return nil default: } if err := settle(call); err != nil { return err } if cmd.Process != nil { _ = cmd.Process.Release() } close(stopped) return nil } if producer.Context != nil { go func() { select { case <-stopped: return case <-producer.Context.Done(): } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() _ = lifecycleRetry(cleanup, stop) }() } lifecycleCheck(t, "tenant ready", start.Ready(ctx, cmd)) prefix, err := hostbox.GenerationPrefix(*admitted.Claim.Holder) lifecycleCheck(t, "generation prefix", err) rep, err := start.Replicate(ctx, prefix) lifecycleCheck(t, "replicate", err) current, err := runtime.Read(ctx, route.Host) lifecycleCheck(t, "read publication", err) _, err = start.Publish(ctx, current) lifecycleCheck(t, "publish", err) lifecycleCheck(t, "healthy", start.Healthy(ctx, 0, cmd)) transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", route.Socket) }} defer transport.CloseIdleConnections() response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://tenant/") lifecycleCheck(t, "serve", err) raw, err := io.ReadAll(response.Body) response.Body.Close() lifecycleCheck(t, "read tenant", err) if response.StatusCode != 200 || string(raw) != "lifecycle tenant" { t.Fatalf("tenant response %d %q", response.StatusCode, raw) } return func() { lifecycleCheck(t, "stop tenant", lifecycleRetry(ctx, stop)) lifecycleCheck(t, "stop replication", rep.Close(ctx)) current, err = runtime.Read(ctx, route.Host) lifecycleCheck(t, "read stopped session", err) lifecycleCheck(t, "stop session", lifecycleRetry(ctx, func(ctx context.Context) error { return wake.Stop(ctx, current) })) } } func lifecycleInstallUnits(t *testing.T) { t.Helper() for _, name := range []string{"carlos.slice", "carlos-instances.slice", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-instance@.service", "carlos-system-recover.service"} { raw, err := unitsFS.ReadFile("units/" + name) lifecycleCheck(t, "unit template", err) path := filepath.Join("/etc/systemd/system", name) f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) lifecycleCheck(t, "exclusive unit", err) _, err = f.Write(raw) f.Close() lifecycleCheck(t, "install unit", err) t.Cleanup(func() { exec.Command("systemctl", "stop", name).Run() os.Remove(path) exec.Command("systemctl", "daemon-reload").Run() }) } t.Cleanup(func() { exec.Command("systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, lifecycleSibling)).Run() exec.Command("systemctl", "stop", instancectl.SliceName(lifecycleSibling)).Run() exec.Command("systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, lifecycleHost)).Run() exec.Command("systemctl", "stop", instancectl.SliceName(lifecycleHost)).Run() for _, name := range []string{instancectl.UnitName(instancectl.RoleRun, lifecycleHost), instancectl.SliceName(lifecycleHost), instancectl.UnitName(instancectl.RoleRun, lifecycleSibling), instancectl.SliceName(lifecycleSibling)} { os.RemoveAll(filepath.Join("/run/systemd/system", name+".d")) os.RemoveAll(filepath.Join("/run/systemd/system.control", name+".d")) } }) } type lifecycleActorEvidence struct { box, console hostactors.Report identity []byte } func (e lifecycleActorEvidence) BoxReport(context.Context, string, hostactors.Box) (hostactors.Report, error) { return e.box, nil } func (e lifecycleActorEvidence) ConsoleReport(context.Context, string, hostactors.Console) (hostactors.Report, error) { return e.console, nil } func (e lifecycleActorEvidence) LocalConsole(context.Context) (hostactors.Report, error) { return e.console, nil } func (e lifecycleActorEvidence) Identity(context.Context, string) ([]byte, error) { return e.identity, nil } func (e lifecycleActorEvidence) Fence(context.Context, hostactors.Observed, string, time.Time) (bool, error) { return false, nil } func lifecycleCreate(t *testing.T, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer, rec provision.Record) func() error { t.Helper() ctx := t.Context() signer := release.RootSigner(key) directory := hostactors.Directory{Store: raw, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket} lifecycleCheck(t, "publish test deployment floor", hostfloor.Publish(ctx, raw, signer, actor.ControlBucket, hostfloor.Floor{Min: 1, MinRelease: "v2.0.0"})) floor, err := hostfloor.Read(ctx, raw, directory.PublicKey, actor.ControlBucket) lifecycleCheck(t, "read deployment floor", err) roster, err := directory.Read(ctx) lifecycleCheck(t, "read actor roster", err) if roster.Roster.Publication != "published" { roster.Roster.Publication, roster.Roster.FloorSHA256 = "published", floor.Digest roster.Roster.Regions = map[string]hostactors.Region{actor.Region: {Account: actor.Cloud.Account, AWSRegion: actor.Cloud.Region, Namespaces: []string{actor.Namespace}, Pools: []string{"default"}}} roster.Roster.Consoles["console"] = hostactors.Console{Enrollment: strings.Repeat("d", 32), InstanceID: "console", Namespaces: []string{actor.Namespace}, DeploymentBucket: actor.ControlBucket} _, err = directory.Replace(ctx, signer, roster, roster.Roster) lifecycleCheck(t, "publish actors", err) } controller, authority, control := lifecycleController(t, raw, key, actor, pointer) data := controller.Store appID, err := control.EnrollNewApp(ctx, rec.Account, rec.App) lifecycleCheck(t, "enroll app", err) before, err := hostclaim.Read(ctx, data, rec.Host) lifecycleCheck(t, "read before registration", err) id := "create-" + rec.App start, err := control.RegisterStart(ctx, rec.Account, rec.App, appID, id, hostops.Registration{Kind: "create", Host: rec.Host, Owner: hostops.RegistrationOwner{Console: "console", Enrollment: strings.Repeat("d", 32), Controller: "console-controller"}, AppID: appID, Epoch: 1, Claim: hostops.SavedObservation(before)}, time.Now()) lifecycleCheck(t, "register create", err) rec.Acquisition = id requested := rec requested.CreatedAt = time.Time{} record, err := hostops.PrepareInstance(requested, rec.CreatedAt, park.PlacementRegion(rec.Region)) lifecycleCheck(t, "prepare create record", err) incarnation, err := record.Incarnation() lifecycleCheck(t, "create incarnation", err) lifecycleCheck(t, "bind registration", start.Bind(ctx, rec.CreatedAt, incarnation, time.Now())) createCtx := hostops.WithStartHandle(start.Lifecycle(ctx), start) observed, err := authority.StartCreate(createCtx, before, record, id, "runner") lifecycleCheck(t, "production begin create", err) lifecycleCheck(t, "production record effects", authority.RecordEffects(createCtx, observed, record)) created := false return func() error { if created { return nil } _, err := controller.Complete(createCtx, rec.Host, id) if err != nil { return err } err = control.SettleRegistration(ctx, rec.Account, rec.App, appID, id, time.Now()) created = err == nil return err } } func TestPlan14LifecycleCreateController(t *testing.T) { store := release.NewDirStore(t.TempDir()) key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) actor := hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Namespace: "platform", Region: "ie", ControlBucket: "deployment", Cloud: hostbox.CloudIdentity{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}} roster := hostactors.Roster{V: 1, HomeRegion: "ie", DeploymentBucket: actor.ControlBucket, Revision: 1, Boxes: map[string]hostactors.Box{actor.Box: {BoxID: actor.Box, Enrollment: actor.Enrollment, Namespace: actor.Namespace, Label: actor.Box, Region: actor.Region, Pool: "default", Cloud: hostactors.Cloud{Provider: actor.Cloud.Provider, Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} raw, err := release.SignRootEnvelope(release.RootSigner(key), roster) lifecycleCheck(t, "sign roster", err) _, err = store.Put(t.Context(), hostactors.Key, raw, "") lifecycleCheck(t, "write roster", err) platform := release.AdoptPlatform() pointer := release.SystemPointer{Schema: 1, Component: "router", Channel: "stable", Spread: "0s", PromotedAt: time.Now(), Current: release.SystemRelease{Version: "v2.0.0", ReleaseTag: "v2.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{platform: {Key: release.SystemArtifactKey("v2.0.0", platform), SHA256: strings.Repeat("a", 64)}}}} lifecycleCreate(t, store, key, actor, pointer, provision.Record{Schema: 1, Host: lifecycleHost, Account: "bab", App: "lifecycle", Channel: "stable", Region: "", CreatedAt: time.Now().UTC()}) record, err := hostops.ReadRecord(t.Context(), store, provision.RecordKey("bab", "lifecycle", lifecycleHost)) lifecycleCheck(t, "read created record", err) if record.Holder.Acquisition == "" { t.Fatal("create did not retain acquisition") } } func lifecycleController(t *testing.T, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer) (hostops.Controller, *hostops.ProductionAuthority, *hostops.ControlWriter) { t.Helper() ctx := t.Context() signer := release.RootSigner(key) directory := hostactors.Directory{Store: raw, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket} floor, err := hostfloor.Read(ctx, raw, directory.PublicKey, actor.ControlBucket) lifecycleCheck(t, "controller floor", err) platform := release.AdoptPlatform() pointer.Current.ConsoleArtifacts = map[string]release.SystemArtifact{platform: {Key: "bootstrap/console/v2.0.0/carlos-" + platform, SHA256: pointer.Current.Artifacts[platform].SHA256}} signed, err := release.SignRootEnvelope(signer, pointer) lifecycleCheck(t, "sign actor release", err) report := hostactors.Report{Component: "router", GeneratedAt: time.Now(), Actor: actor.Box, Enrollment: actor.Enrollment, InstanceID: actor.Cloud.InstanceID, Release: "v2.0.0", ReleaseTag: "v2.0.0", Platform: platform, BinarySHA256: pointer.Current.Artifacts[platform].SHA256, MovesProtocol: 1, BootID: "external-box-boot", FloorSHA256: floor.Digest, FloorGuardReady: true, HostClaimEnabled: true} console := report console.Component, console.Actor, console.InstanceID, console.Enrollment, console.ProcessID = "console", "console", "console", strings.Repeat("d", 32), "console-process" control, err := hostops.NewControlWriter(raw, directory.PublicKey, time.Now) lifecycleCheck(t, "control writer", err) check, err := hostops.NewProductionMutationCheck(raw, control) lifecycleCheck(t, "mutation check", err) records, err := hostops.NewConditionalRecordStore(raw, directory.PublicKey) lifecycleCheck(t, "record boundary", err) data, err := hostops.NewObservedMutationBoundary(true, records, check, nil) lifecycleCheck(t, "mutation boundary", err) directory.Store = data authority := &hostops.ProductionAuthority{HomeRegion: "ie", Store: data, Control: control, Directory: directory, ConsoleID: "console", Enrollment: console.Enrollment, Namespace: actor.Namespace, Signer: signer, Now: time.Now, Evidence: lifecycleActorEvidence{report, console, signed}} controller := hostops.Controller{Store: data, Control: control, Authority: authority, Now: time.Now, NewID: func() string { var id [24]byte if _, err := rand.Read(id[:]); err != nil { t.Fatal(err) } return hex.EncodeToString(id[:]) }} return controller, authority, control } func TestPlan14LifecycleControlProcessPayload(t *testing.T) { lifecycleStackTrace(t) if os.Getenv("PLAN14_LIFECYCLE_CONTROL") != "1" { t.Fatal("controller helper") } lifecycleEnvironment(t, os.Getenv("CARLOS_S3_ENDPOINT")) _, gid := generationRunnerRegistryUser(t) store := release.NewDirStore(lifecycleHome + "/deployment") lifecycleControlFaults(t) reg, err := openServiceRegistry("/data/registry.db") lifecycleCheck(t, "registry", err) t.Cleanup(func() { reg.Close() }) resources := ClaimRuntimeResources{Mode: "instance-control", Registry: reg, DataDir: "/data", OptDir: "/opt/carlos", UnitsDir: "/etc/systemd/system"} ctx := context.WithValue(t.Context(), claimEntryKey{}, claimEntry{Enabled: true, FlagRead: true, Store: store, Resources: resources}) assembly, err := OpenProductionClaimAssembly(ctx, store, resources, lifecycleOwners{t}) lifecycleCheck(t, "root assembly", err) t.Cleanup(func() { assembly.Close(context.Background()) }) ctx = WithClaimAssembly(ctx, assembly) lifecycleCheck(t, "launch verifier", registerProductionLaunchVerifier()) root, err := NewProductionRootClaimStop(ctx) lifecycleCheck(t, "root constructor", err) wrapped, err := WithProductionClaimLaunch(ctx, root) lifecycleCheck(t, "launch constructor", err) rootHTTP := wrapped.(ClaimRootWithLaunch) // lifecycleOwners opens no cleanup enrollment, so production wiring // leaves the stop numeral bound unset (every stop refuses); the fixture // bounds by its identity's roster revision instead. rootHTTP.Root.Stop.Coordinator.VerifiedRosterRevision = func(ctx context.Context) (uint64, error) { actor, err := assembly.Identity.Verify(ctx) return actor.Revision, err } rootHTTP.Root.Stop.Coordinator.Kernel = &lifecycleStopTrace{claimUnitStopKernel: rootHTTP.Root.Stop.Coordinator.Kernel.(claimUnitStopKernel), t: t} rootHTTP.Launch.Kernel = lifecycleKernelTrace{rootHTTP.Launch.Kernel, t} rootHTTP.Launch.Units = lifecycleUnitsTrace{rootHTTP.Launch.Units, t} begin := rootHTTP.Launch.generations.beginUpdate rootHTTP.Launch.generations.beginUpdate = func(ctx context.Context, s *rootGenerationSource, b release.GenerationBinding) (func(string, error, func() error) error, func() error, error) { lifecycleCrash(t, "reservation") finish, close, err := begin(ctx, s, b) t.Log("generation update opener", err) return finish, close, err } t.Cleanup(func() { rootHTTP.Root.executionRuntime.Close(context.Background()) }) deadline := time.Now().Add(time.Minute) for { out, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "MainPID", "--value").Output() if strings.TrimSpace(string(out)) != "" && strings.TrimSpace(string(out)) != "0" { break } if time.Now().After(deadline) { t.Fatal("edge startup timeout") } time.Sleep(20 * time.Millisecond) } if rootHTTP.Root.servingParks != nil { if p, ok := rootHTTP.Root.servingParks.authority.(*servingParkRoot); ok { fs := p.ledger.fs.(*servingParkFS) for _, name := range []string{"namespace-parent.json", "namespace.json", "current-namespace.json"} { raw, err := p.ledger.state.Read(name) t.Log("namespace before activation", name, string(raw), err) } parent, id, e := fs.parent() t.Log("namespace current parent", id, e) if e == nil { ph, pe := fs.handle(parent) t.Log("namespace parent proof", ph, pe) d, did, de := fs.privateDir(parent, ".serving-parked") t.Log("namespace current root", did, de) if de == nil { dh, he := fs.handle(d) t.Log("namespace root proof", dh, he) d.Close() } parent.Close() } } } startup, finishStartup := context.WithTimeout(ctx, 2*time.Minute) defer finishStartup() if err := activateClaimRoot(ctx, startup, rootHTTP); err != nil { lifecycleActivationDiagnostic(t, ctx) lifecycleCheck(t, "production activation gate", err) } finishStartup() listener, err := net.Listen("unix", controlsock.Path) lifecycleCheck(t, "root socket", err) lifecycleCheck(t, "root socket mode", os.Chmod(controlsock.Path, 0660)) lifecycleCheck(t, "root socket group", os.Chown(controlsock.Path, 0, gid)) controlCtx, cancelControl := context.WithCancel(ctx) defer cancelControl() control := &instancectl.Server{ClaimStop: lifecycleControlTrace{rootHTTP, t}, PeerOK: func(c net.Conn) error { _, err := rootHTTP.Root.Stop.Authenticate(ctx, c); return err }, Logf: t.Logf} done := make(chan error, 1) go func() { done <- control.Serve(controlCtx, listener) }() defer func() { cancelControl(); lifecycleCheck(t, "control shutdown", <-done) }() lifecycleCheck(t, "controller ready", os.WriteFile("/run/carlos-control/lifecycle-ready", []byte("ready"), 0644)) <-ctx.Done() } func lifecycleTraceWake(t *testing.T, w *ClaimWakeAssembly) { if old := w.AfterAdmission; old != nil { w.AfterAdmission = func(ctx context.Context, p, o hostclaim.Observed) error { e := old(ctx, p, o) t.Log("after admission", e) return e } } if old := w.Prelaunch; old != nil { w.Prelaunch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) (*activator.RoleLaunch, error) { l, e := old(ctx, o, work, in) t.Log("prelaunch", e) return l, e } } if dispatch, ok := w.Inputs.(ClaimOwnerDispatch); ok { if serving, ok := dispatch.Serving.Inputs.(*claimServingOwners); ok && serving.emptyRecovery != nil { check := serving.emptyRecovery serving.emptyRecovery = func(ctx context.Context, o hostclaim.Observed, a []hostclaim.AdmissionRecord) (bool, error) { yes, err := check(ctx, o, a) t.Logf("empty recovery holder=%+v admissions=%+v yes=%v err=%v", o.Claim.Holder, a, yes, err) return yes, err } if f, ok := serving.Sources.Files.(*movesFiles); ok { f.emptyRecovery = serving.emptyRecovery } } } if old := w.ServingPrerequisites; old != nil { w.ServingPrerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed) error { e := old(ctx, rt, o) t.Log("files prerequisites", e) return e } } if old := w.recoveryPrior; old != nil { w.recoveryPrior = func(ctx context.Context, o hostclaim.Observed) (hostclaim.Observed, bool, error) { p, b, e := old(ctx, o) t.Logf("recovery prior=%+v pending=%v error=%v", p.Claim, b, e) return p, b, e } } recover := w.RecoverArtifacts if recover != nil { w.RecoverArtifacts = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { e := recover(ctx, o, in) t.Log("recover artifacts", e) return e } } prepare := w.PrepareState if prepare != nil { w.PrepareState = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) error { e := prepare(ctx, o, work, in) t.Log("prepare state", e) return e } } before := w.BeforeLegacyInstall if before != nil { w.BeforeLegacyInstall = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { e := before(ctx, o, in, work) t.Logf("legacy install empty=%v admission=%+v error=%v", in.Empty, in.Admission, e) return e } } enroll := w.BeforeInstallLocked if enroll != nil { w.BeforeInstallLocked = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { e := enroll(ctx, o, in) t.Log("enroll", e) return e } } } func lifecycleRetry(ctx context.Context, fn func(context.Context) error) error { bounded, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() for { err := fn(bounded) if err == nil { return nil } select { case <-bounded.Done(): return errors.Join(err, bounded.Err()) case <-time.After(100 * time.Millisecond): } } }