//go:build linux && rootrunner package host import ( "context" "crypto/ed25519" "encoding/json" "errors" "os" "os/exec" "path/filepath" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) func lifecycleDeleteDriver(t *testing.T, phase string, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer) func() { t.Helper() var operation *hostops.OperationHandle var controller hostops.Controller ctx := t.Context() serial := 0 return func() { if phase != "2" && phase != "crash" { t.Fatal("unexpected deletion before reboot") } if operation == nil { var control *hostops.ControlWriter controller, _, control = lifecycleController(t, raw, key, actor, pointer) admission, err := hostops.ReadCanonicalAdmission(ctx, raw, "bab", "lifecycle") lifecycleCheck(t, "delete admission", err) operation, err = control.BeginExclusive(ctx, "bab", "lifecycle", admission.Value.AppID, hostops.AppOperation{ID: "delete-lifecycle", Kind: hostops.OpDeleting, AppID: admission.Value.AppID, Intent: map[string]string{"host": lifecycleHost}}, hostops.RegistrationOwner{Console: "console", Enrollment: strings.Repeat("d", 32), Controller: "console-controller"}) lifecycleCheck(t, "exclusive deletion", err) ctx = operation.Lifecycle(ctx) } obs, err := hostclaim.Read(ctx, controller.Store, lifecycleHost) lifecycleCheck(t, "delete observation", err) if obs.Claim.Free() { return } switch obs.Claim.Op.Kind { case hostclaim.KindServing: _, err = controller.BeginRetire(ctx, obs.Claim.Holder.Key, "runner", obs.Claim.Holder.Incarnation) case hostclaim.KindDeleting: serial++ _, err = operation.Runner(time.Now).Run(ctx, hostops.Effect{ID: "retire-" + strconv.Itoa(serial), Resource: "record:" + lifecycleHost, Action: "retire"}, func(call context.Context) (string, error) { e := controller.Retire(call, lifecycleHost, obs.Claim.Op.ID) if errors.Is(e, hostops.ErrPending) { current, rerr := hostclaim.Read(call, controller.Store, lifecycleHost) if rerr == nil && current.Claim.Op.Kind == hostclaim.KindParking { return "parked", nil } return "", errors.Join(hostops.ErrNotApplied, e) } return "retired", e }) case hostclaim.KindParking: if len(obs.Claim.Obligations) == 0 { err = controller.Free(ctx, lifecycleHost, obs.Claim.Op.ID) } default: t.Fatal("unexpected retirement phase", obs.Claim.Op.Kind) } if err != nil && !errors.Is(err, hostops.ErrPending) && !errors.Is(err, hostops.ErrNotApplied) { lifecycleCheck(t, "delete transition", err) } } } func lifecycleVerifyRetiredPins(t *testing.T, store release.Store, key ed25519.PrivateKey) { t.Helper() ctx := t.Context() lifecycleActiveSibling(t) recordKey := provision.RecordKey("bab", "lifecycle", lifecycleHost) _, _, err := store.Get(ctx, recordKey) if !errors.Is(err, os.ErrNotExist) { t.Fatal("deleted record retained", err) } settleDeadline := time.Now().Add(time.Minute) for { raw, e := os.ReadFile(filepath.Join(rootparticipation.Namespace, lifecycleHost+".json")) lifecycleCheck(t, "final participation record", e) var record execlease.ExecutionRecord lifecycleCheck(t, "final participation decode", json.Unmarshal(raw, &record)) if record.Status == "settled" { break } if time.Now().After(settleDeadline) { t.Fatalf("retired participation remains %s", record.Status) } time.Sleep(100 * time.Millisecond) } journals, err := hostbox.SavedRetirementJournals(claimRetireJournalDir("", "")) lifecycleCheck(t, "final retirement journals", err) if len(journals) == 0 { t.Fatal("retirement has no durable journal") } for _, j := range journals { if j.Inventory.Host != lifecycleHost || !j.Complete { t.Fatal("retirement journal incomplete", j.ID) } for _, step := range j.Steps { if !step.Done { t.Fatal("retirement artifact incomplete", step.Artifact) } if step.Artifact.Kind == hostbox.ArtifactPath { a := step.Artifact if _, e := os.Lstat(filepath.Join("/data", a.Name)); !errors.Is(e, os.ErrNotExist) { t.Fatal("retired source remains", e) } parked := filepath.Join(".parked", j.Inventory.Captured.Incarnation, lifecycleHost, j.ID, a.Root, a.Name) id, e := hostbox.CaptureInventoryIdentity(a.Identity.MountPoint, parked) t.Logf("parked mount evidence before=%+v after=%+v", a.Identity, id) if e != nil || !a.Identity.Same(id) { t.Fatal("parked artifact lost its recorded mount or identity", e) } } } } source := productionRootGenerationPins().(generationPinSource) retire := source.retire source.retire = func(ctx context.Context, g *release.RootGenerations) error { err := retire(ctx, g) t.Log("pins retirement", err) if err != nil { lifecycleRetirementDiagnostic(t, ctx, g) lifecycleCrashDiagnostic(t) } return err } reconcile := source.reconcile source.reconcile = func(ctx context.Context, r []release.GenerationReservation) (release.GenerationEvidence, error) { e, err := reconcile(ctx, r) t.Log("pins reconciliation", err) if err != nil { inv, readErr := generationInventoryAt(ctx, rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") t.Log("pins raw inventory", readErr) inv.Handovers, readErr = productionGenerationHandovers(ctx) t.Log("pins handovers", readErr) inv.Retirements, readErr = productionGenerationRetirements(ctx) t.Log("pins retirements", readErr) _, readErr = generationRetirements(ctx, inv) t.Log("pins retirement map", readErr) _, readErr = productionGenerationRegistryInventory(ctx, inv) t.Log("pins registry", readErr) lifecycleInventoryDiagnostic(t, ctx, inv, r) } return e, err } lock, err := rootstate.Open(rootReleasePrivate, false, false) lifecycleCheck(t, "release lock", err) defer lock.Close() unknown := "plan14-pin-unknown.service" stopUnknown := func() { exec.Command("systemctl", "stop", unknown).Run() } t.Cleanup(stopUnknown) out, err := exec.Command("systemd-run", "--unit="+unknown, "--slice=carlos-instances.slice", "--property=Type=exec", "/bin/sleep", "60").CombinedOutput() lifecycleCheck(t, "unrecorded containment probe: "+string(out), err) if err := rootstate.WithRelease(ctx, lock, func() error { _, err := source.StagedPins(ctx, release.OptDir()); return err }); err == nil { t.Fatal("unrecorded managed process did not prevent generation certification") } stopUnknown() lifecycleCheck(t, "production pins after actual retirement", rootstate.WithRelease(ctx, lock, func() error { pins, err := source.StagedPins(ctx, release.OptDir()) if err != nil { return err } pinned, err := pins.Refers(filepath.Dir(release.VersionPath(release.OptDir(), "bab", "v1", "lifecycle"))) if err != nil { return err } if !pinned { t.Fatal("active sibling failed to pin the shared generation tree") } sibling, err := pins.Refers(release.VersionPath(release.OptDir(), "bab", "v1", "sibling")) if err != nil || !sibling { t.Fatal("active sibling executable is not pinned", err) } retired, err := pins.Refers(release.VersionPath(release.OptDir(), "bab", "v1", "lifecycle")) if err != nil || retired { t.Fatal("retired executable still has a direct pin", err) } return nil })) lifecycleActiveSibling(t) body := []byte("signed sibling") manifest := release.Manifest{App: "sibling", Version: "v9", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} lifecycleCheck(t, "ship sibling", release.Ship(ctx, store, "bab", manifest, map[string][]byte{release.AdoptPlatform(): body})) lifecycleCheck(t, "promote sibling", release.Promote(ctx, store, key, "bab", "sibling", "stable", "v9")) lifecycleCheck(t, "stage sibling", release.StageArtifact(release.OptDir(), "bab", "v9", "sibling", body)) path := release.StagedManifestPath(release.OptDir(), "bab", "v9", "sibling") lifecycleCheck(t, "damage sibling manifest", os.WriteFile(path, []byte("broken sibling manifest"), 0644)) lifecycleCheck(t, "repair sibling with production pins", rootstate.WithRelease(ctx, lock, func() error { proof := release.NewRootProofGate(filepath.Join(t.TempDir(), "policy.json"), filepath.Join(t.TempDir(), "bakes")) _, e := release.StageHeldVerifiedWithPins(ctx, store, key.Public().(ed25519.PublicKey), nil, release.OptDir(), registry.Route{Host: "sibling.example", Account: "bab", App: "sibling", Kind: "instance", Channel: "stable", Status: "landing", HoldOp: "repair"}, "v9", proof, source) return e })) repaired, err := os.ReadFile(path) if err != nil || string(repaired) == "broken sibling manifest" { t.Fatal("sibling repair failed", err) } _, _, err = store.Get(ctx, recordKey) if !errors.Is(err, os.ErrNotExist) { t.Fatal("record resurrected", err) } }