//go:build linux && rootrunner package host import ( "bytes" "context" "crypto/ed25519" "encoding/base64" "encoding/json" "errors" "github.com/carlosframework/platform/internal/hostfloor" "golang.org/x/sys/unix" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) func init() { if os.Getenv("PLAN14_R9_LEGACY_MUTATOR") == "1" { os.Stdout.Write([]byte("ready\n")) var b [1]byte os.Stdin.Read(b[:]) os.Exit(0) } } func TestPlan14InitialMigrationExclusionRootRunner(t *testing.T) { testRootMigrationBootstrap(t, false, false) } func plan14InitialMigrationExclusion(t *testing.T, store release.Store, r ClaimRuntimeResources, owners ClaimOwnerBindings, actor hostbox.Actor, journals *hostbox.JournalService, launches *hostbox.LaunchJournal) { for _, args := range [][]string{{"adopt"}, {"system", "apply"}} { cmd := exec.Command(os.Args[0], args...) cmd.Env = append(os.Environ(), "PLAN14_R9_LEGACY_MUTATOR=1") stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) } stdout, err := cmd.StdoutPipe() if err != nil { t.Fatal(err) } if err := cmd.Start(); err != nil { t.Fatal(err) } t.Cleanup(func() { stdin.Close(); cmd.Process.Kill(); cmd.Wait() }) var b [6]byte if _, err := stdout.Read(b[:]); err != nil { t.Fatal(err) } if err := migrateRootClaimAuthority(t.Context(), store, r, owners, actor, journals, launches); err == nil { t.Fatal("migration overlapped legacy mutator", args) } if _, err := os.Stat(filepath.Join(rootparticipation.Namespace, "census.json")); !os.IsNotExist(err) { t.Fatal("census created before quiescence", err) } stdin.Close() cmd.Wait() } old := rootMigrationQuiesce defer func() { rootMigrationQuiesce = old }() rootMigrationQuiesce = func(context.Context) (func(), error) { if err := verifyOfflineRootMigration(t.Context()); err != nil { t.Fatal(err) } return nil, os.ErrPermission } if err := migrateRootClaimAuthority(t.Context(), store, r, owners, actor, journals, launches); !errors.Is(err, os.ErrPermission) { t.Fatal("exclusion witness", err) } before, err := os.ReadDir(rootReleasePrivate) if err != nil { t.Fatal(err) } for _, flag := range []string{"", "1"} { t.Setenv("CARLOS_HOST_CLAIM", flag) plan14NoCensusCommands(t) if err := withClaimRootRelease(t.Context(), false, func() (rootstate.Store, error) { t.Fatal("release opened after aborted migration") return nil, os.ErrPermission }, func() error { return nil }); err != nil { t.Fatal(err) } for _, recover := range []bool{false, true} { f := newApplyFixture(t, "canary", "v1.0.0") f.a.participation = nil f.a.rootMoves = rootMovesFloorIdentity if err := f.a.runWithRelease(t.Context(), recover); err != nil { t.Fatal("legacy apply after aborted migration", recover, err) } } } after, err := os.ReadDir(rootReleasePrivate) if err != nil || len(after) != len(before) { t.Fatal("new release state after abort", err, before, after) } } func TestPlan14RepairExecutorEndToEndRootRunner(t *testing.T) { opt := generationRootRunnerDir(t) pub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub)) id := rootFloorIdentity{Deployment: "deploy", Box: "box", Enrollment: "en"} f, err := openRootMovesFloors(opt, id, true) if err != nil { t.Fatal(err) } defer f.close() bound := moves.Floor{Min: 99, MinRelease: "v99.0.0"} if err := f.reconstruct(t.Context(), rootFloorReconstruction{Identity: id, Floor: &bound, Source: "raised bound", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, f)}); err != nil { t.Fatal(err) } for _, path := range []string{systemBinaryPath(opt), systemPrevPath(opt), filepath.Join(systemGuardDir(opt), "carlos"), filepath.Join(systemRootDir(opt), "staging", "carlos")} { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { t.Fatal(err) } rootFloorSignRunnerBytes(t, opt, path, "v2.0.0", []byte("#!/bin/sh\nprintf 'v2.0.0\\n'\n"), key) } if err := verifyRootMigrationFloors(opt, id); err != nil { t.Fatal("repair controller cannot start", err) } a := systemApplier{optDir: opt, dataDir: t.TempDir(), rootMoves: func() (rootFloorIdentity, bool, error) { return id, true, nil }, ops: productionSysOps()} for _, path := range []string{systemBinaryPath(opt), filepath.Join(systemGuardDir(opt), "carlos")} { if out, err := a.guardRun(t.Context(), time.Second, path, "version"); err != nil || out == "" { t.Fatal("certified helper blocked", out, err) } } if _, err := a.guardRun(t.Context(), time.Second, filepath.Join(systemRootDir(opt), "staging", "carlos"), "version"); err == nil { t.Fatal("below-floor replacement executed") } if err := a.floorAllowsForward("v2.0.0"); err != nil { t.Fatal("installed repair executor cannot finish recovery", err) } j := rootFloorJudge{floors: f, current: systemBinaryPath(opt), prev: systemPrevPath(opt), binary: func(p string) (rootFloorBinary, error) { return readRootFloorBinary(opt, p) }, idle: func(context.Context) error { return nil }} if err := j.judge(t.Context()); err != nil { t.Fatal(err) } var v rootFloorVerdict raw, err := f.public.Read("prev.json") if err != nil { t.Fatal(err) } if err := json.Unmarshal(raw, &v); err != nil { t.Fatal(err) } v.Prev.File.Device++ raw, _ = json.Marshal(v) if err := f.public.Write("prev.json", raw); err != nil { t.Fatal(err) } rootFloorSignRunnerBytes(t, opt, systemBinaryPath(opt), "v3.0.0", []byte("#!/bin/sh\nexit 99\n"), key) selected, err := rootFloorBootCandidate(t.Context(), j, true, func(context.Context, string) error { return nil }, func(context.Context) error { return nil }) if err != nil || selected != systemPrevPath(opt) { t.Fatal("renumbered signed prev lost recovery", selected, err) } if out, err := a.guardRun(t.Context(), time.Second, selected, "version"); err != nil || out == "" { t.Fatal("selected recovery image failed", out, err) } for _, recovery := range []bool{false, true} { f := newApplyFixture(t, "canary", "v2.0.0") f.a.optDir = opt f.a.rootMoves = a.rootMoves f.a.ops.run = productionSysOps().run f.edgeStartedThisBoot = false rootFloorSignRunnerBytes(t, opt, systemBinaryPath(opt), "v2.0.0", []byte("#!/bin/sh\nexit 99\n"), key) prevDir := filepath.Join(f.a.stagingDir(), "prev") unitDir := filepath.Join(prevDir, "etc", "systemd", "system") if err := os.MkdirAll(unitDir, 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(unitDir, "carlos-edge.service"), []byte("[Unit]\n"), 0644); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(prevDir, "etc", "tmpfiles.d"), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(prevDir, "etc", "tmpfiles.d", "carlos.conf"), nil, 0644); err != nil { t.Fatal(err) } if err := writeJSONDurable(filepath.Join(prevDir, "plan.json"), bootstrapPlan{}, 0600); err != nil { t.Fatal(err) } if err := writeJSONDurable(f.a.file("journal.json"), systemJournal{Phase: systemPhaseInstalled, Target: "v2.0.0", PrevBuild: "v2.0.0", StartedAt: f.now}, 0600); err != nil { t.Fatal(err) } if err := f.a.Run(t.Context(), recovery); err != nil { t.Fatalf("production recovery=%v: %v", recovery, err) } if journal, err := readSystemJournal(opt); err != nil || journal != nil { t.Fatal("repair journal not closed", journal, err) } if result := f.result(); result == nil || result.Outcome != systemOutcomeRolledBack { t.Fatal("repair result", result) } if out, err := f.a.guardRun(t.Context(), time.Second, systemBinaryPath(opt), "version"); err != nil || out == "" { t.Fatal("repaired current cannot execute", out, err) } } } func TestPlan14PendingInventoryStartupRootRunner(t *testing.T) { if err := os.Mkdir("/var/lib/carlos-release", 0700); err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll("/var/lib/carlos-release") }) for _, mode := range []string{"instance-control", "system"} { t.Run(mode, func(t *testing.T) { t.Setenv("PLAN14_R9_PENDING", mode) runServingParkPeerNamespace(t, false, false, "seed-construction") }) } } type r9PendingBoundary struct { rootstate.Store failed bool } func (s *r9PendingBoundary) Write(name string, raw []byte) error { if err := s.Store.Write(name, raw); err != nil { return err } if !s.failed { s.failed = true return os.ErrPermission } return nil } func filesSeedPendingConstruction(t *testing.T, store release.Store, resources ClaimRuntimeResources, owners ClaimOwnerBindings, actor hostbox.Actor) { if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { t.Fatal(err) } for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} { if err := os.Mkdir(path, 0755); err == nil { t.Cleanup(func() { os.RemoveAll(path) }) } else if !os.IsExist(err) { t.Fatal(err) } } emptyUnits := t.TempDir() if err := unix.Mount(emptyUnits, "/etc/systemd/system", "", unix.MS_BIND, ""); err != nil { t.Fatal(err) } t.Cleanup(func() { unix.Unmount("/etc/systemd/system", 0) }) host := os.Getenv("PLAN14_R10_INSTALL_HOST") if host == "" { host = recoveryHost } f := newRecoveryFixture(t, host) obs := f.read() obs.Claim.Op.Exec = actor.Box obs.Claim.Op.Participants = []string{actor.Box} obs.Claim.Touched.Boxes = []string{actor.Box} claimBody, err := json.Marshal(obs.Claim) if err != nil { t.Fatal(err) } claimKey, err := hostclaim.Key(obs.Host) if err != nil { t.Fatal(err) } if _, err := f.claims.Put(t.Context(), claimKey, claimBody, obs.ETag); err != nil { t.Fatal(err) } h := obs.Claim.Holder plan14PutRecord(t, f.claims, provision.Record{Schema: 1, Host: obs.Host, Account: h.Account, App: h.App, Channel: "stable", CreatedAt: f.now}) keys, err := f.claims.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { raw, _, err := f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err = store.Put(t.Context(), key, raw, ""); err != nil { t.Fatal(err) } } obs, err = hostclaim.Read(t.Context(), store, obs.Host) if err != nil { t.Fatal(err) } _, etag, err := store.Get(t.Context(), h.Key) if err != nil { t.Fatal(err) } resources.DataDir = "/data" resources.UnitsDir = "/etc/systemd/system" reg, err := openServiceRegistry("/data/registry.db") if err != nil { t.Fatal(err) } defer reg.Close() journal, err := openClaimRootJournal() if err != nil { t.Fatal(err) } defer journal.Close() journals, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } if os.Getenv("PLAN14_R10_RETIRED") != "" { if err := checkClaimJournalMigration(t.Context(), journals, filepath.Join("/data", ".hostclaims")); err != nil { t.Fatal("journal startup before installation", err) } } state, err := openClaimRootAuthority() if err != nil { t.Fatal(err) } defer state.Close() boundary := &r9PendingBoundary{Store: state, failed: os.Getenv("PLAN14_R10_RETIRED") != ""} authority := hostguard.NewRootAuthority(boundary) roots := map[string]string{"data": "/data", "opt": resources.OptDir, "units": "/etc/systemd/system"} guard := hostguard.Guard{Root: authority, DataDir: "/data", Claims: hostguard.StoreReader{Store: store}} census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots, SharedPaths: claimGlobalSharedPaths(resources)} root := ClaimRootHTTP{history: productionCreationHistory, Journals: journals, Launches: journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: store}}} _, client := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { out, err := root.commitRootInventory(ctx, q) t.Logf("pending commit: %v", err) return out, err }) _, err = client.WriteExpected(t.Context(), hostbox.StopInventory{Complete: true, Host: obs.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1"}, hostbox.InventoryExpectation{RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: obs.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, ClaimETag: obs.ETag, Op: obs.Claim.Op.ID, Phase: obs.Claim.Op.Phase}}) if os.Getenv("PLAN14_R10_RETIRED") != "" { if err != nil { t.Fatal("complete installation", err) } if obs.Host == "sibling.example" { if err := guard.With(t.Context(), obs.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, func(*hostclaim.Claim) error { return nil }, func() error { return nil }); err != nil { t.Fatal("sibling admission", err) } return } want := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} in, err := authority.Installation(t.Context(), obs.Host, want) if err != nil { t.Fatal(err) } interrupted := os.Getenv("PLAN14_R10_RETIRED") == "interrupted" err = guard.RetireInstallationWith(t.Context(), obs.Host, want, func(*hostclaim.Claim) error { return nil }, func() error { if interrupted { return os.ErrPermission } return nil }) if interrupted != errors.Is(err, os.ErrPermission) { t.Fatal("retirement boundary", err) } state.Close() reopened, err := openClaimRootAuthority() if err != nil { t.Fatal(err) } defer reopened.Close() authority = hostguard.NewRootAuthority(reopened) retained, err := authority.Installation(t.Context(), obs.Host, want) if err != nil || retained != in { t.Fatal("retired identity lost", retained, in, err) } guard.Root = authority if err := guard.With(t.Context(), obs.Host, want, func(*hostclaim.Claim) error { return nil }, func() error { t.Fatal("retired admitted"); return nil }); err == nil { t.Fatal("retired admitted") } return } if err == nil || !boundary.failed { t.Fatal("pending boundary not reached", err) } if pending, err := authority.PendingInstallations(); err != nil || len(pending) != 1 { t.Fatal("pending intent not durable", len(pending), err) } if err := verifyRootMigrationStartup(t.Context(), rootFloorIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment}, journals.Inventory); err == nil { t.Fatal("pending state admitted new work") } } func TestPlan14GuardRecoveryLegacyPolicyRootRunner(t *testing.T) { if os.Getenv("PLAN14_R9_GUARD_CHILD") == "1" { RunSystemGuardExec([]string{"recover"}) t.Fatal("guard returned") } dir := generationRootRunnerDir(t) t.Setenv("CARLOS_OPT_DIR", dir) t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "deploy") pub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub)) remote := filepath.Join(dir, "remote") t.Setenv("CARLOS_DEPLOYMENT_DIR", remote) floor, _ := json.Marshal(hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}) auth, err := release.SignRootEnvelope(release.RootSigner(key), hostfloor.Authorization{Schema: 1, Purpose: "moves-protocol", DeploymentBucket: "deploy", Key: hostfloor.Key, SHA256: hostfloor.Digest(floor)}) if err != nil { t.Fatal(err) } policy, _ := json.Marshal(hostfloor.Policy{Schema: 1, DeploymentBucket: "deploy", Mode: "enforced", Floor: floor, Authorization: auth}) if err := os.MkdirAll(systemGuardDir(dir), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(systemGuardDir(dir), "policy.json"), policy, 0644); err != nil { t.Fatal(err) } store := release.NewDirStore(remote) other, _ := json.Marshal(hostfloor.Floor{Min: 2, MinRelease: "v2.0.0"}) otherAuth, err := release.SignRootEnvelope(release.RootSigner(key), hostfloor.Authorization{Schema: 1, Purpose: "moves-protocol", DeploymentBucket: "deploy", Key: hostfloor.Key, SHA256: hostfloor.Digest(other)}) if err != nil { t.Fatal(err) } if _, err := store.Put(t.Context(), hostfloor.Key, other, ""); err != nil { t.Fatal(err) } if _, err := store.Put(t.Context(), hostfloor.AuthKey, otherAuth, ""); err != nil { t.Fatal(err) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { key := strings.TrimPrefix(r.URL.Path, "/deploy/") raw, _, err := store.Get(r.Context(), key) if err != nil { http.Error(w, "missing", http.StatusNotFound) return } w.Write(raw) })) defer server.Close() for k, v := range map[string]string{"CARLOS_S3_ENDPOINT": server.URL, "AWS_REGION": "eu-west-1", "AWS_DEFAULT_REGION": "eu-west-1", "AWS_ACCESS_KEY_ID": "test", "AWS_SECRET_ACCESS_KEY": "test", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true"} { t.Setenv(k, v) } binary, _ := os.ReadFile("/usr/bin/echo") if err := os.WriteFile(systemBinaryPath(dir), binary, 0755); err != nil { t.Fatal(err) } for _, flag := range []string{"", "1"} { cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14GuardRecoveryLegacyPolicyRootRunner$") cmd.Env = append(os.Environ(), "PLAN14_R9_GUARD_CHILD=1", "CARLOS_HOST_CLAIM="+flag) out, err := cmd.CombinedOutput() if err == nil || !strings.Contains(string(out), hostfloor.ErrFloor.Error()) { t.Fatal("legacy recovery ignored reachable policy mismatch", string(out), err) } } } const r9FloorRebootDir = "/var/lib/carlos-rootrunner-r9-floor" func TestRootRebootFloorPrevAuthorizationPhase1(t *testing.T) { if err := os.Mkdir(r9FloorRebootDir, 0755); err != nil { t.Fatal(err) } pub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub)) id := rootFloorIdentity{"deploy", "box", "en"} f, err := openRootMovesFloors(r9FloorRebootDir, id, true) if err != nil { t.Fatal(err) } defer f.close() bound := moves.Floor{Min: 1, MinRelease: "v2.0.0"} if err := f.reconstruct(t.Context(), rootFloorReconstruction{Identity: id, Floor: &bound, Source: "accepted history", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, f)}); err != nil { t.Fatal(err) } for _, p := range []string{systemBinaryPath(r9FloorRebootDir), systemPrevPath(r9FloorRebootDir)} { rootFloorSignRunnerBytes(t, r9FloorRebootDir, p, "v2.0.0", []byte("#!/bin/sh\nprintf recovered\n"), key) } j := rootFloorJudge{floors: f, current: systemBinaryPath(r9FloorRebootDir), prev: systemPrevPath(r9FloorRebootDir), binary: func(p string) (rootFloorBinary, error) { return readRootFloorBinary(r9FloorRebootDir, p) }, idle: func(context.Context) error { return nil }} if err := j.judge(t.Context()); err != nil { t.Fatal(err) } rootFloorSignRunnerBytes(t, r9FloorRebootDir, j.current, "v3.0.0", []byte("#!/bin/sh\nexit 99\n"), key) if err := os.MkdirAll(systemRootDir(r9FloorRebootDir), 0755); err != nil { t.Fatal(err) } if err := writeFileDurable(filepath.Join(systemRootDir(r9FloorRebootDir), "journal.json"), []byte("{}"), 0600); err != nil { t.Fatal(err) } boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil { t.Fatal(err) } if err := writeFileDurable(filepath.Join(r9FloorRebootDir, "boot"), boot, 0600); err != nil { t.Fatal(err) } if err := writeFileDurable(filepath.Join(r9FloorRebootDir, "pub"), []byte(base64.StdEncoding.EncodeToString(pub)), 0600); err != nil { t.Fatal(err) } } func TestRootRebootFloorPrevAuthorizationPhase2(t *testing.T) { defer os.RemoveAll(r9FloorRebootDir) before, err := os.ReadFile(filepath.Join(r9FloorRebootDir, "boot")) if err != nil { t.Fatal(err) } now, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil || bytes.Equal(before, now) { t.Fatal("actual reboot required", err) } pub, err := os.ReadFile(filepath.Join(r9FloorRebootDir, "pub")) if err != nil { t.Fatal(err) } t.Setenv("CARLOS_RELEASE_PUBKEY", string(pub)) f, err := openRootMovesFloors(r9FloorRebootDir, rootFloorIdentity{"deploy", "box", "en"}, false) if err != nil { t.Fatal(err) } defer f.close() raw, err := f.public.Read("prev.json") if err != nil { t.Fatal(err) } var verdict rootFloorVerdict if err := json.Unmarshal(raw, &verdict); err != nil { t.Fatal(err) } prev, err := readRootFloorBinary(r9FloorRebootDir, systemPrevPath(r9FloorRebootDir)) if err != nil { t.Fatal(err) } verdict.Prev.File.Device = prev.File.Device + 1 raw, _ = json.Marshal(verdict) if err := f.public.Write("prev.json", raw); err != nil { t.Fatal(err) } j := rootFloorJudge{floors: f, current: systemBinaryPath(r9FloorRebootDir), prev: systemPrevPath(r9FloorRebootDir), binary: func(p string) (rootFloorBinary, error) { return readRootFloorBinary(r9FloorRebootDir, p) }} journal, err := rootFloorJournalPresent(r9FloorRebootDir) if err != nil || !journal { t.Fatal("interrupted update missing", err) } selected, err := rootFloorBootCandidate(t.Context(), j, journal, func(context.Context, string) error { return nil }, func(context.Context) error { return nil }) if err != nil || selected != j.prev { t.Fatal("accepted signed prev rejected after reboot", selected, err) } out, err := exec.Command(selected, "system", "apply", "--recover").CombinedOutput() if err != nil || string(out) != "recovered" { t.Fatal(string(out), err) } if err := os.WriteFile(j.prev, []byte("substitution"), 0755); err != nil { t.Fatal(err) } if err := rootFloorPrevVerdict(j); err == nil { t.Fatal("substituted prev accepted") } }