package host import ( "encoding/json" "errors" "os" "os/user" "path/filepath" "strconv" "strings" "testing" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" ) // plan9aNewBody is the target build's executable in the M1-4 fixture. var plan9aNewBody = []byte("#!/bin/sh\necho carlos new-build\n") // plan9aTargetFixture is guardRecoveryFixture with a reservation whose // target is not the image on disk: current and recovery are the old build, // the target is new-build, which has its own signed identity. func plan9aTargetFixture(t *testing.T) (*applyFixture, string) { t.Helper() f, id := guardRecoveryFixture(t) a := f.a old, err := os.ReadFile(systemBinaryPath(a.optDir)) if err != nil { t.Fatal(err) } oldHash, newHash := hostfloor.Digest(old), hostfloor.Digest(plan9aNewBody) sg := release.RootSigner(f.priv) rel := release.SystemRelease{Version: "new-build", ReleaseTag: "v1.2.4", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{a.platform: {Key: release.SystemArtifactKey("new-build", a.platform), SHA256: newHash}}} if _, err := release.PromoteSystem(t.Context(), f.store, sg, "router", "plan9a", rel, release.SystemPromoteOptions{}); err != nil { t.Fatal(err) } identity, _, err := f.store.Get(t.Context(), release.SystemPointerKey("router", "plan9a")) if err != nil { t.Fatal(err) } if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", "v1.2.4.json"), identity, 0644); err != nil { t.Fatal(err) } tx, err := a.guardTransaction() if err != nil { t.Fatal(err) } var roster hostactors.Roster if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil { t.Fatal(err) } h := roster.Activations[id] h.Hashes = map[string][]string{tx.Actor: {oldHash, newHash, oldHash}} roster.Activations[id] = h if err := roster.Validate("bucket"); err != nil { t.Fatal(err) } if tx.Roster, err = release.SignRootEnvelope(sg, roster); err != nil { t.Fatal(err) } raw, _ := json.Marshal(tx) if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), raw, 0644); err != nil { t.Fatal(err) } return f, id } func plan9aAbsent(t *testing.T, paths ...string) { t.Helper() for _, p := range paths { if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s present: %v", p, err) } } } // M1-4: a transaction that never changed the box (refused or deferred // before the first destructive step, or abandoned at the rename) records no // completion. Root's marker and transaction go, so the box is not wedged, // and the console's reservation stays held for a retry or an operator. func TestPlan9AUnchangedTransactionRecordsNoCompletion(t *testing.T) { t.Parallel() for _, state := range []string{"prepared", "staging-failed", "late-participation", "abandoned-install"} { t.Run(state, func(t *testing.T) { f, id := plan9aTargetFixture(t) a := f.a marker := filepath.Join(a.floor.Dir, updateBegunFile) record := filepath.Join(a.dataDir, hostexecDir, "pending.json") if state != "prepared" { if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } } switch state { case "late-participation": // The late re-check before the first destructive step runs // under CARLOS_REGION_HA (systemparticipation.go). a.regionHA = true if err := os.WriteFile(record, []byte(`{"state":"pending"}`), 0600); err != nil { t.Fatal(err) } if err := a.beginExecutableChange(t.Context(), "new-build", f.now); !errors.Is(err, errSystemDeferred) { t.Fatalf("late participation = %v", err) } case "abandoned-install": if err := a.journal(systemJournal{Phase: systemPhaseInstalling, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } a.removeJournal() } before, err := os.ReadFile(systemBinaryPath(a.optDir)) if err != nil { t.Fatal(err) } for range 2 { if err := a.Run(t.Context(), true); err != nil { t.Fatalf("unchanged transaction wedged recovery: %v", err) } } plan9aAbsent(t, filepath.Join(a.floor.Dir, "completed.json"), marker, filepath.Join(a.floor.Dir, "activation.json")) if after, _ := os.ReadFile(systemBinaryPath(a.optDir)); string(after) != string(before) { t.Fatal("binary changed") } if len(f.calls) != 0 { t.Fatal("recovery executed something", f.calls) } if state == "late-participation" { if _, err := os.Stat(record); err != nil { t.Fatal("root removed a controller record", err) } if err := os.Remove(record); err != nil { t.Fatal(err) } } // The retry begins the same reservation again. if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatalf("retry blocked: %v", err) } }) } } // M1-4: the target on disk is a finished transaction. func TestPlan9AForwardFinishRecordsCompletion(t *testing.T) { f, id := plan9aTargetFixture(t) a := f.a if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } if err := a.journal(systemJournal{Phase: systemPhaseGated, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } if err := os.WriteFile(systemBinaryPath(a.optDir), plan9aNewBody, 0755); err != nil { t.Fatal(err) } if err := a.closeGuardJournal(t.Context()); err != nil { t.Fatal(err) } if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { t.Fatalf("completion = %q, %v", raw, err) } plan9aAbsent(t, filepath.Join(a.floor.Dir, updateBegunFile)) } // M1-4: a backward recovery restores the old image and still completes the // transaction, because it rolled back. This pins the call-site wiring. func TestPlan9ABackwardRecoveryRecordsCompletion(t *testing.T) { f, id := plan9aTargetFixture(t) a := f.a old, err := os.ReadFile(systemBinaryPath(a.optDir)) if err != nil { t.Fatal(err) } if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } if err := os.WriteFile(systemPrevPath(a.optDir), old, 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(systemBinaryPath(a.optDir), plan9aNewBody, 0755); err != nil { t.Fatal(err) } // The install staged carlos.prev's plan; recovery reuses it rather than // exec'ing a binary (floor.Run needs root-owned paths). prevDir := filepath.Join(a.stagingDir(), "prev") units := filepath.Join(prevDir, "etc", "systemd", "system") if err := os.MkdirAll(units, 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(units, "carlos-edge.service"), []byte("[Unit]\n"), 0644); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(prevDir, "etc", "tmpfiles.d"), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(prevDir, "etc", "tmpfiles.d", "carlos.conf"), []byte("d /run/carlos\n"), 0644); err != nil { t.Fatal(err) } if err := writeJSONDurable(filepath.Join(prevDir, "plan.json"), bootstrapPlan{Enabled: []string{"carlos-edge.service"}}, 0600); err != nil { t.Fatal(err) } if err := a.journal(systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } f.edgeStartedThisBoot = false if err := a.Run(t.Context(), true); err != nil { t.Fatal(err) } if got, _ := os.ReadFile(systemBinaryPath(a.optDir)); string(got) != string(old) { t.Fatal("old image not restored") } if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack { t.Fatalf("result = %+v", r) } if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { t.Fatalf("rolled-back transaction not completed: %q, %v", raw, err) } plan9aAbsent(t, filepath.Join(a.floor.Dir, updateBegunFile), a.file("journal.json")) } // M1-4: a crash after the rollback was recorded and the journal removed, // before completion, still completes on the old image. func TestPlan9ARolledBackMarkerCompletesAfterCrash(t *testing.T) { f, id := plan9aTargetFixture(t) a := f.a if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } if err := hostfloor.MarkRolledBack(a.floor.Dir, id); err != nil { t.Fatal(err) } if err := a.Run(t.Context(), true); err != nil { t.Fatal(err) } if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { t.Fatalf("completion = %q, %v", raw, err) } } // M1-4: closing a rollback under someone else's marker refuses and keeps // the journal; it never completes the local transaction instead. func TestPlan9ARollbackCloseRefusesForeignMarker(t *testing.T) { f, _ := plan9aTargetFixture(t) a := f.a marker := filepath.Join(a.floor.Dir, updateBegunFile) if err := os.WriteFile(marker, []byte(`{"id":"other"}`), 0644); err != nil { t.Fatal(err) } if err := a.journal(systemJournal{Phase: systemPhaseUnits, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } if err := a.closeGuardRollback(t.Context()); err == nil { t.Fatal("foreign marker accepted") } if _, err := os.Stat(a.file("journal.json")); err != nil { t.Fatal("journal discarded", err) } plan9aAbsent(t, filepath.Join(a.floor.Dir, "completed.json")) } // M1-3: guard-arm refuses while root's update state is anywhere in the // guard directory, including main's region marker and its temporary file. func TestPlan9AGuardArmRefusesUpdateState(t *testing.T) { for _, name := range []string{updateBegunFile, updateBegunFile + ".tmp", updateBegunFile + ".old"} { t.Run(name, func(t *testing.T) { guard, root := t.TempDir(), t.TempDir() if err := os.WriteFile(filepath.Join(guard, name), []byte(`{"target":"v2"}`), 0644); err != nil { t.Fatal(err) } err := guardArmIdle(guard, root) if !errors.Is(err, hostfloor.ErrFloor) || !strings.Contains(err.Error(), name) { t.Fatalf("guard-arm with %s = %v", name, err) } }) } guard, root := t.TempDir(), t.TempDir() for _, name := range []string{"policy.json", "carlos", ".guard-123", "completed.json"} { if err := os.WriteFile(filepath.Join(guard, name), nil, 0644); err != nil { t.Fatal(err) } } if err := guardArmIdle(guard, root); err != nil { t.Fatalf("settled guard directory refused: %v", err) } if err := guardArmIdle(filepath.Join(guard, "absent"), root); err != nil { t.Fatalf("absent guard directory refused: %v", err) } if err := os.WriteFile(filepath.Join(root, "journal.json"), []byte("{}"), 0600); err != nil { t.Fatal(err) } if err := guardArmIdle(guard, root); !errors.Is(err, hostfloor.ErrFloor) { t.Fatalf("open journal = %v", err) } } // M1-2: guard-arm creates the service-owned controller hint directory. // Root census and launch completeness independently authorize an update. func TestPlan9AGuardArmCreatesControllerDir(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a if err := os.RemoveAll(filepath.Join(a.dataDir, hostexecDir)); err != nil { t.Fatal(err) } verify := a.floor.Participation.VerifyLaunches a.floor.Participation.VerifyLaunches = nil if err := a.floor.Begin(t.Context(), id, a.dataDir); err == nil { t.Fatal("precondition: update began without root completeness") } me, err := user.Current() if err != nil { t.Skip(err) } original := lookupServiceUser t.Cleanup(func() { lookupServiceUser = original }) var asked []string lookupServiceUser = func(name string) (*user.User, error) { asked = append(asked, name); return me, nil } if err := guardArmControllers(a.dataDir); err != nil { t.Fatal(err) } if len(asked) != 1 || asked[0] != serviceUser { t.Fatalf("looked up %v, want %s", asked, serviceUser) } fi, err := os.Lstat(filepath.Join(a.dataDir, hostexecDir)) if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o770 { t.Fatalf("controller dir = %v, %v", fi, err) } if err := a.floor.Begin(t.Context(), id, a.dataDir); err == nil { t.Fatal("agent directory replaced root completeness") } a.floor.Participation.VerifyLaunches = verify if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatalf("armed update still refused: %v", err) } lookupServiceUser = func(string) (*user.User, error) { return nil, user.UnknownUserError(serviceUser) } if err := guardArmControllers(t.TempDir()); err == nil { t.Fatal("arming proceeded without the service account") } lookupServiceUser = func(string) (*user.User, error) { return &user.User{Uid: "x", Gid: strconv.Itoa(os.Getgid())}, nil } if err := guardArmControllers(t.TempDir()); err == nil { t.Fatal("arming accepted a malformed uid") } } // I-2: an edge restart inside an apply pulls in the recover unit. On an // armed box guard-exec recover hands the journal to the lock holder at once // instead of waiting out systemRecoverLockWait. func TestPlan9AGuardRecoverYieldsToHeldApplyLock(t *testing.T) { opt := t.TempDir() if held, err := recoverLockHeld(opt); err != nil || held { t.Fatalf("no lock file = %v, %v", held, err) } plan9aAbsent(t, filepath.Join(systemRootDir(opt), systemLockFile)) if err := os.MkdirAll(systemRootDir(opt), 0755); err != nil { t.Fatal(err) } lock, err := os.OpenFile(filepath.Join(systemRootDir(opt), systemLockFile), os.O_CREATE|os.O_RDWR, 0600) if err != nil { t.Fatal(err) } defer lock.Close() if held, err := recoverLockHeld(opt); err != nil || held { t.Fatalf("free lock = %v, %v", held, err) } if err := flockExclusiveNonblock(lock); err != nil { t.Skipf("flock unsupported: %v", err) } if held, err := recoverLockHeld(opt); err != nil || !held { t.Fatalf("held lock = %v, %v", held, err) } lock.Close() if held, err := recoverLockHeld(opt); err != nil || held { t.Fatalf("released lock = %v, %v", held, err) } again, err := os.OpenFile(filepath.Join(systemRootDir(opt), systemLockFile), os.O_RDWR, 0) if err != nil { t.Fatal(err) } defer again.Close() if err := flockExclusiveNonblock(again); err != nil { t.Fatalf("probe left the lock held: %v", err) } }