package host import ( "context" "encoding/json" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "os" "path/filepath" "strings" "testing" ) func TestFloorRefusalKeepsRecoveryJournalAndExecutesNothing(t *testing.T) { for _, phase := range []string{systemPhaseBackingUp, systemPhaseInstalled, systemPhaseGated} { t.Run(phase, func(t *testing.T) { f := newApplyFixture(t, "stable", "old-build") f.a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation} _ = os.MkdirAll(f.a.rootDir(), 0700) if err := f.a.journal(systemJournal{Phase: phase, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } if err := f.a.Run(context.Background(), true); err == nil { t.Fatal("missing policy accepted") } if _, err := os.Stat(filepath.Join(f.a.rootDir(), "journal.json")); err != nil { t.Fatal("journal discarded", err) } if len(f.calls) != 0 { t.Fatal("ran before guard", f.calls) } }) } } func TestUnpublishedCompletionAndRecovery(t *testing.T) { f := newApplyFixture(t, "stable", "old-build") f.a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation} raw, _ := json.Marshal(hostfloor.Policy{Schema: 1, DeploymentBucket: "bucket", Mode: "unpublished"}) if err := hostfloor.Durable(filepath.Join(f.a.floor.Dir, "policy.json"), raw, 0644); err != nil { t.Fatal(err) } if err := f.a.beginGuard(t.Context(), nil); err != nil { t.Fatal(err) } if err := f.a.endGuard(t.Context()); err != nil { t.Fatal(err) } if err := f.a.checkGuardRecovery(t.Context()); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(f.a.floor.Dir, "activation.json")); !os.IsNotExist(err) { t.Fatal(err) } } func guardRecoveryFixture(t *testing.T) (*applyFixture, string) { t.Helper() f := newApplyFixture(t, "stable", "old-build") a := f.a a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation} a.guardOpen = os.Open // Isolate the state machine from root-owned path checks. sg := release.RootSigner(f.priv) floor, _ := json.Marshal(hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"}) auth, err := release.SignRootEnvelope(sg, hostfloor.Authorization{Schema: 1, Purpose: "moves-protocol", DeploymentBucket: "bucket", Key: hostfloor.Key, SHA256: hostfloor.Digest(floor)}) if err != nil { t.Fatal(err) } policy, _ := json.Marshal(hostfloor.Policy{Schema: 1, DeploymentBucket: "bucket", Mode: "armed", Floor: floor, Authorization: auth}) if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "policy.json"), policy, 0644); err != nil { t.Fatal(err) } body, err := os.ReadFile(systemBinaryPath(a.optDir)) if err != nil { t.Fatal(err) } hash := hostfloor.Digest(body) rel := release.SystemRelease{Version: "old-build", ReleaseTag: "v1.2.3", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{a.platform: {Key: release.SystemArtifactKey("old-build", a.platform), SHA256: hash}}} if _, err := release.PromoteSystem(t.Context(), f.store, sg, "router", a.channel, rel, release.SystemPromoteOptions{}); err != nil { t.Fatal(err) } identity, _, err := f.store.Get(t.Context(), release.SystemPointerKey("router", a.channel)) if err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(a.floor.Dir, "identities"), 0755); err != nil { t.Fatal(err) } if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", "v1.2.3.json"), identity, 0644); err != nil { t.Fatal(err) } id := strings.Repeat("a", 32) enrollment := strings.Repeat("b", 32) box := hostactors.Box{Enrollment: enrollment, Namespace: "test", BoxID: "box", Label: "box", Region: "region", Pool: "pool", Cloud: hostactors.Cloud{Provider: "aws", Account: "account", Region: "region", InstanceID: "instance"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} h := hostactors.Reservation{ID: id, FloorSHA256: hostfloor.Digest(floor), Actors: map[string]string{"box:box": enrollment}, Releases: map[string]hostactors.Report{"box:box": {Enrollment: enrollment}}, Hashes: map[string][]string{"box:box": {hash, hash, hash}}} roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Boxes: map[string]hostactors.Box{"box": box}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{id: h}} if err := roster.Validate("bucket"); err != nil { t.Fatal(err) } wire, err := release.SignRootEnvelope(sg, roster) if err != nil { t.Fatal(err) } tx, _ := json.Marshal(guardTransaction{Roster: wire, ID: id, Actor: "box:box", Enrollment: enrollment}) if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), tx, 0644); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(a.dataDir, ".hostexec"), 0755); err != nil { t.Fatal(err) } return f, id } func TestGuardRecoveryEveryTerminalCrashState(t *testing.T) { for _, state := range []string{"prepared", "staging-failed", "journal-removed", "completion-written", "marker-removed"} { t.Run(state, func(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a marker := filepath.Join(a.floor.Dir, "update-begun.json") completed := filepath.Join(a.floor.Dir, "completed.json") if state != "prepared" { if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } } if state == "journal-removed" { if err := a.journal(systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil { t.Fatal(err) } } if state == "completion-written" || state == "marker-removed" { if err := hostfloor.Durable(completed, []byte(id), 0644); err != nil { t.Fatal(err) } } if state == "marker-removed" { if err := hostfloor.RemoveDurable(marker); err != nil { t.Fatal(err) } } if err := a.Run(t.Context(), true); err != nil { t.Fatal(err) } if err := a.Run(t.Context(), true); err != nil { t.Fatal("replay", err) } raw, err := os.ReadFile(completed) if err != nil || string(raw) != id { t.Fatal("completion absent", err, string(raw)) } if _, err := os.Stat(marker); !os.IsNotExist(err) { t.Fatal("marker stranded", err) } if len(f.calls) != 0 { t.Fatal("recovery executed a binary", f.calls) } if err := a.floor.Begin(t.Context(), "next", a.dataDir); err != nil { t.Fatal("subsequent update blocked", err) } }) } } func TestGuardRecoveryRetainsMarkerOnInvalidImageOrMarker(t *testing.T) { for _, bad := range []string{"image", "marker", "completion-io"} { t.Run(bad, func(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } marker := filepath.Join(a.floor.Dir, "update-begun.json") switch bad { case "image": if err := os.WriteFile(systemBinaryPath(a.optDir), []byte("wrong executable"), 0755); err != nil { t.Fatal(err) } case "marker": if err := os.WriteFile(marker, []byte(`{"id":"other"}`), 0644); err != nil { t.Fatal(err) } case "completion-io": if err := os.Mkdir(filepath.Join(a.floor.Dir, "completed.json"), 0755); err != nil { t.Fatal(err) } } if err := a.Run(t.Context(), true); err == nil { t.Fatal("unsafe recovery succeeded") } if _, err := os.Stat(marker); err != nil { t.Fatal("denial erased", err) } if bad == "completion-io" { if err := os.Remove(filepath.Join(a.floor.Dir, "completed.json")); err != nil { t.Fatal(err) } if err := a.Run(t.Context(), true); err != nil { t.Fatal("retry failed", err) } } }) } } func TestGuardRecoveryRefusesControllersAfterBegin(t *testing.T) { t.Parallel() states := []struct { name string malformed bool unreadable bool }{ {name: "pending"}, {name: "active"}, {name: "lost"}, {name: "malformed", malformed: true}, {name: "unreadable", unreadable: true}, } paths := []struct { name string journal bool marker bool }{ {name: "journal-backing-up", journal: true, marker: true}, {name: "terminal-marker", marker: true}, {name: "terminal-no-marker"}, } for _, state := range states { for _, path := range paths { t.Run(state.name+"-"+path.name, func(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } marker := filepath.Join(a.floor.Dir, "update-begun.json") binary := systemBinaryPath(a.optDir) before, err := os.ReadFile(binary) if err != nil { t.Fatal(err) } if path.journal { if err := a.journal(systemJournal{Phase: systemPhaseBackingUp, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { t.Fatal(err) } } controllers := filepath.Join(a.dataDir, ".hostexec") if state.unreadable { if err := os.RemoveAll(controllers); err != nil { t.Fatal(err) } if err := os.WriteFile(controllers, []byte("not a directory"), 0644); err != nil { t.Fatal(err) } } else if state.malformed { if err := os.WriteFile(filepath.Join(controllers, "host.json"), []byte("{"), 0600); err != nil { t.Fatal(err) } } else { raw, err := json.Marshal(map[string]string{"state": state.name}) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(controllers, "host.json"), raw, 0600); err != nil { t.Fatal(err) } } if !path.marker { if err := hostfloor.RemoveDurable(marker); err != nil { t.Fatal(err) } } if err := a.Run(t.Context(), true); err == nil { t.Fatal("unsafe recovery succeeded") } if path.marker { if _, err := os.Stat(marker); err != nil { t.Fatal("marker discarded", err) } } else if _, err := os.Stat(marker); !os.IsNotExist(err) { t.Fatal("marker unexpectedly present", err) } if path.journal { if _, err := os.Stat(a.file("journal.json")); err != nil { t.Fatal("journal discarded", err) } } if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) { t.Fatal("completion written", err) } after, err := os.ReadFile(binary) if err != nil || string(after) != string(before) { t.Fatal("binary changed", err) } if len(f.calls) != 0 { t.Fatal("recovery executed a binary", f.calls) } }) } } } func TestGuardRejectsOldPointerForUnpublishedActivation(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a ctx := t.Context() sg := release.RootSigner(f.priv) old, _, err := release.GetSystemPointer(ctx, f.store, f.pub, "router", a.channel) if err != nil { t.Fatal(err) } tx, err := a.guardTransaction() if err != nil { t.Fatal(err) } var roster hostactors.Roster if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil { t.Fatal(err) } publication, err := release.PrepareSystemPromotion(ctx, f.store, sg, "router", a.channel, old.Current, release.SystemPromoteOptions{}, id) if err != nil { t.Fatal(err) } h := roster.Activations[id] h.Publication = &publication h.RequestSHA256 = strings.Repeat("d", 64) h.UpdateMode = "promote" roster.Activations[id] = h raw, err := release.SignRootEnvelope(sg, roster) if err != nil { t.Fatal(err) } if _, err := f.store.Put(ctx, hostactors.Key, raw, ""); err != nil { t.Fatal(err) } identity, _, err := f.store.Get(ctx, release.SystemPointerKey("router", a.channel)) if err != nil { t.Fatal(err) } if err := release.PublishSystemIdentity(ctx, f.store, f.pub, identity); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(a.etcDir, "carlos"), 0755); err != nil { t.Fatal(err) } enrollment, _ := json.Marshal(hostfloor.Enrollment{Box: "box", Enrollment: tx.Enrollment, InstanceID: "instance"}) if err := hostfloor.Durable(filepath.Join(a.etcDir, "carlos", "host-actor.json"), enrollment, 0644); err != nil { t.Fatal(err) } if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json")); err != nil { t.Fatal(err) } if err := a.prepareGuard(ctx, old); err == nil { t.Fatal("old same-target pointer consumed unpublished activation") } if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); !os.IsNotExist(err) { t.Fatalf("refusal wrote transaction: %v", err) } current, err := release.PublishSystemPublication(ctx, f.store, f.pub, publication) if err != nil { t.Fatal(err) } if err := a.prepareGuard(ctx, current); err != nil { t.Fatal("published activation refused", err) } } // M8: a dead pending controller is recovered by root in the no-journal // branches too, not only while a journal is open. Recovery that leaves the // controller in place still refuses. func TestPlan9M8GuardRecoveryRecoversPendingWithoutJournal(t *testing.T) { for _, path := range []struct { name string marker bool }{{"terminal-marker", true}, {"terminal-no-marker", false}} { for _, recovers := range []bool{true, false} { name := path.name + "-recovered" if !recovers { name = path.name + "-still-live" } t.Run(name, func(t *testing.T) { f, id := guardRecoveryFixture(t) a := f.a if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { t.Fatal(err) } marker := filepath.Join(a.floor.Dir, "update-begun.json") state := filepath.Join(a.dataDir, ".hostexec", "host.json") if err := os.WriteFile(state, []byte(`{"state":"pending"}`), 0600); err != nil { t.Fatal(err) } if !path.marker { if err := hostfloor.RemoveDurable(marker); err != nil { t.Fatal(err) } } calls := 0 old := recoverPendingControllers t.Cleanup(func() { recoverPendingControllers = old }) recoverPendingControllers = func(_ context.Context, dataDir, guardDir string) error { calls++ if dataDir != a.dataDir || guardDir != a.floor.Dir { t.Errorf("recovery dirs %s %s", dataDir, guardDir) } if recovers { return os.Remove(state) } return nil } err := a.Run(t.Context(), true) if calls == 0 { t.Fatal("root never tried to recover the pending controller") } completed := filepath.Join(a.floor.Dir, "completed.json") if !recovers { if err == nil { t.Fatal("a controller that survived recovery was settled") } if _, statErr := os.Stat(completed); !os.IsNotExist(statErr) { t.Fatal("completion written", statErr) } return } if err != nil { t.Fatal(err) } if raw, err := os.ReadFile(completed); err != nil || string(raw) != id { t.Fatal("completion absent", err, string(raw)) } if len(f.calls) != 0 { t.Fatal("recovery executed a binary", f.calls) } }) } } }