package host import ( "context" "crypto/sha256" "encoding/hex" "errors" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // Settings plan 4, Astra round 2 finding 1 (ruling R-PLANNED-RECOVERY): the // pre-exec write's revalidation admits the planner's checked recovery or // fallback onto the good build, for that transition only, and still writes // the record. These run a real activator wake (ServeHTTP, the planner, the // held wake check and every exec through startPinnedAt and step 2b) with the // real build-state hook (buildStates.beforeExec) and heldTarget. // plannedProc is a fake process: Stop ends it. type plannedProc struct{ exited atomic.Bool } func (p *plannedProc) Stop(context.Context) error { p.exited.Store(true); return nil } func (p *plannedProc) Exited() bool { return p.exited.Load() } type plannedLease struct{} func (plannedLease) Replicate(context.Context, string, time.Duration) (activator.Rep, error) { return plannedRep{}, nil } func (plannedLease) Release(context.Context) error { return nil } type plannedRep struct{} func (plannedRep) Err() error { return nil } func (plannedRep) Close(context.Context) error { return nil } // plannedBackend is the smallest exec backend a planned wake runs on: a // lease and restore that always succeed, staged generations (a build in // fail exits during its health check, with its process; any other answers // HTTP 200 on a real socket), a generation kill, the live link's version, // and the executable identity step 2b records. type plannedBackend struct { socket string live string fail map[string]bool mu sync.Mutex started []string killed []uint64 procs map[uint64]*plannedProc } func (b *plannedBackend) Acquire(context.Context, registry.Route) (activator.Lease, error) { return plannedLease{}, nil } func (b *plannedBackend) Restore(context.Context, registry.Route) (bool, error) { return false, nil } func (b *plannedBackend) Seed(context.Context, registry.Route) error { return nil } func (b *plannedBackend) Start(context.Context, registry.Route) (activator.Proc, string, error) { return nil, "", errors.New("a planned wake never starts the live link") } func (b *plannedBackend) NextDue(context.Context, registry.Route) (int64, error) { return 0, nil } func (b *plannedBackend) HasSidecar(registry.Route) bool { return false } func (b *plannedBackend) StartSidecar(context.Context, registry.Route) (activator.Proc, error) { return nil, errors.New("no sidecar") } func (b *plannedBackend) StartWorker(context.Context, registry.Route, map[string]string) (activator.Proc, string, error) { return nil, "", errors.New("no worker") } func (b *plannedBackend) StartStaged(ctx context.Context, _ registry.Route, g activator.Gen) (activator.Gen, activator.Proc, error) { // The exec seam, as ExecBackend's: the caller's gate and the wake's // pending record (amadan #272), before anything starts. if err := activator.RunExecGate(ctx); err != nil { return g, nil, err } b.mu.Lock() defer b.mu.Unlock() b.started = append(b.started, g.Build) p := &plannedProc{} if b.procs == nil { b.procs = map[uint64]*plannedProc{} } b.procs[g.N] = p if b.fail[g.Build] { return g, p, errors.New("the process exited during its health check") } g.Socket = b.socket return g, p, nil } func (b *plannedBackend) GenTier(registry.Route, string) cgroup.Limit { return cgroup.TierDefault } func (b *plannedBackend) GenReady(registry.Route, activator.Gen, activator.Proc) {} func (b *plannedBackend) RaiseGenParent(context.Context, registry.Route, cgroup.Limit, cgroup.Limit) error { return nil } func (b *plannedBackend) RestoreGenParent(context.Context, registry.Route, cgroup.Limit) error { return nil } func (b *plannedBackend) GenSwitched(registry.Route, activator.Gen, activator.Proc) error { return nil } func (b *plannedBackend) KillGen(_ context.Context, _ registry.Route, n uint64) error { b.mu.Lock() defer b.mu.Unlock() b.killed = append(b.killed, n) if p := b.procs[n]; p != nil { p.exited.Store(true) } return nil } func (b *plannedBackend) LiveVersion(string) (string, error) { return b.live, nil } func (b *plannedBackend) LiveIdentity(context.Context, registry.Route) (release.StagedIdentity, error) { return release.StagedIdentity{}, errors.New("a planned wake never captures the live link") } func (b *plannedBackend) StartGenExe(context.Context, registry.Route, uint64, bool, release.StagedIdentity) (activator.Proc, activator.Gen, error) { return nil, activator.Gen{}, errors.New("a planned wake never starts the live link") } func (b *plannedBackend) SeedExe(context.Context, registry.Route, release.StagedIdentity) error { return nil } func (b *plannedBackend) builds() []string { b.mu.Lock() defer b.mu.Unlock() return append([]string(nil), b.started...) } // healthySocket serves HTTP 200 on a unix socket under a short directory // (a socket path is bounded). func healthySocket(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp("", "pr") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) path := filepath.Join(dir, "s") l, err := net.Listen("unix", path) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })} go srv.Serve(l) //nolint:errcheck t.Cleanup(func() { srv.Close() }) return path } // plannedStaged writes a staged build and returns its identity (the // activator verifies the bytes before every staged exec). func plannedStaged(t *testing.T, dir, ver string) release.StagedIdentity { t.Helper() body := []byte("build " + ver) path := filepath.Join(dir, ver) if err := os.WriteFile(path, body, 0o755); err != nil { t.Fatal(err) } sum := sha256.Sum256(body) return release.StagedIdentity{Version: ver, StagedPath: path, Digest: hex.EncodeToString(sum[:])} } // plannedWake is a manual instance with good build A (GoodVersion), a // planner that answers plan, and the real build-state hook and held wake // check over b's store and registry. The activator leases at the hibernate // prefix of bucket "hib", which b reads there too. func plannedWake(t *testing.T, live string, fail map[string]bool, plan func(good, t release.StagedIdentity) activator.WakePlan) (*activator.Service, *plannedBackend, *buildStates, release.Store) { t.Helper() b, s, reg := testBuildStates(t) b.prefix = func(h string) string { return activator.HibernatePrefix("", h) } dir := t.TempDir() good, next := plannedStaged(t, dir, "A"), plannedStaged(t, dir, "T") if err := reg.SetGoodBuild(astraHost, registry.GoodBuild{}, registry.GoodBuild{Version: "A", Digest: good.Digest}); err != nil { t.Fatal(err) } // Root's install stamp names the live build, as root's repoint leaves it. if err := reg.SetVersions(astraHost, live, ""); err != nil { t.Fatal(err) } setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable", InstallRequest: 1}) be := &plannedBackend{socket: healthySocket(t), live: live, fail: fail} svc := activator.New(activator.Config{Bucket: "hib", Idle: time.Hour}, be, reg) svc.Planner = func(context.Context, registry.Route) activator.WakePlan { return plan(good, next) } svc.OnGen = func(activator.GenEvent) {} svc.BeforeExec = b.beforeExec svc.HeldTarget = b.heldTarget t.Cleanup(func() { svc.Shutdown(context.Background()) }) return svc, be, b, s } // serve sends one request through the activator: it wakes the instance. func serve(t *testing.T, svc *activator.Service, reg *registry.Registry) *httptest.ResponseRecorder { t.Helper() rt, err := reg.Get(astraHost) if err != nil || rt == nil { t.Fatalf("route: %v", err) } rec := httptest.NewRecorder() req := httptest.NewRequest(http.MethodGet, "http://"+astraHost+"/", nil) proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("served")) }) //nolint:errcheck if !svc.ServeHTTP(rec, req, *rt, proxy) { t.Fatal("the activator did not take the request") } return rec } func durable(t *testing.T, b *buildStates, s release.Store) *release.BuildState { t.Helper() st, _, err := getState(context.Background(), s, release.BuildStateKey(astraHost), astraHost) if err != nil { t.Fatal(err) } return st } // The overlap wake's fallback (Astra round 2 finding 1's scenario): a manual // instance with good build A and Update now resolved to T. Its first wake // records T (completing R1) and T fails its health check. The planner's // fallback A must record and run, though root has not reported T failed and // the durable state's last_executed is T. func TestTheOverlapFallbackAfterTFailsIsRecordedAndRuns(t *testing.T) { svc, be, b, s := plannedWake(t, "T", map[string]bool{"T": true}, func(good, next release.StagedIdentity) activator.WakePlan { return activator.WakePlan{Boot: next, Mode: activator.ModeOverlap, FirstWake: true, Fallback: &good} }) if err := put(t, b, context.Background(), astraHost, release.BuildState{LastExecuted: "A"}); err != nil { t.Fatal(err) } setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}}) decided(t, b, release.Consumed{Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Reason: string(release.ReasonRequest), Desired: "T", Detail: "T"}) rec := serve(t, svc, b.reg) if rec.Code != http.StatusOK || rec.Body.String() != "served" { t.Fatalf("the wake was not served by the fallback: %d %q; generations %v", rec.Code, rec.Body.String(), be.builds()) } if got := be.builds(); len(got) != 2 || got[0] != "T" || got[1] != "A" { t.Fatalf("generations %v, want T then the good build A", got) } st := durable(t, b, s) if st == nil || st.LastExecuted != "A" || st.Completed != 1 { t.Fatalf("durable state %+v; want A recorded before it ran, R1 completed by T's run", st) } } // A planned recovery boot onto the good build (decideWake's recovery: the // failure book blocks T) before root's consumed file names T failed: the // durable state's last_executed is T, and the planner chose A in T's place. // The record admits A for that transition and the wake runs it. func TestAPlannedRecoveryBeforeRootRecordsTheFailureRuns(t *testing.T) { svc, be, b, s := plannedWake(t, "T", nil, func(good, _ release.StagedIdentity) activator.WakePlan { return activator.WakePlan{Boot: good, Mode: activator.ModeOverlap} }) tState := release.BuildState{LastExecuted: "T", Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Completed: 1} if err := put(t, b, context.Background(), astraHost, tState); err != nil { t.Fatal(err) } setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &tState}) decided(t, b, release.Consumed{Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Reason: string(release.ReasonRequest), Desired: "T", Detail: "T"}) rec := serve(t, svc, b.reg) if rec.Code != http.StatusOK || rec.Body.String() != "served" { t.Fatalf("the recovery boot was not served: %d %q; generations %v", rec.Code, rec.Body.String(), be.builds()) } if got := be.builds(); len(got) != 1 || got[0] != "A" { t.Fatalf("generations %v, want the good build A alone", got) } if st := durable(t, b, s); st == nil || st.LastExecuted != "A" || st.Completed != 1 { t.Fatalf("durable state %+v; want A recorded, R1 still completed", st) } } // The negative (GoodVersion is not exempt in general; Astra round 1 finding // 1): the wake decided on a stale copy that names the good build A, which is // installed, so the planner plans A as an ordinary wake, not in place of // anything. The durable state says the data last ran T. With no planner // selection the record refuses A and nothing runs. The same hook, called // directly with no selection, or with a selection for another transition, // refuses A too. func TestTheGoodBuildWithoutThePlannersSelectionIsRefused(t *testing.T) { svc, be, b, s := plannedWake(t, "A", nil, func(good, _ release.StagedIdentity) activator.WakePlan { return activator.WakePlan{Boot: good, Mode: activator.ModeOverlap} }) if err := put(t, b, context.Background(), astraHost, release.BuildState{LastExecuted: "T"}); err != nil { t.Fatal(err) } setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}}) decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "A", Detail: "A"}) rec := serve(t, svc, b.reg) if rec.Code == http.StatusOK { t.Fatalf("the good build ran over T's data with no planner selection: %q; generations %v", rec.Body.String(), be.builds()) } if got := be.builds(); len(got) != 0 { t.Fatalf("generations %v, want none", got) } if st := durable(t, b, s); st == nil || st.LastExecuted != "T" { t.Fatalf("durable state %+v; want T untouched", st) } ctx := context.Background() if err := b.beforeExec(ctx, astraHost, "A"); err == nil || !strings.Contains(err.Error(), "T") { t.Fatalf("beforeExec(A) with no selection = %v; want a stale refusal naming T", err) } other := activator.WithPlannedSelection(ctx, activator.PlannedSelection{Build: "A", InPlaceOf: "B"}) if err := b.beforeExec(other, astraHost, "A"); err == nil { t.Fatal("beforeExec(A) with a selection made in place of another build was admitted") } wrong := activator.WithPlannedSelection(ctx, activator.PlannedSelection{Build: "C", InPlaceOf: "T"}) if err := b.beforeExec(wrong, astraHost, "A"); err == nil { t.Fatal("beforeExec(A) with a selection of another build was admitted") } if st := durable(t, b, s); st == nil || st.LastExecuted != "T" { t.Fatalf("durable state %+v; want T untouched", st) } }