package console // The operator's resolve route answers three actions. With the claim on, // declare-lost and readmit are refused before any effect (the refused fixture // in claimFixtures, and the readmit refusal below). reset-wake-retry is Plan // 6b's operator reset: it writes one host claim (a lifecycle CAS root) under // its own reset-wake-retry admission registration, never an app permit. // // It is a lifecycle variant of the route: driven foreign and open like a // door, but its only effect is the claim root, so its open run is judged by // claimLifecycleOpenProblems (admission written, exactly its own claim moved, // no app effect) instead of claimOpenProblems. import ( "context" "errors" "fmt" "go/ast" "go/parser" "go/token" "net/http" "slices" "sort" "strconv" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/release" ) const claimResolveKey = "route POST /api/cli/ops/hosts/{host}/resolve" // claimLifecycleVariant is one more request on a registered entry whose // fixture covers another of its actions. Its only effect is the host claim on // host, written under apps[0]'s own registration grant. type claimLifecycleVariant struct { name string apps []claimAppRef // apps[0] holds the claim on host and registers host string config func(*Config) setup func(t *testing.T, cx *claimCase) do func(t *testing.T, cx *claimCase) (int, string) // check proves from the world that the foreign runs did nothing // (open=false) and the open run did exactly the action (open=true). check func(t *testing.T, cx *claimCase, open bool) string } var claimLifecycleVariants = map[string][]claimLifecycleVariant{ claimResolveKey: {claimWakeResetVariant()}, } // claimResolveActions is every action handleAPIHostResolve dispatches on, // from source (TestClaimHostResolveActions), with what covers it. var claimResolveActions = map[string]string{ "declare-lost": "claimFixtures' refused fixture: claimHostResolveUnsupportedMsg, foreign and open", "readmit": "TestClaimHostResolveActions: the same refusal, foreign and open", "reset-wake-retry": "claimLifecycleVariants: the claim write under its own reset-wake-retry registration", } // claimVariantCensus: every variant names a registered entry that also has a // fixture, and carries its app, host, request and world check. func claimVariantCensus(entries []claimEntry, fixtures map[string]claimFixture, variants map[string][]claimLifecycleVariant) []string { got := map[string]bool{} for _, e := range entries { got[e.Kind+" "+e.Pattern] = true } var problems []string for key, vs := range variants { if !got[key] { problems = append(problems, fmt.Sprintf("lifecycle variants name %s, which is not registered", key)) } if _, ok := fixtures[key]; !ok { problems = append(problems, fmt.Sprintf("lifecycle variants name %s, which has no claim fixture", key)) } for _, v := range vs { if v.name == "" || len(v.apps) == 0 || v.host == "" || v.do == nil || v.check == nil { problems = append(problems, fmt.Sprintf("lifecycle variant %s %q has no name, app, host, request or check", key, v.name)) } } } sort.Strings(problems) return problems } func runClaimLifecycleVariants(t *testing.T) { keys := make([]string, 0, len(claimLifecycleVariants)) for k := range claimLifecycleVariants { keys = append(keys, k) } sort.Strings(keys) for _, key := range keys { for _, v := range claimLifecycleVariants[key] { t.Run(key+" "+v.name, func(t *testing.T) { for _, p := range runClaimLifecycleVariant(t, v) { t.Error(p) } }) } } } // runClaimLifecycleVariant drives one variant like a door: setup, one // foreign-exclusive run per affected incarnation (each must answer 409 and // leave nothing), then the open run. func runClaimLifecycleVariant(t *testing.T, v claimLifecycleVariant) []string { t.Helper() var opts []func(*Config) if v.config != nil { opts = append(opts, v.config) } cx := newClaimCaseWith(t, opts...) if v.setup != nil { v.setup(t, cx) } cx.quiesce() run := func() (claimWindow, string) { var body string w := cx.measureRun(t, v.apps, func() int { status, out := v.do(t, cx) body = out return status }) return w, body } var problems []string for _, ref := range v.apps { restore := cx.foreign(t, ref) w, body := run() label := "foreign exclusive on " + ref.String() if w.status != http.StatusConflict { problems = append(problems, fmt.Sprintf("%s: answered %d %q, want the 409 refusal", label, w.status, body)) } problems = append(problems, claimForeignProblems(ref, v.apps, w)...) if p := v.check(t, cx, false); p != "" { problems = append(problems, label+": "+p) } restore() } w, body := run() problems = append(problems, claimLifecycleOpenProblems(v, w, body)...) if p := v.check(t, cx, true); p != "" { problems = append(problems, "open run: "+p) } return problems } // claimLifecycleOpenProblems: with every incarnation open, the request must // register with apps[0]'s admission, move exactly its own claim root (which // the production classifier admitted), and reach no app effect at all. func claimLifecycleOpenProblems(v claimLifecycleVariant, w claimWindow, body string) []string { var problems []string say := func(format string, args ...any) { problems = append(problems, "open run: "+fmt.Sprintf(format, args...)) } if w.status != http.StatusOK { say("the authenticated valid request answered %d %q", w.status, body) } key, err := hostclaim.Key(v.host) if err != nil { say("host %s: %v", v.host, err) } admitted, moved := false, false for _, e := range w.events { switch { case e.admissionWrite(v.apps[0]): admitted = true case e.Control: case e.Err != nil: say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) case e.NonApp == claimLifecycleRoot && e.Key == key: moved = true case e.NonApp == claimLifecycleRoot: say("raw %s moved another lifecycle root", e) case e.Unscoped || len(e.Scopes) != 0: say("raw %s is an app effect; this action writes only its claim", e) } } for ref, changed := range w.sqlChanged { say("SQL rows of %s changed in %v", ref, changed) } if !admitted { say("no write to %s's admission, so the request never registered with the gate", v.apps[0]) } if !moved { say("the claim %s did not move", key) } return problems } // ---- reset-wake-retry ------------------------------------------------------- const ( claimResetEpisode = "claim-reset-episode" claimResetNext = "claim-reset-next" claimResetRequest = "claim-reset-request" ) var claimResetAttempt = hostclaim.WakeRetryAttempt{ID: "claim-reset-attempt", Box: "box-ie-default", Enrollment: "claim-reset-enrollment", BootID: "claim-reset-boot", ControllerID: "claim-reset-controller", InvocationID: "claim-reset-invocation", LaunchInputs: "claim-reset-inputs"} // claimResetStops records the attempts the door asked the box to stop. type claimResetStops struct { mu sync.Mutex seen []hostclaim.WakeRetryAttempt } func (s *claimResetStops) stop(_ context.Context, _ hostclaim.Observed, a hostclaim.WakeRetryAttempt) error { s.mu.Lock() s.seen = append(s.seen, a) s.mu.Unlock() return nil } func (s *claimResetStops) list() []hostclaim.WakeRetryAttempt { s.mu.Lock() defer s.mu.Unlock() return slices.Clone(s.seen) } func (s *claimResetStops) clear() { s.mu.Lock() s.seen = nil s.mu.Unlock() } // claimUnusedClaims is the handle the reset door is configured with. Every // call fails: the mounted door must read and write claims only through the // console's guarded data boundary. type claimUnusedClaims struct{} var errClaimUnusedClaims = errors.New("claim fixture: the reset door used its configured raw claims handle") var _ release.Store = claimUnusedClaims{} func (claimUnusedClaims) Get(context.Context, string) ([]byte, string, error) { return nil, "", errClaimUnusedClaims } func (claimUnusedClaims) Put(context.Context, string, []byte, string) (string, error) { return "", errClaimUnusedClaims } func (claimUnusedClaims) List(context.Context, string) ([]string, error) { return nil, errClaimUnusedClaims } func (claimUnusedClaims) ListDirs(context.Context, string) ([]string, error) { return nil, errClaimUnusedClaims } func (claimUnusedClaims) Delete(context.Context, string) error { return errClaimUnusedClaims } func claimWakeResetDoor(stops *claimResetStops) func(*Config) { return func(cfg *Config) { cfg.WakeRetryReset = &WakeRetryResetDoor{Claims: claimUnusedClaims{}, Authority: func(context.Context) error { return nil }, StopAttempt: stops.stop, NewID: func() string { return claimResetNext }} } } // stageWakeReset stages hostLossWorld's serving claim for bab/jam with one // reserved wake retry attempt, and the reset request for it in cx.vars. func (cx *claimCase) stageWakeReset(t *testing.T) hostclaim.Observed { t.Helper() ctx := context.Background() cx.enroll(t, "bab", "jam") cx.hostLossWorld(t) o, err := hostclaim.Read(ctx, cx.rel, claimLossHost) if err != nil { t.Fatal(err) } inc := o.Claim.Holder.Incarnation if o, err = hostclaim.Update(ctx, cx.rel, o, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{ HolderIncarnation: inc, EpisodeID: claimResetEpisode, Attempt: claimResetAttempt, Now: time.Now()})); err != nil { t.Fatalf("reserve the wake retry attempt on %s: %v", claimLossHost, err) } cx.vars["reset"] = `{"action":"reset-wake-retry","incarnation":` + strconv.Quote(inc) + `,"episode":"` + claimResetEpisode + `","request_id":"` + claimResetRequest + `"}` cx.vars["reset-etag"] = o.ETag cx.vars["reset-incarnation"] = inc return o } func claimWakeResetVariant() claimLifecycleVariant { stops := &claimResetStops{} return claimLifecycleVariant{ name: "reset-wake-retry", apps: claimRefs("bab/jam"), host: claimLossHost, config: claimWakeResetDoor(stops), setup: func(t *testing.T, cx *claimCase) { stops.clear() cx.stageWakeReset(t) }, do: claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", ""), check: func(t *testing.T, cx *claimCase, open bool) string { ctx := context.Background() o, err := hostclaim.Read(ctx, cx.rel, claimLossHost) if err != nil || o.Claim == nil || o.Claim.Holder == nil { return fmt.Sprintf("read the claim: %v", err) } stopped := stops.list() if !open { if len(stopped) != 0 { return fmt.Sprintf("stopped %v before the gate refused", stopped) } if o.ETag != cx.vars["reset-etag"] { return "the claim moved" } return "" } if len(stopped) != 1 || stopped[0] != claimResetAttempt { return fmt.Sprintf("stopped %v, want exactly the reserved attempt", stopped) } h := o.Claim.Holder r := h.WakeRetry if h.Incarnation != cx.vars["reset-incarnation"] || r == nil || r.EpisodeID != claimResetNext || r.State != "open" || r.Attempt != nil || !strings.Contains(r.ResetRequest, `"RequestID":"`+claimResetRequest+`"`) { return fmt.Sprintf("the claim was not reset: holder %s retry %+v", h.Incarnation, r) } // The grant came from the reset's own registration: the holder's // exact incarnation and the observation the door read. adm, err := hostops.ReadCanonicalAdmission(ctx, cx.rel, "bab", "jam") if err != nil { return fmt.Sprintf("read the admission: %v", err) } for _, reg := range adm.Value.Registrations { if reg.Kind == wakeRetryResetKind && reg.Host == claimLossHost && reg.Incarnation == h.Incarnation && reg.Claim.ETag == cx.vars["reset-etag"] { return "" } } return fmt.Sprintf("no %s registration for %s at the observed claim: %+v", wakeRetryResetKind, h.Incarnation, adm.Value.Registrations) }, } } // TestClaimHostResolveActions pins, from source, every action the resolve // route dispatches on, and drives the readmit refusal the census names. func TestClaimHostResolveActions(t *testing.T) { fs := token.NewFileSet() f, err := parser.ParseFile(fs, "apihostclaim.go", nil, 0) if err != nil { t.Fatal(err) } got := map[string]bool{} for _, d := range f.Decls { fn, ok := d.(*ast.FuncDecl) if !ok || fn.Name.Name != "handleAPIHostResolve" { continue } ast.Inspect(fn.Body, func(n ast.Node) bool { b, ok := n.(*ast.BinaryExpr) if !ok || (b.Op != token.EQL && b.Op != token.NEQ) { return true } sel, ok := b.X.(*ast.SelectorExpr) lit, lok := b.Y.(*ast.BasicLit) if !ok || !lok || sel.Sel.Name != "Action" || lit.Kind != token.STRING { return true } action, err := strconv.Unquote(lit.Value) if err != nil { t.Fatal(err) } got[action] = true return true }) } for action := range got { if strings.TrimSpace(claimResolveActions[action]) == "" { t.Errorf("the resolve route dispatches on %q, which claimResolveActions does not cover", action) } } for action := range claimResolveActions { if !got[action] { t.Errorf("claimResolveActions names %q, which the resolve route no longer dispatches on", action) } } if f := claimFixtures[claimResolveKey].refused; f == nil || f.msg != claimHostResolveUnsupportedMsg { t.Errorf("%s is not the declare-lost refusal", claimResolveKey) } if !slices.ContainsFunc(claimLifecycleVariants[claimResolveKey], func(v claimLifecycleVariant) bool { return v.name == "reset-wake-retry" }) { t.Errorf("%s has no reset-wake-retry lifecycle variant", claimResolveKey) } readmit := &claimRefusalFixture{ apps: claimRefs("bab/jam"), setup: func(t *testing.T, cx *claimCase) { cx.enroll(t, "bab", "jam") cx.hostLossWorld(t) }, do: claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", `{"action":"readmit","box":"`+claimLossBox+`","enrollment":"claim-readmit","revision":1,"loss_ids":["claim-readmit-loss"]}`), status: http.StatusConflict, msg: claimHostResolveUnsupportedMsg, } for _, p := range runClaimRefusal(t, readmit) { t.Error(p) } } // TestClaimWakeResetWritesOnlyUnderItsGrant: the configured raw handle never // survives ConfigureClaimRuntime, and the same reset written through the // console's data boundary without the reset's registration grant is refused. func TestClaimWakeResetWritesOnlyUnderItsGrant(t *testing.T) { stops := &claimResetStops{} cx := newClaimCaseWith(t, claimWakeResetDoor(stops)) if d := cx.srv.cfg.WakeRetryReset; d == nil || d.Claims == nil { t.Fatal("ConfigureClaimRuntime dropped the reset door") } else if _, raw := d.Claims.(claimUnusedClaims); raw { t.Fatalf("the reset door kept its configured claims handle after ConfigureClaimRuntime") } o := cx.stageWakeReset(t) ctx := context.Background() _, err := hostclaim.Update(ctx, cx.srv.cfg.Releases, o, hostclaim.ResetWakeRetry(hostclaim.WakeRetryReset{HolderIncarnation: o.Claim.Holder.Incarnation, ObservedEpisode: claimResetEpisode, NewEpisodeID: claimResetNext, OperatorRequest: "claim-ungranted", AttemptSettled: true})) if err == nil { t.Fatal("a reset with no lifecycle grant wrote the claim through Config.Releases") } fresh, err := hostclaim.Read(ctx, cx.rel, claimLossHost) if err != nil || fresh.ETag != o.ETag { t.Fatalf("the refused reset moved the claim: %v", err) } if len(stops.list()) != 0 { t.Fatal("a direct claim write stopped an attempt") } } func TestClaimWakeResetRegistrationSurvivesSweepAndDrain(t *testing.T) { stops := &claimResetStops{} cx := newClaimCaseWith(t, claimWakeResetDoor(stops)) cx.stageWakeReset(t) var deletion *hostops.OperationHandle var resetID, appID string control := cx.srv.cfg.ClaimAuthority.Control cx.srv.cfg.WakeRetryReset.StopAttempt = func(ctx context.Context, observed hostclaim.Observed, attempt hostclaim.WakeRetryAttempt) error { adm, err := hostops.ReadCanonicalAdmission(ctx, cx.rel, "bab", "jam") if err != nil { t.Fatal(err) } appID = adm.Value.AppID for id, reg := range adm.Value.Registrations { if reg.Kind == hostops.RegistrationWakeRetryReset { resetID = id } } if resetID == "" { t.Fatal("stop started without registration") } if err := control.SettleRegistration(ctx, "bab", "jam", appID, resetID, time.Now()); !errors.Is(err, hostops.ErrPending) { t.Fatalf("sweep removed reset during stop: %v", err) } owner := hostops.RegistrationOwner{Console: claimFxConsole, Enrollment: claimFxConsoleEnrollment, Controller: "concurrent-delete"} deletion, err = control.BeginExclusive(ctx, "bab", "jam", appID, hostops.AppOperation{ID: "concurrent-delete", Kind: hostops.OpDeleting, AppID: appID}, owner) if err != nil { t.Fatal(err) } if err := deletion.Drain(ctx, time.Now(), nil); !errors.Is(err, hostops.ErrPending) { t.Fatalf("delete drained reset during stop: %v", err) } return stops.stop(ctx, observed, attempt) } status, body := claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", "")(t, cx) if status != http.StatusOK { t.Fatalf("reset after concurrent sweep/drain: %d %s", status, body) } o, err := hostclaim.Read(t.Context(), cx.rel, claimLossHost) if err != nil || o.Claim.Holder.WakeRetry.EpisodeID != claimResetNext || len(stops.list()) != 1 { t.Fatalf("reset did not complete: %+v %v", o, err) } if err := deletion.Drain(t.Context(), time.Now(), nil); err != nil { t.Fatalf("finished reset blocked deletion: %v", err) } adm, err := hostops.ReadAdmission(t.Context(), cx.rel, "bab", "jam", appID) if err != nil { t.Fatal(err) } if _, ok := adm.Value.Registrations[resetID]; ok { t.Fatal("finished reset registration survived drain") } } func TestClaimWakeResetCancelsWhenRenewalStalls(t *testing.T) { started := make(chan struct{}) ctx, closeRegistration := keepWakeResetRegistration(t.Context(), time.Now(), 100*time.Millisecond, time.Millisecond, func(ctx context.Context, _ time.Time) error { close(started) <-ctx.Done() return ctx.Err() }) defer closeRegistration() select { case <-started: case <-time.After(time.Second): t.Fatal("renewal never started") } select { case <-ctx.Done(): case <-time.After(time.Second): t.Fatal("stalled renewal outlived confirmed registration") } }