package hostops import ( "context" "encoding/json" "errors" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // The J17 console move grant: a console-initiated cut, abort or retry is one // typed ControlWriter transition bound to the exact claim, registration and // record the door read. These tests drive it adversarially. // newJ17World is newC7World's moving claim before the seal, with the record // at ready (pinned inputs), the move registration in place and the claim's // obligations empty. func newJ17World(t *testing.T, mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)) *c7World { t.Helper() return newC7World(t, func(c *hostclaim.Claim, rec *provision.Record, adm *AppAdmission) { c.Op.Phase = hostclaim.PhaseMoving c.Obligations = nil rec.Move.State, rec.Move.CutPhase = provision.MoveReady, "" rec.Move.Inputs = &provision.MoveInputs{Release: "r1", BundleETags: map[string]string{"b": "e"}} if mutate != nil { mutate(c, rec, adm) } }) } func j17Observe(t *testing.T, w *c7World) ConsoleMoveObservation { t.Helper() o, err := ObserveConsoleMove(context.Background(), w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } return o } func j17Record(t *testing.T, w *c7World) (provision.Record, string) { t.Helper() rec, tag, err := provision.GetRecordETag(context.Background(), w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } return *rec, tag } func TestJ17ObservationBindsExactIdentities(t *testing.T) { w := newJ17World(t, nil) o := j17Observe(t, w) _, tag := j17Record(t, w) claim := w.claim(t) if o.Account != "bab" || o.App != "app" || o.AppID != "canonical" || o.Host != w.host || o.Incarnation != w.inc || o.OpID != "move" || o.ClaimETag != claim.ETag || o.RecordETag != tag || o.RecordSeq != 0 { t.Fatalf("observation = %+v", o) } // Another app's door cannot observe this host's move. if _, err := ObserveConsoleMove(context.Background(), w.raw, "bab", "other", w.host); !errors.Is(err, ErrIdentity) { t.Fatalf("cross-app observation = %v", err) } } func TestJ17CutAbortRetryWriteOnlyTheirTypedTransition(t *testing.T) { ctx := context.Background() for _, tc := range []struct { name string action ConsoleMoveAction setup func(*hostclaim.Claim, *provision.Record, *AppAdmission) want func(*provision.Record) }{ {"cut", ConsoleMoveCut, nil, func(r *provision.Record) { if err := provision.MoveGuardedCut(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil { panic(err) } }}, {"abort", ConsoleMoveAbort, nil, func(r *provision.Record) { if err := provision.MoveGuardedAbort(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil { panic(err) } }}, {"retry", ConsoleMoveRetry, func(c *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseProbing, "source" c.Op.Intent["rollback"] = "rb_abcdefghijklmnopqrstuvwxyz" rec.Region = "ie" rec.Move.State, rec.Move.StuckReason = provision.MoveStuck, provision.StuckRecoveryProbeAttempts rec.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", TargetPhase: provision.RbTargetReleased, SourcePhase: provision.RbSourceProbing, StuckAt: provision.RbSourceProbing} }, func(r *provision.Record) { if err := provision.MoveRetryRecovery(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil { panic(err) } }}, } { t.Run(tc.name, func(t *testing.T) { w := newJ17World(t, tc.setup) before, _ := j17Record(t, w) claim := w.claim(t) got, err := w.control.ConsoleMove(ctx, j17Observe(t, w), tc.action) if err != nil { t.Fatal(err) } after, _ := j17Record(t, w) want := before mv := *before.Move want.Move = &mv tc.want(&want) want.Move.UpdatedAt, after.Move.UpdatedAt = time.Time{}, time.Time{} want.RecordSeq = before.RecordSeq + 1 if !reflect.DeepEqual(want, after) { t.Fatalf("record:\n%+v\nwant\n%+v", after.Move, want.Move) } if got == nil || got.Move == nil || got.Move.State != after.Move.State || got.RecordSeq != after.RecordSeq { t.Fatalf("returned record = %+v", got) } if c := w.claim(t); c.ETag != claim.ETag { t.Fatal("a console record transition wrote the claim") } }) } } // Expiry and closed admission do not revoke an accepted move: its console // transitions still drain it. func TestJ17ExpiredRegistrationAndClosedAdmissionStillDrain(t *testing.T) { w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, adm *AppAdmission) { r := adm.Registrations["move"] r.Expires = time.Now().Add(-time.Hour) adm.Registrations["move"] = r adm.State = "closed" }) w.clock = time.Now().Add(time.Hour) if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); err != nil { t.Fatal(err) } } func TestJ17RefusesInvalidRegistrationBeforeAnyWrite(t *testing.T) { for name, mutate := range map[string]func(*AppAdmission){ "missing": func(a *AppAdmission) { delete(a.Registrations, "move") }, "other kind": func(a *AppAdmission) { r := a.Registrations["move"]; r.Kind = "start"; a.Registrations["move"] = r }, "other host": func(a *AppAdmission) { r := a.Registrations["move"] r.Host = "other.example" a.Registrations["move"] = r }, "other inc": func(a *AppAdmission) { r := a.Registrations["move"] r.Incarnation = strings.Repeat("f", 64) a.Registrations["move"] = r }, "other epoch": func(a *AppAdmission) { a.Epoch = 2 }, "other appid": func(a *AppAdmission) { r := a.Registrations["move"]; r.AppID = "other"; a.Registrations["move"] = r }, } { t.Run(name, func(t *testing.T) { w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { mutate(a) }) before, tag := j17Record(t, w) if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, ErrPending) { t.Fatalf("err = %v", err) } after, tag2 := j17Record(t, w) if tag != tag2 || !reflect.DeepEqual(before, after) { t.Fatal("refused transition wrote the record") } }) } } // Every field of the observation is authority-bearing: a changed one is // refused before any write, whatever else still matches. func TestJ17RefusesTamperedObservation(t *testing.T) { for name, tamper := range map[string]func(*ConsoleMoveObservation){ "account": func(o *ConsoleMoveObservation) { o.Account = "zzz" }, "app": func(o *ConsoleMoveObservation) { o.App = "other" }, "app id": func(o *ConsoleMoveObservation) { o.AppID = "other" }, "host": func(o *ConsoleMoveObservation) { o.Host = "other.example" }, "incarnation": func(o *ConsoleMoveObservation) { o.Incarnation = strings.Repeat("f", 64) }, "op": func(o *ConsoleMoveObservation) { o.OpID = "move-2" }, "claim etag": func(o *ConsoleMoveObservation) { o.ClaimETag = "\"stale\"" }, "record etag": func(o *ConsoleMoveObservation) { o.RecordETag = "\"stale\"" }, "record seq": func(o *ConsoleMoveObservation) { o.RecordSeq = 9 }, "empty": func(o *ConsoleMoveObservation) { *o = ConsoleMoveObservation{} }, } { t.Run(name, func(t *testing.T) { w := newJ17World(t, nil) o := j17Observe(t, w) tamper(&o) _, tag := j17Record(t, w) if _, err := w.control.ConsoleMove(context.Background(), o, ConsoleMoveAbort); err == nil { t.Fatal("tampered observation accepted") } if _, tag2 := j17Record(t, w); tag != tag2 { t.Fatal("refused transition wrote the record") } }) } } func TestJ17RefusesUnknownOrChangedAction(t *testing.T) { w := newJ17World(t, nil) o := j17Observe(t, w) _, tag := j17Record(t, w) for _, a := range []ConsoleMoveAction{"", "declare-lost", "rollback", "cut ", "CUT", "abort;cut"} { if _, err := w.control.ConsoleMove(context.Background(), o, a); !errors.Is(err, ErrIdentity) { t.Fatalf("action %q: %v", a, err) } } // A legal action the record's state does not admit: the provision // refusal is returned unchanged and nothing is written. if _, err := w.control.ConsoleMove(context.Background(), o, ConsoleMoveRetry); err == nil { t.Fatal("retry of a moving claim accepted") } if _, tag2 := j17Record(t, w); tag != tag2 { t.Fatal("refused action wrote the record") } } // The claim's operation decides which actions exist at all. func TestJ17ActionMustMatchTheClaimOperation(t *testing.T) { w := newJ17World(t, func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { c.Op.Phase = hostclaim.PhaseSealed }) if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, hostclaim.ErrWrongState) { t.Fatalf("abort of a sealed cut = %v", err) } w = newJ17World(t, func(c *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) { c.Op.Kind = hostclaim.KindLanding rec.Move.State = provision.MoveStuck }) if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveRetry); !errors.Is(err, hostclaim.ErrWrongState) { t.Fatalf("retry of a landing = %v", err) } } func TestJ17CrossHostAndCrossOp(t *testing.T) { ctx := context.Background() a := newJ17World(t, nil) b := newJ17World(t, nil) // b's observation names b's host, not a's: a's writer cannot act on it // because a's bucket has no such claim; and a forged pairing of a's host // with b's identities is refused. ob := j17Observe(t, b) oa := j17Observe(t, a) ob.Host = oa.Host if _, err := a.control.ConsoleMove(ctx, ob, ConsoleMoveAbort); err == nil { t.Fatal("cross-host observation accepted") } // The claim names another op (a successor move under the same // registration name is impossible; another op id is refused). c := newJ17World(t, nil) oc := j17Observe(t, c) if _, err := hostclaim.Update(ctx, c.raw, c.claim(t), func(cl *hostclaim.Claim) error { cl.Op.Intent["admission_id"] = "other" return nil }); err != nil { t.Fatal(err) } if _, err := ObserveConsoleMove(ctx, c.raw, "bab", "app", c.host); !errors.Is(err, ErrIdentity) { t.Fatalf("observed an op whose admission id is not its own id: %v", err) } oc.ClaimETag = c.claim(t).ETag if _, err := c.control.ConsoleMove(ctx, oc, ConsoleMoveAbort); !errors.Is(err, ErrIdentity) { t.Fatalf("an op whose admission id is not its own id = %v", err) } d := newJ17World(t, func(_ *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) { rec.Move.ID = "move-2" }) if _, err := ObserveConsoleMove(ctx, d.raw, "bab", "app", d.host); !errors.Is(err, ErrIdentity) { t.Fatalf("record for another move observed: %v", err) } } // A successor incarnation holding the host refuses the predecessor's // observation even when the move id is reused. func TestJ17SuccessorIncarnationRefused(t *testing.T) { ctx := context.Background() w := newJ17World(t, nil) o := j17Observe(t, w) obs := w.claim(t) if _, err := hostclaim.Update(ctx, w.raw, obs, func(c *hostclaim.Claim) error { c.Holder.CreatedAt = c.Holder.CreatedAt.Add(time.Second) inc, err := hostclaim.Incarnation("bab", "app", w.host, c.Holder.CreatedAt) c.Holder.Incarnation = inc return err }); err != nil { t.Fatal(err) } o.ClaimETag = w.claim(t).ETag if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, ErrIdentity) { t.Fatalf("successor = %v", err) } } func TestJ17StaleClaimAndRecordRefused(t *testing.T) { ctx := context.Background() w := newJ17World(t, nil) o := j17Observe(t, w) // An unrelated claim write (a chore acknowledgement) since the read. obs := w.claim(t) if _, err := hostclaim.Update(ctx, w.raw, obs, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("stale claim = %v", err) } w = newJ17World(t, nil) o = j17Observe(t, w) if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw, "bab", "app", w.host, func(r *provision.Record) error { r.Move.Reason = "x"; return nil }); err != nil { t.Fatal(err) } if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, release.ErrCASConflict) { t.Fatalf("stale record = %v", err) } } // A changed record sequence changes content-addressed ETags too. Even a // forged current ETag cannot make an older sequence observation valid. func TestJ17ForgedETagCannotHideOldSequence(t *testing.T) { ctx := context.Background() w := newJ17World(t, nil) seqCtx := provision.WithRecordSequence(ctx) if _, err := provision.SetRecordMove(seqCtx, w.raw, "bab", "app", w.host, func(r *provision.Record) error { return nil }); err != nil { t.Fatal(err) } o := j17Observe(t, w) body, tag, err := w.raw.Get(ctx, w.key) if err != nil { t.Fatal(err) } // Two writes that restore the same bytes apart from the counter. for i := 0; i < 2; i++ { if _, err := provision.SetRecordMove(seqCtx, w.raw, "bab", "app", w.host, func(r *provision.Record) error { return nil }); err != nil { t.Fatal(err) } } // Forge an observation whose ETag matches the current bytes but whose // sequence is the earlier read. _, tag3 := j17Record(t, w) o.RecordETag = tag3 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, release.ErrCASConflict) { t.Fatalf("ABA = %v (tag %s body %d)", err, tag, len(body)) } } // A lost answer: the transition landed, the caller retries with the same // observation. The retry is refused and the record stays as the first // write left it; the door's answer is to read the state. func TestJ17LostAnswerRetryIsFenced(t *testing.T) { ctx := context.Background() w := newJ17World(t, nil) o := j17Observe(t, w) if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); err != nil { t.Fatal(err) } after, tag := j17Record(t, w) if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); err == nil { t.Fatal("replayed observation accepted") } again, tag2 := j17Record(t, w) if tag != tag2 || !reflect.DeepEqual(after, again) { t.Fatal("replay wrote the record") } } // The registration is checked again at the final conditional write: one // removed between the transition's reads and its Put refuses the Put. func TestJ17RegistrationCheckedAtTheFinalWrite(t *testing.T) { ctx := context.Background() w := newJ17World(t, nil) o := j17Observe(t, w) hook := &j17RecordHook{Store: w.raw, key: w.key} hook.beforePut = func() { a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } delete(a.Value.Registrations, "move") b, _ := json.Marshal(a.Value) if _, err := w.raw.Put(ctx, a.Key, b, a.ETag); err != nil { t.Fatal(err) } } cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock }) if err != nil { t.Fatal(err) } _, tag := j17Record(t, w) if _, err := cw.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, ErrPending) { t.Fatalf("late registration removal = %v", err) } if _, tag2 := j17Record(t, w); tag != tag2 { t.Fatal("record written after the registration was removed") } } // j17RecordHook runs beforePut once, before the second admission read // used by the final claim and registration check. type j17RecordHook struct { release.Store key string reads int beforePut func() } func (h *j17RecordHook) Get(ctx context.Context, key string) ([]byte, string, error) { if strings.HasSuffix(key, "/admission.json") { h.reads++ if h.reads == 2 && h.beforePut != nil { f := h.beforePut h.beforePut = nil f() } } return h.Store.Get(ctx, key) } // No exported method of the typed writers takes or returns a store, a // callback or a mutation function: there is nothing to inject. func TestJ17NoRawStoreOrCallbackEscapes(t *testing.T) { storeType := reflect.TypeOf((*release.Store)(nil)).Elem() for _, typ := range []reflect.Type{reflect.TypeOf(&ControlWriter{})} { for i := 0; i < typ.NumMethod(); i++ { m := typ.Method(i) if !strings.Contains(m.Name, "Move") && !strings.Contains(m.Name, "Loss") { continue } ft := m.Type check := func(t2 reflect.Type) { if t2.Kind() == reflect.Func || t2 == storeType || t2.Implements(storeType) { t.Errorf("%s exposes %s", m.Name, t2) } } for j := 1; j < ft.NumIn(); j++ { check(ft.In(j)) } for j := 0; j < ft.NumOut(); j++ { check(ft.Out(j)) } } } // The observation carries values only. ot := reflect.TypeOf(ConsoleMoveObservation{}) for i := 0; i < ot.NumField(); i++ { if k := ot.Field(i).Type.Kind(); k != reflect.String && k != reflect.Uint64 { t.Errorf("observation field %s is %s", ot.Field(i).Name, k) } } } // A context carrying a forged or foreign lifecycle grant adds nothing. func TestJ17ContextGrantsAreIgnored(t *testing.T) { ctx := context.Background() w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { delete(a.Registrations, "move") }) ctx = withLifecycleGrant(ctx, lifecycleGrant{kind: "recovery", w: w.control}) ctx = context.WithValue(ctx, lifecycleRecordKey{}, lifecycleRecordPermit{Observation: w.claim(t)}) if _, err := w.control.ConsoleMove(ctx, j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, ErrPending) { t.Fatalf("injected grant = %v", err) } }