package hostops import ( "context" "crypto/ed25519" "encoding/json" "errors" "os" "strconv" "testing" "time" "github.com/carlosframework/platform/internal/release" ) type recoverFixture struct { ctx context.Context raw *release.DirStore w *ControlWriter now time.Time owner RegistrationOwner op string tag string at int64 } func newRecoverFixture(t *testing.T) *recoverFixture { t.Helper() f := &recoverFixture{ctx: t.Context(), raw: release.NewDirStore(t.TempDir()), now: time.Unix(1_800_000_000, 0), op: "restore-1"} f.owner = RegistrationOwner{Console: "console-a", Enrollment: "enrollment-a", Controller: f.op} var err error if f.w, err = NewControlWriter(f.raw, make(ed25519.PublicKey, ed25519.PublicKeySize), func() time.Time { return f.now }); err != nil { t.Fatal(err) } a := AppAdmission{V: 1, AppID: "inc-a", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]Registration{}, Effects: map[string]Effect{}} body, _ := json.Marshal(a) if _, err := f.raw.Put(f.ctx, "apps/bab/demo/admission.json", body, ""); err != nil { t.Fatal(err) } deleted := f.now.Add(-time.Hour).UTC() tomb, _ := json.Marshal(release.AppTombstone{Schema: 1, Account: "bab", App: "demo", DeletedAt: deleted, Actor: "p"}) if f.tag, err = f.raw.Put(f.ctx, release.AppTombstoneKey("bab", "demo"), tomb, ""); err != nil { t.Fatal(err) } f.at = deleted.Unix() return f } // begin is the restore issuer's BeginExclusive, exactly as the console runs it. func (f *recoverFixture) begin(t *testing.T) *OperationHandle { t.Helper() h, err := f.w.BeginExclusive(f.ctx, "bab", "demo", "inc-a", AppOperation{ID: f.op, Kind: OpAppMutation, AppID: "inc-a", Phase: "issued", Intent: map[string]string{"action": "restore", RestoreActionStarted: strconv.FormatInt(f.now.Unix(), 10)}}, f.owner) if err != nil { t.Fatal(err) } return h } // issue runs the restore issuer up to its permit in the console's order: // BeginExclusive, PrepareRestore (the receipt), then Issue. func (f *recoverFixture) issue(t *testing.T) (*OperationHandle, context.Context) { t.Helper() h := f.begin(t) if err := h.PrepareRestore(f.ctx, f.tag, f.at); err != nil { t.Fatal(err) } pctx, _, err := h.Runner(func() time.Time { return f.now }).Issue(f.ctx, Effect{ID: f.op, Resource: OpAppMutation, Action: "restore"}) if err != nil { t.Fatal(err) } return h, pctx } // stageEffect writes one effect of the restore operation straight into the // raw admission object: a crashed issuer's leftovers, staged as world state. func (f *recoverFixture) stageEffect(t *testing.T, state string) { t.Helper() obs, err := ReadAdmission(f.ctx, f.raw, "bab", "demo", "inc-a") if err != nil { t.Fatal(err) } v := obs.Value v.Effects[f.op] = Effect{ID: f.op, OperationID: f.op, AppID: "inc-a", Resource: OpAppMutation, Action: "restore", Owner: f.owner, State: state} v.Revision++ body, _ := json.Marshal(v) if _, err := f.raw.Put(f.ctx, obs.Key, body, obs.ETag); err != nil { t.Fatal(err) } } func (f *recoverFixture) admission(t *testing.T) AppAdmission { t.Helper() a, err := ReadAdmission(f.ctx, f.raw, "bab", "demo", "inc-a") if err != nil { t.Fatal(err) } return a.Value } func (f *recoverFixture) marker(t *testing.T) string { t.Helper() _, tag, err := f.raw.Get(f.ctx, release.AppTombstoneKey("bab", "demo")) if errors.Is(err, os.ErrNotExist) { return "" } if err != nil { t.Fatal(err) } return tag } func requireRestored(t *testing.T, a AppAdmission, op, tag string) { t.Helper() if a.State != "open" || a.Op != nil || a.LastRestore == nil || a.LastRestore.ID != op || a.LastRestore.TombstoneETag != tag || a.LastRestore.State != "complete" || a.Effects[op].State != "complete" { t.Fatalf("not restored: %+v %+v", a, a.LastRestore) } } func TestRecoverRestoreAfterReceiptBeforeDelete(t *testing.T) { f := newRecoverFixture(t) f.issue(t) // The issuer crashed. Another console finishes the receipt at any age. out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) if err != nil || out != RestoreRestored { t.Fatalf("recover = %q %v", out, err) } if f.marker(t) != "" { t.Fatal("receipt not replayed") } requireRestored(t, f.admission(t), f.op, f.tag) } func TestRecoverRestoreAfterDeleteBeforeCompletion(t *testing.T) { f := newRecoverFixture(t) f.issue(t) if err := release.DeleteIfMatch(f.ctx, f.raw, release.AppTombstoneKey("bab", "demo"), f.tag); err != nil { t.Fatal(err) } out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) if err != nil || out != RestoreRestored { t.Fatalf("recover = %q %v", out, err) } requireRestored(t, f.admission(t), f.op, f.tag) } // The receipt precedes Issue, so a crash between them leaves a prepared // receipt and no effect. It is finished (never abandoned), at any age. // Needs ControlWriter.RecoverMutation to accept a prepared receipt with no // effect (control-work adaptation; integration-notes §C). func TestRecoverRestoreReceiptWithoutEffect(t *testing.T) { f := newRecoverFixture(t) h := f.begin(t) if err := h.PrepareRestore(f.ctx, f.tag, f.at); err != nil { t.Fatal(err) } if err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", f.op, f.now.Add(time.Hour), time.Minute); !errors.Is(err, ErrPending) { t.Fatalf("a saved receipt was abandoned: %v", err) } out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) if err != nil || out != RestoreRestored { t.Fatalf("recover = %q %v", out, err) } if f.marker(t) != "" { t.Fatal("receipt not replayed") } a := f.admission(t) if a.State != "open" || a.Op != nil || a.LastRestore == nil || a.LastRestore.State != "complete" || a.Epoch != 2 { t.Fatalf("not restored: %+v %+v", a, a.LastRestore) } } func TestRecoverRestoreWithoutReceiptAbandonsAndFencesTheIssuer(t *testing.T) { f := newRecoverFixture(t) h := f.begin(t) // crashed or slow before PrepareRestore: nothing issued if _, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(time.Minute), 2*time.Minute); !errors.Is(err, ErrPending) { t.Fatalf("young restore abandoned: %v", err) } out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(3*time.Minute), 2*time.Minute) if err != nil || out != RestoreAbandoned { t.Fatalf("recover = %q %v", out, err) } a := f.admission(t) if a.State != "open" || a.Op != nil || a.LastRestore != nil || len(a.Effects) != 0 || a.Epoch != 2 { t.Fatalf("not abandoned with a new epoch: %+v", a) } // The stale issuer is fenced at every step it has left. if err := h.PrepareRestore(f.ctx, f.tag, f.at); !errors.Is(err, ErrPending) { t.Fatalf("stale receipt: %v", err) } if _, _, err := h.Runner(func() time.Time { return f.now }).Issue(f.ctx, Effect{ID: f.op, Resource: OpAppMutation, Action: "restore"}); err == nil { t.Fatal("stale issuer issued an effect after the abandon") } if f.marker(t) != f.tag { t.Fatal("marker changed") } } // A prepared effect never issued its call: abandoning supersedes it. An // issued or uncertain effect may have deleted the marker: never abandoned, // at any age, receipt or not. func TestAbandonRestoreOnlyReopensUnissuedEffects(t *testing.T) { for _, state := range []string{"prepared", "issued", "uncertain"} { t.Run(state, func(t *testing.T) { f := newRecoverFixture(t) f.begin(t) f.stageEffect(t, state) err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", f.op, f.now.Add(24*time.Hour), time.Minute) a := f.admission(t) if state == "prepared" { if err != nil || a.State != "open" || a.Effects[f.op].State != "complete" || a.Effects[f.op].ResultReference != "superseded" { t.Fatalf("prepared effect not superseded: %v %+v", err, a) } return } if !errors.Is(err, ErrPending) || a.State != "closed" || a.Op == nil || a.Effects[f.op].State != state { t.Fatalf("%s effect abandoned: %v %+v", state, err, a) } if out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(24*time.Hour), time.Minute); !errors.Is(err, ErrPending) || out != RestoreNone { t.Fatalf("recover reopened a %s effect: %q %v", state, out, err) } }) } } func TestRecoverRestoreLeavesOtherOperationsAlone(t *testing.T) { f := newRecoverFixture(t) if _, err := f.w.BeginExclusive(f.ctx, "bab", "demo", "inc-a", AppOperation{ID: "del", Kind: OpDeleting, AppID: "inc-a"}, f.owner); err != nil { t.Fatal(err) } before, _, err := f.raw.Get(f.ctx, "apps/bab/demo/admission.json") if err != nil { t.Fatal(err) } out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(time.Hour), time.Minute) if err != nil || out != RestoreNone { t.Fatalf("recover = %q %v", out, err) } if err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", "del", f.now.Add(time.Hour), time.Minute); !errors.Is(err, ErrPending) { t.Fatalf("abandoned a delete: %v", err) } after, _, err := f.raw.Get(f.ctx, "apps/bab/demo/admission.json") if err != nil || string(after) != string(before) { t.Fatal("admission changed") } }