package historyreadback import ( "bytes" "context" "crypto/ed25519" "encoding/json" "errors" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/safefile" ) type workflowLedger string func (s workflowLedger) Read(k string) ([]byte, error) { return os.ReadFile(filepath.Join(string(s), k)) } func (s workflowLedger) Write(k string, b []byte) error { return safefile.WriteFile(filepath.Join(string(s), k), b, 0600) } func (s workflowLedger) Remove(k string) error { return os.Remove(filepath.Join(string(s), k)) } func (s workflowLedger) Names() ([]string, error) { rows, e := os.ReadDir(string(s)) var names []string for _, r := range rows { names = append(names, r.Name()) } return names, e } func (s workflowLedger) Lock(context.Context, string) (func(), error) { return func() {}, nil } func (s workflowLedger) Close() error { return nil } type workflowCrashStore struct { release.Store boundary string } func (s workflowCrashStore) Put(ctx context.Context, key string, raw []byte, tag string) (string, error) { request := strings.HasSuffix(key, "/request.json") if !request && s.boundary == "before grant" { os.Exit(76) } got, err := s.Store.Put(ctx, key, raw, tag) if err == nil && ((!request && s.boundary == "after grant") || (request && s.boundary == "after request")) { os.Exit(76) } return got, err } func TestPlan14HistoryWorkflowProcessCrash(t *testing.T) { if boundary := os.Getenv("PLAN14_WORKFLOW_CRASH"); boundary != "" { dir := os.Getenv("PLAN14_WORKFLOW_DIR") raw, err := os.ReadFile(filepath.Join(dir, "saved.json")) if err != nil { t.Fatal(err) } var saved struct { Op Operation Pub ed25519.PublicKey Now time.Time } if err = json.Unmarshal(raw, &saved); err != nil { t.Fatal(err) } d := hostactors.Directory{Store: workflowCrashStore{Store: release.NewDirStore(filepath.Join(dir, "store")), boundary: boundary}, PublicKey: saved.Pub, Bucket: "bucket"} if err = saved.Op.Deliver(t.Context(), d, saved.Now); err != nil { t.Fatal(err) } t.Fatal("missed crash") } for _, boundary := range []string{"before grant", "after grant", "after request"} { t.Run(boundary, func(t *testing.T) { f := newFixture(t) dir := t.TempDir() store := release.NewDirStore(filepath.Join(dir, "store")) roster, _, err := f.directory.Store.Get(t.Context(), hostactors.Key) if err != nil { t.Fatal(err) } if _, err = store.Put(t.Context(), hostactors.Key, roster, ""); err != nil { t.Fatal(err) } f.directory.Store = store ledger := workflowLedger(t.TempDir()) h := &release.RootHistory{Store: ledger} resolve := func(release.HistoryBinding) (string, error) { return "canonical", nil } before, err := h.MigrationStatus(f.scope.Binding, resolve) if err != nil { t.Fatal(err) } g := release.HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: before.Identity, Revision: before.Revision, Operation: "operator", IssuedAt: f.now, Account: "abc", App: "web", Hosts: []release.HistoryReconstructionHost{before.Hosts[0].HistoryReconstructionHost}, Holds: map[string]release.HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: f.now}}} op, err := PrepareOperation(t.Context(), f.directory, f.signer, f.scope, &g, nil, f.now) if err != nil { t.Fatal(err) } saved := struct { Op Operation Pub ed25519.PublicKey Now time.Time }{op, f.directory.PublicKey, f.now} raw, err := json.Marshal(saved) if err != nil { t.Fatal(err) } path := filepath.Join(dir, "saved.json") if err = safefile.WriteFile(path, raw, 0600); err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14HistoryWorkflowProcessCrash$", "-test.count=1") cmd.Dir = t.TempDir() cmd.Env = append(os.Environ(), "PLAN14_WORKFLOW_CRASH="+boundary, "PLAN14_WORKFLOW_DIR="+dir) out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 76 { t.Fatalf("child %v: %s", err, out) } // A new operator process reads the saved messages; a new root opener reads // its own ledger. Neither depends on an in-memory successful response. again, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if !bytes.Equal(raw, again) { t.Fatal("crash changed saved authorization") } if err = json.Unmarshal(again, &saved); err != nil { t.Fatal(err) } if err = saved.Op.Deliver(t.Context(), f.directory, f.now); err != nil { t.Fatal(err) } if _, err = saved.Op.Read(t.Context(), f.directory, f.now); err == nil { t.Fatal("delivery completed operation") } h = &release.RootHistory{Store: ledger} if err = h.ReceiveReconstruction(t.Context(), store, f.directory.PublicKey, f.scope.Binding, f.now, resolve); err != nil { t.Fatal(err) } if err = Respond(t.Context(), f.directory, f.scope, f.key, f.now, func(context.Context) (release.HistoryMigrationStatus, error) { return h.MigrationStatus(f.scope.Binding, resolve) }); err != nil { t.Fatal(err) } got, err := saved.Op.Read(t.Context(), f.directory, f.now) if err != nil { t.Fatal(err) } if !got.Status.AppComplete || got.Status.Hosts[0].Programs.Complete { t.Fatal("wrong category migration", got.Status) } if err = saved.Op.Deliver(t.Context(), f.directory, f.now); err != nil { t.Fatal(err) } if err = h.ReceiveReconstruction(t.Context(), store, f.directory.PublicKey, f.scope.Binding, f.now, resolve); err != nil { t.Fatal(err) } final, err := h.ReconstructionStatus() if err != nil || final.Revision != g.Revision+1 { t.Fatal("replay advanced revision", final, err) } }) } }