package host import ( "context" "errors" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) func plan14InventoryProcessRoot(t *testing.T, dir string) (ClaimRootHTTP, *rollbackDisk) { t.Helper() store := release.NewDirStore(filepath.Join(dir, "deployment")) journal, err := hostbox.OpenLaunchJournal(filepath.Join(dir, "launches.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { journal.Close() }) journals, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } disk := &rollbackDisk{dir: filepath.Join(dir, "root")} guard := hostguard.Guard{DataDir: filepath.Join(dir, "data"), LockDir: filepath.Join(dir, "locks"), Claims: hostguard.StoreReader{Store: store}, Root: hostguard.NewRootAuthority(disk)} actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Revision: 1, Binding: "1", Region: "ie"} roots := map[string]string{"data": guard.DataDir, "opt": filepath.Join(dir, "opt"), "units": filepath.Join(dir, "units")} reg := restartTestRegistry(t) census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots} return ClaimRootHTTP{Journals: journals, Launches: journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: store}}}, disk } func TestPlan14InventoryProcessCrash(t *testing.T) { t.Parallel() if phase := os.Getenv("PLAN14_INVENTORY_CRASH"); phase != "" { root, disk := plan14InventoryProcessRoot(t, os.Getenv("PLAN14_INVENTORY_DIR")) writes := 0 disk.beforeWrite = func(string, []byte) { if phase == "database" && writes == 1 { os.Exit(73) } } disk.afterWrite = func(string, []byte) { writes++ if phase == "intent" && writes == 1 || phase == "snapshot" && writes == 2 { os.Exit(73) } } obs, err := hostclaim.Read(t.Context(), root.Stop.Coordinator.Evidence, recoveryHost) if err != nil { t.Fatal(err) } h := obs.Claim.Holder _, etag, err := root.Stop.Coordinator.Evidence.Get(t.Context(), h.Key) if err != nil { t.Fatal(err) } _, client := hostbox.RemoteJournals(root.commitRootInventory) _, err = client.WriteExpected(t.Context(), hostbox.StopInventory{Complete: true, Host: obs.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: "en", DirectoryRevision: "1"}, hostbox.InventoryExpectation{RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: obs.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box-1", Enrollment: "en", ClaimETag: obs.ETag, Op: obs.Claim.Op.ID, Phase: obs.Claim.Op.Phase}}) t.Fatal("crash boundary not reached", phase, err) } for _, phase := range []string{"intent", "database", "snapshot"} { t.Run(phase, func(t *testing.T) { dir := t.TempDir() for _, name := range []string{"root", "data", "opt", "units", "locks", "child"} { if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil { t.Fatal(err) } } f := newRecoveryFixture(t) obs := f.read() h := obs.Claim.Holder plan14PutRecord(t, f.claims, provision.Record{Schema: 1, Host: obs.Host, Account: h.Account, App: h.App, Channel: "stable", CreatedAt: f.now}) store := release.NewDirStore(filepath.Join(dir, "deployment")) keys, err := f.claims.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { body, _, err := f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err = store.Put(t.Context(), key, body, ""); err != nil { t.Fatal(err) } } cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14InventoryProcessCrash$") cmd.Env = append(os.Environ(), "PLAN14_INVENTORY_CRASH="+phase, "PLAN14_INVENTORY_DIR="+dir, "TMPDIR="+filepath.Join(dir, "child")) output, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 73 { t.Fatalf("child: %v %s", err, output) } root, _ := plan14InventoryProcessRoot(t, dir) root.Journals.Now = func() time.Time { return time.Now().Add(25 * time.Hour) } if err = root.recoverRootInventory(t.Context()); err != nil { t.Fatal(err) } captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} view, err := root.Stop.Coordinator.Guard.Root.View(t.Context(), obs.Host, captured) if err != nil || view.Installation.InventoryRevision != 1 { t.Fatal(view, err) } _, revision, err := root.Journals.Inventory.Snapshot(t.Context(), obs.Host, captured) if err != nil || revision != 1 { t.Fatal(revision, err) } if err = root.recoverRootInventory(t.Context()); err != nil { t.Fatal("replay", err) } }) } }