package host import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/rootstate" "github.com/carlosframework/platform/internal/schedule" "time" ) func TestPlan14TimerIntentionBeforeMaterialization(t *testing.T) { plan14TimerIntention(t, func(t *testing.T) rootstate.Store { return &rollbackDisk{dir: t.TempDir()} }) } type timerSyncFault struct { rootstate.Store phase string } func (s *timerSyncFault) Write(name string, body []byte) error { if err := s.Store.Write(name, body); err != nil { return err } pending := strings.Contains(string(body), `"PendingInstallation"`) if s.phase == "pending" && pending || s.phase == "complete" && !pending { s.phase = "" return os.ErrInvalid } return nil } func plan14TimerIntention(t *testing.T, open func(*testing.T) rootstate.Store) { for _, attack := range []string{"none", "already-materialized", "late-intention", "changed-registration", "changed-row", "omit-intention", "crash-after-intention", "crash-after-row", "intention-sync", "materialization-sync", "replace", "remove", "replace-sync", "remove-sync", "replace-crash", "remove-crash", "replace-advanced-crash", "remove-advanced-crash", "producer", "bootstrap"} { t.Run(attack, func(t *testing.T) { t.Setenv("CARLOS_HOST_CLAIM", "1") f := newRecoveryFixture(t) o := f.read() h := o.Claim.Holder plan14PutRecord(t, f.claims, provision.Record{Schema: 1, Host: o.Host, Account: h.Account, App: h.App, Channel: "stable", CreatedAt: f.now}) _, etag, _ := f.claims.Get(t.Context(), h.Key) journals, err := hostbox.NewJournalService(f.journal) if err != nil { t.Fatal(err) } registryPath := filepath.Join(t.TempDir(), "registry.db") reg, err := registry.Open(registryPath) if err != nil { t.Fatal(err) } defer reg.Close() sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%d\x00%s", o.Host, h.Incarnation, h.Stamp, "box-1"))) owner := registry.ClaimOwnership{ClaimIncarnation: h.Incarnation, ClaimStamp: h.Stamp, ClaimOwner: hex.EncodeToString(sum[:]), ClaimRowID: "route"} var bootstrap *registry.Timer if attack == "bootstrap" { if err := reg.UpsertTimer(registry.Timer{Host: o.Host, Name: "daily", Kind: "schedule", Path: "/daily", Spec: "every 1h0m0s", Attempts: 2, NextAt: 17}); err != nil { t.Fatal(err) } bootstrap, err = reg.GetTimer(o.Host, "daily") if err != nil { t.Fatal(err) } owned := owner owned.ClaimRowID = bootstrap.Registration if err := reg.AdoptClaimTimer(t.Context(), *bootstrap, owned); err != nil { t.Fatal(err) } } if err := reg.Upsert(registry.Route{Host: o.Host, Account: h.Account, App: h.App, DBPath: provision.NestedDBPath(f.guard.DataDir, o.Host), ClaimOwnership: owner}); err != nil { t.Fatal(err) } in := hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: "en", DirectoryRevision: "1", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: o.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}}} artifacts := hostguard.InstallationArtifacts{Schema: 1, StateDirectory: true} if bootstrap != nil { artifacts.Timers = bootstrap.Name in.Entries = append(in.Entries, hostbox.StopInventoryEntry{Kind: "timer", ArtifactID: bootstrap.Registration, ManagedRoot: "registry", RelativePath: o.Host + "/" + bootstrap.Name, RowIdentity: bootstrap.Registration, Ownership: owner.ClaimOwner, Materialization: "materialized"}) } if err := journals.Inventory.WriteStopInventory(t.Context(), in); err != nil { t.Fatal(err) } g := f.guard state := &timerSyncFault{Store: open(t)} defer state.Close() g.Root = hostguard.NewRootAuthority(state) if err := g.EstablishRoot(t.Context(), hostguard.RootInstallation{Artifacts: artifacts, Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "box-1", Enrollment: "en", Migration: 1, InventoryRevision: 1}, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Revision: 1, Binding: "1"} root := ClaimRootHTTP{Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: g, Evidence: f.claims}}, Journals: journals, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Roots: map[string]string{"data": g.DataDir}, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Census: &hostbox.EmptyInstallationCensus{}}} var last hostbox.JournalRequest _, client := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { raw, err := json.Marshal(q) if err != nil { t.Fatal(err) } q = hostbox.JournalRequest{} if err := json.Unmarshal(raw, &q); err != nil { t.Fatal(err) } if q.Op == "inventory-version" { out, err := journals.Inventory.SnapshotRequest(ctx, q.Target) out.V, out.ID = 1, q.ID return out, err } last = q return root.commitRootInventory(ctx, q) }) expect := hostbox.InventoryExpectation{Revision: 1, RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}} write := func() error { _, err := client.WriteExpected(t.Context(), in, expect) if err == nil { expect.Revision++ } return err } timer := registry.Timer{Host: o.Host, Name: "daily", Kind: "schedule", Spec: "every 1h0m0s", Path: "/daily", Registration: "planned", ClaimOwnership: owner} timer.ClaimRowID = timer.Registration if attack == "producer" || attack == "bootstrap" { serviceGuard := f.guard if err := serviceGuard.Activate(t.Context(), o.Host, actor.Box, nil); err != nil { t.Fatal(err) } serviceGuard.FenceView = g.Root.View adapter := hostbox.ReconcileAdapter{Registry: reg, Inventory: client, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Reconciler: hostbox.Reconciler{Guard: serviceGuard, Claims: f.claims}} source := hostbox.WorkSource{Guard: serviceGuard, Box: actor.Box, Region: "ie", Now: time.Now, Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }, Timer: adapter.MutateTimer} work, err := source.Select(t.Context(), o.Host, "timers") if err != nil { t.Fatal(err) } producer := schedule.SelectedTimers{Store: reg, Work: work, Context: t.Context(), Host: o.Host} timer.Registration = "" timer.ClaimOwnership = registry.ClaimOwnership{} if err := producer.UpsertTimer(timer); err != nil { t.Fatal("producer registration", err) } first, err := reg.GetTimer(o.Host, timer.Name) if err != nil { t.Fatal(err) } if err := producer.UpsertTimer(timer); err != nil { t.Fatal("producer replacement", err) } second, err := reg.GetTimer(o.Host, timer.Name) if err != nil || first.Registration == second.Registration { t.Fatal("producer identity", second, err) } if err := producer.DeleteTimer(o.Host, timer.Name); err != nil { t.Fatal("producer withdrawal", err) } if _, err := reg.GetTimer(o.Host, timer.Name); !errors.Is(err, os.ErrNotExist) { t.Fatal("producer deletion", err) } if err := producer.UpsertTimer(timer); err != nil { t.Fatal("producer registration after withdrawal", err) } imported := timer imported.Registration = "imported" imported.Attempts, imported.LastAt, imported.NextAt, imported.CreatedAt = 3, 7, 11, 13 if err := producer.ReplaceTimers(o.Host, []registry.Timer{imported}); err != nil { t.Fatal("import", err) } if err := producer.ReplaceTimers(o.Host, []registry.Timer{imported}); err != nil { t.Fatal("import replay", err) } got, err := reg.GetTimer(o.Host, imported.Name) if err != nil || got.Registration != "imported" || got.Attempts != 3 || got.LastAt != 7 || got.NextAt != 11 || got.CreatedAt != 13 { t.Fatal("import identity and bookkeeping", got, err) } return } row := func() { t.Helper() if err := reg.ReplaceTimers(o.Host, []registry.Timer{timer}); err != nil { t.Fatal(err) } } in.Entries = append(in.Entries, hostbox.StopInventoryEntry{Kind: "timer", ArtifactID: "timer", ManagedRoot: "registry", RelativePath: o.Host + "/daily", RowIdentity: timer.Registration, Ownership: owner.ClaimOwner, Materialization: "intended"}) if attack == "already-materialized" || attack == "late-intention" { row() if attack == "already-materialized" { in.Entries[1].Materialization = "materialized" } if err := write(); err == nil { t.Fatal("adopted timer without prior root intention") } return } if attack == "intention-sync" { state.phase = "pending" if err := write(); err == nil { t.Fatal("acknowledged unsynced intention") } if err := root.recoverRootInventory(t.Context()); err != nil { t.Fatal("recover intended write", err) } if _, err := root.commitRootInventory(t.Context(), last); err != nil { t.Fatal("replay recovered intention", err) } expect.Revision++ } else if err := write(); err != nil { t.Fatal("record intention", err) } if attack == "crash-after-intention" { if err := write(); err != nil { t.Fatal("resume absent intended row", err) } } row() if attack == "crash-after-row" { if err := write(); err != nil { t.Fatal("resume intended exact row", err) } } in.Entries[1].Materialization = "materialized" if attack == "materialization-sync" { state.phase = "complete" } switch attack { case "changed-registration", "changed-row": db, err := sql.Open("sqlite", registryPath) if err != nil { t.Fatal(err) } column := "registration" if attack == "changed-row" { column = "claim_row_id" } _, err = db.Exec("UPDATE timers SET "+column+"='successor' WHERE host=? AND name=?", o.Host, timer.Name) db.Close() if err != nil { t.Fatal(err) } case "omit-intention": in.Entries = in.Entries[:1] } err = write() if attack == "materialization-sync" { if err == nil { t.Fatal("acknowledged unsynced materialization") } _, err = root.commitRootInventory(t.Context(), last) if err == nil { expect.Revision++ } } valid := attack == "none" || attack == "crash-after-intention" || attack == "crash-after-row" || attack == "intention-sync" || attack == "materialization-sync" || strings.HasPrefix(attack, "replace") || strings.HasPrefix(attack, "remove") if (err == nil) != valid { t.Fatalf("materialization valid=%t: %v", valid, err) } if valid { if err := write(); err != nil { t.Fatal("materialized replay", err) } if strings.HasPrefix(attack, "replace") || strings.HasPrefix(attack, "remove") { mutation := hostbox.TimerMutation{Timer: timer, Expected: timer.Registration, Remove: strings.HasPrefix(attack, "remove")} if mutation.Remove { in.Entries[1].TimerRetired = true } else { mutation.Timer.Registration = "replacement" mutation.Timer.ClaimRowID = "replacement" in.Entries[1].RowIdentity = "replacement" } if strings.HasSuffix(attack, "sync") { state.phase = "complete" } if strings.HasSuffix(attack, "crash") { calls := 0 journals.Now = func() time.Time { calls++ if calls > 1 { return time.Now().Add(2 * hostbox.JournalRequestWindow) } return time.Now() } } revision, err := client.WriteTimerExpected(t.Context(), in, expect, mutation) if strings.HasSuffix(attack, "crash") { if err == nil { t.Fatal("journal fault did not interrupt mutation") } got, getErr := reg.GetTimer(timer.Host, timer.Name) if mutation.Remove { if !errors.Is(getErr, os.ErrNotExist) { t.Fatal("did not reach row removal", getErr) } } else if getErr != nil || got.Registration != mutation.Timer.Registration { t.Fatal("did not reach row replacement", got, getErr) } journals.Now = time.Now if strings.Contains(attack, "advanced") { key, _ := hostclaim.Key(o.Host) body, etag, err := f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err := f.claims.Put(t.Context(), key, append(body, '\n'), etag); err != nil { t.Fatal(err) } } if err := root.recoverRootInventory(t.Context()); err != nil { t.Fatal(err) } if strings.Contains(attack, "advanced") { snapshot, err := journals.Inventory.SnapshotRequest(t.Context(), expect.Target) if err != nil || snapshot.Revision != expect.Revision+1 || snapshot.Inventory.Entries[1] != in.Entries[1] { t.Fatal("advanced claim lost pending effect", snapshot, err) } pending, err := g.Root.PendingInstallations() if err != nil || len(pending) != 0 { t.Fatal("effect still pending", pending, err) } return } reply, replayErr := root.commitRootInventory(t.Context(), last) err, revision = replayErr, reply.Revision } if strings.HasSuffix(attack, "sync") { if err == nil { t.Fatal("acknowledged failed sync") } reply, replayErr := root.commitRootInventory(t.Context(), last) err, revision = replayErr, reply.Revision } if err != nil || revision != expect.Revision+1 { t.Fatal("timer mutation", revision, err) } expect.Revision = revision got, err := reg.GetTimer(o.Host, timer.Name) if mutation.Remove { if !errors.Is(err, os.ErrNotExist) { t.Fatal("timer survived", got, err) } } else if err != nil || got.Registration != "replacement" { t.Fatal("replacement missing", got, err) } if _, err := root.commitRootInventory(t.Context(), last); err != nil { t.Fatal("mutation replay", err) } } got, err := journals.Inventory.ReadStopInventory(t.Context(), o.Host) if err != nil || len(got.Entries) != 2 || got.Entries[1] != in.Entries[1] { t.Fatal("lost exact timer", got, err) } } }) } }