//go:build linux package host import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "testing" "time" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) func filesSeedCombinedRecovery(t *testing.T, install func(*servingParkDiskFixture) rootstate.Store) { t.Helper() f := newServingParkDiskFixture(t) state := install(f) var parks []servingParkRecord for i := 0; i < 200; i++ { f.base.Host = fmt.Sprintf("park-%03d.example.com", i) f.target.Host = f.base.Host if err := os.MkdirAll(filepath.Dir(provision.NestedDBPath(f.data, f.base.Host)), 0700); err != nil { t.Fatal(err) } f.put("", fmt.Sprintf("retained-%03d", i)) auth, receipts := f.evidence(10, true, "missing") r, err := f.park(auth, receipts) if err != nil { t.Fatal(i, err) } if i != 0 { f.complete(auth) } parks = append(parks, r) } ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() maintenance := &servingParkRuntime{authority: &servingParkRoot{ledger: f.ledger}, ctx: ctx, cancel: cancel, idle: make(chan struct{})} root := ClaimRootHTTP{servingParks: maintenance} exclusion := &release.ReleaseExclusion{Open: func() (rootstate.Store, error) { return &plan14ReleaseStore{}, nil }} ledger := rootparticipation.Ledger{State: state, ExcludeLaunches: exclusion.With, VerifyLaunches: func(ctx context.Context, _ uint64) error { return exclusion.Check(ctx) }} raw, _ := json.Marshal(rootparticipation.Census{Schema: 1, Box: "box", Enrollment: "en", Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) if err := state.Write("census.json", raw); err != nil { t.Fatal(err) } child := exec.Command("sleep", "60") if err := child.Start(); err != nil { t.Fatal(err) } defer func() { child.Process.Kill(); child.Wait() }() boot, err := execlease.BootID() if err != nil { t.Fatal(err) } start, err := claimLaunchProcessStart("/proc", child.Process.Pid) if err != nil { t.Fatal(err) } peer := execlease.ControllerProcess{KernelBootID: boot, PID: child.Process.Pid, Start: start} q := execlease.ExecutionRequest{Schema: 1, ID: "begin", Operation: "executionBegin", Host: "crashed.example.com", Incarnation: "inc", Stamp: 1, Session: 1, Op: "move"} cleaned, settled := 0, 0 cfg := execlease.RootBrokerConfig{State: state, Remote: release.NewDirStore(t.TempDir()), Clock: execlease.SystemClock{}, BootID: boot, Identity: "old-root", Alive: execlease.ControllerAlive, Authority: func(context.Context, execlease.ExecutionRequest, execlease.ControllerProcess) (execlease.ExecutionAuthority, error) { return execlease.ExecutionAuthority{Box: "box", Enrollment: "en", Account: "bab", App: "jam", Installation: "inc/1/box/en", Migration: 1, ClaimRevision: "claim", RecordRevision: "record", DirectoryRevision: "1", Kind: "landing", Phase: "checking"}, nil }, Cleanup: func(ctx context.Context, r execlease.ExecutionRecord) error { if execlease.ControllerAlive(ctx, r.Peer) == nil { return execlease.ErrState } cleaned++ return nil }, Settle: func(context.Context, execlease.ExecutionRecord) error { settled++; return nil }, Exclude: ledger.Acquire, Recovered: ledger.RecoverAcquisition} broker, err := execlease.NewRootBroker(cfg) if err != nil { t.Fatal(err) } if _, err := broker.Execute(ctx, peer, q); err != nil { t.Fatal(err) } if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.BeginUpdate(ctx, "update") }); err == nil { t.Fatal("live participation allowed update") } if err := child.Process.Kill(); err != nil { t.Fatal(err) } child.Wait() if err := broker.Close(ctx); err != nil { t.Fatal(err) } cleaned = 0 cfg.Identity = "replacement-root" broker, err = execlease.NewRootBroker(cfg) if err != nil { t.Fatal(err) } defer broker.Close(context.Background()) root.Execution = broker update, _ := applyHappy(t) update.a.participation = &ledger originalRecovery := recoverPendingControllers t.Cleanup(func() { recoverPendingControllers = originalRecovery }) recoverPendingControllers = func(call context.Context, _, _ string) error { return recoverRootExecutionBeforeUpdate(call, broker, root) } if err := update.a.controllersRecovered(ctx); err != nil { t.Fatal(err) } if cleaned != 1 || settled != 1 { t.Fatal("controller debt not settled exactly once", cleaned, settled) } if maintenance.recoveryAfter == "" { t.Fatal("first page lost its cursor") } if err := maintenance.recoverPending(ctx); err != nil { t.Fatal(err) } if maintenance.recoveryAfter != "" { t.Fatal("second page did not complete") } if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.BeginUpdate(ctx, "update") }); err != nil { t.Fatal("retained parks blocked update", err) } if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.EndUpdateWith(ctx, "update", nil) }); err != nil { t.Fatal(err) } if err := update.a.runWithRelease(ctx, false); err != nil { t.Fatal(err) } if r := update.result(); r == nil || r.Outcome != systemOutcomeApplied { t.Fatal("update did not apply", r) } for i, r := range parks { f.retained(r.Authorization, fmt.Sprintf("retained-%03d", i), r.ReapAfter) } } func TestFilesSeedCombinedRecoveryAllowsUpdate(t *testing.T) { filesSeedCombinedRecovery(t, func(*servingParkDiskFixture) rootstate.Store { return &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}} }) }