//go:build linux && rootrunner package host import ( "bytes" "context" "crypto/ed25519" "encoding/json" "fmt" "net" "os" "os/exec" "path/filepath" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) type seedCrashFixture struct { Claims, Data, Locks, Root, Parks, Journal, Registry, Retirement, Signal string Actor hostbox.Actor Now time.Time } func rootFilesSeedWitnessFor(t *testing.T, binary string) filesSeedLiveWitness { if strings.Contains(t.Name(), "Socket") { return filesSeedLiveWitness{seed: rootFilesSeedLifetime} } if strings.Contains(t.Name(), "WholeExecutorKilledRerun") { return filesSeedLiveWitness{seed: func(t *testing.T, p *servingPreparation, e *filesSeedExecutor) { rootFilesSeedCrash(t, binary, p, e) }} } return rootFilesSeedLiveWitness(t, binary) } func rootFilesSeedCrash(t *testing.T, binary string, p *servingPreparation, e *filesSeedExecutor) { f := p.f root := t.TempDir() saved := seedCrashFixture{Claims: filepath.Join(root, "claims"), Data: f.f.guard.DataDir, Locks: f.f.guard.LockDir, Root: f.rootDisk.dir, Parks: "/var/lib/carlos-claims/authority/serving-parks", Journal: f.f.journalPath, Registry: f.p.root.Reconcile.Registry.Path(), Retirement: f.p.root.Reconcile.Retirement.JournalDir, Signal: filepath.Join(root, "precommit"), Actor: f.actor, Now: f.f.now} claims := release.NewDirStore(saved.Claims) keys, err := f.f.claims.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { body, _, err := f.f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err = claims.Put(t.Context(), key, body, ""); err != nil { t.Fatal(err) } } // The seed selection registry and the handover registry share the same route here. rt, err := e.registry.Get(recoveryHost) if err != nil || rt == nil { t.Fatal(err) } savedBody, _ := json.Marshal(saved) config := filepath.Join(root, "fixture.json") if err := os.WriteFile(config, savedBody, 0600); err != nil { t.Fatal(err) } run := func(crash bool) error { cmd := exec.Command(binary, "-test.run=^TestRootFilesSeedWholeExecutorPayload$", "-test.v", "-test.timeout=40s") cmd.Env = append(os.Environ(), "FILES_SEED_CRASH_CONFIG="+config, fmt.Sprintf("FILES_SEED_CRASH=%v", crash)) cmd.Stdin = os.Stdin var out bytes.Buffer cmd.Stdout, cmd.Stderr = &out, &out err := cmd.Run() t.Log(out.String()) return err } err = run(true) var status syscall.WaitStatus if exit, ok := err.(*exec.ExitError); ok { status = exit.Sys().(syscall.WaitStatus) } if !status.Signaled() || status.Signal() != syscall.SIGKILL { t.Fatal("seed did not die at precommit", err) } if _, err := os.Stat(saved.Signal); err != nil { t.Fatal("never reached materialization", err) } before, err := hostclaim.Read(t.Context(), claims, recoveryHost) if err != nil || before.Claim.Holder.WakeRetry.State != "attempting" || before.Claim.Holder.WakeRetry.AttemptsReserved != 1 { t.Fatal("crash lost retry debt", err) } data := f.f.data.(*leasetest.VersionedBucket) first := movesFilesPrefix(before.Claim.Holder.Location, before.Claim.Holder.Generation) if data.LatestID(f.actor.Bucket, hostfiles.ManifestKey(first)) != "" { t.Fatal("crashed seed committed") } // Model expiry of the dead owner's remote lease; no production TTL is changed. versions, err := data.ListVersions(t.Context(), f.actor.Bucket, before.Claim.Holder.Location+"/") if err != nil { t.Fatal(err) } for _, v := range versions { if strings.HasSuffix(v.Key, "/lock.json") && v.IsLatest { body, err := data.GetVersion(t.Context(), f.actor.Bucket, v.Key, v.VersionID) if err != nil { t.Fatal(err) } var lock map[string]any if err := json.Unmarshal(body, &lock); err != nil { t.Fatal(err) } lock["expires_at"] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) expired, _ := json.Marshal(lock) data.Put(f.actor.Bucket, v.Key, expired) } } if err := run(false); err == nil { t.Fatal("rerun bypassed settlement backoff") } settled, err := hostclaim.Read(t.Context(), claims, recoveryHost) if err != nil || settled.Claim.Holder.WakeRetry.State != "open" || settled.Claim.Holder.WakeRetry.AttemptsReserved != 1 || settled.Claim.Holder.WakeRetry.Attempt != nil { t.Fatal("rerun did not settle crashed attempt before backoff", err) } saved.Now = saved.Now.Add(time.Minute) savedBody, _ = json.Marshal(saved) if err := os.WriteFile(config, savedBody, 0600); err != nil { t.Fatal(err) } if err := run(false); err != nil { t.Fatal("rerun after backoff", err) } after, err := hostclaim.Read(t.Context(), claims, recoveryHost) if err != nil || after.Claim.Holder.WakeRetry.State != "open" || after.Claim.Holder.WakeRetry.AttemptsReserved != 2 || after.Claim.Holder.Generation == before.Claim.Holder.Generation { t.Fatal("rerun did not settle and reserve once", err) } final := movesFilesPrefix(after.Claim.Holder.Location, after.Claim.Holder.Generation) if data.LatestID(f.actor.Bucket, hostfiles.ManifestKey(final)) == "" { t.Fatal("rerun did not commit real admitted prefix") } etag := after.ETag if err := run(false); err != nil { t.Fatal(err) } after, err = hostclaim.Read(t.Context(), claims, recoveryHost) if err != nil || after.ETag != etag { t.Fatal("established rerun changed claim", err) } } func TestRootFilesSeedWholeExecutorPayload(t *testing.T) { path := os.Getenv("FILES_SEED_CRASH_CONFIG") if path == "" { t.Fatal("invoke through killed executor witness") } body, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var saved seedCrashFixture if err := json.Unmarshal(body, &saved); err != nil { t.Fatal(err) } t.Setenv("AWS_REGION", "eu-west-1") t.Setenv("AWS_DEFAULT_REGION", "eu-west-1") claims := release.NewDirStore(saved.Claims) data, err := lease.NewVersionedS3(t.Context(), map[string]string{saved.Actor.Bucket: "eu-west-1"}) if err != nil { t.Fatal(err) } journal, err := hostbox.OpenLaunchJournal(saved.Journal) if err != nil { t.Fatal(err) } defer journal.Close() roots := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: saved.Root}} rf := &recoveryFixture{t: t, claims: claims, evidence: claims, journal: journal, journalPath: saved.Journal, data: data, bucket: saved.Actor.Bucket, signer: ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)), now: saved.Now, seq: os.Getpid() * 100000} rf.guard = hostguard.Guard{DataDir: saved.Data, LockDir: saved.Locks, Claims: hostguard.StoreReader{Store: claims}, Root: hostguard.NewRootAuthority(roots)} rf.kernel = &recoveryKernel{journal: journal} crash := os.Getenv("FILES_SEED_CRASH") == "true" if !crash { rf.now = rf.now.Add(time.Minute) } reg, err := registry.Open(saved.Registry) if err != nil { t.Fatal(err) } defer reg.Close() row, err := reg.Get(recoveryHost) if err != nil || row == nil { t.Fatal(err) } state, err := rootstate.Open(saved.Parks, false, false) if err != nil { t.Fatal(err) } defer state.Close() journals, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } f := &servingRootFixture{t: t, f: rf, rootDisk: roots, db: row.DBPath, actor: saved.Actor} f.p = &servingParkRoot{ledger: &servingParkLedger{state: state, fs: newServingParkFS(saved.Data, state), journal: journal, now: func() time.Time { return rf.now }}, empty: plan9D5Fixture(t).owners} coordinator := rf.root(nil) coordinator.Kernel = &noProcessKernelFake{} coordinator.Inventory = journals.Inventory f.p.root = ClaimRootHTTP{Roots: map[string]string{"data": saved.Data}, Journals: journals, Launches: journal, Stop: ClaimStopHTTP{Coordinator: coordinator, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return rf.peer, nil }}, InventoryActor: func(context.Context) (hostbox.Actor, error) { return f.actor, nil }, RequireNewWork: func(context.Context) error { return nil }} f.p.root.Reconcile.Registry = reg f.p.root.Reconcile.Inventory = journals.Inventory f.p.root.Reconcile.Box, f.p.root.Reconcile.Enrollment = f.actor.Box, f.actor.Enrollment f.p.root.Reconcile.Retirement.JournalDir = saved.Retirement p := newServingPreparation(t, f, "boot", fmt.Sprintf("seed-%d", os.Getpid())) a := p.assembly a.Data = data a.Identity = handoverRecoveryIdentity(t, saved.Actor) source := &movesFilesSource{DataDir: saved.Data, Routes: reg, newFiles: func(ctx context.Context, _, _ string) (release.Store, error) { return release.NewS3Store(ctx, saved.Actor.Bucket) }} a.Wake.seedFactory = func(ctx context.Context, op *filesSeedOperation, token *activator.SeedExclusion) (*activator.ClaimRuntime, *ClaimWakeAssembly, error) { runtime, wake, err := newFilesSeedRuntime(ctx, a, p.source, p.writer, op, token) if err != nil { return runtime, wake, err } prepare := runtime.Prepare runtime.Prepare = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) (activator.ClaimStart, error) { start, err := prepare(ctx, rt, o, policy) if err == nil && crash { start.Materialize = func(context.Context) error { if err := os.WriteFile(saved.Signal, []byte(o.Claim.Holder.Generation), 0600); err != nil { return err } syscall.Kill(os.Getpid(), syscall.SIGKILL) select {} } } return start, err } return runtime, wake, nil } executor := &filesSeedExecutor{assembly: &a, service: &activator.Service{}, registry: reg, source: source} if err := executor.run(t.Context(), recoveryHost, os.Stdout); err != nil { t.Fatal(err) } }