//go:build linux package host import ( "bytes" "context" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) func runFilesSeedRealPreparation(t *testing.T, p *servingPreparation, live ...filesSeedLiveWitness) { t.Helper() f := p.f f.actor.Bucket = f.f.bucket f.actor.Cloud.Region = "eu-west-1" f.actor.HomeRegion = "ie" f.actor.FleetHomeRegion = "ie" p.local.Actor = f.actor rt := registry.Route{Host: f.f.hostName(), Kind: "instance", Hibernate: true, Account: "bab", App: "jam", DBPath: f.db, Socket: provision.NestedSocket(f.f.hostName())} reg, err := registry.Open(filepath.Join(t.TempDir(), "seed.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) if err := reg.Upsert(rt); err != nil { t.Fatal(err) } if err := reg.SetNextDue(rt.Host, 777); err != nil { t.Fatal(err) } filesDir := filepath.Join(filepath.Dir(f.db), rt.Host) if err := os.MkdirAll(filesDir, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(filesDir, "payload"), []byte("real seeded bytes"), 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(filepath.Dir(f.db), rt.Host+".env"), []byte("KEY=value\n"), 0600); err != nil { t.Fatal(err) } source := &movesFilesSource{DataDir: f.f.guard.DataDir, Routes: reg, newFiles: func(context.Context, string, string) (release.Store, error) { return casBucketStore{b: f.f.data.(*leasetest.VersionedBucket), bucket: f.f.bucket}, nil }} a := p.assembly a.Data = f.f.data.(*leasetest.VersionedBucket) a.Identity = handoverRecoveryIdentity(t, f.actor) a.Wake.seedFactory = func(ctx context.Context, op *filesSeedOperation, token *activator.SeedExclusion) (*activator.ClaimRuntime, *ClaimWakeAssembly, error) { runtime, wake, err := newFilesSeedRuntime(ctx, a, p.source, p.writer, op, token) if err != nil { return runtime, wake, err } if f.f.guard.DataDir != "/data" { // Local fixture omits only installed /data launch-shape preflight. runtime.Prerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) error { if err := wake.ServingPrerequisites(ctx, rt, o); err != nil { return err } in, err := wake.Inputs.Select(ctx, rt, o, policy) if err != nil { return err } if err := wake.stopPredecessor(ctx, o); err != nil { return err } held, err := lease.Acquire(ctx, f.f.bucket, o.Claim.Holder.Location, wake.LeaseRegion) if err != nil { return err } live, cancel := context.WithCancel(ctx) done := make(chan struct{}) go func() { <-live.Done(); close(done) }() wake.pending[rt.Host] = &claimWakeAttempt{owner: wake, observed: o, inputs: in, held: held, ctx: live, cancel: cancel, done: done} return nil } } return runtime, wake, nil } openedSelection, err := source.Open(t.Context(), f.f.claims, f.f.data.(*leasetest.VersionedBucket), f.actor) if err != nil { t.Fatal(err) } classification, err := inspectFilesSeed(t.Context(), f.f.claims, openedSelection.(*movesFiles), f.f.read(), f.actor, &rt) if err != nil { t.Fatal("classification", err) } if _, err := selectFilesSeedInputs(t.Context(), p.owners, rt, classification); err != nil { t.Fatal("input selection", err) } executor := &filesSeedExecutor{assembly: &a, service: &activator.Service{}, registry: reg, source: source} for _, witness := range live { if witness.seed != nil { witness.seed(t, p, executor) return } } var output bytes.Buffer if err := executor.run(t.Context(), rt.Host, &output); err != nil { t.Fatalf("seed: %v\n%s", err, output.String()) } seeded := f.f.read() if seeded.Claim.Origins == nil || seeded.Claim.Holder.WakeRetry.AttemptsReserved != 1 || seeded.Claim.Holder.WakeRetry.State != "open" { t.Fatal("seed did not settle one real admission", seeded) } if len(executor.pending) != 0 { t.Fatal("seed retained its exclusion") } before := seeded.ETag if err := executor.run(t.Context(), rt.Host, &output); err != nil { t.Fatal("established rerun", err) } if f.f.read().ETag != before { t.Fatal("established rerun mutated claim") } opened, err := source.Open(t.Context(), f.f.claims, f.f.data.(*leasetest.VersionedBucket), f.actor) if err != nil { t.Fatal(err) } ordinary := opened.(*movesFiles) for wake := 0; wake < 2; wake++ { f.f.now = f.f.now.Add(time.Minute) normal := newServingPreparation(t, f, "boot", "controller", func(w *ClaimWakeAssembly) { w.Inputs.(*claimServingOwners).Sources.Files = ordinary w.ServingPrerequisites = servingFilesPrerequisites(f.f.claims, ordinary) for _, witness := range live { witness.configure(w) } }) for _, witness := range live { if witness.prepare != nil { witness.prepare(normal) } } admitted, start := normal.prepare(t) if err := start.Restore(t.Context()); err != nil { t.Fatal("cold restore", wake, err) } if err := start.Reserve(t.Context(), 0); err != nil { t.Fatal(err) } if err := start.Materialize(t.Context()); err != nil { t.Fatal("ordinary C8", wake, err) } prefix := movesFilesPrefix(admitted.Claim.Holder.Location, admitted.Claim.Holder.Generation) manifest, found, err := hostfiles.Load(t.Context(), ordinary.Files, prefix) if err != nil || !found { t.Fatal(err) } if _, ok := manifest.Entries["payload"]; !ok { t.Fatal("files lost at cold wake") } b, err := os.ReadFile(filepath.Join(filesDir, "payload")) if err != nil || string(b) != "real seeded bytes" { t.Fatal("restored files", string(b), err) } for _, witness := range live { witness.serve(t, normal, admitted, start) } healthy := false for _, witness := range live { healthy = healthy || witness.healthy } if !healthy { if err := start.Failed(t.Context(), 0); err != nil { t.Fatal(err) } } if err := normal.runtime.FailedAdmission(t.Context(), admitted, hostclaim.ErrWrongState); err != nil { t.Fatal(err) } if ordinary.Seed != nil { t.Fatal("ordinary provider armed") } } healthy := false for _, witness := range live { healthy = healthy || witness.healthy } if healthy { if f.f.read().Claim.Holder.WakeRetry != nil { t.Fatal("health did not clear retry debt") } } else if f.f.read().Claim.Holder.WakeRetry.AttemptsReserved != 3 { t.Fatal("attempt budget changed") } t.Log(output.String()) } type filesSeedLiveWitness struct { host string shared bool start func(*testing.T, *servingPreparation, hostclaim.Observed, activator.ClaimStart) func() healthy bool prepare func(*servingPreparation) seed func(*testing.T, *servingPreparation, *filesSeedExecutor) configure func(*ClaimWakeAssembly) serve func(*testing.T, *servingPreparation, hostclaim.Observed, activator.ClaimStart) } func TestFilesSeedRealOriginPreparation(t *testing.T) { runFilesSeedOrdinaryPreparation(t, "real-seed", "same-controller") } func TestFilesSeedReportsSkippedPaths(t *testing.T) { witness := filesSeedLiveWitness{seed: func(t *testing.T, p *servingPreparation, executor *filesSeedExecutor) { host := p.f.f.hostName() dir := filepath.Join(filepath.Dir(p.f.db), host) large, err := os.Create(filepath.Join(dir, "large.bin")) if err != nil { t.Fatal(err) } err = large.Truncate(hostfiles.MaxFileBytes + 1) large.Close() if err != nil { t.Fatal(err) } if err := os.Symlink("payload", filepath.Join(dir, "link\nname")); err != nil { t.Fatal(err) } var output bytes.Buffer if err := executor.run(t.Context(), host, &output); err != nil { t.Fatalf("seed: %v\n%s", err, output.String()) } for _, want := range []string{`skipped file "large.bin": exceeds 64 MiB`, `skipped file "link\nname": not a regular file.`, `: seeded.`} { if !strings.Contains(output.String(), want) { t.Fatalf("missing %q in output:\n%s", want, output.String()) } } }} runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", nil, []filesSeedLiveWitness{witness}) } func TestFilesSeedPreparationChangedController(t *testing.T) { witness := filesSeedLiveWitness{configure: func(*ClaimWakeAssembly) {}, serve: func(*testing.T, *servingPreparation, hostclaim.Observed, activator.ClaimStart) {}} witness.prepare = func(p *servingPreparation) { p.local.ControllerID, p.local.PID, p.local.ProcessStart = "new-controller", 88, "new-start" p.f.f.peer = StopPeer{UID: 2001, PID: 88, ProcessStart: "new-start", BootID: p.local.BootID, ControllerID: "new-controller"} p.f.p.root.Stop.Coordinator.CarlosUID = 2001 p.wake.Retries.BootID, p.wake.Retries.ControllerID = p.local.BootID, "new-controller" if len(p.wake.pending) != 0 { t.Fatal("rebuilding active fixture") } p.wake.pending = nil var err error p.runtime, err = p.wake.Build(t.Context(), p.source, p.writer) if err != nil { t.Fatal(err) } } runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", nil, []filesSeedLiveWitness{witness}) }