package host import ( "context" "encoding/base64" "errors" "fmt" "reflect" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // Plan 5 Task 9b (moves-after-Plan-12 Task 8; rulings Q1, Q12, Q15): the // moves wake source is registered right after the parks, with both // switches on, and its registration is what opens the one gate // (movesWakeSourceRegistered), so the parks arm in the same agent start as // the wake owners and never before. // mwSecretsKey is a CARLOS_SECRETS_KEY for the registered source's env // report key and held render: a valid P-256 scalar. var mwSecretsKey = base64.StdEncoding.EncodeToString([]byte("0123456789abcdef0123456789abcdef")) // mwEnv is an environment with the two switches as given. func mwEnv(claim, phase1 bool) func(string) string { vals := map[string]string{"CARLOS_SECRETS_KEY": mwSecretsKey} if claim { vals["CARLOS_HOST_CLAIM"] = "1" } if phase1 { vals[moves.SwitchEnv] = "on" } return func(k string) string { return vals[k] } } // mwReadAccess is the claim's move read access as a fixture: the route and // launch it answers, and the held works it selects, counted. type mwReadAccess struct { route registry.Route launch ClaimServingLaunch report moves.EnvReport works []string } func (a *mwReadAccess) HeldWork(_ context.Context, _, action string, _ hostguard.Captured) (hostwrite.Work, error) { a.works = append(a.works, action) return plan9D5Work{}, nil } func (a *mwReadAccess) Route(_ context.Context, host string) (*registry.Route, error) { if host != a.route.Host { return nil, hostbox.ErrChanged } rt := a.route return &rt, nil } func (a *mwReadAccess) Launch(_ context.Context, host string) (ClaimServingLaunch, moves.EnvReport, error) { if host != a.route.Host { return ClaimServingLaunch{}, moves.EnvReport{}, hostbox.ErrChanged } return a.launch, a.report, nil } // mwArm arms a moveCfg for a box whose placement is placement over the // claim's engine in engineRegion, with the fleet's home binding home. func mwArm(t *testing.T, view func(context.Context, hostclaim.Observed) (park.View, error), engineRegion, home, placement string) (moveCfg, error) { t.Helper() auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil } providers := ClaimProviders{moveParks: &claimMoveParks{ local: claimMoveLocalAuthorizer(func(context.Context) error { return nil }, auth), engine: park.Engine{Bucket: "data", Region: engineRegion, HomeRegion: home, Leases: (&renewTable{}).leases(), View: view, Authorize: auth}, leases: hostbox.ReplicaParkLeaseSet, check: func(context.Context) error { return nil }}} c := moveCfg{OwnBucket: "data", Region: placement} err := armMovesParks(t.Context(), &c, providers) return c, err } // TestBothFlagsOnRegistersAndArms (Q1, Q15): with CARLOS_HOST_CLAIM and // CARLOS_MOVES_PHASE1 both on, the agent's two registrations, the parks and // then the wake source, open the gate: the claim's registered move source // is this box's moves source, and the moves arms take the claim's engine. A // named box arms over its own engine. A home box arms only when the claim // grants it the @home alias (claim Plan 13's signed fleet home // designation); without it the home box stays unarmed. The parks alone do // not open the gate, and a second registration is refused. func TestBothFlagsOnRegistersAndArms(t *testing.T) { withMovesWakeSource(t, false) ctx := t.Context() env := mwEnv(true, true) hib := activator.Config{Bucket: "data", Region: "eu-west-1"} store := release.NewDirStore(t.TempDir()) roster := func(context.Context) (string, error) { return "ie", nil } parks, err := registerProductionMovesParks(ctx, env, store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)) if err != nil { t.Fatal(err) } if parks.View == nil || parks.Evidence == nil || claimMoveSources.Parks == nil { t.Fatalf("the parks are not registered: %+v", parks) } var dep *ClaimDependencyError if err := movesWakeSourceRegistered(); !errors.As(err, &dep) { t.Fatalf("the parks alone opened the gate: %v", err) } if c, err := mwArm(t, parks.View, "ie", "", "ie"); !errors.As(err, &dep) || parkHooksArmed(c) { t.Fatalf("armed on the parks alone: %v", err) } if err := registerProductionMovesWakeSource(env, parks, t.TempDir()); err != nil { t.Fatal(err) } if err := movesWakeSourceRegistered(); err != nil { t.Fatalf("the gate is shut after the wake source registered: %v", err) } reg, ok := claimMoveSources.Source.(*movesWakeSourceRegistration) if !ok { t.Fatalf("the claim's move source is %T, not this box's moves source", claimMoveSources.Source) } if _, ok := reg.ClaimProductionMoveSources.(*movesWakeSources); !ok { t.Fatalf("the registered source is %T", reg.ClaimProductionMoveSources) } // A named box arms over its own engine. if c, err := mwArm(t, parks.View, "ie", "", "ie"); err != nil || !parkHooksArmed(c) || c.ParkRunHeld == nil || c.ParkLocalAuthorize == nil { t.Fatalf("a named box did not arm: %v", err) } // A home box with the @home grant arms, and keeps the home binding. c, err := mwArm(t, parks.View, "ie", "ie", "") if err != nil || !parkHooksArmed(c) || c.ParkHomeRegion != "ie" { t.Fatalf("a home box the claim grants @home did not arm: %v (%q)", err, c.ParkHomeRegion) } // Without the grant a home box stays unarmed, over its named engine or // over one that names itself @home. for _, engine := range []string{"ie", park.HomeRegion} { if c, err := mwArm(t, parks.View, engine, "", ""); !errors.Is(err, hostbox.ErrChanged) || parkHooksArmed(c) { t.Fatalf("a home box without the grant (engine %q) armed: %v", engine, err) } } // One registration per process: a second is refused and the gate stays. if err := registerProductionMovesWakeSource(env, parks, t.TempDir()); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("a second registration: %v", err) } if err := movesWakeSourceRegistered(); err != nil { t.Fatalf("a refused second registration shut the gate: %v", err) } } // TestEitherFlagOffRegistersNothing (ruling H1, Q1): with either switch // off, or both, neither registration does anything, the gate stays shut, // the moves arms stay unarmed, and the claim keeps its named refusals. func TestEitherFlagOffRegistersNothing(t *testing.T) { hib := activator.Config{Bucket: "data", Region: "eu-west-1"} roster := func(context.Context) (string, error) { return "ie", nil } f := newVwFixture(t) live := movesParkSources{View: f.view("ie"), Evidence: movesRetirementEvidence{Records: f.records(), Now: time.Now}} for name, env := range map[string]func(string) string{ "claim off": mwEnv(false, true), "phase 1 off": mwEnv(true, false), "both off": mwEnv(false, false), } { t.Run(name, func(t *testing.T) { withMovesWakeSource(t, false) store := release.NewDirStore(t.TempDir()) parks, err := registerProductionMovesParks(t.Context(), env, store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)) if err != nil || parks.View != nil || parks.Evidence != nil { t.Fatalf("the parks: %+v %v", parks, err) } // Off, even a complete park source registers no wake source. for _, p := range []movesParkSources{parks, live} { if err := registerProductionMovesWakeSource(env, p, t.TempDir()); err != nil { t.Fatal(err) } } if claimMoveSources.Source != nil || claimMoveSources.Parks != nil { t.Fatalf("registered: source %v parks %v", claimMoveSources.Source, claimMoveSources.Parks) } var dep *ClaimDependencyError if err := movesWakeSourceRegistered(); !errors.As(err, &dep) || !strings.Contains(err.Error(), movesWakeSourceContract) { t.Fatalf("the gate: %v", err) } if c, err := mwArm(t, live.View, "ie", "", "ie"); !errors.As(err, &dep) || parkHooksArmed(c) { t.Fatalf("armed with a switch off: %v", err) } }) } } // TestTheClaimsParkViewIsTheMovesView (Q15): the wake source's View and // VerifyRetirement are never read by the claim, because the parks 9a // registered first stay in place; both delegate to the same park sources, // so whichever the claim reads, the answer is the moves view's (and the // moves evidence's). The claim's regional view, which every host's park // chores and the at-rest sweep read, is that view too. func TestTheClaimsParkViewIsTheMovesView(t *testing.T) { withMovesWakeSource(t, false) f := newVwFixture(t) evidence := &countingEvidence{RegionalEvidence: movesRetirementEvidence{Records: f.records(), Now: time.Now}} parks, err := newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), evidence) if err != nil { t.Fatal(err) } if err := registerMovesParks(parks); err != nil { t.Fatal(err) } registered := claimMoveSources.Parks if err := registerProductionMovesWakeSource(mwEnv(true, true), parks, t.TempDir()); err != nil { t.Fatal(err) } if claimMoveSources.Parks != registered { t.Fatal("the wake source replaced the parks 9a registered") } f.marker(t, "cur", f.inc) obs := vwObs(&hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}}) direct, err := movesParkView(f.records(), f.bucket, "data", "ie")(t.Context(), obs) if err != nil || direct.AuthoritativeRegion != "ie" || !direct.MarkerKnown || direct.MarkerGeneration != "cur" { t.Fatalf("the moves view: %+v %v", direct, err) } src := claimMoveSources.Source for name, view := range map[string]func(context.Context, hostclaim.Observed) (park.View, error){ "the claim's park view": claimMoveSources.Parks.View, "the wake source's View": src.View, "the registered parks": parks.View, "the claim's regional view": productionClaimRegional{Identity: handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "ie"}), Parks: claimMoveSources.Parks}.View, } { got, err := view(t.Context(), obs) if err != nil { t.Fatalf("%s: %v", name, err) } got.GenerationOf, direct.GenerationOf = nil, nil got.ArchivedIncarnations, got.RefusedLocations = nil, nil if !reflect.DeepEqual(got, direct) { t.Fatalf("%s is not the moves view:\n got %+v\nwant %+v", name, got, direct) } } retired := vwObs(&hostclaim.Claim{}) if _, _, err := src.VerifyRetirement(t.Context(), retired, nil); evidence.calls != 1 { t.Fatalf("the wake source's VerifyRetirement did not delegate to the moves evidence: %d calls (%v)", evidence.calls, err) } if _, _, err := claimMoveSources.Parks.Evidence.VerifyRetirement(t.Context(), retired, nil); evidence.calls != 2 { t.Fatalf("the claim's evidence is not the moves evidence: %d calls (%v)", evidence.calls, err) } } // countingEvidence counts VerifyRetirement calls. type countingEvidence struct { hostbox.RegionalEvidence calls int } func (e *countingEvidence) VerifyRetirement(ctx context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) { e.calls++ return e.RegionalEvidence.VerifyRetirement(ctx, o, v) } // TestRegisteredSourceOpensTheMovesOwnersWithTheClaimsFiles (Task 8's // controller overrides): the registered source's Open returns the moves // owner set the claim binds (the landing, committed and recovery owners, // with Operations, LossRecovery and LossSources), over the claim's read // access, the actor's bucket and region, and Task 7b's held release adapter; // it opens no files provider of its own. The C8 provider the recovery // owners establish through is the one Task 6's registration binds for the // claim's serving owners: the same value. A second provider is a wiring // error. func TestRegisteredSourceOpensTheMovesOwnersWithTheClaimsFiles(t *testing.T) { withMovesWakeSource(t, false) ctx := t.Context() f := newVwFixture(t) parks, err := newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), movesRetirementEvidence{Records: f.records(), Now: time.Now}) if err != nil { t.Fatal(err) } if err := registerMovesParks(parks); err != nil { t.Fatal(err) } env := mwEnv(true, true) dataDir := t.TempDir() if err := registerProductionMovesWakeSource(env, parks, dataDir); err != nil { t.Fatal(err) } src := claimMoveSources.Source.(*movesWakeSourceRegistration).ClaimProductionMoveSources.(*movesWakeSources) src.Getenv = env src.markers = func(context.Context, string, string) (ClaimMarkerStore, error) { return plan9D5Markers{f.bucket}, nil } actor := hostbox.Actor{Box: "ie-box", Region: "ie", Bucket: "data", Enrollment: "enr", Cloud: hostbox.CloudIdentity{Region: "eu-west-1"}} store := release.NewDirStore(t.TempDir()) data := leasetest.NewVersionedBucket() access := &mwReadAccess{route: registry.Route{Host: vwHost}} out := ClaimOwnerBindings{Data: data} // OpenClaimOwners hands Open its move read access; here the fixture's. owners, files, err := claimMoveSources.Source.Open(ctx, store, data, actor, access) if err != nil || files != nil { t.Fatalf("open: files %v, %v", files, err) } if ok, err := claimMovesC1(env, owners); !ok || err != nil { t.Fatalf("the owner set is not complete: %v %v", ok, err) } if owners.LossRecovery == nil || owners.LossSources == nil || owners.Operations == nil || owners.ProbeReady != nil { t.Fatalf("owners %+v", owners) } if reflect.TypeOf(owners.LossRecovery) != reflect.TypeOf(hostops.MoveLossRecoveryOwner()) { t.Fatalf("the loss owner is %T", owners.LossRecovery) } m, ok := owners.Inputs.(*movesProbeOwners) if !ok { t.Fatalf("the inputs are %T", owners.Inputs) } if !m.valid() || m.Box != "ie-box" || m.Region != "ie" || m.Bucket != "data" || m.Data != data || m.Claims != store || m.DataDir != movesDataDir(dataDir) || m.Files != nil || m.Release == nil || m.Page == nil { t.Fatalf("the probe owners: %+v", m) } ops, ok := owners.Operations.(*movesProbeAttempts) if !ok || ops.Claims != store || ops.DataDir != m.DataDir || ops.Admitted == nil || ops.RecoveryAttempt == nil { t.Fatalf("the attempts: %#v", owners.Operations) } // The read access is the claim's: route, launch and held works. if rt, err := m.Route(vwHost); err != nil || rt.Host != vwHost { t.Fatalf("route %v %v", rt, err) } if _, err := m.Works(ctx, vwHost, movesActionHeldConfig, hostguard.Captured{}); err != nil || !reflect.DeepEqual(access.works, []string{movesActionHeldConfig}) { t.Fatalf("held work %v %v", access.works, err) } // The claim's C8 provider binds the serving owners and the recovery // owners alike: one value. out.Moves = owners provider := &movesFiles{} closeFiles, err := openClaimFilesManifest(ctx, env, &fakeFilesSource{files: provider}, store, data, actor, &out, func(context.Context) error { return nil }) if err != nil || closeFiles == nil { t.Fatalf("files: %v", err) } if out.Serving.Files != ClaimFilesManifest(provider) || m.Files != ClaimFilesManifest(provider) { t.Fatalf("the recovery owners' files %v, the serving owners' %v", m.Files, out.Serving.Files) } // Another provider for the same owners is refused, never swapped in. again := ClaimOwnerBindings{Moves: owners} if _, err := openClaimFilesManifest(ctx, env, &fakeFilesSource{files: &movesFiles{}}, store, data, actor, &again, func(context.Context) error { return nil }); !errors.Is(err, hostwrite.ErrUnavailable) || m.Files != ClaimFilesManifest(provider) { t.Fatalf("a second provider: %v, files %v", err, m.Files) } // Without the read access, or the secrets key, Open refuses: the wake // is then refused by name, never half-wired. if _, _, err := src.Open(ctx, store, data, actor, nil); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("no read access: %v", err) } src.Getenv = mwEnv(true, true) noKey := *src noKey.Getenv = func(k string) string { if k == "CARLOS_SECRETS_KEY" { return "" } return env(k) } if _, _, err := noKey.Open(ctx, store, data, actor, access); err == nil { t.Fatal("opened without the env report key") } if _, _, err := src.Open(ctx, store, data, hostbox.Actor{Box: "ie-box"}, access); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("a bare actor: %v", err) } } // mwDataSource is a moves source that records the bucket the claim hands // its Open. type mwDataSource struct { parkRegistrationWake data lease.VersionedStore } func (s *mwDataSource) Open(_ context.Context, _ release.Store, data lease.VersionedStore, _ hostbox.Actor, _ ClaimMoveReadAccess) (ClaimMovesOwners, ClaimFilesManifest, error) { s.data = data return ClaimMovesOwners{}, nil, nil } // mwBucket is the claim's data bucket, recording each listing's context. // When started is set, a listing closes it once it has reached the bucket // and then holds until its context ends. The test sets started before the // listing's goroutine starts, and reads the counters only after that // goroutine's result has been received, so neither side races. type mwBucket struct { *leasetest.VersionedBucket lists int deadlines []time.Time started chan struct{} } func (b *mwBucket) ListVersions(ctx context.Context, bucket, prefix string) ([]lease.ObjectVersion, error) { b.lists++ d, _ := ctx.Deadline() b.deadlines = append(b.deadlines, d) if b.started != nil { close(b.started) <-ctx.Done() return nil, ctx.Err() } return b.VersionedBucket.ListVersions(ctx, bucket, prefix) } // TestMovesOwnersReadTheBucketThroughTheRequestStore (Task 8 fix round 1, // Astra and platform-1e): the moves owners get the claim's data bucket as // the serving owners do, through hostbox.RequestStore: every request is // bounded by the claim's request limit, the box's identity is checked // before each one (a refused identity issues no later bucket call), and // the operation's own context still cancels a request in flight. func TestMovesOwnersReadTheBucketThroughTheRequestStore(t *testing.T) { inner := &mwBucket{VersionedBucket: leasetest.NewVersionedBucket()} inner.AddBucket("data") var refuse error checks := 0 check := func(context.Context) error { checks++; return refuse } src := &mwDataSource{} out := ClaimOwnerBindings{Data: inner} openProductionClaimMoveWake(t.Context(), release.NewDirStore(t.TempDir()), hostbox.Actor{Box: "ie-box"}, src, &out, check) if out.wakeUnavailable != nil || src.data == nil { t.Fatalf("open: %v", out.wakeUnavailable) } if _, ok := src.data.(hostbox.RequestStore); !ok { t.Fatalf("the moves owners' bucket is %T, not the claim's request store", src.data) } before := time.Now() if _, err := src.data.ListVersions(t.Context(), "data", "instances/"); err != nil { t.Fatal(err) } if inner.lists != 1 || checks != 1 || inner.deadlines[0].IsZero() || inner.deadlines[0].After(time.Now().Add(hostbox.RequestLimit)) || inner.deadlines[0].Before(before.Add(hostbox.RequestLimit-time.Minute)) { t.Fatalf("the request's deadline %v (lists %d, checks %d), want the request limit", inner.deadlines, inner.lists, checks) } refuse = hostbox.ErrChanged if _, err := src.data.ListVersions(t.Context(), "data", "instances/"); !errors.Is(err, hostbox.ErrChanged) || inner.lists != 1 { t.Fatalf("a refused identity: %v, %d bucket calls", err, inner.lists) } if _, err := src.data.GetVersion(t.Context(), "data", "k", "v"); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a refused identity's read: %v", err) } refuse = nil // A request in flight: wait (bounded) until it has reached the // bucket, or has ended early, then cancel the operation. started := make(chan struct{}) inner.started = started ctx, cancel := context.WithCancel(t.Context()) defer cancel() done := make(chan error, 1) go func() { _, err := src.data.ListVersions(ctx, "data", "instances/"); done <- err }() select { case <-started: case err := <-done: t.Fatalf("the request ended before it reached the bucket: %v", err) case <-time.After(10 * time.Second): t.Fatal("the request never reached the bucket") } cancel() select { case err := <-done: if !errors.Is(err, context.Canceled) { t.Fatalf("a cancelled operation's request: %v", err) } case <-time.After(10 * time.Second): t.Fatal("the operation's cancellation did not reach the request") } if inner.lists != 2 { t.Fatalf("%d bucket calls, want the first and the cancelled one", inner.lists) } } // TestMovesSourceOpenRefusalIsLoggedOnce (Task 8 fix round 1, // platform-1e): a moves source that does not open keeps every wake on the // box refused (fail closed), and the cause is logged once per agent start, // so an operator can see why the box serves nothing. func TestMovesSourceOpenRefusalIsLoggedOnce(t *testing.T) { var lines []string oldLogf, oldOnce := claimMoveWakeLogf, claimMoveWakeLogOnce claimMoveWakeLogf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } claimMoveWakeLogOnce = new(sync.Once) t.Cleanup(func() { claimMoveWakeLogf, claimMoveWakeLogOnce = oldLogf, oldOnce }) cause := errors.New("moves: secrets key: not a 32-byte key") for i := 0; i < 3; i++ { out := ClaimOwnerBindings{} openProductionClaimMoveWake(t.Context(), nil, hostbox.Actor{}, &parkRegistrationWake{failure: cause}, &out, func(context.Context) error { return nil }) if !errors.Is(out.wakeUnavailable, cause) { t.Fatalf("open %d: the refusal is not kept: %v", i, out.wakeUnavailable) } } if len(lines) != 1 || !strings.Contains(lines[0], cause.Error()) || !strings.Contains(lines[0], "every wake") { t.Fatalf("logged %q, want the cause once", lines) } } // TestBindClaimMovesFilesRefusesASecondProvider (Task 8 fix round 1, // platform-1e): the claim's C8 provider binds the moves owners once; the // same provider again changes nothing, and another is refused and never // swapped in. A binding without moves owners, or without a provider, does // nothing. func TestBindClaimMovesFilesRefusesASecondProvider(t *testing.T) { m := &movesProbeOwners{} first, second := &movesFiles{Box: "first"}, &movesFiles{Box: "second"} out := ClaimOwnerBindings{Moves: m.MovesOwners(&movesProbeAttempts{})} out.Serving.Files = first if err := bindClaimMovesFiles(&out); err != nil || m.Files != ClaimFilesManifest(first) { t.Fatalf("the first provider: %v, files %v", err, m.Files) } if err := bindClaimMovesFiles(&out); err != nil || m.Files != ClaimFilesManifest(first) { t.Fatalf("the same provider again: %v", err) } out.Serving.Files = second if err := bindClaimMovesFiles(&out); !errors.Is(err, hostwrite.ErrUnavailable) || m.Files != ClaimFilesManifest(first) { t.Fatalf("a second provider: %v, files %v", err, m.Files) } if err := bindClaimMovesFiles(&ClaimOwnerBindings{Serving: ClaimServingSources{Files: second}}); err != nil { t.Fatalf("no moves owners: %v", err) } if err := bindClaimMovesFiles(&ClaimOwnerBindings{Moves: out.Moves}); err != nil || m.Files != ClaimFilesManifest(first) { t.Fatalf("no provider: %v", err) } }