package hostbox import ( "context" "errors" "fmt" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/release" ) type enrollmentFunc func(context.Context, string, string, time.Duration) error func (f enrollmentFunc) VerifyRegional(ctx context.Context, region, bucket string, limit time.Duration) error { return f(ctx, region, bucket, limit) } type evidenceFunc func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) func (f evidenceFunc) VerifyRetirement(ctx context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) { return f(ctx, o, v) } type workerLease struct{} func (workerLease) Check(context.Context) error { return nil } func (workerLease) Release(context.Context) error { return nil } func (workerLease) Close() {} type workerLeases struct{} func (workerLeases) Acquire(context.Context, string, string) (park.HeldLease, error) { return workerLease{}, nil } func TestRegionalWorkerRealEngineCensusSweepAndBound(t *testing.T) { ctx := t.Context() evidenceAt := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) host := "app.example" inc, err := hostclaim.Incarnation("acct", "app", host, evidenceAt) if err != nil { t.Fatal(err) } location := "instances/" + host + "/" + inc bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") bucket.PageSize = 1 bucket.Put("data", location+"/0000/db", []byte("old")) // The moves namespace participates in the top-level census independently. bucket.Put("data", "moves/"+host+"/landing/lock.json", []byte("lease")) claims := release.NewDirStore(t.TempDir()) _, err = hostclaim.Update(ctx, claims, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error { c.Holder = &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: evidenceAt, Location: location, Stamp: 1, Session: 1} c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"} c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}} return nil }) if err != nil { t.Fatal(err) } seq := 0 var limits []time.Duration engine := park.Engine{ Store: bucket, Claims: claims, Bucket: "data", Region: "ie", Leases: workerLeases{}, Now: func() time.Time { return evidenceAt.Add(18 * time.Minute) }, NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) }, View: func(_ context.Context, o hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil }, Authorize: func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error { if o.Park != nil { limits = append(limits, o.Park.RequestLimit) } return nil }, } engine.VerifyMovesSweep = func(_ context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (park.MoveSweepProof, error) { if len(versions) == 0 { t.Fatal("missing census") } return park.MoveSweepProof{Host: o.Host, ClaimETag: o.ETag, MoveID: "retired", Incarnation: inc, RetiredAt: evidenceAt}, nil } worker := RegionalWorker{ Regional: Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }}, Enrollment: enrollmentFunc(func(_ context.Context, region, gotBucket string, limit time.Duration) error { if region != "ie" || gotBucket != "data" { t.Fatalf("binding = %s/%s", region, gotBucket) } if limit != 120*time.Second { t.Fatalf("enforced request limit = %v", limit) } return nil }), Evidence: evidenceFunc(func(_ context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (string, time.Time, error) { if o.ETag == "" || len(versions) != 2 { t.Fatalf("evidence observation = etag %q, versions %d", o.ETag, len(versions)) } return inc, evidenceAt, nil }), } if err := worker.Run(ctx); !errors.Is(err, park.ErrPending) { t.Fatalf("new moves sweep must wait from listing time: %v", err) } if len(limits) == 0 { t.Fatal("authorization did not see a scoped chore") } for _, limit := range limits { if limit != RequestLimit { t.Fatalf("request limit = %v", limit) } } } func TestRegionalWorkerRefusesMissingAuthorityEvidenceAndUnknownKinds(t *testing.T) { base := RegionalWorker{} if _, err := base.Census(t.Context()); !errors.Is(err, ErrUnsupported) { t.Fatalf("missing providers: %v", err) } claims := release.NewDirStore(t.TempDir()) host := "unknown.example" _, err := hostclaim.Update(t.Context(), claims, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error { c.Obligations = []hostclaim.Obligation{{ID: "id", Op: "op", Kind: "park-superseded", Region: "ie", Owner: "region:ie"}} return nil }) if err == nil { // Claim validation itself currently refuses this pending-decision kind, // which is the desired boundary. If it becomes schema-valid, runHost's // explicit kind check below remains the adapter refusal. w := RegionalWorker{Regional: Regional{Engine: park.Engine{Claims: claims, Region: "ie", Bucket: "data"}}, Enrollment: enrollmentFunc(func(context.Context, string, string, time.Duration) error { return nil })} if got := w.runHost(t.Context(), host); !errors.Is(got, ErrUnsupported) { t.Fatalf("unknown kind: %v", got) } } } func TestRegionalWorkerCensusIncludesQueuedClaimWithoutResidue(t *testing.T) { for name, namespaces := range map[string][]string{ "direct-only": nil, "evidence": {"evidence/fence/receipt.json"}, "exec": {"exec/controller.json"}, "both": {"evidence/fence/receipt.json", "exec/controller.json"}, } { t.Run(name, func(t *testing.T) { claims := release.NewDirStore(t.TempDir()) host := "queued.example" created := time.Unix(1, 0).UTC() inc, err := hostclaim.Incarnation("bab", "app", host, created) if err != nil { t.Fatal(err) } _, err = hostclaim.Update(t.Context(), claims, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error { c.Holder = &hostclaim.Holder{Family: "instance", Key: "apps/bab/app/instances/" + host + ".json", Account: "bab", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/" + host + "/" + inc, Stamp: 1} c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "op"} c.Obligations = []hostclaim.Obligation{{ID: "id", Op: "op", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "retire", Park: &hostclaim.ParkScope{Location: c.Holder.Location, Incarnation: inc, EvidenceAt: time.Unix(1, 0).UTC(), RequestLimit: RequestLimit}}} c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}} return nil }) if err != nil { t.Fatal(err) } for _, namespace := range namespaces { for _, nestedHost := range []string{host, "nested-only.example"} { key := "control/hosts/" + nestedHost + "/" + namespace if _, err := claims.Put(t.Context(), key, []byte(`{"not_a_claim":true}`), ""); err != nil { t.Fatal(err) } } } keys, err := claims.List(t.Context(), "control/hosts/") if err != nil || len(keys) != 1+2*len(namespaces) { t.Fatalf("recursive claim-store listing = %v, %v", keys, err) } bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") w := RegionalWorker{ Regional: Regional{Engine: park.Engine{Claims: claims, Store: bucket, Bucket: "data", Region: "ie"}}, Enrollment: enrollmentFunc(func(context.Context, string, string, time.Duration) error { return nil }), Evidence: evidenceFunc(func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) { return "", time.Time{}, errors.New("unused") }), } hosts, err := w.Census(t.Context()) if err != nil || len(hosts) != 1 || hosts[0] != host { t.Fatalf("census = %v, %v", hosts, err) } }) } } func TestRegionalWorkerLoopRunsStartupAndJoinsCancellation(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) claims := release.NewDirStore(t.TempDir()) bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") started := make(chan struct{}, 1) w := RegionalWorker{ Regional: Regional{Engine: park.Engine{Claims: claims, Store: bucket, Bucket: "data", Region: "ie"}}, Enrollment: enrollmentFunc(func(context.Context, string, string, time.Duration) error { select { case started <- struct{}{}: default: } return nil }), Evidence: evidenceFunc(func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) { return "", time.Time{}, errors.New("unused") }), } done := make(chan error, 1) go func() { done <- w.Loop(ctx, nil, nil) }() <-started cancel() if err := <-done; !errors.Is(err, context.Canceled) { t.Fatalf("loop = %v", err) } } func TestRegionalWorkerDoesNotBindEvidenceToLateVersion(t *testing.T) { host := "late.example" inc := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" claims := release.NewDirStore(t.TempDir()) if _, err := hostclaim.Update(t.Context(), claims, hostclaim.Observed{Host: host, Absent: true}, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") key := "instances/" + host + "/old/data" bucket.Put("data", key, []byte("evidenced")) seq := 0 engine := park.Engine{ Store: bucket, Claims: claims, Bucket: "data", Region: "ie", Leases: workerLeases{}, Now: func() time.Time { return time.Unix(2000, 0).UTC() }, NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) }, View: func(_ context.Context, obs hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, } w := RegionalWorker{ Regional: Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }}, Enrollment: enrollmentFunc(func(context.Context, string, string, time.Duration) error { return nil }), Evidence: evidenceFunc(func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) { bucket.Put("data", key, []byte("late")) return inc, time.Unix(1, 0).UTC(), nil }), } if err := w.Run(t.Context()); !errors.Is(err, park.ErrChanged) { t.Fatalf("Run = %v", err) } obs, err := hostclaim.Read(t.Context(), claims, host) if err != nil || len(obs.Claim.Obligations) != 0 { t.Fatalf("late residue queued under old evidence: %+v, %v", obs.Claim.Obligations, err) } } func TestRegionalWorkerJoinsHolderlessSweepFromAnotherRegion(t *testing.T) { ctx := t.Context() host := "joining.example" claims := release.NewDirStore(t.TempDir()) at := time.Unix(1, 0).UTC() inc := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" uk := hostclaim.Obligation{ID: "uk-chore", Op: "original-op", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "uk-retire", Park: &hostclaim.ParkScope{Location: "instances/" + host + "/" + inc, Incarnation: inc, EvidenceAt: at, RequestLimit: RequestLimit}} if _, err := hostclaim.Update(ctx, claims, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "original-op", Intent: map[string]string{"park_sweep": "v1"}} c.Obligations = []hostclaim.Obligation{uk} return nil }); err != nil { t.Fatal(err) } bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") bucket.Put("data", "instances/"+host+"/"+inc+"/old", []byte("residue")) seq := 0 seen := false engine := park.Engine{Claims: claims, Store: bucket, Bucket: "data", Region: "ie", Leases: workerLeases{}, Now: func() time.Time { return at.Add(18 * time.Minute) }, NewID: func() string { seq++; return fmt.Sprintf("new%d", seq) }, View: func(_ context.Context, o hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: o.ETag, Region: "ie"}, nil }, Authorize: func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error { if o.Op != "original-op" { t.Fatalf("replaced operation: %+v", o) } seen = true return nil }} w := RegionalWorker{Regional: Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }}, Enrollment: enrollmentFunc(func(context.Context, string, string, time.Duration) error { return nil }), Evidence: evidenceFunc(func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) { return inc, at, nil })} if err := w.Run(ctx); err != nil { t.Fatal(err) } obs, err := hostclaim.Read(ctx, claims, host) if err != nil || !seen || obs.Claim.Op.ID != "original-op" { t.Fatalf("join: seen=%v obs=%+v err=%v", seen, obs, err) } found := false for _, o := range obs.Claim.Obligations { if o.ID == uk.ID { found = true } } if !found { t.Fatal("other region's chore lost") } }