package hostops import ( "context" "crypto/ed25519" "crypto/rand" "encoding/json" "errors" "fmt" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostevidence" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/release" ) func TestAdoptedHostDiscoveryThroughProductionRegionalWorker(t *testing.T) { ctx := context.Background() pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } store := release.NewDirStore(t.TempDir()) at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) enrollment := strings.Repeat("a", 32) roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 3, RequestLimitNS: int64(hostbox.RequestLimit), Regions: map[string]hostactors.Region{"ie": {}, "uk": {}}, Boxes: map[string]hostactors.Box{ "box": {CensusPublicKey: boxPub, Enrollment: enrollment, Namespace: "platform", BoxID: "box", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-box"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}, }, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} live := roster.Boxes["box"] live.BoxID = "live" live.Label = "live" live.Region = "uk" live.Enrollment = strings.Repeat("b", 32) live.Cloud.Region = "uk" live.Cloud.InstanceID = "i-live" roster.Boxes["live"] = live signed, err := release.SignRootEnvelope(release.RootSigner(key), roster) if err != nil { t.Fatal(err) } if _, err = store.Put(ctx, hostactors.Key, signed, ""); err != nil { t.Fatal(err) } host := "adopted.example" nh := hostclaim.NewHolder{Acquisition: "acquisition", Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Host: host, CreatedAt: at, Region: "uk"} seq := 0 next := func() string { seq++; return fmt.Sprintf("id%d", seq) } obs, err := hostclaim.Update(ctx, store, hostclaim.Observed{Host: host, Absent: true}, hostclaim.Adopt(nh, "live", []string{"ie"}, []string{"box"}, "discovery", next)) if err != nil { t.Fatal(err) } // Completing the local chore must not erase the non-serving writer census. for _, ob := range obs.Claim.Obligations { if ob.Kind == "park-local" { obs, err = hostclaim.Update(ctx, store, obs, hostclaim.CompleteObligation(ob.Op, ob.ID)) if err != nil { t.Fatal(err) } } } inc := obs.Claim.Holder.Incarnation receipt := StopReceipt{V: 1, ID: "empty", Host: host, Incarnation: inc, OperationID: "discovery", Phase: "discovery", ObligationID: "box-census", Box: "box", Enrollment: enrollment, BootID: "boot", ControllerID: "controller", ClaimGen: obs.Claim.Gen, DirectoryRevision: "3", Kind: "empty", EvidenceAt: at, FenceID: "census-fence", InventorySHA256: strings.Repeat("d", 64), LocalJournalSHA256: strings.Repeat("e", 64), ProducerDrainDigest: strings.Repeat("f", 64), RequestLimitNS: int64(hostbox.RequestLimit)} raw, err := json.Marshal(receipt) if err != nil { t.Fatal(err) } if err = hostevidence.Publish(ctx, store, boxKey, "control/hosts/"+host+"/evidence/discovery/box/empty.json", raw); err != nil { t.Fatal(err) } authority := ProductionAuthority{Store: store, Directory: hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}, Now: func() time.Time { return at.Add(time.Hour) }} bucket := leasetest.NewVersionedBucket() bucket.AddBucket("data") checks := 0 regional := hostbox.Regional{Owner: func(context.Context, hostclaim.Observed) error { return nil }, Engine: park.Engine{ Store: hostbox.RequestStore{Store: bucket, Check: func(context.Context) error { return nil }}, Claims: store, Bucket: "data", Region: "ie", Leases: discoveryLeases{}, Now: authority.Now, NewID: next, View: func(_ context.Context, o hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil }, Authorize: func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error { checks++ if ob.Park == nil { return ErrIdentity } return nil }, }} worker, err := NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority) if err != nil { t.Fatal(err) } hosts, err := worker.Census(ctx) if err != nil || len(hosts) != 1 || hosts[0] != host { t.Fatalf("census=%v %v", hosts, err) } // At-rest sweeping may remain pending; discovery itself must complete. if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) { t.Fatal(err) } after, err := hostclaim.Read(ctx, store, host) if err != nil { t.Fatal(err) } if len(after.Claim.Obligations) != 0 || !after.Claim.Touched.Complete || checks == 0 { t.Fatalf("discovery stuck: %+v checks=%d", after.Claim, checks) } if after.Claim.Op.Exec != "live" || after.Claim.Op.Kind != hostclaim.KindServing { t.Fatal("discovery stopped live serving holder") } // A pre-bound Moves park keeps its own prior verifier. It must not ask // the live destination for a delete-style stop receipt. move := hostclaim.Obligation{ID: "move-park", Op: "move", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "move-retire", Park: &hostclaim.ParkScope{Location: "moves/" + host + "/move", Incarnation: inc, EvidenceAt: at, RequestLimit: hostbox.RequestLimit}} after, err = hostclaim.Update(ctx, store, after, func(c *hostclaim.Claim) error { c.Obligations = append(c.Obligations, move); return nil }) if err != nil { t.Fatal(err) } deny := true moveChecks := 0 regional.Engine.Authorize = func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error { moveChecks++ if deny || ob.Op != "move" || ob.Park == nil || *ob.Park != *move.Park { return ErrIdentity } return nil } worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority) if err != nil { t.Fatal(err) } if err = worker.Run(ctx); !errors.Is(err, ErrIdentity) { t.Fatalf("move verifier bypassed: %v", err) } after, err = hostclaim.Read(ctx, store, host) if err != nil || len(after.Claim.Obligations) != 1 { t.Fatalf("denied move changed chore: %v", err) } deny = false if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) { t.Fatal(err) } after, err = hostclaim.Read(ctx, store, host) if err != nil || len(after.Claim.Obligations) != 0 || moveChecks < 2 { t.Fatalf("bound move stuck: %v checks=%d", err, moveChecks) } generation := hostclaim.Obligation{ID: "generation-park", Op: "session:1", Kind: "park-generation", Region: "ie", Generation: "G1", Owner: "region:ie", Retirement: "generation-retire"} scope := hostclaim.ParkScope{Location: after.Claim.Holder.Location, Incarnation: inc, EvidenceAt: at, RequestLimit: hostbox.RequestLimit} after, err = hostclaim.Update(ctx, store, after, func(c *hostclaim.Claim) error { c.Obligations = append(c.Obligations, generation); return nil }) if err != nil { t.Fatal(err) } regional.Engine.Authorize = func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error { if ob.ID != generation.ID || ob.Op != generation.Op || ob.Generation != generation.Generation || ob.Park == nil || *ob.Park != scope { return ErrIdentity } return nil } worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority) if err != nil { t.Fatal(err) } if err = worker.Run(ctx); !errors.Is(err, ErrContract) { t.Fatalf("unbound old chore accepted without its verifier: %v", err) } after, err = hostclaim.Read(ctx, store, host) if err != nil || len(after.Claim.Obligations) != 1 || after.Claim.Obligations[0].Park != nil { t.Fatalf("missing verifier changed chore: %v", err) } fallback := &discoveryPreviousEvidence{Host: host, Obligation: generation, Scope: scope} worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority, fallback) if err != nil { t.Fatal(err) } if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) { t.Fatal(err) } after, err = hostclaim.Read(ctx, store, host) if err != nil || len(after.Claim.Obligations) != 0 || fallback.ObligationCalls != 1 || fallback.RetirementCalls != 1 { t.Fatalf("lost previous verifier: %v obligation=%d retirement=%d", err, fallback.ObligationCalls, fallback.RetirementCalls) } // Either marker selects the new evidence contract; neither can silently // become an old-generation request, even with a previous verifier present. provider := worker.Evidence.(hostbox.RegionalObligationEvidence) for _, marked := range []hostclaim.Obligation{ {ID: "marked", Op: "delete", Kind: "park-replica", Region: "ie", Owner: "region:ie", ParkTarget: scope.Location}, {ID: "marked", Op: "delete", Kind: "park-replica", Region: "ie", Owner: "region:ie", EvidenceRefs: map[string]hostclaim.RetirementEvidenceRef{"box": {Box: "box"}}}, } { if _, err = provider.VerifyObligation(ctx, after, marked); err == nil { t.Fatal("marked retirement borrowed old evidence") } } if fallback.ObligationCalls != 1 { t.Fatal("marked work reached old verifier") } } type discoveryPreviousEvidence struct { Host string Obligation hostclaim.Obligation Scope hostclaim.ParkScope ObligationCalls, RetirementCalls int } func (e *discoveryPreviousEvidence) VerifyObligation(_ context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) { e.ObligationCalls++ want := e.Obligation if obs.Host != e.Host || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Holder.Incarnation != e.Scope.Incarnation || ob.ID != want.ID || ob.Op != want.Op || ob.Kind != want.Kind || ob.Generation != want.Generation || ob.Region != want.Region || ob.Owner != want.Owner || ob.Retirement != want.Retirement || ob.Park != nil || ob.ParkTarget != "" || len(ob.EvidenceRefs) != 0 { return hostclaim.ParkScope{}, ErrIdentity } return e.Scope, nil } func (e *discoveryPreviousEvidence) VerifyRetirement(_ context.Context, obs hostclaim.Observed, _ []lease.ObjectVersion) (string, time.Time, error) { e.RetirementCalls++ if obs.Host != e.Host { return "", time.Time{}, ErrIdentity } return "", time.Time{}, ErrPending } type discoveryEnrollment struct{} func (discoveryEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error { return nil } type discoveryLease struct{} func (discoveryLease) Check(context.Context) error { return nil } func (discoveryLease) Release(context.Context) error { return nil } func (discoveryLease) Close() {} type discoveryLeases struct{} func (discoveryLeases) Acquire(context.Context, string, string) (park.HeldLease, error) { return discoveryLease{}, nil }