//go:build linux package host import ( "context" "encoding/json" "errors" "net/http/httptest" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/park" ) func TestPlan14OrdinaryGenerationParks(t *testing.T) { for _, tc := range []string{"valid", "real-leases", "moves", "foreign-region", "foreign-box", "current-generation", "unknown-generation", "missing-receipt", "altered-receipt", "foreign-enrollment", "unbound-root", "changed-time", "changed-location", "unknown-obligation"} { t.Run(tc, func(t *testing.T) { f, _, _, in := servingRetirementFixture(t) first := servingAdmit(t, f, &in, "G") receipt := servingStop(t, f, first) current := servingAdmit(t, f, &in, "G2") current, err := hostclaim.Update(t.Context(), f.claims, current, hostclaim.PublishGeneration("", "", current.Claim.Holder.Session, "G2", "ie", f.newID)) if err != nil { t.Fatal(err) } var ob hostclaim.Obligation for _, x := range current.Claim.Obligations { if x.Generation == "G" { ob = x } } if ob.ID == "" { t.Fatal("missing publication obligation") } identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) identity.HomeRegion = "ie" if tc == "foreign-enrollment" { receipt.Target.Enrollment = "another-enrollment" var proof StopReceipt if err := json.Unmarshal(receipt.Bytes, &proof); err != nil { t.Fatal(err) } proof.Enrollment = receipt.Target.Enrollment receipt.Bytes = mustMarshalStopReceipt(proof) } guard := f.guard guard.FenceView = func(_ context.Context, host string, c hostguard.Captured) (hostguard.FenceView, error) { receipts := []hostguard.FenceReceipt{receipt} if tc == "missing-receipt" { receipts = nil } if tc == "altered-receipt" { receipts[0].Bytes = []byte("forged") } return hostguard.FenceView{Schema: 1, Host: host, Captured: c, Installation: hostguard.RootInstallation{Host: host, Migration: 1}, Receipts: receipts}, nil } data := leasetest.NewVersionedBucket() data.AddBucket("data") p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Guard: guard, Data: data}}} switch tc { case "moves": current.Claim.Op.Kind = hostclaim.KindMoving case "foreign-region": ob.Region, ob.Owner = "uk", "region:uk" case "foreign-box": current.Claim.Holder.Admissions[0].Box = "other" case "current-generation": ob.Generation = "G2" case "unknown-generation": ob.Generation = "unknown" case "unknown-obligation": ob.ID = "unknown" case "unbound-root": p.Local.assembly.Guard.FenceView = nil case "changed-time", "changed-location": scope, err := p.VerifyObligation(t.Context(), current, ob) if err != nil { t.Fatal(err) } if tc == "changed-time" { scope.EvidenceAt = scope.EvidenceAt.Add(1) } else { scope.Location += "/other" } ob.Park = &scope } if tc == "foreign-region" || tc == "current-generation" || tc == "unknown-generation" { for i := range current.Claim.Obligations { if current.Claim.Obligations[i].ID == ob.ID { current.Claim.Obligations[i] = ob } } } scope, err := p.VerifyObligation(t.Context(), current, ob) if tc == "valid" || tc == "real-leases" { if err != nil || scope.Location != current.Claim.Holder.Location || !scope.EvidenceAt.Equal(receipt.EvidenceAt) || scope.RequestLimit != hostbox.RequestLimit { t.Fatal(scope, err) } ob.Park = &scope p.Local.Identity, p.Local.lifetime = identity, t.Context() p.Local.Resources.Mode = "agent" p.Local.executionCall = func(context.Context, execlease.ExecutionRequest) (execlease.ExecutionResponse, error) { return execlease.ExecutionResponse{}, errors.New("stopped current writer cannot execute") } if err := p.Authorize(t.Context(), current, ob); err != nil { t.Fatal("cleanup required writer execution permission", err) } if _, err = p.VerifyObligation(t.Context(), current, ob); err != nil { t.Fatal("bound replay", err) } if _, err = p.VerifyMovesSweep(t.Context(), current, nil); err == nil { t.Fatal("moves sweep enabled") } freed := current claim := *current.Claim freed.Claim = &claim claim.RetiredIncarnations = []string{claim.Holder.Incarnation} claim.Holder, claim.Op = nil, hostclaim.Op{} claim.QuietAfter = map[string]time.Time{"ie": receipt.EvidenceAt} if _, err = p.VerifyMovesSweep(t.Context(), freed, nil); err == nil { t.Fatal("free claim enabled moves sweep") } location := current.Claim.Holder.Location oldSnapshot := location + "/g-G/0009/snapshot.ltx" oldVersion := data.Put("data", oldSnapshot, []byte("old snapshot")) data.Put("data", location+"/g-G2/0009/snapshot.ltx", []byte("current snapshot")) keptFile := location + "/files/g-G/objects/0000000000000001-aa" keptVersion := data.Put("data", keptFile, []byte("kept")) filesManifest(t, func(k string, b []byte) { data.Put("data", k, b) }, location, "G2", "kept.txt="+keptFile) marker, err := json.Marshal(map[string]any{"host": current.Host, "incarnation": current.Claim.Holder.Incarnation, "op": "G2"}) if err != nil { t.Fatal(err) } data.Put("data", location+"/generation.json", marker) engine := park.Engine{HomeRegion: "ie", Store: data, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return f.now.Add(park.SignatureAge + hostbox.RequestLimit + time.Second) }, NewID: f.newID, View: p.View, Authorize: p.Authorize} if tc == "real-leases" { handler, _ := leasetest.NewS3Server(data) server := httptest.NewServer(handler) defer server.Close() for key, value := range map[string]string{"CARLOS_S3_ENDPOINT": server.URL, "AWS_ENDPOINT_URL_S3": server.URL, "AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_CONFIG_FILE": "/nonexistent", "AWS_SHARED_CREDENTIALS_FILE": "/nonexistent"} { t.Setenv(key, value) } engine.Leases = hostbox.AggregateLeases{Leases: hostbox.RenewingLeases{Region: "eu-west-1"}, Resolve: p.LeaseSet} } work := park.Work{Host: current.Host, ID: ob.ID, Op: ob.Op} if err := engine.Bind(t.Context(), work, scope); err != nil { t.Fatal("bind generation", err) } pinnedMarker, err := json.Marshal(map[string]any{"host": current.Host, "incarnation": current.Claim.Holder.Incarnation, "op": "G2", "snapshot_key": oldSnapshot, "snapshot_version": oldVersion}) if err != nil { t.Fatal(err) } data.Put("data", location+"/generation.json", pinnedMarker) if err := engine.Run(t.Context(), work); !errors.Is(err, park.ErrUnsafe) { t.Fatal("database pin did not protect its generation", err) } if _, err := data.GetVersion(t.Context(), "data", oldSnapshot, oldVersion); err != nil { t.Fatal("pinned database snapshot was deleted", err) } data.Put("data", location+"/generation.json", marker) if err := engine.Run(t.Context(), work); err != nil { t.Fatal("retire generation", err) } left, err := data.ListVersions(t.Context(), "data", location+"/g-G/") if err != nil || len(left) != 0 { t.Fatal("predecessor remains", left, err) } left, err = data.ListVersions(t.Context(), "data", location+"/g-G2/") if err != nil || len(left) != 1 { t.Fatal("current generation changed", left, err) } if body, err := data.GetVersion(t.Context(), "data", keptFile, keptVersion); err != nil || string(body) != "kept" { t.Fatal("referenced file was retired with its database generation", err) } } else if err == nil { t.Fatal("accepted", tc) } }) } } func TestPlan14OrdinaryRetirementRequiresProof(t *testing.T) { t.Parallel() for _, target := range []string{"holder", "empty-staging", "staging-residue", "foreign"} { t.Run(target, func(t *testing.T) { f, _, _, in := servingRetirementFixture(t) o := servingAdmit(t, f, &in, "G") identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) identity.HomeRegion = "ie" data := leasetest.NewVersionedBucket() data.AddBucket("data") ob := hostclaim.Obligation{ID: "park", Op: "delete", Kind: "park-replica", Owner: "region:ie", Region: "ie", Retirement: "retire", ParkTarget: o.Claim.Holder.Location} want := hostops.ErrIdentity switch target { case "empty-staging", "staging-residue": ob.ParkTarget = "moves/" + o.Host data.Put("data", ob.ParkTarget+"/lock.json", []byte("lock")) if target == "staging-residue" { data.Put("data", ob.ParkTarget+"/other/g-G/object", []byte("deferred move")) want = park.ErrUnsafe } case "foreign": ob.ParkTarget += "/another" want = park.ErrUnsafe } o.Claim.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "delete"} o.Claim.Obligations = []hostclaim.Obligation{ob} if err := o.Claim.Validate(); err != nil { t.Fatal(err) } p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Data: data, RetirementAuthority: &hostops.ProductionAuthority{}}}} if err := p.authorizeOrdinaryRetirement(t.Context(), o, ob); !errors.Is(err, want) { t.Fatal("unsigned retirement or deferred target accepted", err) } }) } } func TestPlan14OrdinaryParkView(t *testing.T) { t.Parallel() f, _, _, in := servingRetirementFixture(t) o := servingAdmit(t, f, &in, "G") identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) identity.HomeRegion = "ie" data := leasetest.NewVersionedBucket() data.AddBucket("data") p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Data: data}}} v, err := p.View(t.Context(), o) if err != nil { t.Fatal(err) } live, err := park.LiveGenerations(o.Claim, v, o.Claim.Holder.Location) if err != nil || !live["G"] || !live[hostclaim.RootGeneration] || v.RefusedLocations["moves/"+o.Host] == nil { t.Fatal(live, err) } p.Identity.HomeRegion = "" v, err = p.View(t.Context(), o) if err != nil || v.HomeRegion != "" || v.AuthoritativeRegion != "ie" { t.Fatal("named home placement", v, err) } p.Identity.HomeRegion = "ie" key := o.Claim.Holder.Location + "/g-G/0009/snapshot.ltx" version := data.Put("data", key, []byte("retained snapshot")) marker, err := json.Marshal(map[string]any{"host": o.Host, "incarnation": o.Claim.Holder.Incarnation, "op": "G", "snapshot_key": key, "snapshot_version": version}) if err != nil { t.Fatal(err) } data.Put("data", o.Claim.Holder.Location+"/generation.json", marker) data.Put("data", key, []byte("newer bytes")) v, err = p.View(t.Context(), o) if err != nil || !v.Pinned[park.Ref{Key: key, VersionID: version}] || v.MarkerGeneration != "G" { t.Fatal("lost exact snapshot pin", v, err) } unavailable := errors.New("version read unavailable") data.FailOn = func(op, key string) error { if op == "get" { return unavailable } return nil } if _, err := p.View(t.Context(), o); !errors.Is(err, unavailable) { t.Fatal("reduced live set on failed read", err) } data.FailOn = nil for _, kind := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindRelocating, hostclaim.KindRestoring} { o.Claim.Op.Kind = kind if _, err := p.View(t.Context(), o); err == nil { t.Fatal("move view enabled", kind) } } }