package hostactors import ( "context" "encoding/json" "errors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "strings" "testing" "time" ) // writeFence records ref's termination as observed by a separate regional // writer box. The writer sits in another pool, so no census scope requires // it; a box never witnesses its own termination (M1-5). func writeFence(t *testing.T, d Directory, o *Observed, ref string, now time.Time, invalid bool) { t.Helper() b := o.Roster.Boxes[ref] writer := b writer.BoxID, writer.Label, writer.Pool = "fence-writer", "fence-writer", "workers" writer.Enrollment = strings.Repeat("9", 32) writer.Cloud.InstanceID = "i-fence-writer" o.Roster.Boxes["fence-writer"] = writer state := "terminated" if invalid { state = "stopped" } f := map[string]any{"v": 1, "box": ref, "enrollment": b.Enrollment, "cloud": b.Cloud, "state": state, "observed_at": now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "fence-writer", "enrollment": writer.Enrollment}} raw, _ := json.Marshal(f) if err := release.CreateExact(context.Background(), d.Store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { t.Fatal(err) } } func TestTerminationFenceExcludesActiveBeforeRosterUpdate(t *testing.T) { for _, invalid := range []bool{false, true} { t.Run(map[bool]string{false: "valid", true: "invalid"}[invalid], func(t *testing.T) { d, o, e, f, scope, now := censusFixture(t) writeFence(t, d, &o, "box", now, invalid) reports, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now) if invalid { if err == nil { t.Fatal("invalid fence treated as absence or exclusion") } } else if err != nil || len(reports) != 1 { t.Fatal(reports, err) } if _, err := d.Lookup(context.Background(), "box"); err == nil { t.Fatal("fenced or invalid actor eligible") } if _, err := d.CheckBoxStart(context.Background(), "box"); err == nil { t.Fatal("fenced or invalid start allowed") } scope.Participants = []string{"box"} if _, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now); err == nil { t.Fatal("fenced participant allowed") } }) } } func boundConsole(o *Observed, e *reportsFixture) { c := o.Roster.Consoles["console-1"] c.Compute = &ComputeBinding{Box: "box", Enrollment: e.box.Enrollment} o.Roster.Consoles["console-1"] = c e.console.Compute = c.Compute e.box.ConsoleProcesses = []ConsoleProcess{{ConsoleID: "console-1", Enrollment: c.Enrollment, ProcessID: e.console.ProcessID, BinarySHA256: e.console.BinarySHA256, GuardFloorSHA256: e.box.FloorSHA256, AllLaunchPathsGuarded: true}} } func TestConsoleFenceRequiresExactProcessMachineAndBoot(t *testing.T) { for _, fault := range []string{"good", "missing", "boot", "enrollment", "process", "ambiguous", "console-enrollment", "console-instance"} { t.Run(fault, func(t *testing.T) { d, o, e, f, scope, now := censusFixture(t) boundConsole(&o, &e) // The starter lives elsewhere; only the bound console can be excluded. scope.ConsoleID = "starter" c := o.Roster.Consoles["console-1"] c.Compute = nil c.InstanceID = "starter" c.Enrollment = "dddddddddddddddddddddddddddddddd" o.Roster.Consoles["starter"] = c switch fault { case "missing": c = o.Roster.Consoles["console-1"] c.Compute = nil o.Roster.Consoles["console-1"] = c case "console-enrollment": e.console.Enrollment = "wrong" case "console-instance": e.console.InstanceID = "wrong" case "boot": e.console.BootID = "other" case "enrollment": e.console.Compute = &ComputeBinding{Box: "box", Enrollment: "other"} case "process": e.box.ConsoleProcesses[0].ProcessID = "other" case "ambiguous": e.box.ConsoleProcesses = append(e.box.ConsoleProcesses, e.box.ConsoleProcesses[0]) } // Validate the binding independently, then exercise full census below with exact objects. _, err := verifyCompute(context.Background(), o, e, "console-1", o.Roster.Consoles["console-1"], e.console) if (err == nil) != (fault == "good") { t.Fatal(fault, err) } if fault != "good" { return } writeFence(t, d, &o, "box", now, false) for id, cc := range o.Roster.Consoles { r := e.console if id == "starter" { r.Actor = id r.Enrollment = cc.Enrollment r.InstanceID = id r.Compute = nil } raw, _ := json.Marshal(r) if err := release.CreateExact(context.Background(), d.Store, "status/consoles/"+id+".json", raw); err != nil { t.Fatal(err) } } local := e.console local.Actor = "starter" local.Enrollment = c.Enrollment local.InstanceID = "starter" local.Compute = nil exact := ExactEvidence{Store: d.Store, PublicKey: d.PublicKey, Boxes: e.BoxReport, Self: func(context.Context) (Report, error) { return local, nil }} reports, err := Census(context.Background(), o, exact, d.PublicKey, f, scope, now) if err != nil || len(reports) != 1 || reports["console:starter"].Actor != "starter" { t.Fatal(reports, err) } }) } } func TestRelocationOnlyNewestGuardedConsoleMayBeStale(t *testing.T) { for _, fault := range []string{"good", "ordinary", "guard", "floor", "binding", "future", "newer-malformed", "newer-image", "other-operation"} { t.Run(fault, func(t *testing.T) { d, o, e, f, scope, now := censusFixture(t) boundConsole(&o, &e) e.console.GeneratedAt = now.Add(-time.Hour) scope.boxRelocation = true scope.StarterBox = "box" scope.ConsoleID = "" switch fault { case "ordinary", "other-operation": scope.boxRelocation = false scope.ConsoleID = "console-1" case "guard": e.box.ConsoleProcesses[0].AllLaunchPathsGuarded = false case "floor": e.box.ConsoleProcesses[0].GuardFloorSHA256 = "wrong" case "binding": e.console.Compute = nil case "future": e.console.GeneratedAt = now.Add(time.Minute) case "newer-image": e.console.BinarySHA256 = "wrong" } raw, _ := json.Marshal(e.console) if fault == "newer-malformed" { raw = []byte("{") } if err := release.CreateExact(context.Background(), d.Store, "status/consoles/console-1.json", raw); err != nil { t.Fatal(err) } exact := ExactEvidence{Store: d.Store, PublicKey: d.PublicKey, Boxes: e.BoxReport, Self: func(context.Context) (Report, error) { return e.box, nil }} _, err := Census(context.Background(), o, exact, d.PublicKey, f, scope, now) if (err == nil) != (fault == "good") { t.Fatal(fault, err) } }) } } type brokenFenceStore struct{ release.Store } func (s brokenFenceStore) Get(ctx context.Context, key string) ([]byte, string, error) { if strings.HasPrefix(key, "control/box-fences/") { return nil, "", errors.New("fence read unavailable") } return s.Store.Get(ctx, key) } func TestFenceAuthorityErrorsAndMismatchesRefuse(t *testing.T) { for _, fault := range []string{"read", "malformed", "enrollment", "cloud", "writer", "future"} { t.Run(fault, func(t *testing.T) { d, o, e, f, scope, now := censusFixture(t) if fault == "read" { d.Store = brokenFenceStore{d.Store} e.d = d } else { writeFence(t, d, &o, "box", now, false) key := "control/box-fences/" + o.Roster.Boxes["box"].Enrollment + ".json" raw, etag, err := d.Store.Get(context.Background(), key) if err != nil { t.Fatal(err) } var record map[string]any if err := json.Unmarshal(raw, &record); err != nil { t.Fatal(err) } switch fault { case "enrollment": record["enrollment"] = "wrong" case "cloud": record["cloud"] = Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "wrong"} case "writer": record["written_by"] = map[string]string{"box": "unknown", "enrollment": "wrong"} case "future": record["observed_at"] = now.Add(time.Minute) } raw, _ = json.Marshal(record) if fault == "malformed" { raw = []byte("{") } // Test-only mutation creates corrupt authority without allowing production overwrite. if _, err = d.Store.Put(context.Background(), key, raw, etag); err != nil { t.Fatal(err) } } if _, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now); err == nil { t.Fatal("bad authority admitted census") } if _, err := d.Lookup(context.Background(), "box"); err == nil { t.Fatal("bad authority granted eligibility") } if _, err := d.CheckBoxStart(context.Background(), "box"); err == nil { t.Fatal("bad authority granted start") } }) } } func TestStartRevalidatesReservedActorEligibility(t *testing.T) { for _, fault := range []string{"good", "quarantined", "retired", "fenced", "malformed", "unreadable", "console-quarantined", "console-fenced"} { t.Run(fault, func(t *testing.T) { ctx := t.Context() d, sg, o := fixture(t) floor := hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"} if err := hostfloor.Publish(ctx, d.Store, sg, d.Bucket, floor); err != nil { t.Fatal(err) } f, err := hostfloor.Read(ctx, d.Store, d.PublicKey, d.Bucket) if err != nil { t.Fatal(err) } r := Clone(o.Roster) r.Publication, r.FloorSHA256 = "published", f.Digest actor, enrollment := "box:box", r.Boxes["box"].Enrollment if strings.HasPrefix(fault, "console-") { actor, enrollment = "console:console-1", strings.Repeat("c", 32) r.Consoles["console-1"] = Console{Enrollment: enrollment, InstanceID: "console-1", Namespaces: []string{"platform"}, DeploymentBucket: d.Bucket, Compute: &ComputeBinding{Box: "box", Enrollment: r.Boxes["box"].Enrollment}} } o, err = d.Replace(ctx, sg, o, r) if err != nil { t.Fatal(err) } before, err := hostclaim.Read(ctx, d.Store, "shop.example.com") if err != nil { t.Fatal(err) } h := Reservation{ID: "hold", Host: before.Host, OpID: "operation", ExpectedClaimETag: before.ETag, FloorSHA256: f.Digest, Actors: map[string]string{actor: enrollment}, Releases: map[string]Report{actor: {Enrollment: enrollment, GeneratedAt: time.Now()}}} o, err = d.Reserve(ctx, sg, o, h, false) if err != nil { t.Fatal(err) } switch fault { case "quarantined", "console-quarantined": _, err = d.Quarantine(ctx, sg, o, "box", o.Roster.Boxes["box"].Enrollment, "loss") case "retired": r = Clone(o.Roster) b := r.Boxes["box"] b.State = "retired" r.Boxes["box"] = b _, err = d.Replace(ctx, sg, o, r) case "fenced", "console-fenced", "malformed": writeFence(t, d, &o, "box", time.Now(), fault == "malformed") case "unreadable": d.Store = brokenFenceStore{d.Store} } if err != nil { t.Fatal(err) } called := false _, err = d.Start(ctx, before, h.ID, func(*hostclaim.Claim) error { called = true; return errors.New("stop before writing") }) if err == nil || called != (fault == "good") { t.Fatalf("transition called=%v, error=%v", called, err) } after, err := hostclaim.Read(ctx, d.Store, before.Host) if err != nil || before.ETag != after.ETag || before.Absent != after.Absent { t.Fatal("refusal changed claim", err) } }) } }