package hostactors import ( "context" "crypto/ed25519" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" "github.com/carlosframework/platform/internal/termfence" ) // plan9World is a published deployment with three active boxes: a and b // hold an executable activation reservation, w is a regional fence writer. type plan9World struct { d Directory sg release.Signer floor hostfloor.Verified now time.Time hash string target string } const plan9Activation = "act" func plan9Fixture(t *testing.T, edit func(*Roster)) (plan9World, Observed) { t.Helper() ctx := context.Background() pub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } store := release.NewDirStore(t.TempDir()) sg := release.RootSigner(key) w := plan9World{sg: sg, now: time.Now().UTC(), hash: strings.Repeat("a", 64), target: strings.Repeat("b", 64)} rel := release.SystemRelease{Version: "v1.0.0", Source: "release:v1.0.0", ReleaseTag: "v1.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.0.0", "linux-amd64"), SHA256: w.hash}}} if _, err := release.PromoteSystem(ctx, store, sg, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil { t.Fatal(err) } wire, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable")) if err != nil { t.Fatal(err) } if err := release.PublishSystemIdentity(ctx, store, pub, wire); err != nil { t.Fatal(err) } if err := hostfloor.Publish(ctx, store, sg, "bucket", hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}); err != nil { t.Fatal(err) } w.d = Directory{Store: store, PublicKey: pub, Bucket: "bucket"} if w.floor, err = hostfloor.Read(ctx, store, pub, "bucket"); err != nil { t.Fatal(err) } r := Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Publication: "published", FloorSHA256: w.floor.Digest, Boxes: map[string]Box{}, Consoles: map[string]Console{}, Holds: map[string]Reservation{}, Activations: map[string]Reservation{}} for i, ref := range []string{"a", "b", "w"} { r.Boxes[ref] = Box{Enrollment: strings.Repeat(string(rune('1'+i)), 32), Namespace: "platform", BoxID: ref, Label: ref, Region: "ie", Pool: "default", Cloud: Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("e", 64), State: "active", LossIDs: []string{}} } h := Reservation{ID: plan9Activation, FloorSHA256: w.floor.Digest, Actors: map[string]string{}, Releases: map[string]Report{}, Hashes: map[string][]string{}} for _, ref := range []string{"a", "b"} { en := r.Boxes[ref].Enrollment h.Actors["box:"+ref] = en h.Releases["box:"+ref] = Report{Enrollment: en} h.Hashes["box:"+ref] = []string{w.hash, w.target, w.hash} } r.Activations[plan9Activation] = h if edit != nil { edit(&r) } raw, err := release.SignRootEnvelope(sg, r) if err != nil { t.Fatal(err) } if _, err := store.Put(ctx, Key, raw, ""); err != nil { t.Fatal(err) } w.report(t, "a", true) o, err := w.d.Read(ctx) if err != nil { t.Fatal(err) } return w, o } // report publishes ref's private status as a box that finished (or did not // finish) the reserved activation on its current image. func (w plan9World) report(t *testing.T, ref string, complete bool) { t.Helper() o, err := w.d.Read(context.Background()) if err != nil { t.Fatal(err) } b := o.Roster.Boxes[ref] box := status.Box{GeneratedAt: w.now, Metrics: &status.BoxMetrics{Enrollment: b.Enrollment, InstanceID: b.Cloud.InstanceID, Release: "v1.0.0", ReleaseTag: "v1.0.0", Platform: "linux-amd64", BinarySHA256: w.hash, MovesProtocol: 1, BootID: "boot", FloorSHA256: w.floor.Digest, FloorGuardReady: true, SystemChannel: "stable", ActivationID: plan9Activation, ActivationComplete: complete}} raw, err := json.Marshal(box) if err != nil { t.Fatal(err) } _, etag, _ := w.d.Store.Get(context.Background(), "status/boxes/"+ref+".json") if _, err := w.d.Store.Put(context.Background(), "status/boxes/"+ref+".json", raw, etag); err != nil { t.Fatal(err) } } // fence creates target's termination fence, observed at the given time and // written by writer, exactly as a regional worker persists it. func (w plan9World) fence(t *testing.T, target, writer string, at time.Time, state string) { t.Helper() o, err := w.d.Read(context.Background()) if err != nil { t.Fatal(err) } b, by := o.Roster.Boxes[target], o.Roster.Boxes[writer] f := termfence.Record{V: 1, Box: target, Enrollment: b.Enrollment, Cloud: b.Cloud, State: state, ObservedAt: at.UTC().Format(time.RFC3339Nano), Via: termfence.ViaDescribe, RequestID: "request", WrittenBy: termfence.WrittenBy{Box: writer, Enrollment: by.Enrollment}} raw, err := json.Marshal(f) if err != nil { t.Fatal(err) } if err := release.CreateExact(context.Background(), w.d.Store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { t.Fatal(err) } } // quarantine marks ref quarantined in the roster edit and returns the loss // record a root-signed operator declaration would have created for it. func plan9Quarantine(r *Roster, ref, loss string) { b := r.Boxes[ref] b.State = "quarantined" b.QuarantineRevision = 1 b.LossIDs = []string{loss} r.Boxes[ref] = b } func (w plan9World) loss(t *testing.T, ref, id string, fencedAt time.Time) { t.Helper() o, err := w.d.Read(context.Background()) if err != nil { t.Fatal(err) } b := o.Roster.Boxes[ref] record := hostclaim.LossRecord{ V: 1, ID: id, State: "prepared", Actor: "operator", AcceptedAt: fencedAt, Host: "shop.bab.oncarlos.com", HolderIncarnation: strings.Repeat("a", 64), HolderStamp: 3, HolderSession: 2, ExpectedClaimGen: 4, ExpectedClaimETag: `"claim"`, OpID: "op", OpKind: hostclaim.KindMoving, OpPhase: "precommit", Box: ref, Enrollment: b.Enrollment, QuarantineRevision: b.QuarantineRevision, CloudObservation: hostclaim.LossCloudObservation{ObservedAt: fencedAt, Account: "123", Region: "eu-west-1", InstanceID: b.Cloud.InstanceID, State: "stopped", RequestID: "request-1"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: fencedAt, ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"lost-target"}`), TimersPotentiallyLost: true}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, } if _, err := hostclaim.CreateLossRecord(context.Background(), w.d.Store, w.sg, record); err != nil { t.Fatal(err) } } // I-4 and D7: a held activation is released only by verified completion of // every reserved actor, except an actor excluded through the valid // enrollment-qualified termination fence or its durable quarantine // authority. Silence, unreadable fence authority and an invalid fence never // exclude. func TestPlan9ReleaseActivationExcludesOnlyFencedOrQuarantinedActors(t *testing.T) { for _, tc := range []struct { name string released bool }{ {"silent", false}, {"incomplete", false}, {"fenced", true}, {"fenced-other-incomplete", false}, {"fence-nonterminal", false}, {"fence-self-written", false}, {"fence-unreadable", false}, {"quarantined", true}, {"quarantined-loss-missing", false}, {"both-complete", true}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() var edit func(*Roster) if strings.HasPrefix(tc.name, "quarantined") { edit = func(r *Roster) { plan9Quarantine(r, "b", "loss-b") } } w, _ := plan9Fixture(t, edit) switch tc.name { case "incomplete": w.report(t, "b", false) case "fenced": w.fence(t, "b", "w", w.now.Add(-time.Minute), "terminated") case "fenced-other-incomplete": w.fence(t, "b", "w", w.now.Add(-time.Minute), "terminated") w.report(t, "a", false) case "fence-nonterminal": w.fence(t, "b", "w", w.now.Add(-time.Minute), "stopped") case "fence-self-written": w.fence(t, "b", "b", w.now.Add(-time.Minute), "terminated") case "fence-unreadable": w.fence(t, "b", "w", w.now.Add(-time.Minute), "terminated") w.d.Store = brokenFenceStore{w.d.Store} case "quarantined": w.loss(t, "b", "loss-b", w.now.Add(-time.Hour)) case "both-complete": w.report(t, "b", true) } err := w.d.ReleaseActivation(ctx, w.sg, plan9Activation, w.now) after, readErr := w.d.Read(ctx) if readErr != nil { t.Fatal(readErr) } _, held := after.Roster.Activations[plan9Activation] if tc.released { if err != nil || held { t.Fatalf("%s: release = %v, held = %v; want released", tc.name, err, held) } return } if err == nil || !held { t.Fatalf("%s: release = %v, held = %v; want refusal and a held reservation", tc.name, err, held) } }) } } // M1-5: require a currently active and unfenced writer. func TestPlan9FenceWriterMustBeActiveAndUnfencedNow(t *testing.T) { observed := time.Now().UTC().Add(-time.Hour) for _, tc := range []struct { name string valid bool }{ {"active-writer", true}, {"self-written", false}, {"writer-fenced-before", false}, {"writer-fenced-same-instant", false}, {"writer-fenced-after", false}, {"writer-fence-malformed", false}, {"writer-fence-unreadable", false}, {"writer-quarantined-before", false}, {"writer-quarantined-after", false}, {"writer-quarantined-loss-missing", false}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() var edit func(*Roster) if strings.HasPrefix(tc.name, "writer-quarantined") { edit = func(r *Roster) { plan9Quarantine(r, "w", "loss-w") } } w, _ := plan9Fixture(t, edit) writer := "w" if tc.name == "self-written" { writer = "b" } w.fence(t, "b", writer, observed, "terminated") o, err := w.d.Read(ctx) if err != nil { t.Fatal(err) } writerFence := "control/box-fences/" + o.Roster.Boxes["w"].Enrollment + ".json" switch tc.name { case "writer-fenced-before": w.fence(t, "w", "a", observed.Add(-time.Minute), "terminated") case "writer-fenced-same-instant": w.fence(t, "w", "a", observed, "terminated") case "writer-fenced-after": w.fence(t, "w", "a", observed.Add(time.Minute), "terminated") case "writer-fence-malformed": if err := release.CreateExact(ctx, w.d.Store, writerFence, []byte("{")); err != nil { t.Fatal(err) } case "writer-fence-unreadable": w.d.Store = plan9KeyFailStore{Store: w.d.Store, key: writerFence} case "writer-quarantined-before": w.loss(t, "w", "loss-w", observed.Add(-time.Minute)) case "writer-quarantined-after": w.loss(t, "w", "loss-w", observed.Add(time.Minute)) } fenced, err := w.d.TerminationFence(ctx, o, "b", time.Now()) if tc.valid != (err == nil && fenced) { t.Fatalf("%s: fenced = %v, err = %v; want valid = %v", tc.name, fenced, err, tc.valid) } if !tc.valid && fenced { t.Fatalf("%s: invalid evidence excluded the box", tc.name) } }) } } type plan9KeyFailStore struct { release.Store key string } func (s plan9KeyFailStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == s.key { return nil, "", errors.New("writer fence unavailable") } return s.Store.Get(ctx, key) } // I2: binding a census key rewrites the roster outside Replace. It must // stay behind the published floor (older binaries strict-decode the roster // and would refuse census_public_key) and bind only an active, unfenced // enrollment, read fresh rather than from a caller-supplied copy.