package hostops import ( "context" "crypto/ed25519" "encoding/json" "errors" "os" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostevidence" "github.com/carlosframework/platform/internal/release" ) func resetTransportFixture(t *testing.T) (WakeResetTransport, WakeResetRequest, hostclaim.Observed, ed25519.PrivateKey) { t.Helper() w, o, clock := wakeResetFixture(t) key := ed25519.NewKeyFromSeed(make([]byte, 32)) boxkey := ed25519.NewKeyFromSeed([]byte(strings.Repeat("b", 32))) h := o.Claim.Holder h.WakeRetry.Attempt.Box = "box" h.WakeRetry.Attempt.Enrollment = strings.Repeat("a", 32) h.WakeRetry.Attempt.BootID = "boot" h.WakeRetry.Attempt.ControllerID = "controller" h.WakeRetry.Attempt.Session = h.Session raw, _ := json.Marshal(o.Claim) etag, err := w.raw.Put(t.Context(), "control/hosts/"+o.Host+".json", raw, o.ETag) if err != nil { t.Fatal(err) } o.ETag = etag r := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, RequestLimitNS: int64(hostbox.RequestLimit), Boxes: map[string]hostactors.Box{"box": {BoxID: "box", Enrollment: strings.Repeat("a", 32), Namespace: "platform", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active", CensusPublicKey: boxkey.Public().(ed25519.PublicKey)}}, Consoles: map[string]hostactors.Console{"console": {InstanceID: "console", Enrollment: strings.Repeat("d", 32), Namespaces: []string{"platform"}, DeploymentBucket: "bucket", Compute: &hostactors.ComputeBinding{Box: "box", Enrollment: strings.Repeat("a", 32)}}}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} wire, err := release.SignRootEnvelope(release.RootSigner(key), r) if err != nil { t.Fatal(err) } if _, err = w.raw.Put(t.Context(), hostactors.Key, wire, ""); err != nil { t.Fatal(err) } transport := WakeResetTransport{Store: w.raw, Directory: hostactors.Directory{Store: w.raw, PublicKey: key.Public().(ed25519.PublicKey), Bucket: "bucket"}, Now: clock} req := WakeResetRequest{V: 1, ID: strings.Repeat("e", 64), Bucket: "bucket", Console: "console", ConsoleEnrollment: strings.Repeat("d", 32), Namespace: "platform", Host: o.Host, ClaimETag: o.ETag, ClaimGen: o.Claim.Gen, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Operation: o.Claim.Op.ID, Phase: o.Claim.Op.Phase, Episode: h.WakeRetry.EpisodeID, Attempt: *h.WakeRetry.Attempt, IssuedAt: clock(), Expires: clock().Add(time.Minute)} owner := RegistrationOwner{Console: req.Console, Enrollment: req.ConsoleEnrollment, Controller: "controller"} _, err = w.RegisterStart(t.Context(), h.Account, h.App, "app", "reset", Registration{Kind: RegistrationWakeRetryReset, Host: o.Host, Owner: owner, AppID: "app", Epoch: 1, Claim: SavedObservation(o), Incarnation: h.Incarnation, CreatedAt: clock()}, clock()) if err != nil { t.Fatal(err) } req.Admission = WakeResetAdmission{Account: h.Account, App: h.App, AppID: "app", Registration: "reset", Epoch: 1, Owner: owner} req.ID = wakeResetID(req) return transport, req, o, boxkey } func TestPlan9ResetRequestRejectsChangedAuthorityAndClaim(t *testing.T) { tr, r, o, _ := resetTransportFixture(t) if err := tr.Authorize(t.Context(), r); err != nil { t.Fatal(err) } for _, change := range []func(*WakeResetRequest){func(r *WakeResetRequest) { r.ConsoleEnrollment = "other" }, func(r *WakeResetRequest) { r.ClaimETag = "other" }, func(r *WakeResetRequest) { r.Incarnation = "other" }, func(r *WakeResetRequest) { r.Attempt.ID = "other" }, func(r *WakeResetRequest) { r.Expires = r.IssuedAt }, func(r *WakeResetRequest) { r.Namespace = "other" }} { copy := r change(©) if tr.Authorize(t.Context(), copy) == nil { t.Fatal("accepted changed binding") } } o.Claim.Holder.WakeRetry.Attempt.ID = "replacement" raw, _ := json.Marshal(o.Claim) if _, err := tr.Store.Put(t.Context(), "control/hosts/"+o.Host+".json", raw, o.ETag); err != nil { t.Fatal(err) } if tr.Authorize(t.Context(), r) == nil { t.Fatal("accepted replaced attempt") } } func TestPlan9ResetLostResponseIsUncertain(t *testing.T) { tr, r, _, _ := resetTransportFixture(t) if err := tr.VerifyReceipt(t.Context(), r); !errors.Is(err, ErrWakeResetUncertain) { t.Fatalf("absence = %v", err) } ctx, cancel := context.WithCancel(t.Context()) cancel() if err := tr.Wait(ctx, r, time.Millisecond); !errors.Is(err, ErrWakeResetUncertain) { t.Fatalf("cancel = %v", err) } } func publishResetProof(t *testing.T, tr WakeResetTransport, r WakeResetRequest, key ed25519.PrivateKey) string { t.Helper() k := "control/hosts/" + r.Host + "/serving-evidence/box/proof.json" proof := StopReceipt{Region: "ie", V: 1, ID: "proof", Host: r.Host, Incarnation: r.Incarnation, InstallationStamp: r.Stamp, WriterSession: r.Session, OperationID: r.Operation, Phase: r.Phase, Box: r.Attempt.Box, Enrollment: r.Attempt.Enrollment, BootID: r.Attempt.BootID, ControllerID: r.Attempt.ControllerID, ClaimETag: r.ClaimETag, ClaimGen: r.ClaimGen, Kind: "stopped", EvidenceAt: tr.Now(), FenceID: "fence", DirectoryRevision: "1", RequestLimitNS: int64(hostbox.RequestLimit), InventorySHA256: strings.Repeat("1", 64), LocalJournalSHA256: strings.Repeat("2", 64), ProducerDrainDigest: strings.Repeat("3", 64), CgroupIdentities: []string{"group"}} raw, _ := json.Marshal(proof) if err := hostevidence.Publish(t.Context(), tr.Store, key, k, raw); err != nil { t.Fatal(err) } return k } func TestPlan9ResetReceiptNeedsRootSignatureAndExactTuple(t *testing.T) { tr, r, _, key := resetTransportFixture(t) k := publishResetProof(t, tr, r, key) if err := tr.VerifyReceipt(t.Context(), r); err != nil { t.Fatal(err) } changed := r changed.Attempt.ControllerID = "other" if tr.VerifyReceipt(t.Context(), changed) == nil { t.Fatal("accepted other controller") } raw, etag, err := tr.Store.Get(t.Context(), k) if err != nil { t.Fatal(err) } raw = append(raw, ' ') if _, err = tr.Store.Put(t.Context(), k, raw, etag); err != nil { t.Fatal(err) } if tr.VerifyReceipt(t.Context(), r) == nil { t.Fatal("accepted forged proof") } } func signedResetRequest(t *testing.T, tr WakeResetTransport, r WakeResetRequest) { t.Helper() key := ed25519.NewKeyFromSeed(make([]byte, 32)) wire, err := release.SignRootEnvelope(release.RootSigner(key), r) if err != nil { t.Fatal(err) } if _, err = tr.Store.Put(t.Context(), r.Key(), wire, ""); err != nil { t.Fatal(err) } } func TestPlan9ResetWorkerReplayCancellationAndExpiry(t *testing.T) { for _, mode := range []string{"dispatch", "cancel", "expiry", "invalid-signature", "completed", "lost-response"} { t.Run(mode, func(t *testing.T) { tr, r, _, key := resetTransportFixture(t) if mode == "expiry" { r.IssuedAt = tr.Now().Add(-2 * time.Minute) r.Expires = tr.Now().Add(-time.Minute) } signedResetRequest(t, tr, r) if mode == "cancel" { raw, _ := json.Marshal(r) wire, err := release.SignRootEnvelope(release.RootSigner(ed25519.NewKeyFromSeed(make([]byte, 32))), WakeResetCancellation{V: 1, RequestSHA256: evidenceDigest(raw)}) if err != nil { t.Fatal(err) } if _, err = tr.Store.Put(t.Context(), r.Key()+".cancel", wire, ""); err != nil { t.Fatal(err) } } if mode == "invalid-signature" { raw, tag, _ := tr.Store.Get(t.Context(), r.Key()) raw[0] = '[' if _, err := tr.Store.Put(t.Context(), r.Key(), raw, tag); err != nil { t.Fatal(err) } } if mode == "completed" { publishResetProof(t, tr, r, key) } calls := 0 stop := func(ctx context.Context, o hostclaim.Observed, a hostclaim.WakeRetryAttempt) error { calls++ if !r.matches(o) || a != r.Attempt { t.Fatal("wrong attempt") } publishResetProof(t, tr, r, key) if mode == "lost-response" { return errors.New("connection lost") } return nil } err := tr.Pass(t.Context(), "box", stop) want := 0 if mode == "dispatch" || mode == "lost-response" { want = 1 } if calls != want { t.Fatalf("calls=%d want=%d err=%v", calls, want, err) } if mode == "invalid-signature" && err == nil { t.Fatal("accepted unsigned request") } if mode == "lost-response" && !errors.Is(err, ErrWakeResetUncertain) { t.Fatalf("lost stop = %v", err) } if mode == "dispatch" || mode == "lost-response" || mode == "completed" { if err := tr.Pass(t.Context(), "box", stop); err != nil { t.Fatal(err) } if calls != want { t.Fatal("replayed completed stop") } if err := tr.VerifyReceipt(t.Context(), r); err != nil { t.Fatal(err) } } }) } } func TestPlan9ResetTypedWriterRequiresLiveExactRegistration(t *testing.T) { tr, r, o, _ := resetTransportFixture(t) w, err := NewControlWriter(tr.Store, tr.Directory.PublicKey, tr.Now) if err != nil { t.Fatal(err) } signer := release.RootSigner(ed25519.NewKeyFromSeed(make([]byte, 32))) issue := func(ctx context.Context) (WakeResetRequest, error) { return w.RequestWakeReset(ctx, tr, signer, r.Console, r.ConsoleEnrollment, r.Namespace, o, r.Attempt) } if _, err := issue(t.Context()); !errors.Is(err, ErrIdentity) { t.Fatalf("unregistered issue=%v", err) } b := r.Admission grant := lifecycleGrant{kind: "app", account: b.Account, app: b.App, appID: b.AppID, operation: b.Registration, owner: b.Owner, epoch: b.Epoch, w: w, registration: RegistrationWakeRetryReset, incarnation: r.Incarnation} ctx := withLifecycleGrant(t.Context(), grant) first, err := issue(ctx) if err != nil { t.Fatal(err) } replay, err := issue(ctx) if err != nil || replay != first { t.Fatalf("replay=%+v err=%v", replay, err) } if err = w.CancelWakeReset(t.Context(), tr, signer, first); err != nil { t.Fatal(err) } cancelled, err := tr.cancelled(t.Context(), first) if err != nil || !cancelled { t.Fatalf("cancel=%t %v", cancelled, err) } wrong := grant wrong.incarnation = "another" if _, err := issue(withLifecycleGrant(t.Context(), wrong)); !errors.Is(err, ErrIdentity) { t.Fatalf("other incarnation=%v", err) } } func TestPlan9ResetNewControlObjectsNeverPassDataBoundary(t *testing.T) { for _, key := range []string{WakeResetPrefix + "box/id.json", WakeResetPrefix + "box/id.json.cancel"} { if !IsControlOnlyKey(key) { t.Fatalf("not control-only: %s", key) } } } func TestPlan9ResetArchivePreventsResurrectionAndKeepsProof(t *testing.T) { tr, r, _, boxkey := resetTransportFixture(t) root := ed25519.NewKeyFromSeed(make([]byte, 32)) raw, err := release.SignRootEnvelope(release.RootSigner(root), r) if err != nil { t.Fatal(err) } if _, err = tr.Store.Put(t.Context(), r.Key(), raw, ""); err != nil { t.Fatal(err) } publishResetProof(t, tr, r, boxkey) now := r.Expires.Add(hostbox.RequestLimit + time.Second) tr.Now = func() time.Time { return now } writer := &ControlWriter{raw: tr.Store, pub: root.Public().(ed25519.PublicKey), now: tr.Now} if err = writer.SweepWakeResets(t.Context(), tr); err != nil { t.Fatal(err) } if _, _, err = tr.Store.Get(t.Context(), r.Key()); !errors.Is(err, os.ErrNotExist) { t.Fatalf("expired request retained: %v", err) } if _, _, err = tr.Store.Get(t.Context(), wakeResetArchiveKey(r)); err != nil { t.Fatal(err) } if err = tr.VerifyReceipt(t.Context(), r); err != nil { t.Fatalf("archive lost proof: %v", err) } // A delayed reissue that passed its first archive read cannot dispatch even // if it recreated the original request key with a later expiry. revived := r revived.IssuedAt = now revived.Expires = now.Add(time.Minute) if revived.ID != wakeResetID(revived) { t.Fatal("fixture did not reuse deterministic id") } if err = tr.Authorize(t.Context(), revived); err == nil { t.Fatal("archived request resurrected") } if !IsControlOnlyKey(wakeResetArchiveKey(r)) { t.Fatal("archive not control-only") } } func TestPlan9ResetStopDeadlineIsFairAndExpiryBounded(t *testing.T) { now := time.Now() if got := minResetDeadline(now, now.Add(time.Minute)); !got.Equal(now.Add(hostbox.RequestLimit / 4)) { t.Fatal(got) } if got := minResetDeadline(now, now.Add(time.Second)); !got.Equal(now.Add(time.Second)) { t.Fatal(got) } } func TestPlan9ArchivedResetRejectsLateCancellation(t *testing.T) { tr, r, _, _ := resetTransportFixture(t) root := ed25519.NewKeyFromSeed(make([]byte, 32)) raw, _ := release.SignRootEnvelope(release.RootSigner(root), r) if _, err := tr.Store.Put(t.Context(), wakeResetArchiveKey(r), raw, ""); err != nil { t.Fatal(err) } writer := &ControlWriter{raw: tr.Store, pub: root.Public().(ed25519.PublicKey), now: tr.Now} if err := writer.CancelWakeReset(t.Context(), tr, release.RootSigner(root), r); err == nil { t.Fatal("late cancellation recreated an orphan") } if _, _, err := tr.Store.Get(t.Context(), r.Key()+".cancel"); !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } } type plan9CancelDeleteStore struct { release.Store fail bool deleted []string } func (s *plan9CancelDeleteStore) DeleteIfMatch(ctx context.Context, key, etag string) error { s.deleted = append(s.deleted, key) if s.fail && strings.HasSuffix(key, ".cancel") { return errors.New("cancel delete unavailable") } return release.DeleteIfMatch(ctx, s.Store, key, etag) } func TestPlan9ResetSweepRetainsRequestUntilCancellationRemoved(t *testing.T) { tr, r, _, _ := resetTransportFixture(t) root := ed25519.NewKeyFromSeed(make([]byte, 32)) raw, _ := release.SignRootEnvelope(release.RootSigner(root), r) if _, err := tr.Store.Put(t.Context(), r.Key(), raw, ""); err != nil { t.Fatal(err) } writer := &ControlWriter{raw: tr.Store, pub: root.Public().(ed25519.PublicKey), now: tr.Now} if err := writer.CancelWakeReset(t.Context(), tr, release.RootSigner(root), r); err != nil { t.Fatal(err) } now := r.Expires.Add(hostbox.RequestLimit + time.Second) tr.Now = func() time.Time { return now } writer.now = tr.Now store := &plan9CancelDeleteStore{Store: tr.Store, fail: true} writer.raw = store if err := writer.SweepWakeResets(t.Context(), tr); err == nil { t.Fatal("ignored cancellation deletion failure") } if _, _, err := tr.Store.Get(t.Context(), r.Key()); err != nil { t.Fatalf("removed retry discovery: %v", err) } if len(store.deleted) != 1 || store.deleted[0] != r.Key()+".cancel" { t.Fatalf("wrong deletion order: %v", store.deleted) } store.fail = false store.deleted = nil if err := writer.SweepWakeResets(t.Context(), tr); err != nil { t.Fatal(err) } if len(store.deleted) != 2 || store.deleted[0] != r.Key()+".cancel" || store.deleted[1] != r.Key() { t.Fatalf("wrong successful order: %v", store.deleted) } }