package hostresolve import ( "context" "crypto/ed25519" "crypto/rand" "encoding/json" "errors" "fmt" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/hostquarantine" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" "strings" "testing" "time" ) type failClaimStore struct { release.Store fail bool crash string } func (s *failClaimStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { if s.fail && strings.HasPrefix(key, "control/hosts/") { return "", errors.New("claim write unavailable") } if s.crash == "acceptance" && strings.HasPrefix(key, "control/host-loss-acceptances/") || s.crash == "before-quarantine" && key == hostactors.Key || s.crash == "loss-create" && strings.HasPrefix(key, "control/host-loss/") { return "", errors.New("crash before durable write") } next, err := s.Store.Put(ctx, key, raw, etag) if err == nil && s.crash == "after-quarantine" && key == hostactors.Key { return "", errors.New("crash after durable quarantine") } return next, err } func resolveFixture(t *testing.T) (Service, *failClaimStore, *time.Time, string) { t.Helper() ctx := context.Background() pub, key, _ := ed25519.GenerateKey(rand.Reader) store := &failClaimStore{Store: release.NewDirStore(t.TempDir())} signer := release.RootSigner(key) now := time.Now().UTC() d := hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"} if err := d.Initialize(ctx, signer); err != nil { t.Fatal(err) } o, err := d.Read(ctx) if err != nil { t.Fatal(err) } r := hostactors.Clone(o.Roster) r.Boxes["dead"] = hostactors.Box{Enrollment: strings.Repeat("a", 32), Namespace: "platform", BoxID: "dead", Label: "dead-label", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-dead"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active"} if _, err = d.Replace(ctx, signer, o, r); err != nil { t.Fatal(err) } host := "shop.bab.oncarlos.com" observed, _ := hostclaim.Read(ctx, store, host) n := 0 _, err = hostclaim.Update(ctx, store, observed, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie"}, "live", nil, []string{"dead"}, "discovery", func() string { n++; return fmt.Sprintf("id-%d", n) })) if err != nil { t.Fatal(err) } return Service{Directory: d, Signer: signer, Now: func() time.Time { return now }}, store, &now, host } func submitted(t *testing.T, s Service, now *time.Time, host string) hostfence.Submission { t.Helper() in, err := s.Inspect(context.Background(), "operator", host, "dead-label") if err != nil { t.Fatal(err) } *now = now.Add(2 * time.Second) return hostfence.Submission{Challenge: in.Challenge, Confirmation: in.Details.ID, Observation: hostfence.CloudObservation{ObservedAt: in.Details.IssuedAt.Add(time.Second), Account: "123", Region: "eu-west-1", InstanceID: "i-dead", State: "stopped", RequestID: "request"}, Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: in.Details.Consequences.DestroyedStorageAssertion}} } func TestLostWiringPreservesServingAuthorityAndRetryEvidence(t *testing.T) { ctx := context.Background() s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) before, _ := hostclaim.Read(ctx, store, host) after, err := s.Declare(ctx, "operator", host, in) if err != nil { t.Fatal(err) } if after.Claim.Op.Exec != before.Claim.Op.Exec || after.Claim.Holder.Stamp != before.Claim.Holder.Stamp || len(after.Claim.Obligations) != 0 { t.Fatal(after.Claim) } loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || loss.Record.State != "committed" { t.Fatal(loss, err) } first := loss.Record.FencedAt *now = now.Add(time.Hour) if _, err = s.Declare(ctx, "operator", host, in); err != nil { t.Fatal("accepted retry expired", err) } loss, _ = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if !loss.Record.FencedAt.Equal(first) { t.Fatal("retry moved fence time") } if _, err = s.Declare(ctx, "other-person", host, in); err == nil { t.Fatal("actor changed") } } func TestFailedClaimWriteLeavesGlobalQuarantine(t *testing.T) { s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) store.fail = true if _, err := s.Declare(context.Background(), "operator", host, in); err == nil { t.Fatal("expected write error") } actor, err := s.Directory.Lookup(context.Background(), "dead") if err != nil || actor.State != "quarantined" { t.Fatal(actor, err) } store.fail = false if _, err := s.Declare(context.Background(), "operator", host, in); err != nil { t.Fatal(err) } } func TestDeclareCrashAndChallengeExpiryAllowRetryAndReadmission(t *testing.T) { for _, crash := range []string{"acceptance", "before-quarantine", "after-quarantine", "loss-create"} { t.Run(crash, func(t *testing.T) { ctx := t.Context() s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) acceptedAt := *now before, err := hostclaim.Read(ctx, store, host) if err != nil { t.Fatal(err) } store.crash = crash if _, err := s.Declare(ctx, "operator", host, in); err == nil { t.Fatal("crash not reached") } unchanged, err := hostclaim.Read(ctx, store, host) if err != nil || unchanged.ETag != before.ETag { t.Fatal("claim changed before quarantine completion", err) } roster, err := s.Directory.Read(ctx) if err != nil { t.Fatal(err) } b := roster.Roster.Boxes["dead"] if crash == "acceptance" || crash == "before-quarantine" { if b.State != "active" || len(b.LossIDs) != 0 { t.Fatal("quarantined before acceptance", b) } } else if b.State != "quarantined" || len(b.LossIDs) != 1 || b.LossIDs[0] != in.Confirmation { t.Fatal("deny not retained", b) } if _, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation); !errors.Is(err, hostclaim.ErrLossRecordAbsent) { t.Fatal("loss unexpectedly materialized", err) } if crash != "acceptance" { if _, err := s.readAcceptance(ctx, in.Confirmation); err != nil { t.Fatal("acceptance missing", err) } } store.crash = "" if crash == "before-quarantine" { // An unrelated roster CAS must not strand accepted evidence. if _, err := s.Directory.Replace(ctx, s.Signer, roster, hostactors.Clone(roster.Roster)); err != nil { t.Fatal(err) } } *now = now.Add(time.Hour) if _, err := hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, *now); err == nil { t.Fatal("challenge did not expire") } if crash == "acceptance" { if _, err := s.Declare(ctx, "operator", host, in); err == nil { t.Fatal("unaccepted expired submission allowed") } return } if _, err := s.Declare(ctx, "operator", host, in); err != nil { t.Fatal("accepted retry expired", err) } loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || loss.Record.State != "committed" || !loss.Record.AcceptedAt.Equal(acceptedAt) || !loss.Record.FencedAt.Equal(acceptedAt) { t.Fatal("acceptance changed", loss, err) } prepareReadmission(t, s, *now, loss.Record) roster, err = s.Directory.Read(ctx) if err != nil { t.Fatal(err) } b = roster.Roster.Boxes["dead"] if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err != nil { t.Fatal("orphan blocked readmission", err) } actor, err := s.Directory.Lookup(ctx, "dead") if err != nil || actor.State != "active" { t.Fatal("readmission failed", actor, err) } }) } } func prepareReadmission(t *testing.T, s Service, now time.Time, loss hostclaim.LossRecord) { t.Helper() ctx := t.Context() store := s.Directory.Store floor := hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"} if err := hostfloor.Publish(ctx, store, s.Signer, s.Directory.Bucket, floor); err != nil { t.Fatal(err) } f, err := hostfloor.Read(ctx, store, s.Directory.PublicKey, s.Directory.Bucket) if err != nil { t.Fatal(err) } hash := strings.Repeat("d", 64) rel := release.SystemRelease{Version: "build123", ReleaseTag: floor.MinRelease, MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("build123", "linux-amd64"), SHA256: hash}}} if _, err := release.PromoteSystem(ctx, store, s.Signer, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil { t.Fatal(err) } wire, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable")) if err != nil { t.Fatal(err) } if err := release.PublishSystemIdentity(ctx, store, s.Directory.PublicKey, wire); err != nil { t.Fatal(err) } receipt := hostquarantine.RetirementCompletion{V: 1, Box: loss.Box, Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, QuarantineRevision: loss.QuarantineRevision, BootID: "new-boot", CompletedAt: now, Complete: true, InventorySHA256: strings.Repeat("e", 64), Entries: []hostquarantine.RetirementEntry{{Host: loss.Host, Incarnation: loss.HolderIncarnation, Stamp: loss.HolderStamp, RetirementID: "retirement", JournalSHA256: strings.Repeat("f", 64)}}} if err := receipt.Validate(); err != nil { t.Fatal(err) } raw, err := json.Marshal(receipt) if err != nil { t.Fatal(err) } key, err := hostquarantine.RetirementKey(loss.Box, loss.Enrollment, loss.QuarantineRevision) if err != nil { t.Fatal(err) } if err := release.CreateExact(ctx, store, key, raw); err != nil { t.Fatal(err) } report := status.Box{GeneratedAt: now, Label: "dead-label", Metrics: &status.BoxMetrics{Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, Release: rel.Version, ReleaseTag: rel.ReleaseTag, Platform: "linux-amd64", BinarySHA256: hash, MovesProtocol: 1, BootID: receipt.BootID, FloorSHA256: f.Digest, FloorGuardReady: true, HostClaimEnabled: true, RetirementReceiptSHA256: hostfloor.Digest(raw)}} raw, err = json.Marshal(report) if err != nil { t.Fatal(err) } if err := release.CreateExact(ctx, store, "status/boxes/"+loss.Box+".json", raw); err != nil { t.Fatal(err) } }