package hostresolve import ( "context" "encoding/json" "errors" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/hostquarantine" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) func TestPreparedLossClaimProgressAllowsRetiredReadmission(t *testing.T) { for _, retired := range []bool{false, true} { t.Run(map[bool]string{false: "missing-retirement", true: "retired"}[retired], func(t *testing.T) { ctx := t.Context() s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) store.fail = true if _, err := s.Declare(ctx, "operator", host, in); err == nil { t.Fatal("claim failure not reached") } store.fail = false prepared, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || prepared.Record.State != "prepared" { t.Fatal(prepared, err) } before, err := hostclaim.Read(ctx, store, host) if err != nil { t.Fatal(err) } // The retirement owner completes the exact local chore while quarantined. chore := before.Claim.Obligations[0] progressed, err := hostclaim.Update(ctx, store, before, hostclaim.CompleteObligation(chore.Op, chore.ID)) if err != nil { t.Fatal(err) } if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) { t.Fatal("stale declaration must refuse", err) } conflicted, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || conflicted.Record.State != "conflicted" { t.Fatal("stale loss not settled", conflicted, err) } want := prepared.Record want.State = "conflicted" if !reflect.DeepEqual(want, conflicted.Record) { t.Fatal("accepted history changed") } if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) { t.Fatal("conflicted retry changed outcome", err) } unchanged, err := hostclaim.Read(ctx, store, host) if err != nil || unchanged.ETag != progressed.ETag { t.Fatal("stale consequences applied", err) } prepareReadmission(t, s, *now, conflicted.Record) if !retired { key, _ := hostquarantine.RetirementKey("dead", conflicted.Record.Enrollment, conflicted.Record.QuarantineRevision) raw, etag, err := store.Get(ctx, key) if err != nil { t.Fatal(err) } var receipt hostquarantine.RetirementCompletion if err = json.Unmarshal(raw, &receipt); err != nil { t.Fatal(err) } receipt.Entries = []hostquarantine.RetirementEntry{} if err := receipt.Validate(); err != nil { t.Fatal(err) } raw, _ = json.Marshal(receipt) if _, err = store.Put(ctx, key, raw, etag); err != nil { t.Fatal(err) } digest := hostfloor.Digest(raw) raw, etag, err = store.Get(ctx, "status/boxes/dead.json") if err != nil { t.Fatal(err) } var report status.Box if err = json.Unmarshal(raw, &report); err != nil { t.Fatal(err) } report.Metrics.RetirementReceiptSHA256 = digest raw, _ = json.Marshal(report) if _, err = store.Put(ctx, "status/boxes/dead.json", raw, etag); err != nil { t.Fatal(err) } } roster, err := s.Directory.Read(ctx) if err != nil { t.Fatal(err) } b := roster.Roster.Boxes["dead"] err = s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs) if (err == nil) != retired { t.Fatal("readmission must require retirement of conflicted loss", err) } after, err := s.Directory.Read(ctx) if err != nil { t.Fatal(err) } state := "quarantined" if retired { state = "active" } if after.Roster.Boxes["dead"].State != state || !reflect.DeepEqual(after.Roster.Boxes["dead"].LossIDs, b.LossIDs) { t.Fatal("state or loss history changed", after) } retained, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || !reflect.DeepEqual(retained.Record, conflicted.Record) { t.Fatal("readmission erased history", err) } }) } } type fenceReadmitStore struct { release.Store unreadable bool writes int } func (s *fenceReadmitStore) Get(ctx context.Context, key string) ([]byte, string, error) { if s.unreadable && strings.HasPrefix(key, "control/box-fences/") { return nil, "", errors.New("fence unavailable") } return s.Store.Get(ctx, key) } func (s *fenceReadmitStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { s.writes++ return s.Store.Put(ctx, key, raw, etag) } func TestReadmitRefusesTerminationAuthorityBeforeMutation(t *testing.T) { for _, fault := range []string{"terminated", "malformed", "wrong-writer", "unreadable"} { t.Run(fault, func(t *testing.T) { ctx := t.Context() s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) if _, err := s.Declare(ctx, "operator", host, in); err != nil { t.Fatal(err) } loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil { t.Fatal(err) } prepareReadmission(t, s, *now, loss.Record) roster, err := s.Directory.Read(ctx) if err != nil { t.Fatal(err) } b := roster.Roster.Boxes["dead"] fence := map[string]any{"v": 1, "box": "dead", "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated", "observed_at": *now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "dead", "enrollment": b.Enrollment}} if fault == "wrong-writer" { fence["written_by"] = map[string]string{"box": "unknown", "enrollment": b.Enrollment} } raw, _ := json.Marshal(fence) if fault == "malformed" { raw = []byte("{") } if fault != "unreadable" { if err := release.CreateExact(ctx, store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { t.Fatal(err) } } before, err := hostclaim.Read(ctx, store, host) if err != nil { t.Fatal(err) } wrapped := &fenceReadmitStore{Store: store, unreadable: fault == "unreadable"} s.Directory.Store = wrapped if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err == nil { t.Fatal("fence authority ignored") } if wrapped.writes != 0 { t.Fatal("mutation before fence refusal", wrapped.writes) } after, err := hostclaim.Read(ctx, store, host) if err != nil || after.ETag != before.ETag { t.Fatal("per-host quarantine changed", err) } unchanged, err := s.Directory.Read(ctx) if err != nil || unchanged.ETag != roster.ETag { t.Fatal("global quarantine changed", err) } if unchanged.Roster.Boxes["dead"].State != "quarantined" || len(after.Claim.LossRefs) != 1 { t.Fatal("quarantine or history removed") } }) } } type lossBookkeepingStore struct { release.Store fail bool } func (s *lossBookkeepingStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { if s.fail && etag != "" && strings.HasPrefix(key, "control/host-loss/") { return "", errors.New("loss bookkeeping unavailable") } return s.Store.Put(ctx, key, raw, etag) } func TestPreparedLossCommitReferenceSurvivesClaimProgress(t *testing.T) { ctx := t.Context() s, store, now, host := resolveFixture(t) in := submitted(t, s, now, host) wrapped := &lossBookkeepingStore{Store: store, fail: true} s.Directory.Store = wrapped if _, err := s.Declare(ctx, "operator", host, in); err == nil { t.Fatal("bookkeeping failure not reached") } loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || loss.Record.State != "prepared" { t.Fatal(loss, err) } committed, err := hostclaim.Read(ctx, store, host) if err != nil || len(committed.Claim.LossRefs) != 1 || committed.Claim.LossRefs[0] != in.Confirmation { t.Fatal("claim not committed", err) } progressed, err := hostclaim.Update(ctx, store, committed, func(*hostclaim.Claim) error { return nil }) if err != nil { t.Fatal(err) } wrapped.fail = false if _, err := s.Declare(ctx, "operator", host, in); err != nil { t.Fatal("successful CAS treated as conflict", err) } loss, err = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) if err != nil || loss.Record.State != "committed" { t.Fatal(loss, err) } after, err := hostclaim.Read(ctx, store, host) if err != nil || after.ETag != progressed.ETag { t.Fatal("commit recovery changed claim", err) } }