package hostresolve import ( "context" "encoding/json" "errors" "reflect" "slices" "testing" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" ) type plan10LossOwner struct { service *Service store *failClaimStore calls []string failPrepare, failFinish, mutate bool sources []hostclaim.WriteSource } func (p *plan10LossOwner) InspectLoss(ctx context.Context, o hostclaim.Observed, record hostfence.RecordObservation, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) { if o.Claim.Op.Kind != hostclaim.KindMoving || box != "dead" || o.Claim.Op.Intent["target"] != box || record.Key != o.Claim.Holder.Key { return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrResolve } actual := hostclaim.LossConsequences{RemoveParticipation: true, WritesPotentiallyLost: true, TimersPotentiallyLost: true, RecoveryIntent: json.RawMessage(`{"kind":"test-move-loss"}`)} if p.mutate { o.Claim.Op.Intent["target"] = "wrong" o.Claim.Holder.Stamp++ } return hostfence.Consequences{Transition: "test-move-loss", PossibleWriteLoss: true, PossibleTimerLoss: true}, actual, nil } func (p *plan10LossOwner) LossWriteSources(context.Context, hostclaim.Observed, hostclaim.LossConsequences) ([]hostclaim.WriteSource, error) { return p.sources, nil } func (p *plan10LossOwner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error { if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["target"] != d.Box || d.Box != "dead" || !d.Consequences.RemoveParticipation { return ErrResolve } return nil } func (p *plan10LossOwner) PrepareLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { p.calls = append(p.calls, "prepare") roster, err := p.service.Directory.Read(ctx) if err != nil || roster.Roster.Boxes[loss.Box].State != "quarantined" { return ErrResolve } durable, err := hostclaim.ReadLossRecord(ctx, p.store, p.service.Directory.PublicKey, loss.ID) if err != nil || durable.Record.State != "prepared" || slices.Contains(o.Claim.LossRefs, loss.ID) { return ErrResolve } if p.failPrepare { return ErrResolve } if p.mutate { o.Claim.Op.Intent["target"] = "wrong" loss.ConfirmedConsequences.RecoveryIntent[0] = 'x' } return nil } func (p *plan10LossOwner) FinishLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { p.calls = append(p.calls, "finish") durable, err := hostclaim.ReadLossRecord(ctx, p.store, p.service.Directory.PublicKey, loss.ID) if err != nil || durable.Record.State != "committed" || !slices.Contains(o.Claim.LossRefs, loss.ID) { return ErrResolve } if p.failFinish { return ErrResolve } if p.mutate { o.Claim.Op.Intent["target"] = "wrong" loss.ConfirmedConsequences.RecoveryIntent[0] = 'x' } return nil } func plan10ResolveMove(t *testing.T) (Service, *plan10LossOwner, hostfence.Submission, string) { t.Helper() s, store, now, host := resolveFixture(t) observed, err := hostclaim.Read(t.Context(), store, host) if err != nil { t.Fatal(err) } _, err = hostclaim.Update(t.Context(), store, observed, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{ID: "move-test", Kind: hostclaim.KindMoving, Phase: hostclaim.PhaseMoving, Exec: "live", Participants: []string{"live", "dead"}, Intent: map[string]string{"source": "live", "target": "dead", "to": "de"}} return nil }) if err != nil { t.Fatal(err) } owner := &plan10LossOwner{store: store} s.Owners = map[string]OperationOwner{hostclaim.KindMoving: owner} owner.service = &s return s, owner, submitted(t, s, now, host), host } func TestPlan10MoveOwnerRequiredBeforeAcceptance(t *testing.T) { s, p, in, host := plan10ResolveMove(t) s.Owners = nil before, _ := s.Directory.Read(t.Context()) if _, err := s.Inspect(t.Context(), "operator", host, "dead"); !errors.Is(err, ErrResolve) { t.Fatal(err) } if _, err := s.Declare(t.Context(), "operator", host, in); !errors.Is(err, ErrResolve) { t.Fatal(err) } after, _ := s.Directory.Read(t.Context()) if after.ETag != before.ETag { t.Fatal("missing owner quarantined") } keys, err := p.store.List(t.Context(), "control/host-loss-acceptances/") if err != nil || len(keys) != 0 { t.Fatal(keys, err) } } func TestPlan10OwnerHookOrderAndReplay(t *testing.T) { for _, fault := range []string{"prepare", "claim", "bookkeeping", "finish"} { t.Run(fault, func(t *testing.T) { s, p, in, host := plan10ResolveMove(t) p.failPrepare = fault == "prepare" p.store.fail = fault == "claim" p.failFinish = fault == "finish" bookkeeping := &lossBookkeepingStore{Store: p.store, fail: fault == "bookkeeping"} s.Directory.Store = bookkeeping if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { t.Fatal("fault not reached") } p.failPrepare = false p.store.fail = false p.failFinish = false bookkeeping.fail = false if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { t.Fatal(err) } if p.calls[len(p.calls)-1] != "finish" { t.Fatal(p.calls) } before, _ := hostclaim.Read(t.Context(), p.store, host) n := len(p.calls) if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { t.Fatal(err) } after, _ := hostclaim.Read(t.Context(), p.store, host) if before.ETag != after.ETag || !reflect.DeepEqual(p.calls[n:], []string{"finish"}) { t.Fatal("retry rewrote claim or re-prepared", p.calls) } }) } } func TestPlan10OwnerReceivesDetachedValues(t *testing.T) { s, p, in, host := plan10ResolveMove(t) p.mutate = true after, err := s.Declare(t.Context(), "operator", host, in) if err != nil { t.Fatal(err) } if after.Claim.Op.Intent["target"] != "dead" { t.Fatal("callback changed claim") } loss, err := hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) if err != nil || !json.Valid(loss.Record.ConfirmedConsequences.RecoveryIntent) { t.Fatal(loss, err) } } func TestPlan10AcceptedOwnerUsesOriginalKind(t *testing.T) { s, p, in, host := plan10ResolveMove(t) if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { t.Fatal(err) } before, _ := hostclaim.Read(t.Context(), p.store, host) _, err := hostclaim.Update(t.Context(), p.store, before, func(c *hostclaim.Claim) error { c.Op.Kind = hostclaim.KindReverting; return nil }) if err != nil { t.Fatal(err) } if _, err = s.Declare(t.Context(), "operator", host, in); err != nil { t.Fatal("retry selected current owner", err) } s.Owners = nil if _, err = s.Declare(t.Context(), "operator", host, in); !errors.Is(err, ErrResolve) { t.Fatal("retry bypassed missing owner", err) } } func TestPlan10LossSourcesFrozenAtAcceptance(t *testing.T) { s, p, in, host := plan10ResolveMove(t) p.sources = []hostclaim.WriteSource{{Reason: "target", Box: "dead", Session: 12, Generation: "generation-12", Region: "de", Baseline: "generation-12"}} p.store.fail = true if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { t.Fatal("fault not reached") } loss, err := hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) if err != nil || !reflect.DeepEqual(loss.Record.WriteSources, p.sources) { t.Fatal(loss, err) } p.sources[0].Session = 99 p.store.fail = false if _, err = s.Declare(t.Context(), "operator", host, in); err != nil { t.Fatal(err) } loss, err = hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) if err != nil || len(loss.Record.WriteSources) != 1 || loss.Record.WriteSources[0].Session != 12 { t.Fatal("retry recomputed loss sources", loss, err) } } func TestPlan10BadSourcesRefuseBeforeQuarantine(t *testing.T) { s, p, in, host := plan10ResolveMove(t) p.sources = []hostclaim.WriteSource{{Reason: "target", Box: "dead"}} before, _ := s.Directory.Read(t.Context()) if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { t.Fatal("invalid frozen sources accepted") } after, _ := s.Directory.Read(t.Context()) if before.ETag != after.ETag || len(p.calls) != 0 { t.Fatal("invalid sources changed authority") } }