package park import ( "context" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" ) func coverageFixture(t *testing.T, foreign bool) *fixture { t.Helper() f := newFixture(t) allowFixtureMoveSweep(f) obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { c.Obligations = nil c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"} c.Holder.Generation = "live" c.Holder.Session = 3 c.Holder.Admissions = []hostclaim.AdmissionRecord{ {Region: "uk", PredecessorRegion: "ie", Predecessor: "dead"}, {Region: "ie", PredecessorRegion: "uk", Predecessor: "dead"}, {Region: "ie", PredecessorRegion: "ie", Predecessor: "unseen"}, {Region: "ie", PredecessorRegion: "ie", Predecessor: "stage-only"}, {Region: "ie", PredecessorRegion: "ie", Predecessor: "old-only"}, {Region: "uk", PredecessorRegion: "ie", Predecessor: ""}, } return nil }) if err != nil { t.Fatal(err) } base := f.ob.Park.Location + "/" for _, key := range []string{ base + "dead/data", base + "0000/data", "moves/" + f.w.Host + "/stage-only/data", "instances/" + f.w.Host + "/" + strings.Repeat("a", 64) + "/old-only/data", } { f.b.Put("data", key, []byte(key)) } f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) { authority := "ie" if foreign { authority = "uk" } return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: authority, MarkerKnown: true, MarkerGeneration: "live", GenerationOf: func(r Ref) (string, bool) { parts := strings.Split(r.Key, "/") g := parts[len(parts)-2] if g == "0000" { g = "" } return g, true }}, nil } return f } func TestSweepQueueCoversOnlyQueuedPredecessors(t *testing.T) { for _, foreign := range []bool{false, true} { name := "generation" if foreign { name = "replica" } t.Run(name, func(t *testing.T) { f := coverageFixture(t, foreign) before := f.b.Snapshot() f.held.failed = true if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) { t.Fatal(err) } obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } if len(obs.Claim.Obligations) == 0 || f.b.Snapshot() != before { t.Fatal("queue boundary did not preserve pending work and storage") } for i, a := range obs.Claim.Holder.Admissions { if want := i == 0 || i == 5; a.Covered != want { t.Errorf("queue admission %d covered=%v, want %v", i, a.Covered, want) } } }) } } func TestSweepAcknowledgementCoversOnlyRetiredPredecessors(t *testing.T) { for _, scenario := range []string{"generation", "replica", "empty", "old-incarnation", "staging"} { t.Run(scenario, func(t *testing.T) { f := coverageFixture(t, scenario == "replica") f.held.failed = true if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) { t.Fatal(err) } obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } var chore hostclaim.Obligation for _, o := range obs.Claim.Obligations { match := o.Park.Location == f.ob.Park.Location && (o.Generation == "dead" || scenario == "replica") if scenario == "old-incarnation" { match = strings.HasSuffix(o.Park.Location, strings.Repeat("a", 64)) } else if scenario == "staging" { match = strings.HasPrefix(o.Park.Location, "moves/") } if match { chore = o break } } if chore.ID == "" { t.Fatal("target chore was not queued") } if scenario == "empty" { versions, err := f.e.list(t.Context(), f.w.Host, chore.Park.Location) if err != nil { t.Fatal(err) } for _, x := range versions { if strings.Contains(x.Key, "/dead/") { if err := f.b.DeleteVersion(t.Context(), "data", x.Key, x.VersionID); err != nil { t.Fatal(err) } } } } _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { later := append([]hostclaim.AdmissionRecord(nil), c.Holder.Admissions...) for i := range later { later[i].Covered = false } c.Holder.Admissions = append(c.Holder.Admissions, later...) return nil }) if err != nil { t.Fatal(err) } f.held.failed = false f.e.After = func(step string) error { if step == "before-completion" { return ErrPending } return nil } w := Work{f.w.Host, chore.ID, chore.Op} if err := f.e.Run(t.Context(), w); !errors.Is(err, ErrPending) { t.Fatal(err) } pending, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } for _, a := range pending.Claim.Holder.Admissions[6:] { if a.Covered { t.Fatal("later admission covered before acknowledgement") } } f.e.After = nil f.b.FailOn = func(op, key string) error { if op == "copy" { t.Error("completion retry copied again") return ErrUnsafe } return nil } if err := f.e.Run(t.Context(), w); err != nil { t.Fatal(err) } completed, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } if _, present := find(completed.Claim, w); present || f.held.released != 1 || completed.Claim.QuietAfter["ie"].IsZero() { t.Fatal("retirement was not acknowledged") } for i, a := range completed.Claim.Holder.Admissions[6:] { want := (i == 0 && (scenario == "generation" || scenario == "replica")) || (i == 5 && scenario == "replica") if a.Covered != want { t.Errorf("ack admission %d covered=%v, want %v", i, a.Covered, want) } } }) } } func TestSweepCoverageSurvivesRetirementRotation(t *testing.T) { f := coverageFixture(t, true) obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "uk", PredecessorRegion: "ie", Predecessor: "late"}) return nil }) if err != nil { t.Fatal(err) } f.held.failed = true if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) { t.Fatal(err) } obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } var w Work for _, o := range obs.Claim.Obligations { if o.Park.Location == f.ob.Park.Location { w = Work{f.w.Host, o.ID, o.Op} } } if w.ID == "" || obs.Claim.Holder.Admissions[6].Covered { t.Fatal("late generation was queued or covered without an object") } f.held.failed = false for _, next := range []string{"late", "last"} { f.e.After = func(step string) error { if step == "manifest" { f.b.Put("data", f.ob.Park.Location+"/"+next+"/data", []byte(next)) } return nil } if err := f.e.Run(t.Context(), w); !errors.Is(err, ErrPending) { t.Fatalf("retirement rotation: %v", err) } } f.e.After = nil if err := f.e.Run(t.Context(), w); err != nil { t.Fatal(err) } completed, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) if err != nil { t.Fatal(err) } if !completed.Claim.Holder.Admissions[6].Covered || completed.Claim.Holder.Admissions[2].Covered { t.Fatalf("coverage lost a retired generation or covered unseen data: %+v", completed.Claim.Holder.Admissions) } if _, present := find(completed.Claim, w); present { t.Fatal("rotated retirement did not complete") } }