package hostclaim import ( "context" "encoding/json" "errors" "fmt" "reflect" "testing" "time" "github.com/carlosframework/platform/internal/release" ) func TestAdmitWriterWarmColdAdmissions(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) stamp := o.Claim.Holder.Stamp if o.Claim.Holder.Session != 0 || o.Claim.Holder.WriterBox != "" { t.Fatalf("legacy session = %d, want zero", o.Claim.Holder.Session) } raw, err := json.Marshal(o.Claim.Holder) if err != nil { t.Fatal(err) } var fields map[string]json.RawMessage if err := json.Unmarshal(raw, &fields); err != nil { t.Fatal(err) } for _, name := range []string{"session", "writer_box", "admissions"} { if _, ok := fields[name]; ok { t.Fatalf("zero %s must be omitted", name) } } o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) if err != nil { t.Fatal(err) } cold := o.Claim.Gen if o.Claim.Holder.WriterBox != "adi-knitting" || o.Claim.Holder.Generation != "gen_a" || o.Claim.Holder.Stamp != cold || o.Claim.Holder.Session != cold || cold <= stamp { t.Fatalf("cold holder = %+v, gen = %d", o.Claim.Holder, cold) } first := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: cold, Box: "adi-knitting", Generation: "gen_a", Mode: "cold"} assertAdmissions(t, o.Claim, []AdmissionRecord{first}) o, err = Update(ctx, s, o, func(c *Claim) error { c.Op.Exec = "boxT" return nil }) if err != nil { t.Fatal(err) } o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: cold, ObservedGeneration: "gen_a", Box: "boxT", Mode: "warm", PredecessorCertified: true})) if err != nil { t.Fatal(err) } warm := o.Claim.Gen if o.Claim.Holder.WriterBox != "boxT" || o.Claim.Holder.Session != warm || warm <= cold || o.Claim.Holder.Generation != "gen_a" || o.Claim.Holder.Stamp != cold { t.Fatalf("warm holder = %+v, gen = %d", o.Claim.Holder, warm) } second := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: warm, Box: "boxT", Generation: "gen_a", Mode: "warm", Predecessor: "gen_a", Covered: true} assertAdmissions(t, o.Claim, []AdmissionRecord{first, second}) o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: warm, ObservedGeneration: "gen_a", Box: "boxT", Mode: "cold", NewGeneration: "gen_b", PredecessorCertified: true})) if err != nil { t.Fatal(err) } if o.Claim.Holder.Generation != "gen_b" || o.Claim.Holder.Stamp != o.Claim.Gen || o.Claim.Holder.Session != o.Claim.Gen { t.Fatalf("second cold holder = %+v, gen = %d", o.Claim.Holder, o.Claim.Gen) } third := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a"} assertAdmissions(t, o.Claim, []AdmissionRecord{first, second, third}) stored, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(stored, o) { t.Fatalf("stored admission = %+v, want %+v", stored, o) } } func assertAdmissions(t *testing.T, c *Claim, want []AdmissionRecord) { t.Helper() if !reflect.DeepEqual(c.Holder.Admissions, want) { t.Fatalf("admissions = %+v, want %+v", c.Holder.Admissions, want) } if _, ok := c.Op.Journal["writers"]; ok { t.Fatal("admission wrote an operation writers journal") } } func TestAdmitWriterPreservesOperation(t *testing.T) { for _, kind := range []string{KindServing, KindMoving, KindLanding, KindReverting, KindRestoring, KindPushing} { for _, mode := range []string{"warm", "cold"} { t.Run(kind+"/"+mode, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o, err := Update(ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: kind, ID: "m1", Phase: "probe", Exec: "boxT", Participants: []string{"boxS", "boxT"}, Intent: map[string]string{"target": "boxT"}, Journal: map[string]string{"source": "unchanged"}, By: "operator", StartedAt: c.Holder.CreatedAt} c.Obligations = []Obligation{{ID: "ob1", Op: "m1", Kind: "park-local", Box: "boxS", Owner: "box:boxS", Retirement: "rt1"}} return nil }) if err != nil { t.Fatal(err) } generation := "" if mode == "cold" { generation = "gen_a" } next, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "m1", ObservedPhase: "probe", Box: "boxT", Mode: mode, NewGeneration: generation, PredecessorCertified: true})) if err != nil { t.Fatal(err) } want := *o.Claim holder := *want.Holder want.Holder = &holder want.Gen++ want.Holder.Session = want.Gen want.Holder.WriterBox = "boxT" if mode == "cold" { want.Holder.Generation = generation want.Holder.Stamp = want.Gen } want.Holder.Admissions = []AdmissionRecord{{Region: "ie", PredecessorRegion: "ie", Session: want.Gen, Box: "boxT", Generation: generation, Mode: mode, Covered: mode == "warm"}} if !reflect.DeepEqual(next.Claim, &want) { t.Fatalf("admission changed fields outside holder writer fields and admissions: got %+v, want %+v", next.Claim, &want) } }) } } } func TestAdmitWriterRefusals(t *testing.T) { for _, name := range []string{"no holder", "no executor", "non-executor", "stale session", "stale generation", "warm with generation", "warm with current generation", "cold without generation", "cold same generation", "unknown mode", "empty mode", "unknown coverage", "warm without certificate", "stale op", "stale phase", "creating", "deleting", "parking", "purging", "placing"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o, err := Update(ctx, s, o, func(c *Claim) error { c.Holder.Session = c.Gen + 1 c.Holder.Generation = "gen_a" switch name { case "no holder": c.Holder = nil c.Op = Op{} case "no executor": c.Op.Exec = "" case "creating", "deleting", "parking", "purging", "placing": c.Op.Kind, c.Op.ID = name, "op1" } return nil }) if err != nil { t.Fatal(err) } session, generation, box, mode, newGeneration := o.Claim.Gen, "gen_a", "adi-knitting", "cold", "gen_b" switch name { case "no executor": box = "" case "non-executor": box = "boxT" case "stale session": session-- case "stale generation": generation = "old" case "warm with generation": mode = "warm" case "warm with current generation": mode, newGeneration = "warm", generation case "cold without generation": newGeneration = "" case "cold same generation": newGeneration = generation case "unknown mode": mode = "hot" case "empty mode": mode = "" } in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: session, ObservedGeneration: generation, Box: box, Mode: mode, NewGeneration: newGeneration, PredecessorCertified: true} switch name { case "warm without certificate": in.Mode, in.NewGeneration, in.PredecessorCertified = "warm", "", false case "unknown coverage": in.Coverage = "remote" case "stale op": in.ObservedOpID = "old" case "stale phase": in.ObservedPhase = "old" } if _, err := Update(ctx, s, o, AdmitWriter(in)); !errors.Is(err, ErrWrongState) { t.Fatalf("admit writer = %v, want ErrWrongState", err) } stored, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(stored, o) { t.Fatalf("refused admission changed stored claim or ETag: got %+v, want %+v", stored, o) } }) } } func TestAdmitWriterCompetingAdmissions(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) before := servingClaim(t, ctx, s) in := Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"} first := AdmitWriter(in) in.NewGeneration = "gen_b" second := AdmitWriter(in) admitted, err := Update(ctx, s, before, first) if err != nil { t.Fatal(err) } if _, err := Update(ctx, s, before, second); !errors.Is(err, ErrConflict) { t.Fatalf("competing admission = %v, want ErrConflict", err) } current := assertAdmissionStored(t, ctx, s, admitted) if _, err := Update(ctx, s, current, second); !errors.Is(err, ErrWrongState) { t.Fatalf("stale evidence after reread = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, admitted) } func TestAdmitWriterLostResponse(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) before := servingClaim(t, ctx, s) admit := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) admitted, err := Update(ctx, s, before, admit) if err != nil { t.Fatal(err) } if _, err := Update(ctx, s, before, admit); !errors.Is(err, ErrConflict) { t.Fatalf("retry after lost response = %v, want ErrConflict", err) } assertAdmissionStored(t, ctx, s, admitted) } func TestAdmitWriterBoundToOperationAndPhase(t *testing.T) { for _, change := range []string{"operation", "phase", "delete"} { t.Run(change, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) stale := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) var err error if change == "delete" { o, err = Update(ctx, s, o, BeginDelete("d1", "console", "etag", time.Now(), nil, nil, "")) } else { o, err = Update(ctx, s, o, func(c *Claim) error { if change == "operation" { c.Op.Kind, c.Op.ID = KindMoving, "m1" } else { c.Op.Phase = "probe" } return nil }) } if err != nil { t.Fatal(err) } if _, err := Update(ctx, s, o, stale); !errors.Is(err, ErrWrongState) { t.Fatalf("admission after %s changed = %v, want ErrWrongState", change, err) } assertAdmissionStored(t, ctx, s, o) if change == "delete" { fresh := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "d1", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) if _, err := Update(ctx, s, o, fresh); !errors.Is(err, ErrWrongState) { t.Fatalf("deleting admission = %v", err) } assertAdmissionStored(t, ctx, s, o) id := 0 o, err = Update(ctx, s, o, ToParking("d1", nil, nil, func() string { id++; return fmt.Sprintf("o%d", id) })) if err != nil { t.Fatal(err) } if _, err := Update(ctx, s, o, fresh); !errors.Is(err, ErrWrongState) { t.Fatalf("parking admission = %v", err) } assertAdmissionStored(t, ctx, s, o) } }) } } func TestAdmitWriterSurvivesOperationReplacement(t *testing.T) { for _, transition := range []string{"delete", "move"} { t.Run(transition, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"})) if err != nil { t.Fatal(err) } writer := *o.Claim.Holder if writer.WriterBox != "adi-knitting" || writer.Session != o.Claim.Gen { t.Fatalf("admitted holder = %+v", writer) } if transition == "delete" { o, err = Update(ctx, s, o, BeginDelete("d1", "console", "etag", time.Now(), nil, nil, "")) } else { o, err = Update(ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindMoving, ID: "m1", Phase: "probe", Exec: "boxT"} return nil }) } if err != nil { t.Fatal(err) } o = assertAdmissionStored(t, ctx, s, o) if !reflect.DeepEqual(*o.Claim.Holder, writer) { t.Fatalf("%s changed writer: %+v", transition, o.Claim.Holder) } if transition == "delete" { return } o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "m1", ObservedPhase: "probe", ObservedSession: writer.Session, ObservedGeneration: writer.Generation, Box: "boxT", Mode: "cold", NewGeneration: "gen_b", RemoteMaxTXID: 42})) if err != nil { t.Fatal(err) } want := append(append([]AdmissionRecord(nil), writer.Admissions...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a", WritesLost: &WritesLost{Box: "adi-knitting", Generation: "gen_a", Session: writer.Session, RemoteMaxTXID: 42}, }) assertAdmissions(t, o.Claim, want) if o.Claim.Holder.WriterBox != "boxT" { t.Fatalf("new writer = %q", o.Claim.Holder.WriterBox) } assertAdmissionStored(t, ctx, s, o) }) } } func assertAdmissionStored(t *testing.T, ctx context.Context, s release.Store, want Observed) Observed { t.Helper() got, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(got, want) { t.Fatalf("stored claim or ETag changed: got %+v, want %+v", got, want) } return got } func TestAdmitWriterKeepStamp(t *testing.T) { for _, name := range []string{"landing", "equal session", "later session", "warm"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) if err != nil { t.Fatal(err) } if name == "landing" || name == "warm" { o, err = Update(ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "m1", Phase: "probe", Exec: "boxT"} c.Holder.Stamp = c.Gen + 1 return nil }) } else if name == "later session" { o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "gen_a", Box: "adi-knitting", Mode: "warm", PredecessorCertified: true})) } if err != nil { t.Fatal(err) } before := *o.Claim.Holder in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: before.Session, ObservedGeneration: before.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "gen_b", KeepStamp: true, PredecessorCertified: true} if name == "warm" { in.Mode, in.NewGeneration = "warm", "" } next, err := Update(ctx, s, o, AdmitWriter(in)) if name != "landing" { if !errors.Is(err, ErrWrongState) { t.Fatalf("KeepStamp admission = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, o) return } if err != nil { t.Fatal(err) } if next.Claim.Holder.Stamp != before.Stamp || next.Claim.Holder.Session != next.Claim.Gen || next.Claim.Holder.Generation != "gen_b" { t.Fatalf("landing admission holder = %+v, prior stamp = %d", next.Claim.Holder, before.Stamp) } want := append(append([]AdmissionRecord(nil), before.Admissions...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: next.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a", }) assertAdmissions(t, next.Claim, want) assertAdmissionStored(t, ctx, s, next) in.ObservedSession, in.ObservedGeneration, in.NewGeneration = next.Claim.Holder.Session, "gen_b", "gen_c" if _, err := Update(ctx, s, next, AdmitWriter(in)); !errors.Is(err, ErrWrongState) { t.Fatalf("restarted probe keeping stamp = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, next) }) } } func TestAdmitWriterCoverage(t *testing.T) { for _, mode := range []string{"cold", "warm"} { for _, certified := range []bool{false, true} { for _, coverage := range []string{"", "sealed", "clean-return"} { if mode == "warm" && !certified { continue } t.Run(fmt.Sprintf("%s/certified=%t/coverage=%s", mode, certified, coverage), func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) before := *o.Claim.Holder in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: before.Session, ObservedGeneration: before.Generation, Box: "adi-knitting", Mode: mode, PredecessorCertified: certified, Coverage: coverage, RemoteMaxTXID: 42} generation := before.Generation if mode == "cold" { in.NewGeneration, generation = "gen_b", "gen_b" } o = updateGenerationTest(t, ctx, s, o, AdmitWriter(in)) last := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "adi-knitting", Generation: generation, Predecessor: "gen_a", Mode: mode, Coverage: coverage, Covered: mode == "warm"} if mode == "cold" && !certified && coverage == "" { last.WritesLost = &WritesLost{Box: "adi-knitting", Generation: "gen_a", Session: before.Session, RemoteMaxTXID: 42} } assertAdmissions(t, o.Claim, append(append([]AdmissionRecord(nil), before.Admissions...), last)) assertAdmissionStored(t, ctx, s, o) }) } } } } func updateGenerationTest(t *testing.T, ctx context.Context, s release.Store, o Observed, tr func(*Claim) error) Observed { t.Helper() next, err := Update(ctx, s, o, tr) if err != nil { t.Fatal(err) } return assertAdmissionStored(t, ctx, s, next) } func TestAdmitWriterCleanReturnPredecessor(t *testing.T) { for _, coverage := range []string{"", "clean-return"} { t.Run(coverage, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} return nil }) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "move", ObservedPhase: "probe", Box: "boxT", Mode: "cold", NewGeneration: "GT"})) target := o.Claim.Holder.Admissions[0] o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Holder.Generation = "GS" c.Op = Op{Kind: KindServing, Exec: "boxS"} return nil }) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: target.Session, ObservedGeneration: "GS", Box: "boxS", Mode: "cold", NewGeneration: "GS2", Coverage: coverage, RemoteMaxTXID: 42})) last := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "boxS", Generation: "GS2", Predecessor: "GT", Mode: "cold", Coverage: coverage} if coverage == "" { last.WritesLost = &WritesLost{Box: "boxT", Generation: "GT", Session: target.Session, RemoteMaxTXID: 42} } assertAdmissions(t, o.Claim, []AdmissionRecord{target, last}) }) } } func TestAdoptAdmitPublishRootGeneration(t *testing.T) { for _, replaceOp := range []bool{false, true} { t.Run(fmt.Sprintf("replaceOp=%t", replaceOp), func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } o = updateGenerationTest(t, ctx, s, o, Adopt(newHolder(), "boxS", nil, nil, "adopt", counter())) if o.Claim.Holder.WriterBox != "boxS" || o.Claim.Holder.Generation != "" || RootGeneration != "@root" { t.Fatalf("adopted holder = %+v", o.Claim.Holder) } for _, generation := range []string{"", RootGeneration} { newID := func() string { t.Fatal("unadmitted publication allocated an ID"); return "" } if _, err := Update(ctx, s, o, PublishGeneration("", "", 0, generation, "ie", newID)); !errors.Is(err, ErrWrongState) { t.Fatalf("publication before admission = %v", err) } assertAdmissionStored(t, ctx, s, o) } if replaceOp { o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} return nil }) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindServing, Exec: "boxT"} return nil }) } o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1", RemoteMaxTXID: 42})) assertAdmissions(t, o.Claim, []AdmissionRecord{{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: o.Claim.Op.Exec, Generation: "G1", Mode: "cold", WritesLost: &WritesLost{Box: "boxS", Session: 0, Generation: "", RemoteMaxTXID: 42}, }}) o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G1", "ie", counter())) if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 1 { t.Fatalf("published claim = %+v", o.Claim) } ob := o.Claim.Obligations[0] if ob.Generation != RootGeneration || ob.Kind != "park-generation" || ob.Region != "ie" || ob.Owner != "region:ie" || ob.Op != fmt.Sprintf("session:%d", o.Claim.Holder.Session) { t.Fatalf("root obligation = %+v", ob) } holder := *o.Claim.Holder o, err = Update(ctx, s, o, CompleteObligation(ob.Op, ob.ID)) if err != nil { t.Fatal(err) } stored, err := Read(ctx, s, testHost) if err != nil || stored.ETag != o.ETag || len(stored.Claim.Obligations) != 0 || !reflect.DeepEqual(*stored.Claim.Holder, holder) || stored.Claim.Op.Kind != KindServing { t.Fatalf("completed root retirement = %+v, %v", stored, err) } }) } } func TestPublishGenerationChain(t *testing.T) { for _, kind := range []string{KindServing, KindMoving, KindLanding, KindReverting, KindRestoring, KindPushing} { t.Run(kind, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) if kind != KindServing { o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: kind, ID: "move", Phase: "probe", Exec: "boxT", Journal: map[string]string{"source": "unchanged"}} return nil }) } for _, generation := range []string{"G1", "G2"} { o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: generation})) } before := o calls := 0 newID := func() string { calls++; return fmt.Sprintf("id%d", calls) } publish := PublishGeneration(o.Claim.Op.ID, o.Claim.Op.Phase, o.Claim.Holder.Session, "G2", "ie", newID) o = updateGenerationTest(t, ctx, s, o, publish) op := before.Claim.Op.ID if op == "" { op = fmt.Sprintf("session:%d", before.Claim.Holder.Session) } want := *before.Claim holder := *want.Holder want.Holder = &holder want.Gen++ holder.Admissions = append([]AdmissionRecord(nil), holder.Admissions...) for i := range holder.Admissions { holder.Admissions[i].Covered = true } want.Obligations = []Obligation{ {ID: "id1", Op: op, Kind: "park-generation", Region: "ie", Generation: RootGeneration, Retirement: "id2", Owner: "region:ie"}, {ID: "id3", Op: op, Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "id4", Owner: "region:ie"}, } if calls != 4 || !reflect.DeepEqual(o.Claim, &want) { t.Fatalf("publication = %+v, want %+v; ID calls %d", o.Claim, &want, calls) } o = updateGenerationTest(t, ctx, s, o, publish) want.Gen++ if calls != 4 || !reflect.DeepEqual(o.Claim, &want) { t.Fatalf("republication changed coverage or chores: %+v; calls %d", o.Claim, calls) } }) } } func TestPublishGenerationRefusals(t *testing.T) { for _, name := range []string{"no holder", "creating", "deleting", "parking", "purging", "placing", "stale op", "stale phase", "stale session", "wrong generation", "empty region", "no matching admission"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1"})) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} if name == "no holder" { c.Holder, c.Op = nil, Op{} } else if knownKinds[name] { c.Op.Kind = name } else if name == "no matching admission" { c.Holder.Generation = "unrecorded" } return nil }) op, phase, session, generation, region := o.Claim.Op.ID, o.Claim.Op.Phase, uint64(0), "G1", "ie" if o.Claim.Holder != nil { session, generation = o.Claim.Holder.Session, o.Claim.Holder.Generation } switch name { case "stale op": op = "" case "stale phase": phase = "" case "stale session": session-- case "wrong generation": generation = "old" case "empty region": region = "" } newID := func() string { t.Fatal("refusal allocated an ID"); return "" } if _, err := Update(ctx, s, o, PublishGeneration(op, phase, session, generation, region, newID)); !errors.Is(err, ErrWrongState) { t.Fatalf("publication = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, o) }) } } func TestPublishGenerationAllAdmissionsInOrder(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Holder.Generation = "G2" c.Holder.Admissions = []AdmissionRecord{ {Region: "ie", PredecessorRegion: "ie", Generation: "old", Predecessor: "already-covered", Covered: true}, {Region: "ie", PredecessorRegion: "ie", Generation: "G1", Predecessor: ""}, {Region: "ie", PredecessorRegion: "ie", Generation: "G1", Predecessor: RootGeneration}, {Region: "ie", PredecessorRegion: "ie", Generation: "G2", Predecessor: "G1"}, {Region: "ie", PredecessorRegion: "ie", Generation: "G2", Predecessor: "G2", Mode: "warm", Covered: true}, {Region: "ie", PredecessorRegion: "ie", Generation: "later", Predecessor: "G2", Mode: "cold"}, {Region: "ie", PredecessorRegion: "ie", Generation: "after-last-match", Predecessor: "later", Mode: "cold"}, {Region: "ie", PredecessorRegion: "uk", Generation: "foreign", Predecessor: "foreign-predecessor", Mode: "cold"}, {Region: "ie", Generation: "empty", Predecessor: "no-objects", Mode: "cold"}, {Region: "ie", PredecessorRegion: "ie", Generation: "duplicate", Predecessor: "later", Mode: "cold"}, {Region: "ie", PredecessorRegion: "ie", Generation: "warm-only", Predecessor: "warm-only", Mode: "warm", Covered: true}, {Region: "ie", PredecessorRegion: "ie", Generation: "last", Predecessor: "final-predecessor", Mode: "cold"}, } c.Obligations = []Obligation{ {ID: "existing", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "ret1", Owner: "region:ie"}, {ID: "other-region", Op: "old-op", Kind: "park-generation", Region: "uk", Generation: RootGeneration, Retirement: "ret2", Owner: "region:uk"}, } return nil }) before := o calls := 0 newID := func() string { calls++; return fmt.Sprintf("id%d", calls) } o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", 0, "G2", "ie", newID)) want := append(append([]Obligation(nil), before.Claim.Obligations...), Obligation{ID: "id1", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: RootGeneration, Retirement: "id2", Owner: "region:ie"}, Obligation{ID: "id3", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: "later", Retirement: "id4", Owner: "region:ie"}, Obligation{ID: "id5", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: "final-predecessor", Retirement: "id6", Owner: "region:ie"}, ) if calls != 6 || !reflect.DeepEqual(o.Claim.Obligations, want) { t.Fatalf("deduplicated chores = %+v, want %+v; calls %d", o.Claim.Obligations, want, calls) } for i, record := range o.Claim.Holder.Admissions { if record.Covered != (i != 5 && i != 7) { t.Fatalf("record %d covered = %t", i, record.Covered) } } } func TestPublishGenerationActiveRoot(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "warm", PredecessorCertified: true})) o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "", "ie", func() string { t.Fatal("active root must not be retired") return "" })) if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 0 { t.Fatalf("active root publication = %+v", o.Claim) } } func TestAdmitWriterTrimsCoveredPrefix(t *testing.T) { for _, name := range []string{"all covered", "uncovered first", "uncovered middle"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) for i := 1; i <= 10; i++ { o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: fmt.Sprintf("G%d", i)})) } if len(o.Claim.Holder.Admissions) != 10 { t.Fatalf("uncovered records were trimmed: %+v", o.Claim.Holder.Admissions) } o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G10", "ie", counter())) if name != "all covered" { o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { i := 0 if name == "uncovered middle" { i = 1 } c.Holder.Admissions[i].Covered = false return nil }) } before := o.Claim.Holder.Admissions o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G10", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G11", PredecessorCertified: true})) // Loss-bearing rows remain until archival, even when covered. drop := 1 if name == "uncovered first" { drop = 0 } want := append(append([]AdmissionRecord(nil), before[drop:]...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: o.Claim.Op.Exec, Generation: "G11", Predecessor: "G10", Mode: "cold"}) assertAdmissions(t, o.Claim, want) }) } } func TestAdmissionRecordJSON(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1", RemoteMaxTXID: 42})) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G1", Box: "adi-knitting", Mode: "cold", NewGeneration: "G2", Coverage: "sealed", PredecessorCertified: true})) o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G2", "ie", counter())) raw, err := json.Marshal(o.Claim.Holder) if err != nil { t.Fatal(err) } var holder map[string]json.RawMessage if err := json.Unmarshal(raw, &holder); err != nil { t.Fatal(err) } var got []map[string]any if err := json.Unmarshal(holder["admissions"], &got); err != nil { t.Fatal(err) } want := []map[string]any{ {"region": "ie", "predecessor_region": "ie", "session": float64(o.Claim.Holder.Admissions[0].Session), "box": "adi-knitting", "generation": "G1", "mode": "cold", "covered": true}, {"region": "ie", "predecessor_region": "ie", "session": float64(o.Claim.Holder.Session), "box": "adi-knitting", "generation": "G2", "predecessor": "G1", "mode": "cold", "coverage": "sealed", "covered": true}, } if !reflect.DeepEqual(got, want) { t.Fatalf("admissions JSON = %s, want %+v", holder["admissions"], want) } } func TestAdmitWriterIgnoresLegacyWriterJournal(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op.Journal = map[string]string{"writers": "not-json", "source": "unchanged"} return nil }) before := o.Claim.Op o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1"})) if !reflect.DeepEqual(o.Claim.Op, before) || len(o.Claim.Holder.Admissions) != 1 { t.Fatalf("legacy journal affected admission: %+v", o.Claim) } } func TestPublishGenerationForeignRegion(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G1", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G2"})) newID := func() string { t.Fatal("foreign-region publication allocated an ID"); return "" } publish := PublishGeneration("", "", o.Claim.Holder.Session, "G2", "uk", newID) o = updateGenerationTest(t, ctx, s, o, publish) for _, record := range o.Claim.Holder.Admissions { if record.Covered || record.PredecessorRegion != "ie" { t.Fatalf("foreign predecessor = %+v", record) } } if len(o.Claim.Obligations) != 0 { t.Fatalf("foreign publication queued chores: %+v", o.Claim.Obligations) } o = updateGenerationTest(t, ctx, s, o, CoverRegionPredecessors("", "", "ie")) for _, record := range o.Claim.Holder.Admissions { if !record.Covered { t.Fatalf("source cleanup left predecessor uncovered: %+v", record) } } before := *o.Claim o = updateGenerationTest(t, ctx, s, o, publish) before.Gen++ if len(o.Claim.Obligations) != 0 || !reflect.DeepEqual(o.Claim, &before) { t.Fatalf("publication after source coverage changed claim: %+v", o.Claim) } } func TestPublishGenerationHoldsNone(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G1", "ie", func() string { t.Fatal("predecessor with no objects allocated an ID") return "" })) if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 0 { t.Fatalf("empty predecessor publication = %+v", o.Claim) } raw, err := json.Marshal(o.Claim.Holder.Admissions[0]) if err != nil { t.Fatal(err) } var fields map[string]json.RawMessage if err := json.Unmarshal(raw, &fields); err != nil { t.Fatal(err) } if string(fields["region"]) != `"ie"` || fields["predecessor_region"] != nil { t.Fatalf("region JSON = %s", raw) } } func TestAdmitWriterRegionRules(t *testing.T) { for _, name := range []string{"empty region", "warm other region", "warm same region", "recorded predecessor region", "first predecessor region", "first warm other region"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) if name != "first predecessor region" && name != "empty region" && name != "first warm other region" { o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) } in := Admission{Region: "uk", PredecessorRegion: "fr", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G2", PredecessorCertified: true} switch name { case "empty region": in.Region = "" case "warm other region", "warm same region", "first warm other region": in.Mode, in.NewGeneration = "warm", "" if name == "warm same region" { in.Region = "ie" } } next, err := Update(ctx, s, o, AdmitWriter(in)) if name == "empty region" || name == "warm other region" || name == "first warm other region" { if !errors.Is(err, ErrWrongState) { t.Fatalf("admission = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, o) return } if err != nil { t.Fatal(err) } assertAdmissionStored(t, ctx, s, next) record := next.Claim.Holder.Admissions[len(next.Claim.Holder.Admissions)-1] wantPredecessorRegion := "ie" if name == "first predecessor region" || name == "first warm other region" { wantPredecessorRegion = "fr" } if record.Region != in.Region || record.PredecessorRegion != wantPredecessorRegion { t.Fatalf("admission regions = %+v, want %q / %q", record, in.Region, wantPredecessorRegion) } }) } } func TestAdmitWriterNeverTrimsForeignUncovered(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) newID := counter() for i := 1; i <= 12; i++ { o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: fmt.Sprintf("G%d", i)})) o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, o.Claim.Holder.Generation, "uk", newID)) } if len(o.Claim.Holder.Admissions) != 12 || o.Claim.Holder.Admissions[0].Generation != "G1" || o.Claim.Holder.Admissions[0].Covered { t.Fatalf("foreign uncovered head was trimmed or covered: %+v", o.Claim.Holder.Admissions) } for _, record := range o.Claim.Holder.Admissions[1:] { if !record.Covered { t.Fatalf("publication stopped at foreign predecessor: %+v", record) } } } func TestCoverRegionPredecessors(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "cleanup", Exec: c.Op.Exec} c.Holder.Admissions = []AdmissionRecord{ {Region: "ie", PredecessorRegion: "ie", Covered: true}, {Region: "uk", PredecessorRegion: "uk"}, {Region: "uk", PredecessorRegion: "ie"}, {Region: "ie"}, {Region: "ie", PredecessorRegion: "ie"}, } c.Obligations = []Obligation{{ID: "source", Op: "move", Kind: "park-replica", Region: "ie", Retirement: "ret", Owner: "region:ie"}} return nil }) want := *o.Claim holder := *want.Holder want.Holder = &holder holder.Admissions = append([]AdmissionRecord(nil), holder.Admissions...) holder.Admissions[2].Covered, holder.Admissions[4].Covered = true, true for i := 0; i < 2; i++ { o = updateGenerationTest(t, ctx, s, o, CoverRegionPredecessors("move", "cleanup", "ie")) want.Gen++ if !reflect.DeepEqual(o.Claim, &want) { t.Fatalf("region coverage = %+v, want %+v", o.Claim, &want) } } } func TestCoverRegionPredecessorsRefusals(t *testing.T) { for _, name := range []string{"stale op", "stale phase", "nil holder", "empty region"} { t.Run(name, func(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "cleanup", Exec: c.Op.Exec} c.Holder.Admissions = []AdmissionRecord{{Region: "uk", PredecessorRegion: "ie"}} if name == "nil holder" { c.Holder, c.Op = nil, Op{} } return nil }) op, phase, region := o.Claim.Op.ID, o.Claim.Op.Phase, "ie" switch name { case "stale op": op = "" case "stale phase": phase = "" case "empty region": region = "" } if _, err := Update(ctx, s, o, CoverRegionPredecessors(op, phase, region)); !errors.Is(err, ErrWrongState) { t.Fatalf("region coverage = %v, want ErrWrongState", err) } assertAdmissionStored(t, ctx, s, o) }) } } func TestValidateAdmissionRegion(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) for _, records := range [][]AdmissionRecord{ {{Region: ""}}, {{Region: "ie"}, {Region: ""}}, {{Region: "", Covered: true}, {Region: "ie"}}, } { o.Claim.Holder.Admissions = records if err := o.Claim.Validate(); !errors.Is(err, ErrMalformed) { t.Fatalf("admissions %+v: Validate = %v, want ErrMalformed", records, err) } } }