//go:build unix package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "net" "net/http" "os" "path/filepath" "slices" "testing" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/release" ) // N6, ruling 56: wireClaimAdmission with the completed-relocation branch. // The first serving wake after a relocation completed on this box hands // over from the probe's existing receipt: before admission the receipt is // found and verified and nothing is stopped; after it, root's // /claim/handover is posted that receipt's target as the prior. Any other // serving prior keeps main's stop and main's local handover. func TestRelocationCompletedHandoverRoutesOnlyTheCompletedPrior(t *testing.T) { ctx := t.Context() f := newStopProbeFixture(t) if _, err := f.stop(); err != nil { t.Fatal(err) } sum := sha256.Sum256(f.receipt) done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) serving := func(journal map[string]string) hostclaim.Observed { c := *f.obs.Claim c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: journal} return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} } completedPrior := serving(map[string]string{hostclaim.JournalCompleted: string(done)}) plainPrior := serving(nil) admitted := func(prior hostclaim.Observed) hostclaim.Observed { c := *prior.Claim h := *c.Holder h.Stamp, h.Session, h.Generation = 40, 40, "g40" c.Holder = &h return hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &c} } // Root's handover endpoint, recording what it is posted. socket := filepath.Join(t.TempDir(), "c.sock") l, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } var posted []coldHandoverRequest srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req coldHandoverRequest if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil { w.WriteHeader(http.StatusBadRequest) return } posted = append(posted, req) w.WriteHeader(http.StatusNoContent) })} go srv.Serve(l) t.Cleanup(func() { srv.Close() }) stops, locals := 0, 0 w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} local := func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) completed := &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"} wireClaimAdmission(w, "dst", local, claimCrossBoxHandover{}, &ClaimRelocationOwners{}, completed) if err := w.BeforeAdmission(ctx, completedPrior); err != nil || stops != 0 { t.Fatalf("before the completed prior's admission: %v (stops %d)", err, stops) } if err := w.AfterAdmission(ctx, completedPrior, admitted(completedPrior)); err != nil { t.Fatalf("after the completed prior's admission: %v", err) } if len(posted) != 1 || posted[0].Prior != f.target || posted[0].Next.Session != 40 || posted[0].Next.Box != "dst" || locals != 0 { t.Fatalf("root was posted %+v (locals %d)", posted, locals) } if err := w.BeforeAdmission(ctx, plainPrior); err != nil || stops != 1 { t.Fatalf("a serving prior without the record: %v (stops %d)", err, stops) } if err := w.AfterAdmission(ctx, plainPrior, admitted(plainPrior)); err != nil || locals != 1 || len(posted) != 1 { t.Fatalf("a serving prior's handover: %v (locals %d, posted %d)", err, locals, len(posted)) } // After the first serving wake the holder has moved on (session, stamp // or generation), while the record still names the stopped probe: that // prior is a serving session like any other, main's stop and main's // local handover, never the probe's receipt again. for name, move := range map[string]func(*hostclaim.Holder){ "session": func(h *hostclaim.Holder) { h.Session = 40 }, "stamp": func(h *hostclaim.Holder) { h.Stamp = 40 }, "generation": func(h *hostclaim.Holder) { h.Generation = "g40" }, } { moved := completedPrior c := *moved.Claim h := *c.Holder move(&h) c.Holder = &h moved.Claim = &c if err := w.BeforeAdmission(ctx, moved); err != nil || stops != 2 { t.Fatalf("%s moved on, before admission: %v (stops %d)", name, err, stops) } if err := w.AfterAdmission(ctx, moved, admitted(moved)); err != nil || locals != 2 || len(posted) != 1 { t.Fatalf("%s moved on, after admission: %v (locals %d, posted %d)", name, err, locals, len(posted)) } stops, locals = 1, 1 } // A record whose stop is not root's receipt is refused before anything. forged := serving(map[string]string{hostclaim.JournalCompleted: string(mustJSON(t, hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(make([]byte, 32))}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}) if err := w.BeforeAdmission(ctx, forged); err == nil || stops != 1 { t.Fatalf("a forged record: %v (stops %d)", err, stops) } // A serving prior on this box whose record cannot be read is refused // before and after admission, never routed to main's stop or local // handover (absence is never evidence). unreadable := serving(map[string]string{hostclaim.JournalCompleted: "{not json"}) if err := w.BeforeAdmission(ctx, unreadable); !errors.Is(err, hostclaim.ErrRelocation) || stops != 1 { t.Fatalf("an unreadable record before admission: %v (stops %d)", err, stops) } if err := w.AfterAdmission(ctx, unreadable, admitted(unreadable)); !errors.Is(err, hostclaim.ErrRelocation) || locals != 1 || len(posted) != 1 { t.Fatalf("an unreadable record after admission: %v (locals %d, posted %d)", err, locals, len(posted)) } } // handoverRoot is root's /claim/handover endpoint on a unix socket, // recording every request it is posted. func handoverRoot(t *testing.T) (string, *[]coldHandoverRequest) { t.Helper() dir, err := os.MkdirTemp(os.TempDir(), "handover-") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) socket := filepath.Join(dir, "c.sock") l, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } posted := &[]coldHandoverRequest{} srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req coldHandoverRequest if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil { w.WriteHeader(http.StatusBadRequest) return } *posted = append(*posted, req) w.WriteHeader(http.StatusNoContent) })} go srv.Serve(l) t.Cleanup(func() { srv.Close() }) return socket, posted } // completedServing is the serving op CompleteRelocation leaves after the // probe stop s of the fixture's relocating claim: the same holder, and // the completed relocation journaled, marked discharged when the // completion pass has run. func (f *stopProbeFixture) completedServing(s hostclaim.ProbeStop, discharged bool) hostclaim.Observed { f.t.Helper() c := *f.obs.Claim c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(mustJSON(f.t, hostclaim.CompletedRelocation{Op: "rel-1", Stop: s, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}} if discharged { if err := hostclaim.MarkCompletedDischarged("rel-1")(&c); err != nil { f.t.Fatal(err) } } return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} } // Ruling 82's reasoning, applied to N6 (Task 9 review): a relocation // completed while region HA was on, and region HA is now off. The claim // assembly then has no relocation owners (and, with moves C1 off too, no // cross-box routing at all), yet the first serving wake must still hand // over from the probe's receipt: main's Wake.Stop would ask a // serving-shaped re-stop of the stopped probe session. Through // OpenProductionClaimAssembly's own wiring (wireProductionAdmission), the // wake's predecessor stop finds and verifies the receipt with nothing // stopped, and root is posted that receipt's target as the prior. A prior // with no completion record, on a box that completed nothing, meets main's // hooks exactly as they were. func TestRelocationCompletedHandoverWithRegionHAOff(t *testing.T) { ctx := t.Context() for _, movesC1 := range []bool{true, false} { for _, discharged := range []bool{false, true} { t.Run(fmt.Sprintf("moves C1 %v, discharged %v", movesC1, discharged), func(t *testing.T) { f := newStopProbeFixture(t) s, err := f.stop() if err != nil { t.Fatal(err) } f.order = nil socket, posted := handoverRoot(t) stops, locals := 0, 0 w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil, &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"}) prior := f.completedServing(s, discharged) admitted := *prior.Claim next := *admitted.Holder next.Stamp, next.Session, next.Generation = 40, 40, "g40" admitted.Holder = &next if err := w.stopPredecessor(ctx, prior); err != nil || stops != 0 || len(f.order) != 0 { t.Fatalf("the first serving wake's predecessor stop: %v (stops %d, root %v)", err, stops, f.order) } if err := w.AfterAdmission(ctx, prior, hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &admitted}); err != nil { t.Fatalf("the first serving wake's handover: %v", err) } if len(*posted) != 1 || (*posted)[0].Prior != f.target || (*posted)[0].Next.Session != 40 || locals != 0 { t.Fatalf("root was posted %+v (locals %d)", *posted, locals) } }) } t.Run(fmt.Sprintf("moves C1 %v, no completion record", movesC1), func(t *testing.T) { // A box that never completed a relocation: an empty guard // journal, a serving prior with no record. f := newStopProbeFixture(t) socket, posted := handoverRoot(t) stops, locals := 0, 0 w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil, &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"}) c := *f.obs.Claim c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}} prior := hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} if err := w.stopPredecessor(ctx, prior); err != nil || stops != 1 { t.Fatalf("main's stop: %v (stops %d)", err, stops) } if err := w.AfterAdmission(ctx, prior, prior); err != nil || locals != 1 || len(*posted) != 0 { t.Fatalf("main's handover: %v (locals %d, posted %d)", err, locals, len(*posted)) } }) } } // C10 and N6's Before verify root's receipt, not only find it: a receipt // in the guard journal whose fence id, phase and digest match the serving // op's record, but whose bytes disagree with their own fence target, is // refused by both. With root's genuine receipt both pass. func TestRelocationCompletedReceiptIsVerifiedByC10AndN6(t *testing.T) { ctx := t.Context() for _, genuine := range []bool{true, false} { t.Run(fmt.Sprintf("genuine %v", genuine), func(t *testing.T) { f := newStopProbeFixture(t) if !genuine { var r StopReceipt if err := json.Unmarshal(f.receipt, &r); err != nil { t.Fatal(err) } r.BootID = "another-boot" // the target says "boot" f.receipt = mustMarshalStopReceipt(r) } s, err := f.stop() if err != nil { t.Fatal(err) } prior := f.completedServing(s, false) check := func(what string, err error) { t.Helper() if genuine && err != nil { t.Fatalf("%s refused root's receipt: %v", what, err) } if !genuine && !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("%s accepted a receipt that does not verify: %v", what, err) } } h := relocationCompletedHandover{Guard: f.a.Guard, Box: "dst"} check("N6's Before", h.Before(ctx, prior)) // C10 over the persisted binding, the serving claim in the store. store := release.NewDirStore(t.TempDir()) key, _ := hostclaim.Key(prior.Host) if _, err := store.Put(ctx, key, mustJSON(t, prior.Claim), ""); err != nil { t.Fatal(err) } f.a.Claims = store done, err := f.a.completedRelocationDischarge(ctx, execlease.Binding{Box: "dst", Host: prior.Host, Incarnation: prior.Claim.Holder.Incarnation, Op: "rel-1"}) if !done { t.Fatalf("C10 fell through to main's cleanup: %v", err) } check("C10", err) }) } } // N6 is now always in front of the admission hooks in claim mode, so // Plan 11's relocating routing reaches its owners through N6's fallback: // TestPlan11Fix1RelocationAdmissionHooks's three writers, with N6 wired as // production wires it. func TestRelocationCompletedHandoverKeepsPlan11Routing(t *testing.T) { for _, writer := range []string{"source", "", "target"} { t.Run(writer, func(t *testing.T) { calls := []string{} w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { calls = append(calls, "local-stop"); return nil }} w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { calls = append(calls, "local-after") return nil } reloc := ClaimRelocationOwners{BeforeAdmission: func(context.Context, hostclaim.Observed) error { calls = append(calls, "relocation-before") return nil }, AfterAdmission: func(context.Context, hostclaim.Observed, hostclaim.Observed) error { calls = append(calls, "relocation-after") return nil }} wireProductionAdmission(w, "target", true, claimCrossBoxHandover{}, &reloc, &relocationCompletedHandover{Box: "target"}) o := plan11Relocating("probing") o.Claim.Op.Exec = "target" o.Claim.Holder.WriterBox = writer if err := w.stopPredecessor(t.Context(), o); err != nil { t.Fatal(err) } if err := w.AfterAdmission(t.Context(), o, o); err != nil { t.Fatal(err) } want := []string{"relocation-before", "relocation-after"} if writer == "target" { want = []string{"local-stop", "local-after"} } if !slices.Equal(calls, want) { t.Fatalf("calls=%v want=%v", calls, want) } }) } }