package host import ( "context" "errors" "os/exec" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) func plan11Relocating(phase string) hostclaim.Observed { exec := "box-2" switch phase { case "starting", "releasing": exec = "box-1" case "pinning": exec = "none" } return hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Session: 9, WriterBox: "box-2"}, Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "r1", Phase: phase, Exec: exec, Participants: []string{"box-1", "box-2"}, Intent: map[string]string{"region": "ie", "source_box": "box-1", "target_box": "box-2"}}}} } func plan11Env(moves, ha string) func(string) string { return func(k string) string { switch k { case "CARLOS_HOST_CLAIM": return "1" case "CARLOS_MOVES_PHASE1": return moves case "CARLOS_REGION_HA": return ha } return "" } } func plan11RelocationOwners(calls *[]string) ClaimRelocationOwners { return ClaimRelocationOwners{BeforeAdmission: func(context.Context, hostclaim.Observed) error { return nil }, AfterAdmission: func(context.Context, hostclaim.Observed, hostclaim.Observed) error { return nil }, ClaimKindOwners: plan9D5Owner{"relocation", calls}.owners(), Operations: &plan9D2Attempts{}, Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }} } // R1 as accepted: ClaimOwnerDispatch's typed Relocation entry receives // relocating observations only at a phase of the relocating session row // (probing, and the probe's session carried into serving). Unbound, the // kind keeps main's refusal; an incomplete entry is a typed refusal naming // region HA; any other relocating phase reaches no owner. func TestPlan11OwnerDispatchRoutesRelocatingToItsTypedEntry(t *testing.T) { var calls []string moves := plan9D5Owner{"moves", &calls}.owners() relocation := plan9D5Owner{"relocation", &calls}.owners() d := ClaimOwnerDispatch{Serving: plan9D5Owner{"serving", &calls}.owners(), Moves: &moves, Relocation: &relocation} all := "select,materialize,empty,publish,admission,publication" want := "relocation." + strings.ReplaceAll(all, ",", ",relocation.") for _, phase := range []string{"probing", "serving"} { calls = nil if err := plan9D5Call(d, plan11Relocating(phase)); err != nil { t.Fatalf("%s: %v", phase, err) } if got := strings.Join(calls, ","); got != want { t.Fatalf("%s routed %s", phase, got) } } for _, phase := range []string{"starting", "releasing", "pinning", "installing", "installed", "probe-failed"} { calls = nil if err := plan9D5Call(d, plan11Relocating(phase)); !errors.Is(err, hostbox.ErrChanged) || len(calls) != 0 { t.Fatalf("%s: %v %v", phase, err, calls) } } // Every move kind still goes to moves, serving to serving. for kind, owner := range map[string]string{hostclaim.KindServing: "serving", hostclaim.KindLanding: "moves", hostclaim.KindRestoring: "moves"} { calls = nil if err := plan9D5Call(d, plan9D5Kind(kind)); err != nil || !strings.HasPrefix(strings.Join(calls, ","), owner+".select") { t.Fatalf("%s: %v %v", kind, err, calls) } } // Unbound: main's ErrUnsupported, exactly as before Plan 11. d.Relocation = nil calls = nil if err := plan9D5Call(d, plan11Relocating("probing")); !errors.Is(err, hostbox.ErrUnsupported) || len(calls) != 0 { t.Fatalf("unbound: %v %v", err, calls) } // Incomplete: a typed refusal naming region HA's provider. d.Relocation = &ClaimKindOwners{Inputs: relocation.Inputs} var refusal *ClaimDependencyError if err := plan9D5Call(d, plan11Relocating("probing")); !errors.As(err, &refusal) || !strings.Contains(refusal.Error(), "region HA") || len(calls) != 0 { t.Fatalf("incomplete: %v %v", err, calls) } // The fenced publisher routes to the relocation inputs' own, never to // the unfenced Publish. d.Relocation = &relocation if _, err := d.PublishFenced(context.Background(), plan11Relocating("probing"), ClaimLaunchInputs{}, ClaimPublication{}); !errors.As(err, &refusal) || len(calls) != 0 { t.Fatalf("fenced publication without a fenced publisher: %v %v", err, calls) } } // R1/R8: the relocation owner set binds only with CARLOS_HOST_CLAIM, // CARLOS_MOVES_PHASE1 and CARLOS_REGION_HA all on; a partial set refuses the // assembly rather than half-wiring it. func TestPlan11RelocationBindsOnlyWithAllFlagsAndACompleteOwnerSet(t *testing.T) { var calls []string complete := plan11RelocationOwners(&calls) for name, tc := range map[string]struct { getenv func(string) string movesC1 bool owners ClaimRelocationOwners want bool err error }{ "ha-off": {plan11Env("1", ""), true, complete, false, nil}, "ha-garbage": {plan11Env("1", "maybe"), true, complete, false, nil}, "moves-c1-off": {plan11Env("", "1"), false, complete, false, nil}, "no-owners": {plan11Env("1", "1"), true, ClaimRelocationOwners{}, false, nil}, "complete": {plan11Env("1", "true"), true, complete, true, nil}, "no-attempts": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: complete.ClaimKindOwners, Policy: complete.Policy}, false, hostwrite.ErrUnavailable}, "no-policy": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: complete.ClaimKindOwners, Operations: complete.Operations}, false, hostwrite.ErrUnavailable}, "no-publication": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: ClaimKindOwners{Inputs: complete.Inputs, AdmissionEvidence: complete.AdmissionEvidence}, Operations: complete.Operations, Policy: complete.Policy}, false, hostwrite.ErrUnavailable}, "probe-only": {plan11Env("1", "1"), true, ClaimRelocationOwners{ProbeReady: func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }}, false, hostwrite.ErrUnavailable}, } { got, err := claimRelocationC1(tc.getenv, tc.movesC1, tc.owners) if got != tc.want || !errors.Is(err, tc.err) { t.Errorf("%s: %v %v", name, got, err) } } } // The constructor's Group D wiring: with relocation unbound the moves // attempts, probe check and policy are exactly Plan 9's; bound, a router // sends landing to moves and relocating to region HA. func TestPlan11GroupDWiringIsPlan9sWithRelocationUnbound(t *testing.T) { var calls []string serving := plan9D5Owner{"serving", &calls}.owners() movesAttempts := &plan9D2Attempts{} moves := ClaimMovesOwners{ClaimKindOwners: plan9D5Owner{"moves", &calls}.owners(), Operations: movesAttempts} off, err := claimGroupDWiring(plan11Env("1", ""), serving, moves, plan11RelocationOwners(&calls)) if err != nil { t.Fatal(err) } if off.Dispatch.Relocation != nil || off.Relocation != nil || off.Operations != OperationAttempts(movesAttempts) || off.ProbeReady == nil { t.Fatalf("relocation leaked with region HA off: %+v", off) } if _, routed := off.Operations.(claimOperationRouter); routed { t.Fatal("router bound with region HA off") } reloc := plan11RelocationOwners(&calls) on, err := claimGroupDWiring(plan11Env("1", "1"), serving, moves, reloc) if err != nil { t.Fatal(err) } if on.Dispatch.Relocation == nil || on.Relocation == nil || on.Dispatch.Moves == nil { t.Fatalf("relocation not bound: %+v", on) } router, ok := on.Operations.(claimOperationRouter) if !ok || router[hostclaim.KindLanding].Attempts != OperationAttempts(movesAttempts) || router[hostclaim.KindRelocating].Attempts != reloc.Operations { t.Fatalf("router: %+v", on.Operations) } if router[hostclaim.KindRelocating].ProbeReady == nil || router[hostclaim.KindLanding].ProbeReady == nil { t.Fatal("probe readiness defaults to the socket check") } for _, kind := range []string{hostclaim.KindReverting, hostclaim.KindRestoring} { if router[kind].Attempts != OperationAttempts(movesAttempts) || router[kind].ProbeReady == nil { t.Fatalf("recovery owner missing with relocation enabled: %s", kind) } } if _, err := claimGroupDWiring(plan11Env("1", "1"), serving, moves, ClaimRelocationOwners{Operations: reloc.Operations}); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("partial relocation set: %v", err) } } // D2 for relocating: the router serves relocating/probing attempts from // region HA's journal and landing attempts from moves', and refuses any // other kind before either journal is touched. func TestPlan11OperationAttemptsRouteByOpKind(t *testing.T) { landing, relocating := &plan9D2Attempts{}, &plan9D2Attempts{} r := claimOperationRouter{hostclaim.KindLanding: {Attempts: landing}, hostclaim.KindRelocating: {Attempts: relocating}} ctx := context.Background() start := func(o hostclaim.Observed) OperationStart { return OperationStart{Admitted: o} } reloc := start(plan11Relocating("probing")) if _, err := r.OpenProbe(ctx, reloc, time.Unix(5, 0)); err != nil { t.Fatal(err) } r.CurrentAttempt(reloc) if err := errors.Join(r.Reserve(ctx, reloc, 1), r.Failed(ctx, reloc, 1), r.Healthy(ctx, reloc, 2, nil, func(context.Context) error { return nil })); err != nil { t.Fatal(err) } if got := strings.Join(relocating.calls, ","); got != "open,current,reserve-1,failed-1,healthy-2" || len(landing.calls) != 0 { t.Fatalf("relocating routed %q, landing %v", got, landing.calls) } land := start(plan9D5Kind(hostclaim.KindLanding)) if err := r.Reserve(ctx, land, 0); err != nil || len(landing.calls) != 1 { t.Fatalf("landing: %v %v", err, landing.calls) } var refusal *ClaimDependencyError for _, o := range []hostclaim.Observed{plan9D5Kind(hostclaim.KindRestoring), plan9D5Kind(hostclaim.KindServing), {Host: recoveryHost}} { s := start(o) _, openErr := r.OpenProbe(ctx, s, time.Unix(5, 0)) err := errors.Join(openErr, r.Reserve(ctx, s, 1), r.Failed(ctx, s, 1), r.Healthy(ctx, s, 1, nil, func(context.Context) error { return nil })) if !errors.As(err, &refusal) { t.Fatalf("unrouted kind: %v", err) } if got := r.CurrentAttempt(s); !reflect.DeepEqual(got, hostclaim.WakeRetryAttempt{}) { t.Fatalf("unrouted CurrentAttempt %+v", got) } } if len(landing.calls) != 1 || len(relocating.calls) != 5 { t.Fatalf("refused kinds reached a journal: %v %v", landing.calls, relocating.calls) } } // The wake's operation owner and spawn action helpers the assembly uses. // With Plan 9's single moves journal, only landing has an owner, exactly as // before; with the router, relocating has region HA's. func TestPlan11WakeOperationOwnerAndProbeAction(t *testing.T) { moves := &plan9D2Attempts{} ready := func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } w := &ClaimWakeAssembly{Operations: moves, ProbeReady: ready} if ops, r := w.operationOwner(plan9D5Kind(hostclaim.KindLanding)); ops != OperationAttempts(moves) || r == nil { t.Fatal("plan 9 landing owner") } for _, o := range []hostclaim.Observed{plan11Relocating("probing"), plan9D5Kind(hostclaim.KindServing), {}} { if ops, r := w.operationOwner(o); ops != nil || r != nil { t.Fatalf("%+v has an owner without the router", o.Claim) } } reloc := &plan9D2Attempts{} w.Operations = claimOperationRouter{hostclaim.KindLanding: {Attempts: moves, ProbeReady: ready}, hostclaim.KindRelocating: {Attempts: reloc, ProbeReady: ready}} if ops, r := w.operationOwner(plan11Relocating("probing")); ops != OperationAttempts(reloc) || r == nil { t.Fatal("relocating owner") } if ops, _ := w.operationOwner(plan9D5Kind(hostclaim.KindLanding)); ops != OperationAttempts(moves) { t.Fatal("landing owner through the router") } if ops, _ := w.operationOwner(plan9D5Kind(hostclaim.KindReverting)); ops != nil { t.Fatal("reverting owner") } if !w.operationKind(plan11Relocating("probing")) || w.operationKind(plan9D5Kind(hostclaim.KindServing)) { t.Fatal("operationKind") } if got := claimSpawnAction(plan11Relocating("probing").Claim); got != "probe" { t.Fatalf("relocating spawn action %q", got) } for _, kind := range []string{hostclaim.KindServing, hostclaim.KindLanding, hostclaim.KindRestoring} { if got := claimSpawnAction(plan9D5Kind(kind).Claim); got != "spawn" { t.Fatalf("%s spawn action %q", kind, got) } } if claimSpawnAction(nil) != "spawn" { t.Fatal("nil claim") } } type plan11Source struct { owners ClaimRelocationOwners refusals []hostbox.PolicyRefusal opened int closed int err error } func (s *plan11Source) Open(context.Context, release.Store, lease.VersionedStore, hostbox.Actor) (ClaimRelocationOwners, []hostbox.PolicyRefusal, error) { s.opened++ return s.owners, s.refusals, s.err } func (s *plan11Source) Close(context.Context) error { s.closed++; return nil } // Production registration: region HA's source registers once; it is opened // only with CARLOS_REGION_HA on, and its refusals are appended after any the // bindings already carry, never replacing them. func TestPlan11ProductionRelocationSourceRegistersOnceAndOpensOnlyWithRegionHA(t *testing.T) { claimRelocationSources.Lock() saved := claimRelocationSources.Source claimRelocationSources.Source = nil claimRelocationSources.Unlock() t.Cleanup(func() { claimRelocationSources.Lock() claimRelocationSources.Source = saved claimRelocationSources.Unlock() }) var calls []string src := &plan11Source{owners: plan11RelocationOwners(&calls)} if err := RegisterProductionClaimRelocationSource(nil); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("nil source: %v", err) } if err := RegisterProductionClaimRelocationSource(src); err != nil { t.Fatal(err) } if err := RegisterProductionClaimRelocationSource(&plan11Source{}); !errors.Is(err, hostwrite.ErrUnavailable) || productionClaimRelocationSource() != ClaimProductionRelocationSource(src) { t.Fatalf("second registration replaced the first: %v", err) } ctx := context.Background() var out ClaimOwnerBindings closeFn, err := openClaimRelocationOwners(ctx, plan11Env("1", ""), src, nil, nil, hostbox.Actor{}, &out) if err != nil || closeFn != nil || src.opened != 0 || !out.Relocation.zero() { t.Fatalf("opened with region HA off: %v %d", err, src.opened) } if closeFn, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), nil, nil, nil, hostbox.Actor{}, &out); err != nil || closeFn != nil { t.Fatalf("nothing registered: %v", err) } first := hostbox.PolicyRefusal(func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }) added := hostbox.PolicyRefusal(func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }) out.PolicyRefusals = []hostbox.PolicyRefusal{first} src.refusals = []hostbox.PolicyRefusal{added} closeFn, err = openClaimRelocationOwners(ctx, plan11Env("1", "1"), src, nil, nil, hostbox.Actor{}, &out) if err != nil || closeFn == nil || src.opened != 1 || out.Relocation.Operations == nil || len(out.PolicyRefusals) != 2 { t.Fatalf("region HA on: %v %d %d", err, src.opened, len(out.PolicyRefusals)) } if err := closeFn(ctx); err != nil || src.closed != 1 { t.Fatal("close is the source's") } src.err = errors.New("open failed") if _, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), src, nil, nil, hostbox.Actor{}, &ClaimOwnerBindings{}); !errors.Is(err, src.err) { t.Fatalf("open error: %v", err) } }