package activator import ( "context" "errors" "fmt" "os" "os/exec" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) func servingRetryFixture(t *testing.T) (release.Store, hostclaim.Observed, *ClaimRuntime, ServingRetries, *time.Time, *int) { t.Helper() ctx := context.Background() st := release.NewDirStore(t.TempDir()) now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) host := "shop.bab.oncarlos.com" in := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: now} body := []byte(`{"schema":1,"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie","created_at":"2026-10-06T12:00:00Z"}`) absent, _ := hostclaim.Read(ctx, st, host) o, err := hostclaim.Update(ctx, st, absent, hostclaim.BeginCreate(in, body, "create", "console", now)) if err != nil { t.Fatal(err) } o, err = hostclaim.Update(ctx, st, o, hostclaim.CompleteCreate("create", "box")) if err != nil { t.Fatal(err) } seq, stops := 0, 0 d := ServingRetries{Claims: st, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", Now: func() time.Time { return now }, NewID: func() string { seq++; return fmt.Sprint("id", seq) }, Inputs: func(context.Context, registry.Route) (string, error) { return "inputs", nil }, StopAttempt: func(context.Context, hostclaim.Observed, hostclaim.WakeRetryAttempt) error { stops++; return nil }, ExactHealth: func(context.Context, *exec.Cmd, hostclaim.WakeRetryAttempt) error { return nil }} base := &ClaimRuntime{Box: "box", Read: func(ctx context.Context, host string) (hostclaim.Observed, error) { return hostclaim.Read(ctx, st, host) }, StopForeign: func(context.Context, string) error { return nil }, Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, StartPolicy) error { return nil }, Admit: func(ctx context.Context, _ registry.Route, o hostclaim.Observed, _ StartPolicy) (hostclaim.Observed, error) { r := retryOf(o) if r == nil || r.Attempt == nil || r.Attempt.Session != 0 { t.Fatal("admission preceded first reservation") } seq++ return hostclaim.Update(ctx, st, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Region: "ie", PredecessorRegion: "ie", Box: "box", Mode: "cold", NewGeneration: fmt.Sprint("gen", seq), PredecessorCertified: true})) }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, StartPolicy) (ClaimStart, error) { return ClaimStart{}, nil }, FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return nil }} return st, o, base, d, &now, &stops } func TestServingRetriesSurviveRestartAndFifthFailure(t *testing.T) { st, o, base, d, now, stops := servingRetryFixture(t) ctx := context.Background() p := StartPolicy{} rt := registry.Route{Host: o.Host} for n := 1; n <= 5; n++ { runtime, err := d.Wrap(base) if err != nil { t.Fatal(err) } if err = runtime.Prerequisites(ctx, rt, o, p); err != nil { t.Fatal(err) } admitted, err := runtime.Admit(ctx, rt, o, p) if err != nil { t.Fatal(err) } if retryOf(admitted).Attempt.Session != admitted.Claim.Holder.Session { t.Fatal("unbound start") } s, err := runtime.Prepare(ctx, rt, admitted, p) if err != nil { t.Fatal(err) } if err = s.Reserve(ctx, 0); err != nil { t.Fatal(err) } if err = s.Failed(ctx, 0); err != nil { t.Fatal(err) } o, err = hostclaim.Read(ctx, st, o.Host) if err != nil { t.Fatal(err) } if retryOf(o).AttemptsReserved != n { t.Fatal("budget reset") } if err = runtime.Prerequisites(ctx, rt, o, p); err == nil { t.Fatal("early retry admitted") } *now = now.Add(time.Second << uint(n-1)) } if retryOf(o).State != "failed" || *stops != 5 { t.Fatalf("retry=%+v stops=%d", retryOf(o), *stops) } runtime, _ := d.Wrap(base) if err := runtime.Prerequisites(ctx, rt, o, p); err == nil { t.Fatal("new controller reset failed latch") } } func TestServingRetriesReadOnlyRefusalSpendsNothing(t *testing.T) { st, o, base, d, _, _ := servingRetryFixture(t) ctx := context.Background() base.Prerequisites = func(context.Context, registry.Route, hostclaim.Observed, StartPolicy) error { return hostbox.ErrUnsupported } runtime, _ := d.Wrap(base) if err := runtime.Prerequisites(ctx, registry.Route{Host: o.Host}, o, StartPolicy{}); err == nil { t.Fatal("missing prerequisite passed") } fresh, _ := hostclaim.Read(ctx, st, o.Host) if fresh.ETag != o.ETag || retryOf(fresh) != nil { t.Fatal("prerequisite spent authority") } } func TestServingRetriesCrashMustStopBeforeNextReservation(t *testing.T) { st, o, base, d, _, stops := servingRetryFixture(t) ctx := context.Background() rt := registry.Route{Host: o.Host} runtime, _ := d.Wrap(base) admitted, err := runtime.Admit(ctx, rt, o, StartPolicy{}) if err != nil { t.Fatal(err) } restarted, _ := d.Wrap(base) if err = restarted.Prerequisites(ctx, rt, admitted, StartPolicy{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatal(err) } fresh, _ := hostclaim.Read(ctx, st, o.Host) if *stops != 1 || retryOf(fresh).AttemptsReserved != 1 || retryOf(fresh).State != "open" { t.Fatalf("not settled: %+v", retryOf(fresh)) } } func TestServingRetriesHealthBindsExactLaunch(t *testing.T) { st, o, base, d, _, _ := servingRetryFixture(t) ctx := context.Background() rt := registry.Route{Host: o.Host} d.ExactHealth = func(context.Context, *exec.Cmd, hostclaim.WakeRetryAttempt) error { return hostbox.ErrChanged } runtime, _ := d.Wrap(base) admitted, err := runtime.Admit(ctx, rt, o, StartPolicy{}) if err != nil { t.Fatal(err) } s, err := runtime.Prepare(ctx, rt, admitted, StartPolicy{}) if err != nil { t.Fatal(err) } process, _ := os.FindProcess(os.Getpid()) if err = s.Healthy(ctx, 0, &exec.Cmd{Process: process}); err == nil { t.Fatal("wrong invocation cleared episode") } fresh, _ := hostclaim.Read(ctx, st, o.Host) if retryOf(fresh) == nil { t.Fatal("health cleared episode") } d.ExactHealth = func(context.Context, *exec.Cmd, hostclaim.WakeRetryAttempt) error { return nil } runtime, _ = d.Wrap(base) s, err = runtime.Prepare(ctx, rt, fresh, StartPolicy{}) if err != nil { t.Fatal(err) } if err = s.Healthy(ctx, 0, &exec.Cmd{Process: process}); err != nil { t.Fatal(err) } fresh, _ = hostclaim.Read(ctx, st, o.Host) if retryOf(fresh) != nil { t.Fatal("exact health did not clear") } } func TestServingDiskRefusalSpendsNoRetry(t *testing.T) { st, o, base, d, _, stops := servingRetryFixture(t) effects := 0 base.Prerequisites = func(context.Context, registry.Route, hostclaim.Observed, StartPolicy) error { effects++; return nil } d.BeforeReserve = func(context.Context, registry.Route, hostclaim.Observed) error { return instancectl.ErrDiskFull } runtime, err := d.Wrap(base) if err != nil { t.Fatal(err) } for i := 0; i < 12; i++ { if err := runtime.Prerequisites(t.Context(), registry.Route{Host: o.Host}, o, StartPolicy{}); !errors.Is(err, instancectl.ErrDiskFull) { t.Fatal(err) } if _, err := runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, StartPolicy{}); !errors.Is(err, instancectl.ErrDiskFull) { t.Fatal(err) } } after, err := hostclaim.Read(t.Context(), st, o.Host) if err != nil || after.ETag != o.ETag || effects != 0 || *stops != 0 { t.Fatal("low disk spent a retry or stopped a writer", err, effects, *stops) } } func TestServingSeedRerunSettlesWithoutReservation(t *testing.T) { for _, fault := range []string{"", "stop-fails", "stale"} { t.Run(fault, func(t *testing.T) { st, o, base, d, _, stops := servingRetryFixture(t) runtime, err := d.Wrap(base) if err != nil { t.Fatal(err) } admitted, err := runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, StartPolicy{}) if err != nil { t.Fatal(err) } expected := admitted.Claim.Holder.WakeRetry.AttemptsReserved if fault == "stop-fails" { d.StopAttempt = func(context.Context, hostclaim.Observed, hostclaim.WakeRetryAttempt) error { return hostbox.ErrChanged } } if fault == "stale" { admitted.ETag = "stale" } d.BeforeReserve = func(context.Context, registry.Route, hostclaim.Observed) error { t.Fatal("settlement required new preparation capacity") return nil } settled, err := d.SettleAbandoned(t.Context(), admitted) if fault != "" { if err == nil { t.Fatal("settled", fault) } return } r := settled.Claim.Holder.WakeRetry if err != nil || r.State != "open" || r.Attempt != nil || r.AttemptsReserved != expected || *stops != 1 { t.Fatal("rerun settlement", r, err, *stops) } again, err := d.SettleAbandoned(t.Context(), settled) if err != nil || again.ETag != settled.ETag || *stops != 1 { t.Fatal("settled replay changed state", err) } fresh, err := hostclaim.Read(t.Context(), st, o.Host) if err != nil || fresh.ETag != settled.ETag { t.Fatal(err) } }) } } func TestServingRetriesReturnOwnClaimWrites(t *testing.T) { st, o, base, d, now, _ := servingRetryFixture(t) var expected hostclaim.Observed calls := 0 base.Prepare = func(_ context.Context, _ registry.Route, o hostclaim.Observed, _ StartPolicy) (ClaimStart, error) { expected = o return ClaimStart{RetryClaimWrite: func(before, after hostclaim.Observed) error { if before.ETag != expected.ETag || after.ETag == before.ETag || after.Claim.Gen != before.Claim.Gen+1 { t.Fatal("write did not chain from Prepare") } expected = after calls++ return nil }}, nil } runtime, err := d.Wrap(base) if err != nil { t.Fatal(err) } rt := registry.Route{Host: o.Host} admitted, err := runtime.Admit(t.Context(), rt, o, StartPolicy{}) if err != nil { t.Fatal(err) } start, err := runtime.Prepare(t.Context(), rt, admitted, StartPolicy{}) if err != nil { t.Fatal(err) } if err := start.Reserve(t.Context(), 0); err != nil { t.Fatal(err) } if calls != 0 { t.Fatal("first reservation invented a write") } if err := start.Failed(t.Context(), 0); err != nil { t.Fatal(err) } *now = now.Add(2 * time.Second) if err := start.Reserve(t.Context(), 1); err != nil { t.Fatal(err) } actual, err := hostclaim.Read(t.Context(), st, o.Host) if err != nil || calls != 2 || expected.ETag != actual.ETag { t.Fatal("lost retry write", calls, err) } }