//go:build linux package activator import ( "context" "errors" "os/exec" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" ) // The start loop of startAdmitted, driven for real (ruling H17, fix round 1). // // What can and cannot be driven. RoleLaunch.Start (rolelaunch_linux.go) // returns ErrClaimIsolationUnavailable unconditionally, until claim Plan 9 // Group D replaces the by-name refusal with the isolated-root launcher. It // is a value method on a concrete type, with no seam, so nothing a test can // supply runs inside it. Two of the loop's hooks run only from inside Start // or after it: beforeStart (which journals the attempt's first start through // OpenProbe and so opens the 30 s window) and Ready. Hence the window can // not be opened, and readiness can not be offered, through this entry point // today. The loop itself, ahead of and around Start, can be driven: the // refusal is an ordinary failed start, and these tests use it as one. // OpenProbe and Ready are tripwires: the day Start stops refusing they fail, // and the skipped test below is to be written then. type loopRun struct { mu sync.Mutex reserved []time.Time failed int } func (r *loopRun) starts() []time.Time { r.mu.Lock() defer r.mu.Unlock() return append([]time.Time(nil), r.reserved...) } func driveLoop(t *testing.T, policy startPolicy) (*loopRun, error) { t.Helper() run := &loopRun{} obs := hostclaim.Observed{Host: "x.example", ETag: "one", Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Location: "instances/x.example/inc", Session: 1, Generation: "one"}, Op: hostclaim.Op{Kind: "landing", ID: "op", Phase: "probing", Exec: "box"}}} work, _ := (hostwrite.Config{}).Select(t.Context(), obs.Host, "spawn") a := &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Read: func(context.Context, string) (hostclaim.Observed, error) { return obs, nil }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { return ClaimStart{RoleLauncher: &RoleLaunch{}, CurrentAttempt: func() hostclaim.WakeRetryAttempt { return hostclaim.WakeRetryAttempt{} }, Host: obs.Host, Work: work, Identity: hostwrite.Identity{Config: "c", Release: "r"}, OpenProbe: func(context.Context, time.Time) (time.Time, error) { t.Fatal("OpenProbe ran: Start no longer refuses, replace the skipped window test") return time.Time{}, nil }, Restore: func(context.Context) error { return nil }, Materialize: func(context.Context) error { return nil }, Build: func() *exec.Cmd { return nil }, Ready: func(context.Context, *exec.Cmd) error { t.Fatal("Ready ran: Start no longer refuses, replace the skipped window test") return nil }, Reserve: func(context.Context, int) error { run.mu.Lock() run.reserved = append(run.reserved, time.Now()) run.mu.Unlock() return nil }, Failed: func(context.Context, int) error { run.mu.Lock(); run.failed++; run.mu.Unlock(); return nil }, Healthy: func(context.Context, int, *exec.Cmd) error { return nil }, Replicate: func(context.Context, string) (Rep, error) { return nil, nil }, Publish: func(context.Context, hostclaim.Observed) (hostclaim.Observed, error) { return obs, nil }, Retain: func(Proc, Rep, error) {}, }, nil }}}} return run, a.startAdmitted(t.Context(), &actEntry{host: obs.Host}, obs, policy) } // Five starts, then the loop gives up, with the delay between starts // doubling: 1, 2, 4 and 8 seconds. The loop's backoff is time.NewTimer // inline (claimwake.go), with no clock or sleeper to substitute, so this // reads real time. It runs on a quarter of the production base // (claimStartBackoff), so 0.25, 0.5, 1 and 2 s, each delay checked to // within 200 ms, which keeps every step apart from its neighbours: the // doubling is what is proven, on 3.75 s of waiting rather than 15. func TestStartAdmittedLoopStopsAfterFiveStartsWithDoublingBackoff(t *testing.T) { if claimStartBackoff != time.Second { t.Fatalf("claimStartBackoff = %v: the production schedule is 1, 2, 4, 8 s", claimStartBackoff) } base := 250 * time.Millisecond shrinkClaimStartBackoff(t, base) run, err := driveLoop(t, startPolicy{MaxStarts: 5, Window: 30 * time.Second, Deadline: time.Now().Add(time.Hour), Probe: true}) if !errors.Is(err, ErrClaimIsolationUnavailable) || errors.Is(err, ErrAttemptOver) { t.Fatalf("the last start's refusal should end the loop: %v", err) } at := run.starts() if len(at) != 5 || run.failed != 5 { t.Fatalf("starts=%d failed=%d, want 5 and 5", len(at), run.failed) } for i, want := range []time.Duration{base, 2 * base, 4 * base, 8 * base} { if gap := at[i+1].Sub(at[i]); gap < want || gap > want+200*time.Millisecond { t.Errorf("delay before start %d = %v, want %v", i+2, gap, want) } } } // The cut deadline ends the attempt while it waits between starts: one // start, then ErrAttemptOver at the deadline, not after the 1 s backoff. func TestStartAdmittedLoopCutDeadlineEndsTheBackoffWait(t *testing.T) { deadline := time.Now().Add(300 * time.Millisecond) run, err := driveLoop(t, startPolicy{MaxStarts: 5, Window: 30 * time.Second, Deadline: deadline, Probe: true}) if !errors.Is(err, ErrAttemptOver) { t.Fatalf("err=%v", err) } if n := len(run.starts()); n != 1 { t.Fatalf("starts=%d, want 1", n) } if late := time.Since(deadline); late > 500*time.Millisecond { t.Fatalf("the attempt ended %v after the deadline", late) } } // A deadline already passed ends the attempt before any start: Reserve is // never reached. func TestStartAdmittedLoopNoStartPastTheCutDeadline(t *testing.T) { run, err := driveLoop(t, startPolicy{MaxStarts: 5, Window: 30 * time.Second, Deadline: time.Now().Add(-time.Second), Probe: true}) if !errors.Is(err, ErrAttemptOver) || len(run.starts()) != 0 { t.Fatalf("err=%v starts=%d", err, len(run.starts())) } } func TestStartAdmittedLoopWindowAndReadinessAtTheBoundary(t *testing.T) { t.Skip("blocked on claim Plan 9 Group D: RoleLaunch.Start (rolelaunch_linux.go) returns ErrClaimIsolationUnavailable " + "unconditionally, so beforeStart/OpenProbe (which opens the 30 s window) and Ready never run through startAdmitted. " + "When Group D replaces that refusal, drive: window expiry before a start (OpenProbe returns a time older than the window), " + "readiness at opened+Window and at the cut deadline rejected with ErrAttemptOver, readiness at opened+Window-1s accepted.") }