package hostrelocate import ( "context" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" ) // releasedAndInstalling drives a drain to installing on dst, released. func releasedAndInstalling(t *testing.T, f *fixture) string { t.Helper() ctx := context.Background() if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { t.Fatal(err) } f.ack("src") f.ack("dst") f.worker(nil) // releasing h := f.claim().Claim.Holder id := f.claim().Claim.Op.ID f.participant(hostclaim.JournalRelease(id, hostclaim.Release{ReleasedBy: "src", Enrollment: enrollment("src"), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) f.worker(nil) // pinning f.worker(nil) // pin f.worker(nil) // installing on dst return id } // Astra round 3 finding 4, ruling 67: a passed probe whose stop receipt // never comes is bounded. ReleaseDeadline after the probe passed, the op // is stuck for an operator; before it, nothing changes; nothing is // released. Why the receipt never comes is the participant's (its two // scenarios are TestParticipantEndsNoProbeWithoutItsLease and // TestParticipantRecordsNoProbeStopWhileRootRefuses); the worker sees // only a probe with no probe_stop. func TestWorkerBoundsAPassedProbeAwaitingItsStop(t *testing.T) { f := newFixture(t) id := releasedAndInstalling(t, f) f.installAndProbe("dst") o := f.claim() f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session, At: f.now})) f.now = f.now.Add(ReleaseDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Kind != hostclaim.KindRelocating { t.Fatalf("stuck before the bound: %+v", o.Claim.Op.Journal) } f.now = f.now.Add(time.Second) o = f.worker(nil) if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Kind != hostclaim.KindRelocating || o.Claim.Op.Exec != "dst" || o.Claim.Op.Journal["probe_stop"] != "" { t.Fatalf("not stuck for an operator, or something released: %+v", o.Claim.Op) } } // A probe journaled before rev 4 has no At: the bound counts from // probing_since plus ProbeDeadline. func TestWorkerBoundsAnUntimedPassedProbeFromProbingSince(t *testing.T) { f := newFixture(t) id := releasedAndInstalling(t, f) f.installAndProbe("dst") o := f.claim() f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session})) f.now = f.now.Add(ProbeDeadline + ReleaseDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" { t.Fatal("stuck before probing_since + ProbeDeadline + ReleaseDeadline") } f.now = f.now.Add(time.Second) if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" { t.Fatalf("not stuck: %+v", o.Claim.Op) } } // Ruling 60: an install that never enters probing (its stamp retired, its // participation denied) is stuck ProbeDeadline after installed_at. func TestWorkerBoundsAnInstallThatNeverProbes(t *testing.T) { f := newFixture(t) releasedAndInstalling(t, f) f.worker(ErrNotYet) // P is not built yet p := f.claim().Claim.Op.Journal["prepared"] f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil }) if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseInstalled || o.Claim.Op.Journal["installed_at"] == "" { t.Fatalf("installed with its start: %s %+v", o.Claim.Op.Phase, o.Claim.Op.Journal) } f.now = f.now.Add(ProbeDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" { t.Fatal("stuck before the bound") } f.now = f.now.Add(time.Second) if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseInstalled { t.Fatalf("not stuck: %+v", o.Claim.Op) } } // Ruling 60: a target that enters probing but never admits (its stamp // retired, root refusing) is probe-failed at the probe deadline, then // stuck ReleaseDeadline later for an operator. A live target is never // replaced automatically while claim ask C1 is open: exec stays dst // throughout. func TestWorkerBoundsARefusedProbe(t *testing.T) { f := newFixture(t) id := releasedAndInstalling(t, f) p := f.claim().Claim.Op.Journal["prepared"] f.worker(ErrNotYet) // P is not built yet f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil }) f.worker(nil) // installed var stamp uint64 if err := json.Unmarshal([]byte(f.claim().Claim.Op.Journal["install_stamp"]), &stamp); err != nil { t.Fatal(err) } f.participant(hostclaim.RecordLocalInstall(id, hostclaim.LocalInstall{Box: "dst", Enrollment: enrollment("dst"), InstallStamp: stamp, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("d", 64)})) f.participant(hostclaim.ToProbing(id, f.now)) f.now = f.now.Add(ProbeDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Phase != hostclaim.PhaseRelocatingProbing { t.Fatalf("probe-failed before the probe deadline: %s", o.Claim.Op.Phase) } f.now = f.now.Add(time.Second) if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseProbeFailed || o.Claim.Op.Exec != "dst" { t.Fatalf("not probe-failed on dst: %s %s", o.Claim.Op.Phase, o.Claim.Op.Exec) } f.now = f.now.Add(ReleaseDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Exec != "dst" { t.Fatalf("stuck or replaced before the release deadline: %+v", o.Claim.Op) } f.now = f.now.Add(time.Second) if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Exec != "dst" || o.Claim.Op.Phase != hostclaim.PhaseProbeFailed { t.Fatalf("not stuck on dst for an operator: %+v", o.Claim.Op) } } // Fix round 1, M3: a timed probe's bound counts from Probe.At, not from // probing_since and not from probing_since plus ProbeDeadline. The probe // passes 7 minutes into probing: ReleaseDeadline after At the op waits, // a second later it is stuck. func TestWorkerBoundsATimedProbeFromItsPassNotProbingSince(t *testing.T) { f := newFixture(t) id := releasedAndInstalling(t, f) f.installAndProbe("dst") // probing_since is f.now passed := f.now.Add(7 * time.Minute) o := f.claim() f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session, At: passed})) f.now = passed.Add(ReleaseDeadline) if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" { t.Fatal("stuck before ReleaseDeadline after the probe passed: the bound counted from probing_since") } f.now = f.now.Add(time.Second) if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" { t.Fatalf("not stuck ReleaseDeadline after the probe passed: %+v", o.Claim.Op) } } // Fix round 1, M2: installed's bound needs its start. A missing or // unreadable installed_at is an error, never a pass with no bound; the op // is neither stuck nor waiting silently. func TestWorkerRefusesAnInstalledWithoutItsStart(t *testing.T) { for name, value := range map[string]*string{"missing": nil, "unreadable": new("yesterday")} { t.Run(name, func(t *testing.T) { f := newFixture(t) releasedAndInstalling(t, f) f.worker(ErrNotYet) // P is not built yet p := f.claim().Claim.Op.Journal["prepared"] f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil }) f.worker(nil) // installed f.participant(func(c *hostclaim.Claim) error { if value == nil { delete(c.Op.Journal, "installed_at") } else { c.Op.Journal["installed_at"] = *value } return nil }) f.now = f.now.Add(ProbeDeadline + time.Hour) o, err := f.w.Step(context.Background(), host) if err == nil || errors.Is(err, ErrNotYet) { t.Fatalf("an installed without its start: %v", err) } if o.Claim.Op.Journal["stuck"] != "" || f.claim().Claim.Op.Phase != hostclaim.PhaseInstalled { t.Fatalf("the op moved: %+v", f.claim().Claim.Op) } }) } } // J3-3: a converted landing waits for an eligible target in starting, the // chooser's refusal returned each pass, and is stuck ReleaseDeadline after // its target's fence was observed, never pinned without a target. func TestWorkerBoundsAConversionWaitingForATarget(t *testing.T) { f := newFixture(t) converted := f.convertedLanding() var fenced time.Time for k, v := range converted.Claim.Op.Journal { if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") { at, err := time.Parse(time.RFC3339Nano, v) if err != nil { t.Fatal(err) } fenced = at } } if fenced.IsZero() { t.Fatalf("the conversion journals no fence observation: %+v", converted.Claim.Op.Journal) } none := errors.New("no eligible box") f.w.Choose = func(context.Context, hostclaim.Observed, []string) (hostclaim.Target, error) { return hostclaim.Target{}, none } f.now = fenced.Add(ReleaseDeadline) if o := f.worker(none); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Intent["target_box"] != "" { t.Fatalf("stuck or named before the bound: %+v", o.Claim.Op) } f.now = f.now.Add(time.Second) o := f.worker(nil) if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseStarting || o.Claim.Op.Journal["pin"] != "" { t.Fatalf("not stuck for an operator: %+v", o.Claim.Op) } } // Absence is never evidence: a converted landing whose fence observation // does not parse, or is gone, has nothing to bound its wait. The chooser's // refusal comes back with that defect named (ErrRefused), long after the // bound would have passed, and nothing is written: it is neither waited on // silently for ever nor stuck on a guess. func TestWorkerRefusesAConversionWaitWithNoReadableFenceObservation(t *testing.T) { for name, change := range map[string]func(map[string]string){ "unparseable": func(j map[string]string) { for k := range j { if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") { j[k] = "not a time" } } }, "unparseable beside a readable one": func(j map[string]string) { for k := range j { if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") { j[k] = "not a time" } } j["fence.spa.observed_at"] = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339Nano) }, "missing": func(j map[string]string) { for k := range j { if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") { delete(j, k) } } }, } { t.Run(name, func(t *testing.T) { f := newFixture(t) converted := f.step(f.convertedLanding(), func(c *hostclaim.Claim) error { change(c.Op.Journal); return nil }) none := errors.New("no eligible box") f.w.Choose = func(context.Context, hostclaim.Observed, []string) (hostclaim.Target, error) { return hostclaim.Target{}, none } f.now = f.now.Add(10 * ReleaseDeadline) o, err := f.w.Step(context.Background(), host) if !errors.Is(err, none) || !errors.Is(err, ErrRefused) { t.Fatalf("step: %v, want the chooser's refusal and the defect", err) } if o.ETag != converted.ETag || f.claim().ETag != converted.ETag { t.Fatalf("the op was written: %+v", f.claim().Claim.Op) } }) } }