package host import ( "context" "os" "path/filepath" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" ) // The box's side of `move resolve --retry` (moves spec §7.8 stuck, §7.10; // Plan 4 Task 7). The console door writes provision.MoveRetry and nothing // else (h.retry is the same transition through the same record writer); the // box then resumes the source's recovery at the phase the stuck rollback // recorded as StuckAt, with the stuck reason gone and the rollback's own // reason kept. A verification runs again and is never bypassed (INV-4); a // probe that ran out of attempts is called again with the whole budget; // nothing earlier than StuckAt runs again. The fixture is Task 5's. // verifications counts how many times the source ran its baseline check. func (h *recoverHarness) verifications() int { n := 0 for _, p := range h.rcHits { if p == "verify" { n++ } } return n } // retried asserts what MoveRetry leaves for the box: rolling back at the // phase the stuck mark named, no stuck reason, no stuck-at, the cause kept. func (h *recoverHarness) retried(t *testing.T, at, cause string) { t.Helper() m := h.read(t, p1Host).Move if m.State != provision.MoveRollingBack || m.StuckReason != "" || m.Rollback == nil || m.Rollback.SourcePhase != at || m.Rollback.StuckAt != "" { t.Fatalf("after the retry: %+v %+v, want rolling_back at %s", m, m.Rollback, at) } if m.Rollback.Reason != cause { t.Fatalf("the retry changed the rollback's cause: %q, want %q", m.Rollback.Reason, cause) } } // TestRetryResumesTheReturnsVerification (J20, INV-4): a clean return whose // files no longer match the source's stopped baseline is stuck at verified. // A retry returns the record to verified, and the box verifies again, once // per retry; unrepaired it is stuck again with the same sentence and nothing // after the check has run; repaired it recovers. The retry never skips the // check, and never runs the hand-over, the claim write or anything before // verified a second time. func TestRetryResumesTheReturnsVerification(t *testing.T) { t.Parallel() h := newRecoverHarness(t, false) rb := h.released(t) cause := h.read(t, p1Host).Move.Rollback.Reason upload := filepath.Join(h.sdir, p1Host, "upload.txt") if err := os.WriteFile(upload, []byte("changed on the source"), 0o644); err != nil { t.Fatal(err) } stuck := func(verified int) { t.Helper() m := h.read(t, p1Host).Move if m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch || m.Rollback.StuckAt != provision.RbSourceVerified { t.Fatalf("not stuck at verified: %+v %+v", m, m.Rollback) } if n := h.verifications(); n != verified { t.Fatalf("verified %d times, want %d (%v)", n, verified, h.rcHits) } if len(h.probes) != 0 { t.Fatalf("probed past a mismatch: %+v", h.probes) } } h.srcPass(t) stuck(1) handed := len(h.srcHanded) if handed != 1 { t.Fatalf("hand-overs before the retry: %v", h.srcHanded) } // Stuck is not driven on: a pass between the stuck mark and the retry // verifies nothing. h.srcPass(t) stuck(1) // The retry alone verifies nothing: it only moves the record. h.retry(t) h.retried(t, provision.RbSourceVerified, cause) if n := h.verifications(); n != 1 { t.Fatalf("the retry verified by itself: %d", n) } h.srcPass(t) stuck(2) h.retry(t) h.srcPass(t) stuck(3) // The operator repairs the source; one more retry reaches opened. if err := os.WriteFile(upload, []byte("hello"), 0o644); err != nil { t.Fatal(err) } h.retry(t) h.retried(t, provision.RbSourceVerified, cause) h.srcPass(t) got := h.opened(t) if got.ID != rb.ID || got.StuckAt != "" || h.read(t, p1Host).Move.StuckReason != "" { t.Fatalf("opened: %+v", got) } if n := h.verifications(); n != 4 { t.Fatalf("verified %d times, want 4 (one per attempt)", n) } if len(h.srcHanded) != handed { t.Fatalf("the retry handed the row over again: %v", h.srcHanded) } if len(h.probes) != 1 { t.Fatalf("probes: %d, want the one after the repair", len(h.probes)) } } // TestRetryGivesTheProbeItsWholeBudgetAgain (J20): a recovery probe that ran // out of attempts is stuck at probing. A retry resumes at probing, so the // box calls the probe again with the full nine attempts (the counter starts // again at 0) and runs nothing before it: no verification, no install, no // claim step. The probe keeps its own count (Task 9 builds it on claim // Plan 10's attempts); the box's journal holds none to reset. func TestRetryGivesTheProbeItsWholeBudgetAgain(t *testing.T) { t.Parallel() h := newRecoverHarness(t, false) h.released(t) h.probe = func(recoveryProbeRequest) (recoveryProbeOutcome, error) { return recoveryProbeExhausted, nil } h.srcPass(t) cause := h.read(t, p1Host).Move.Rollback.Reason stuck := func(probes int) { t.Helper() m := h.read(t, p1Host).Move if m.State != provision.MoveStuck || m.StuckReason != provision.StuckRecoveryProbeAttempts || m.Rollback.StuckAt != provision.RbSourceProbing { t.Fatalf("not stuck at probing: %+v %+v", m, m.Rollback) } if len(h.probes) != probes { t.Fatalf("probe calls: %d, want %d", len(h.probes), probes) } } stuck(1) h.srcPass(t) stuck(1) // stuck is not driven on // The source's files change under the stuck probe, as a live app's would. if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("written by the recovered app"), 0o644); err != nil { t.Fatal(err) } h.retry(t) h.retried(t, provision.RbSourceProbing, cause) h.srcPass(t) stuck(2) h.retry(t) h.probe = nil h.srcPass(t) h.opened(t) if len(h.probes) != 3 { t.Fatalf("probe calls: %d, want 3", len(h.probes)) } for i, req := range h.probes { if req.Attempts != 9 { t.Fatalf("probe call %d carries a budget of %d, want the whole nine", i, req.Attempts) } if !req.Clean || req.Coverage != recoveryCoverageClean || req.SourceGeneration != "cur" { t.Fatalf("probe call %d: %+v", i, req) } } if n := h.verifications(); n != 1 { t.Fatalf("verified %d times, want 1: a retry at probing never verifies again (INV-2)", n) } if len(h.srcHanded) != 1 { t.Fatalf("hand-overs: %v", h.srcHanded) } } // TestRetryResumesADirtyInstallAtLocal: a dirty return's install that found // the source's files occupied is stuck at local (the three install reasons // are legal at local and configured). A retry resumes the shared installer // at local from its journal, not from the start: the pre-move park, run // once, is not run again, and once the operator has cleared the files the // recovery reaches opened. func TestRetryResumesADirtyInstallAtLocal(t *testing.T) { t.Parallel() h := newRecoverHarness(t, true) h.released(t) sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt") h.landFault = func(p string) error { if p == "after-"+landStepRetired { if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil { t.Fatal(err) } return os.WriteFile(sentinel, []byte("not the return's"), 0o644) } return nil } h.srcPass(t) cause := h.read(t, p1Host).Move.Rollback.Reason m := h.read(t, p1Host).Move if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal { t.Fatalf("not stuck at local: %+v %+v", m, m.Rollback) } if len(h.probes) != 0 { t.Fatalf("probed past a refused install: %+v", h.probes) } // Stuck, retried with the sentinel still there: stuck again, the same // sentence, the sentinel untouched. h.retry(t) h.retried(t, provision.RbSourceLocal, cause) h.srcPass(t) if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal { t.Fatalf("an unrepaired retry: %+v %+v", m, m.Rollback) } if body, err := os.ReadFile(sentinel); err != nil || string(body) != "not the return's" { t.Fatalf("the sentinel: %q %v", body, err) } // Repaired: the operator removes what is not the return's. if err := os.Remove(sentinel); err != nil { t.Fatal(err) } h.landFault = nil h.retry(t) h.srcPass(t) rb := h.opened(t) if body, err := os.ReadFile(provision.DefaultDBPath(h.sdir, p1Host) + ".gen"); err != nil || strings.TrimSpace(string(body)) != rb.ID { t.Fatalf("the installed generation marker: %q %v", body, err) } if n := h.verifications(); n != 0 { t.Fatalf("a dirty return verified the baseline %d times; it installs instead", n) } if len(h.probes) != 1 || h.probes[0].Clean || h.probes[0].Coverage != recoveryCoverageSealed || h.probes[0].Attempts != recoveryProbeAttempts { t.Fatalf("probes: %+v", h.probes) } } // TestRetryLeavesTheClaimAndTheRowAlone: the retry is a record write. It // does not touch the host claim or the registry row; the box's next pass // does what the phase asks. func TestRetryLeavesTheClaimAndTheRowAlone(t *testing.T) { t.Parallel() h := newRecoverHarness(t, false) h.released(t) if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed"), 0o644); err != nil { t.Fatal(err) } h.srcPass(t) claim, row := h.claim(t), h.srcRoute h.retry(t) after := h.claim(t) if after.Op.Phase != hostclaim.PhaseClaimed || after.Op.Phase != claim.Op.Phase || after.Op.Exec != claim.Op.Exec { t.Fatalf("the retry changed the claim: %+v -> %+v", claim.Op, after.Op) } if !reflect.DeepEqual(h.srcRoute, row) { t.Fatalf("the retry changed the row: %+v -> %+v", row, h.srcRoute) } // A second retry, now that the move is not stuck, is the provision // refusal the door speaks as its 409. g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetry(g)); err == nil || !strings.Contains(err.Error(), provision.RefusedNotStuck) { t.Fatalf("a retry of a move that is not stuck: %v", err) } } // countEvent counts event in events. func countEvent(events []string, event string) int { n := 0 for _, e := range events { if e == event { n++ } } return n } // Final review I3: a retry's event comes from the box's own pass. After // `move resolve --retry` (the claim-backed MoveRetryRecovery, which advances // the epoch) the record is rolling_back or reverting, which movesPass sends // to the source's recovery or revert before phase1MoveRound; the pass still // logs retry-, once, however many passes see it. func TestRetryEventIsLoggedByThePass(t *testing.T) { t.Parallel() t.Run("a rollback's source recovery", func(t *testing.T) { h := newRecoverHarness(t, false) h.released(t) if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed on the source"), 0o644); err != nil { t.Fatal(err) } h.srcPass(t) if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck { t.Fatalf("not stuck: %+v", m) } h.retry(t) if m := h.read(t, p1Host).Move; m.State != provision.MoveRollingBack || m.RecoveryRetryEpoch != 1 { t.Fatalf("after the retry: %+v", m) } h.srcPass(t) h.srcPass(t) if n := countEvent(h.sEvents, "retry"); n != 1 { t.Fatalf("retry events %d, want one: %v", n, h.sEvents) } }) t.Run("a revert", func(t *testing.T) { h := newRevertHarness(t) h.fromStopped(t) h.arm(t) if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed while stopped"), 0o644); err != nil { t.Fatal(err) } h.pass(t) if m := h.move(t); m.State != provision.MoveStuck { t.Fatalf("not stuck: %+v", m) } g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetryRecovery(g)); err != nil { t.Fatal(err) } if m := h.move(t); m.State != provision.MoveReverting || m.RecoveryRetryEpoch != 1 { t.Fatalf("after the retry: %+v", m) } h.pass(t) h.pass(t) if n := countEvent(h.events, "retry"); n != 1 { t.Fatalf("retry events %d, want one: %v", n, h.events) } }) }