package host import ( "context" "errors" "os" "strings" "testing" "time" ) // The update gate on a claims-on box (CARLOS_HOST_CLAIM). There the edge's // listener binds only after the agent's whole claim startup, about 22 s // after the restart, so a fixed 20 s window rolled every update back and // the rollback too. With claims on the window is 120 s and the gate fails // fast on a crash; with claims off it is exactly the 20 s gate it was. // slowListener makes the fake edge refuse the probe for listenAfter after // every restart, as a claims-on edge does while the claim assembly runs. // It returns the times at which each restart was issued. func slowListener(f *applyFixture, listenAfter time.Duration) *[]time.Time { var restarts []time.Time orig := f.a.ops.systemctl f.a.ops.systemctl = func(args ...string) (string, error) { if args[0] == "restart" && len(args) > 1 && args[1] == systemEdgeUnit { restarts = append(restarts, f.now) } return orig(args...) } f.a.ops.probeEdge = func() error { if len(restarts) == 0 || f.now.Sub(restarts[len(restarts)-1]) < listenAfter { return errors.New("dial tcp 127.0.0.1:80: connect: connection refused") } return nil } return &restarts } func TestSystemGateWaitIsKeyedOnClaimsMode(t *testing.T) { if got := systemGateWaitFor(false); got != 20*time.Second { t.Fatalf("claims-off gate window = %s, want the 20 s it always was", got) } if got := systemGateWaitFor(true); got != 120*time.Second { t.Fatalf("claims-on gate window = %s, want 120 s", got) } } func TestSystemApplyReadsClaimsModeFromTheUnitEnvironment(t *testing.T) { for _, tc := range []struct { value string claims bool wait time.Duration }{{"", false, 20 * time.Second}, {" ", false, 20 * time.Second}, {"1", true, 120 * time.Second}, {"on", true, 120 * time.Second}} { claims, wait := systemApplyGate(func(k string) string { if k == "CARLOS_HOST_CLAIM" { return tc.value } return "" }) if claims != tc.claims || wait != tc.wait { t.Fatalf("CARLOS_HOST_CLAIM=%q: claims=%t wait=%s, want %t %s", tc.value, claims, wait, tc.claims, tc.wait) } } } func TestClaimsOffGateIsUnchangedByAnEdgeThatListensAt22s(t *testing.T) { // Ruling 6: a claims-off box sees no change. The same 22 s edge rolls // back, and its 22 s previous build fails its gate too, as on main. f, req := applyHappy(t) slowListener(f, 22*time.Second) if err := f.a.Run(context.Background(), false); err == nil { t.Fatal("a claims-off 20 s gate passed an edge that listened at 22 s") } r := f.result() if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) { t.Fatalf("result = %+v, want rollback-failed as on main", r) } } func TestClaimsOffGateWaitsItsFullWindowThroughACrash(t *testing.T) { // Claims off never fails fast: a crash-restarting edge still gets the // full 20 s, and the gate's systemctl calls are only is-active. f := newApplyFixture(t, "stable", "v1") f.edgeUp = false f.nrestarts = "3" start := f.now if err := f.a.gate(context.Background(), "v1"); err == nil { t.Fatal("gate passed a failed edge") } if waited := f.now.Sub(start); waited != 20*time.Second { t.Fatalf("claims-off gate gave up after %s, want exactly 20 s", waited) } for _, c := range f.calls { if strings.HasPrefix(c, "systemctl show") { t.Fatalf("claims-off gate read %q inside its wait; it must not", c) } } } func TestClaimsOnGatePassesAnEdgeThatListensAt22s(t *testing.T) { f, req := applyHappy(t) f.a.claims = true restarts := slowListener(f, 22*time.Second) if err := f.a.Run(context.Background(), false); err != nil { t.Fatalf("claims-on gate failed a still-starting edge: %v", err) } r := f.result() if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) { t.Fatalf("result = %+v", r) } if len(*restarts) != 1 { t.Fatalf("edge restarted %d times, want 1 (no rollback)", len(*restarts)) } } func TestClaimsOnGateWaitsUpTo120s(t *testing.T) { f := newApplyFixture(t, "stable", "v1") f.a.claims = true f.probeErr = errors.New("connection refused") start := f.now err := f.a.gate(context.Background(), "v1") if err == nil || !strings.Contains(err.Error(), "connection refused") { t.Fatalf("gate = %v, want the probe's error at the deadline", err) } if waited := f.now.Sub(start); waited != 120*time.Second { t.Fatalf("claims-on gate gave up after %s, want 120 s", waited) } } func TestClaimsOnGateFailsFastOnACrashRestart(t *testing.T) { f := newApplyFixture(t, "stable", "v1") f.a.claims = true f.probeErr = errors.New("connection refused") f.nrestarts = "1" start := f.now err := f.a.gate(context.Background(), "v1") if err == nil || !strings.Contains(err.Error(), "NRestarts=1") { t.Fatalf("gate = %v, want a fail-fast on NRestarts", err) } if waited := f.now.Sub(start); waited != 0 { t.Fatalf("claims-on gate waited %s on a crash-restarted edge, want no wait", waited) } } func TestClaimsOnGateFailsFastOnAFailedOrInactiveEdge(t *testing.T) { for _, state := range []string{"failed", "inactive"} { t.Run(state, func(t *testing.T) { f := newApplyFixture(t, "stable", "v1") f.a.claims = true orig := f.a.ops.systemctl f.a.ops.systemctl = func(args ...string) (string, error) { if args[0] == "is-active" { return state + "\n", errors.New("exit status 3") } return orig(args...) } start := f.now err := f.a.gate(context.Background(), "v1") if err == nil || !strings.Contains(err.Error(), "failing fast: carlos-edge is "+state) { t.Fatalf("gate = %v, want a fail-fast on %s", err, state) } if waited := f.now.Sub(start); waited != 0 { t.Fatalf("claims-on gate waited %s on a %s edge", waited, state) } }) } } func TestClaimsOnGateKeepsWaitingWhileActivating(t *testing.T) { // "activating" with no restart is a process still starting, not a // crash: no fail-fast, the gate passes once it answers. f := newApplyFixture(t, "stable", "v1") f.a.claims = true start := f.now orig := f.a.ops.systemctl f.a.ops.systemctl = func(args ...string) (string, error) { if args[0] == "is-active" && f.now.Sub(start) < 30*time.Second { return "activating\n", errors.New("exit status 3") } return orig(args...) } if err := f.a.gate(context.Background(), "v1"); err != nil { t.Fatalf("gate = %v, want a pass once the edge is active", err) } } func TestClaimsOnRollbackGateUsesTheSameRule(t *testing.T) { // v2 crashes on start; v1, the previous build, needs 22 s to listen. // The install's gate fails fast, and the rollback's gate gives v1 the // claims window, so the rollback succeeds instead of rollback-failed. f, req := applyHappy(t) f.a.claims = true restarts := slowListener(f, 22*time.Second) orig := f.a.ops.systemctl f.a.ops.systemctl = func(args ...string) (string, error) { if args[0] == "is-active" { if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v2") { return "failed\n", errors.New("exit status 3") } } return orig(args...) } err := f.a.Run(context.Background(), false) if err == nil { t.Fatal("a failed gate must exit non-zero") } r := f.result() if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) { t.Fatalf("result = %+v, want rolled-back", r) } if !strings.Contains(r.Reason, "failing fast") { t.Fatalf("reason = %q, want the install gate's fail-fast", r.Reason) } if len(*restarts) != 2 { t.Fatalf("edge restarted %d times, want install + rollback", len(*restarts)) } if d := (*restarts)[1].Sub((*restarts)[0]); d > 5*time.Second { t.Fatalf("install gate took %s to give up on a failed edge, want fail-fast", d) } } func TestClaimsOnRollbackGateFailsFastToo(t *testing.T) { // Both builds crash: the rollback gate fails fast as the install's did, // rather than waiting 120 s on a dead previous build. f, req := applyHappy(t) f.a.claims = true f.nrestarts = "2" f.probeErr = errors.New("connection refused") start := f.now if err := f.a.Run(context.Background(), false); err == nil { t.Fatal("a failed gate must exit non-zero") } r := f.result() if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) { t.Fatalf("result = %+v, want rollback-failed", r) } if waited := f.now.Sub(start); waited >= 20*time.Second { t.Fatalf("install and rollback gates took %s between them, want fail-fast on both", waited) } }