//go:build linux && rootrunner package host import ( "encoding/json" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rollout" "github.com/carlosframework/platform/internal/status" "os" "os/exec" "path/filepath" "syscall" "testing" "time" ) func TestPlan14RootRetryProjectionDelivery(t *testing.T) { if os.Getenv("PLAN14_RETRY_READER") == "1" { report := rootRetryStatus(&status.RolloutReport{Hosts: map[string]status.HostRollout{"h.example.com": {HeldBook: "forged", HeldEpoch: 999}}}) raw, _ := json.Marshal(report) os.Stdout.Write(raw) os.Exit(0) } if os.Geteuid() != 0 { t.Fatal("root required") } if err := os.Mkdir(rootRetryProjection, 0755); err != nil { t.Fatal(err) } defer os.RemoveAll(rootRetryProjection) state := testRootFailureStore(t) book, err := rollout.InitializeRootFailures(state, "box/en/1", nil) if err != nil { t.Fatal(err) } rt := registry.Route{Host: "h.example.com", Account: "bab", App: "km"} if _, err = book.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { t.Fatal(err) } p := rootUpdateSource{book: book, publish: publishRootRetry} if err = p.publishRetry(rootUpdateRecord{Route: rt}, "v1"); err != nil { t.Fatal(err) } exe, err := os.Executable() if err != nil { t.Fatal(err) } cmd := exec.Command(exe, "-test.run=^TestPlan14RootRetryProjectionDelivery$") cmd.Env = append(os.Environ(), "PLAN14_RETRY_READER=1") // /root is private on the runner: the child uses an executable descriptor // opened before dropping credentials, without making the runner binary public. binary, err := os.Open(exe) if err != nil { t.Fatal(err) } defer binary.Close() cmd.Path = "/proc/self/fd/3" cmd.Args[0] = cmd.Path cmd.ExtraFiles = []*os.File{binary} cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: 65534, Gid: 65534}} raw, err := cmd.Output() if err != nil { t.Fatal(err) } var report status.RolloutReport if err = json.Unmarshal(raw, &report); err != nil { t.Fatal(err, string(raw)) } h := report.Hosts[rt.Host] if h.HeldBook != book.ID() || h.HeldEpoch != book.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host) { t.Fatal("agent reported forged retry", h) } legacy, err := rollout.OpenFailureBook(t.TempDir()) if err != nil { t.Fatal(err) } if _, err = legacy.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { t.Fatal(err) } legacyEpoch := legacy.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host) rt.Kind, rt.Status, rt.Version, rt.Channel, rt.Hibernate = "instance", "ready", "v1", "canary/x", true if !rollout.Restrictable(rt) { t.Fatal("fixture route not restrictable") } hold := &failureHolds{book: legacy, get: func(string) (*registry.Route, error) { return &rt, nil }} old := declinedHolds.Swap(hold) defer declinedHolds.Store(old) // A root retry has no effect on the separate local mirror until root's // durable acknowledgement names the exact local snapshot it observed. seed, public, _ := release.GenerateKey() priv, _ := release.ParseSigningKey(seed, "test") pub, _ := release.ParsePublicKey(public, "test") wire, err := release.SignHostRetry(release.RootSigner(priv), rt.Account, rt.App, rt.Host, h.HeldBy, h.HeldBook, h.HeldEpoch, time.Now()) if err != nil { t.Fatal(err) } data := t.TempDir() t.Setenv("CARLOS_DATA_DIR", data) request := release.RestartRequest{Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: rt.Host, Signed: wire}} work := []scopedRestart{{app: [2]string{rt.Account, rt.App}, requests: []release.RestartRequest{request}}} applyRetries([]registry.Route{rt}, work, &restartMemory{pub: pub}) marker := filepath.Join(data, stageRequestMarker) if _, err := os.Stat(marker); err != nil { t.Fatal("root retry was not relayed", err) } if !book.HeldDown(rt.Account, rt.App, "v1", rt.Host) { t.Fatal("agent relay cleared root authority") } if err := os.Remove(marker); err != nil { t.Fatal(err) } work[0].requests[0].Scope.Signed = "forged" applyRetries([]registry.Route{rt}, work, &restartMemory{pub: pub}) if _, err := os.Stat(marker); !os.IsNotExist(err) { t.Fatal("forged retry was relayed", err) } if err = book.VerifiedRetry(pub, wire, rt.Account, rt.App, rt.Host, time.Now()); err != nil { t.Fatal(err) } if book.HeldDown(rt.Account, rt.App, "v1", rt.Host) { t.Fatal("delivered retry did not clear root failure") } saved := rootUpdateRecord{Route: rt, RetryMirror: &rootLegacyRetry{Build: "v1", Book: legacy.ID(), Epoch: legacyEpoch}} if err = p.publishRetry(saved, "v1"); err != nil { t.Fatal(err) } local := &status.RolloutReport{Hosts: map[string]status.HostRollout{rt.Host: {Account: rt.Account, App: rt.App, HeldBy: "v1", HeldBook: legacy.ID(), HeldEpoch: legacyEpoch}}} rootRetryStatus(local) if legacy.HeldDown(rt.Account, rt.App, "v1", rt.Host) { t.Fatal("agent mirror still blocks the root retry") } if _, err = legacy.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { t.Fatal(err) } current := *local current.Hosts = map[string]status.HostRollout{rt.Host: {HeldBy: "v1", HeldBook: legacy.ID(), HeldEpoch: legacy.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host)}} rootRetryStatus(¤t) if !legacy.HeldDown(rt.Account, rt.App, "v1", rt.Host) { t.Fatal("old receipt erased a later local failure") } // A service-written projection cannot deliver a mirror clear. if err = os.Chmod(filepath.Join(rootRetryProjection, rt.Host+".json"), 0666); err != nil { t.Fatal(err) } if report := rootRetryStatus(local); report == nil || report.Unavailable != rolloutBookUnreadable { t.Fatal("hostile retry projection accepted", report) } }