package release import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/updatewords" ) func settingsRoute(t *testing.T, reg *registry.Registry, host, app, channel string, u UpdateSettings, bs BuildStateCopy) { t.Helper() addRoute(t, reg, host, app, channel, true) if err := reg.SetUpdateSettings(host, u.Column()); err != nil { t.Fatal(err) } if err := reg.SetBuildState(host, bs.Column()); err != nil { t.Fatal(err) } } func passWithSettings(t *testing.T, reg *registry.Registry, s Store, pub []byte, opt string, more ...PassOption) string { t.Helper() var out bytes.Buffer withCapturedStdout(t, &out, func() { opts := append([]PassOption{WithUpdateSettings(nil, nil, true)}, more...) if err := Pass(context.Background(), reg, s, pub, opt, nil, opts...); err != nil { t.Fatal(err) } }) return out.String() } // Spec §8 "Root pass": a manual instance on a fresh box installs the build // state's build, not the head. func TestAManualInstanceOnAFreshBoxInstallsItsLastBuild(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}) passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want v1 (the data's build), not the head", v) } } // Spec §8: an existing instance with a replica but no build state is held as // unknown, never moved to the head. func TestAnUnknownBuildInstallsNothing(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}, BuildStateCopy{Known: true, Replica: true}) passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "" { t.Fatalf("live %q for an unknown build", v) } if c, _ := ReadConsumed(opt, "a.example.com"); c == nil || c.Reason != string(ReasonUnknown) { t.Fatalf("consumed = %+v; want reason unknown recorded", c) } } // Spec §8 "Evidence": a pin of a pruned and reused version name, with a // replayed pointer, is refused for lack of a matching record. func TestAPinNeedsAMatchingRecord(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("promoted")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } shipRelease(t, s, "bab", "km", "v2", adoptPlatform, []byte("v2")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v2"); err != nil { t.Fatal(err) } if err := DeleteVersion(ctx, s, "bab", "km", "v1"); err != nil { t.Fatal(err) } shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("nobody promoted these")) settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) out := passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v == "v1" { t.Fatal("the pin installed a reused name's new bytes") } if !strings.Contains(out, updatewords.Text("settings.evidence.no-record")) { t.Fatalf("no refusal:\n%s", out) } } // §4.1, invariant 7: a record whose digest the reshipped manifest does not // match is a refusal, never a fallback to unbound staging. func TestAPinWithAMismatchedRecordIsRefused(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } reship(t, s, "bab", "km", "v1", []byte("not what was promoted")) // the record stays settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) out := passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "" { t.Fatalf("live %q: the pin installed bytes its record does not bind", v) } if _, err := os.Stat(VersionPath(opt, "bab", "v1", "km")); err == nil { t.Fatal("the reshipped bytes were staged") } if !strings.Contains(out, "a.example.com") { t.Fatalf("no refusal line:\n%s", out) } } // Spec §8: a build promoted before the feature is recovered from a staged // copy, refused elsewhere. func TestAPreFeatureBuildIsRecoveredFromAStagedCopy(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } if err := s.Delete(ctx, PromotionRecordKey("bab", "km", "v1")); err != nil { // promoted "before the feature" t.Fatal(err) } shipRelease(t, s, "bab", "km", "v2", adoptPlatform, []byte("v2")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v2"); err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}) passWithSettings(t, reg, s, pub, opt) // no staged copy: refused if v, _ := LiveVersion(opt, "a.example.com"); v == "v1" { t.Fatal("recovered with neither a record nor a staged copy") } if _, err := ensureStaged(ctx, s, opt, "bab", "km", "v1", "seed"); err != nil { // the box already had it t.Fatal(err) } // A staged copy whose bytes changed since it was verified recovers nothing. path := VersionPath(opt, "bab", "v1", "km") good, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("tampered"), 0o755); err != nil { t.Fatal(err) } passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v == "v1" { t.Fatal("recovered from a staged copy that no longer matches its manifest") } if err := os.WriteFile(path, good, 0o755); err != nil { t.Fatal(err) } passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want v1 recovered from the staged copy", v) } } // Spec §8 "Root pass": an install request is consumed exactly once. func TestAnInstallRequestResolvesOnceAndWaitsOnBake(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v0"}}) passWithSettings(t, reg, s, pub, opt) c, _ := ReadConsumed(opt, "a.example.com") if c == nil || c.Resolved == nil || c.Resolved.ID != 1 || c.Resolved.Target != "v1" { t.Fatalf("consumed = %+v; want {1, v1} written", c) } if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want T", v) } shipRelease(t, s, "bab", "km", "v2", adoptPlatform, []byte("v2")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v2"); err != nil { t.Fatal(err) } passWithSettings(t, reg, s, pub, opt) if c2, _ := ReadConsumed(opt, "a.example.com"); c2 == nil || c2.Resolved == nil || c2.Resolved.Target != "v1" { t.Fatalf("the request was resolved again: %+v", c2) } if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want T (v1) kept, not the new head", v) } } // Astra I1 #9: a record-authorized pin installs on a box that cannot read // the channel pointer. func TestAPinInstallsWithoutAChannelPointer(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } if err := s.Delete(ctx, ChannelKey("bab", "km", "canary/x")); err != nil { // the pointer is gone t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want the pin v1 with no pointer", v) } } // Astra I1 #8: a request waiting on bake is reported beside a major hold, // and never replaces it; once the window passes, root resolves it to the // head and installs it. func TestABakingRequestKeepsTheHeldReason(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) ratchetedProductionApp(t, s, ctx, priv, pub, opt, reg) // adopt_test.go: myapp's production head v2 inside its 24h window on this box settingsRoute(t, reg, "held.example.com", "myapp", "production", UpdateSettings{Updates: UpdatesManual, Channel: "production", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}) // A new held instance's first head bakes too (§4.3 rule 3), as rule 4 // does for p.example.com, the fixture's default route. settingsRoute(t, reg, "new.example.com", "myapp", "production", UpdateSettings{Updates: UpdatesManual, Channel: "production"}, BuildStateCopy{Known: true}) passWithSettings(t, reg, s, pub, opt) c, _ := ReadConsumed(opt, "held.example.com") if c == nil || c.Reason != string(ReasonManual) || c.Baking != "v2" || c.Resolved != nil { t.Fatalf("consumed = %+v; want reason manual, the baking head v2 beside it, nothing resolved", c) } if v, _ := LiveVersion(opt, "held.example.com"); v != "v1" { t.Fatalf("live %q while baking, want the held build v1", v) } for _, h := range []string{"new.example.com", "p.example.com"} { if v, _ := LiveVersion(opt, h); v == "v2" { t.Fatalf("%s live on the baking head v2", h) } } // The window passes on this box's own clock. key := channelClockKey("bab", "myapp", "production") st := LoadFirstSeen(FirstSeenPath(opt)) seen, ok := st.Channels[key]["v2"] if !ok { t.Fatal("no first sighting recorded for v2") } st.Channels[key]["v2"] = seen.Add(-31 * time.Hour) if err := st.Save(); err != nil { t.Fatal(err) } passWithSettings(t, reg, s, pub, opt) c, _ = ReadConsumed(opt, "held.example.com") if c == nil || c.Resolved == nil || c.Resolved.ID != 1 || c.Resolved.Target != "v2" || c.Baking != "" || c.Reason != string(ReasonRequest) { t.Fatalf("consumed = %+v; want {1, v2} resolved, nothing baking, reason request", c) } if v, _ := LiveVersion(opt, "held.example.com"); v != "v2" { t.Fatalf("live %q, want the resolved T v2", v) } if v, _ := LiveVersion(opt, "new.example.com"); v != "v2" { t.Fatalf("new.example.com live %q, want the head v2 once eligible", v) } } func TestDefaultSettingsAreTodaysAdoption(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } addRoute(t, reg, "a.example.com", "km", "canary/x", true) // The same box twice: once without the option (today), once with it. The // journal and the outcome must match line for line. optToday := t.TempDir() var today bytes.Buffer withCapturedStdout(t, &today, func() { if err := Pass(ctx, reg, s, pub, optToday, nil); err != nil { t.Fatal(err) } }) out := passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q", v) } if c, _ := ReadConsumed(opt, "a.example.com"); c != nil { t.Fatalf("a default route wrote a consumed file: %+v", c) } if strings.ReplaceAll(out, opt, "OPT") != strings.ReplaceAll(today.String(), optToday, "OPT") { t.Fatalf("journal differs from today's:\nwith settings:\n%s\ntoday:\n%s", out, today.String()) } // A failure book never reaches a default unmanaged route, readable or not. reg2, opt2 := testRegistry(t), t.TempDir() addRoute(t, reg2, "a.example.com", "km", "canary/x", true) broken := func() (FailedBuild, error) { return nil, errors.New("swap-failures.json: unreadable") } var buf bytes.Buffer withCapturedStdout(t, &buf, func() { if err := Pass(ctx, reg2, s, pub, opt2, nil, WithUpdateSettings(broken, func(registry.Route) bool { return false }, true)); err != nil { t.Fatal(err) } }) if v, _ := LiveVersion(opt2, "a.example.com"); v != "v1" { t.Fatalf("live %q: an unreadable book held a default unmanaged route", v) } } // Astra I2 #8: the proof gate checks the build the decision selected, not // the channel head. Pinned to v1 with the head at v2: a proof for v2 alone // installs nothing, and a proof for v1 installs v1. func TestProofGateSeesTheSelectedBuild(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg, bake := t.TempDir(), testRegistry(t), t.TempDir() for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) prover, _, proverPub := proverKey(t) // proof_test.go policy := filepath.Join(t.TempDir(), "proof-policy.json") if err := os.WriteFile(policy, []byte(fmt.Sprintf(`{"schema":1,"require":[{"app":"km","prover":"dev","key":%q}]}`, proverPub)), 0o644); err != nil { t.Fatal(err) } gate := NewProofGate(policy, bake) prove := func(v string) { sha := etagOf([]byte(v)) p, err := SignProof(prover, Proof{App: "km", Platform: adoptPlatform, SHA256: sha, Version: v, Prover: "dev", ActiveSince: time.Now().Add(-time.Hour), ProvenAt: time.Now()}) if err != nil { t.Fatal(err) } b, _ := json.Marshal(p) if _, err := s.Put(ctx, ProofKey("bab", "km", sha, "dev"), b, ""); err != nil { t.Fatal(err) } } pass := func() { var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, nil, BakeDir(bake), WithProofGate(gate), WithUpdateSettings(nil, nil, true)); err != nil { t.Fatal(err) } }) } prove("v2") // the head is proven, the pinned build is not pass() if v, _ := LiveVersion(opt, "a.example.com"); v == "v1" { t.Fatal("the pinned v1 went live on the head's proof") } prove("v1") gate.clear("a.example.com") // drop the 30-second miss cache pass() if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q, want the proven pin v1", v) } } // §4.3 rule 2: only root resolves. An agent pass applies the decision, never // writes the consumed file, and installs nothing a resolution would decide. func TestAnAgentPassNeverResolves(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true}) var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, nil, WithUpdateSettings(nil, nil, false)); err != nil { t.Fatal(err) } }) if c, err := ReadConsumed(opt, "a.example.com"); c != nil || err != nil { t.Fatalf("an agent pass wrote the consumed file: %+v %v", c, err) } if v, _ := LiveVersion(opt, "a.example.com"); v != "" { t.Fatalf("live %q: an agent pass installed an unresolved request's head", v) } passWithSettings(t, reg, s, pub, opt) // root if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q after root's pass, want T v1", v) } // The agent now follows root's resolution from the consumed file. if err := os.Remove(LiveLink(opt, "a.example.com")); err != nil { t.Fatal(err) } withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, nil, WithUpdateSettings(nil, nil, false)); err != nil { t.Fatal(err) } }) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q: the agent did not apply root's resolution", v) } } // §4.3: the failure records outrank every rule, for settings routes. A // default route keeps today's adoption, which never read the book; an // unreadable book holds every settings route and every managed route. func TestTheFailureBookBindsSettingsRoutes(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } settingsRoute(t, reg, "pin.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) addRoute(t, reg, "plain.example.com", "km", "canary/x", true) // cleared.example.com's settings were cleared after root recorded a held // decision: rule 4, byte for byte, and the record follows it. addRoute(t, reg, "cleared.example.com", "km", "canary/x", true) if err := WriteConsumed(opt, Consumed{Host: "cleared.example.com", Desired: "v1", Reason: string(ReasonManual), Detail: "v1"}); err != nil { t.Fatal(err) } type call struct { build, host string served bool } var calls []call book := func() (FailedBuild, error) { return func(acct, app, build, host string, served bool) bool { calls = append(calls, call{build, host, served}) return acct == "bab" && app == "km" && !served }, nil } none := func(registry.Route) bool { return false } passWithSettings(t, reg, s, pub, opt, WithUpdateSettings(book, none, true)) if v, _ := LiveVersion(opt, "pin.example.com"); v != "" { t.Fatalf("live %q: a pin onto a failed build installed", v) } if c, _ := ReadConsumed(opt, "pin.example.com"); c == nil || c.Reason != string(ReasonFailed) || c.Detail != "v1" { t.Fatalf("consumed = %+v; want reason failed naming v1", c) } for _, h := range []string{"plain.example.com", "cleared.example.com"} { if v, _ := LiveVersion(opt, h); v != "v2" { t.Fatalf("%s live %q: the book reached a default route", h, v) } } if c, _ := ReadConsumed(opt, "cleared.example.com"); c == nil || c.Reason != string(ReasonAdopt) || c.Desired != "v2" { t.Fatalf("cleared consumed = %+v; want root's record to follow rule 4", c) } for _, c := range calls { if c.host != "pin.example.com" { t.Fatalf("the book was asked about %s", c.host) } } // served is v == GoodVersion. calls = nil if err := reg.SetGoodBuild("pin.example.com", registry.GoodBuild{}, registry.GoodBuild{Version: "v1", Digest: etagOf([]byte("v1"))}); err != nil { t.Fatal(err) } passWithSettings(t, reg, s, pub, opt, WithUpdateSettings(book, none, true)) if v, _ := LiveVersion(opt, "pin.example.com"); v != "v1" { t.Fatalf("live %q: a build the instance served is not blocked", v) } if len(calls) == 0 || !calls[0].served { t.Fatalf("calls = %+v; want served for the GoodVersion build", calls) } // An unreadable book: nothing for a settings route or a managed default // route, today's adoption for an unmanaged default one; no consumed write. opt2, reg2 := t.TempDir(), testRegistry(t) settingsRoute(t, reg2, "pin.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) addRoute(t, reg2, "managed.example.com", "km", "canary/x", true) addRoute(t, reg2, "plain.example.com", "km", "canary/x", true) broken := func() (FailedBuild, error) { return nil, errors.New("swap-failures.json: unreadable") } managed := func(rt registry.Route) bool { return rt.Host == "managed.example.com" } out := passWithSettings(t, reg2, s, pub, opt2, WithUpdateSettings(broken, managed, true)) for _, h := range []string{"pin.example.com", "managed.example.com"} { if v, _ := LiveVersion(opt2, h); v != "" { t.Fatalf("%s live %q with an unreadable failure book", h, v) } if c, _ := ReadConsumed(opt2, h); c != nil { t.Fatalf("%s: an unreadable book wrote a decision: %+v", h, c) } } if v, _ := LiveVersion(opt2, "plain.example.com"); v != "v2" { t.Fatalf("plain live %q, want today's adoption", v) } if !strings.Contains(out, "failure book unreadable") { t.Fatalf("no line for the unreadable book:\n%s", out) } } // R-MERGE: the consumed-file update is one pure function, the one root runs // and the simulation runs. func TestNextConsumed(t *testing.T) { head := &ChannelPointer{Channel: "stable", Version: "v9"} known := func(st *BuildState) BuildStateCopy { return BuildStateCopy{Known: true, Replica: true, State: st} } manual := UpdateSettings{Updates: UpdatesManual} req := func(id uint64) UpdateSettings { return UpdateSettings{Updates: UpdatesManual, InstallRequest: id} } at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) cases := []struct { name string in TargetInput dec TargetDecision want Consumed }{ { name: "fresh: the decision only", in: TargetInput{Pointer: head, Settings: manual, Build: known(&BuildState{LastExecuted: "v1"})}, dec: TargetDecision{Version: "v1", Reason: ReasonManual, Detail: "v1"}, want: Consumed{Host: "a.example.com", Desired: "v1", Reason: "manual", Detail: "v1"}, }, { name: "a resolution is recorded", in: TargetInput{Pointer: head, Settings: req(3), Build: known(nil)}, dec: TargetDecision{Version: "v9", Reason: ReasonRequest, Resolve: &ResolvedRequest{ID: 3, Target: "v9"}, Detail: "v9"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v9"}, Desired: "v9", Reason: "request", Detail: "v9"}, }, { name: "a failed T completes above the old watermark", in: TargetInput{Pointer: head, Settings: req(3), Build: known(nil), Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v8"}, Completed: 2}}, dec: TargetDecision{Reason: ReasonManual, Fail: 3}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v8"}, Completed: 3, CompletedFailed: true, Reason: "manual"}, }, { name: "a completed T that failed later is marked once, not reopened", in: TargetInput{Pointer: head, Settings: req(3), Build: known(nil), Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v8"}, Completed: 3}}, dec: TargetDecision{Reason: ReasonFailed, Fail: 3, Detail: "v8"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v8"}, Completed: 3, CompletedFailed: true, Reason: "failed", Detail: "v8"}, }, { name: "an older failure never lowers the watermark", in: TargetInput{Pointer: head, Settings: req(5), Build: known(nil), Consumed: &Consumed{Host: "a.example.com", Completed: 5}}, dec: TargetDecision{Reason: ReasonManual, Fail: 4}, want: Consumed{Host: "a.example.com", Completed: 5, Reason: "manual"}, }, { name: "R-TARGET-FAILEDHEAD: resolved and completed failed in one write", in: TargetInput{Pointer: head, Settings: req(4), Build: known(&BuildState{LastExecuted: "v1"})}, dec: TargetDecision{Version: "v1", Reason: ReasonManual, Resolve: &ResolvedRequest{ID: 4, Target: "v9"}, Fail: 4, Detail: "v1"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 4, Target: "v9"}, Completed: 4, CompletedFailed: true, Desired: "v1", Reason: "manual", Detail: "v1"}, }, { name: "baking is reported beside the held reason", in: TargetInput{Pointer: head, HeadHeld: "in its supply-chain bake window", Settings: req(2), Build: known(&BuildState{LastExecuted: "v1"})}, dec: TargetDecision{Version: "v1", Reason: ReasonManual, Detail: "v1"}, want: Consumed{Host: "a.example.com", Desired: "v1", Reason: "manual", Detail: "v1", Baking: "v9"}, }, { name: "baking for a new request beside an older, completed resolution", in: TargetInput{Pointer: head, HeadHeld: "held", Settings: req(2), Build: known(&BuildState{LastExecuted: "v1", Completed: 1}), Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 1, Target: "v1"}, Completed: 1}}, dec: TargetDecision{Version: "v1", Reason: ReasonManual, Detail: "v1"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 1, Target: "v1"}, Completed: 1, Desired: "v1", Reason: "manual", Detail: "v1", Baking: "v9"}, }, { name: "no baking while the build state is unread", in: TargetInput{Pointer: head, HeadHeld: "held", Settings: req(2), Build: BuildStateCopy{}}, dec: TargetDecision{Reason: ReasonUnknown}, want: Consumed{Host: "a.example.com", Reason: "unknown"}, }, { name: "no baking for a fast-tracked request", in: TargetInput{Pointer: head, HeadHeld: "held", Settings: UpdateSettings{Updates: UpdatesManual, InstallRequest: 2, InstallFastTrack: true}, Build: known(nil)}, dec: TargetDecision{Version: "v9", Reason: ReasonRequest, Resolve: &ResolvedRequest{ID: 2, Target: "v9"}, Detail: "v9"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 2, Target: "v9"}, Desired: "v9", Reason: "request", Detail: "v9"}, }, { name: "no baking once R has a T", in: TargetInput{Pointer: head, HeadHeld: "held", Settings: req(2), Build: known(&BuildState{Resolved: &ResolvedRequest{ID: 2, Target: "v8"}}), Consumed: &Consumed{Host: "a.example.com", Baking: "v8"}}, dec: TargetDecision{Version: "v8", Reason: ReasonRequest, Evidence: EvidenceRecord, Detail: "v8"}, want: Consumed{Host: "a.example.com", Desired: "v8", Reason: "request", Detail: "v8"}, }, { name: "no baking for a completed request; a stale At is dropped", in: TargetInput{Pointer: head, HeadHeld: "held", Settings: req(2), Build: known(&BuildState{Completed: 2}), Consumed: &Consumed{Host: "a.example.com", Baking: "v9", Desired: "v1", Reason: "manual", At: at}}, dec: TargetDecision{Version: "v9", Reason: ReasonAdopt, Bake: true, Detail: "v9"}, want: Consumed{Host: "a.example.com", Desired: "v9", Reason: "adopt", Detail: "v9"}, }, { name: "R-RESOLUTION-STATE-WINS: a file disagreeing with a read build state takes its T", in: TargetInput{Pointer: head, Settings: req(3), Build: known(&BuildState{LastExecuted: "v1", Resolved: &ResolvedRequest{ID: 3, Target: "v4"}}), Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v1"}}}, dec: TargetDecision{Version: "v4", Reason: ReasonRequest, Evidence: EvidenceRecord, Detail: "v4"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v4"}, Desired: "v4", Reason: "request", Detail: "v4"}, }, { name: "R-RESOLUTION-STATE-WINS: an unread build state leaves the file alone", in: TargetInput{Pointer: head, Settings: req(3), Build: BuildStateCopy{}, Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v1"}}}, dec: TargetDecision{Reason: ReasonUnknown}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v1"}, Reason: "unknown"}, }, { name: "R-RESOLUTION-STATE-WINS: another request's resolution in the build state changes nothing", in: TargetInput{Pointer: head, Settings: req(3), Build: known(&BuildState{LastExecuted: "v1", Resolved: &ResolvedRequest{ID: 2, Target: "v4"}, Completed: 2}), Consumed: &Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v1"}}}, dec: TargetDecision{Version: "v1", Reason: ReasonRequest, Evidence: EvidenceRecord, Detail: "v1"}, want: Consumed{Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v1"}, Desired: "v1", Reason: "request", Detail: "v1"}, }, { name: "no baking with no readable head", in: TargetInput{HeadHeld: "", Settings: req(2), Build: known(&BuildState{LastExecuted: "v1"})}, dec: TargetDecision{Version: "v1", Reason: ReasonManual, Detail: "v1"}, want: Consumed{Host: "a.example.com", Desired: "v1", Reason: "manual", Detail: "v1"}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { tc.in.Route = registry.Route{Host: "a.example.com", UpdateSettings: tc.in.Settings.Column()} // Every decision names the column it was made for (R-HELD-PURE). tc.want.Settings = SettingsDigest(tc.in.Settings.Column()) // And the route's tenant (R22): a route with no account of its // own is the platform's. tc.want.Account = OpsAccount var before *Consumed if tc.in.Consumed != nil { cp := *tc.in.Consumed before = &cp } got := NextConsumed(tc.in, tc.dec) if !reflect.DeepEqual(got, tc.want) { t.Fatalf("NextConsumed =\n %+v\nwant\n %+v", got, tc.want) } if before != nil && !reflect.DeepEqual(*tc.in.Consumed, *before) { t.Fatalf("NextConsumed changed its input: %+v", *tc.in.Consumed) } if got.Resolved != nil && tc.in.Consumed != nil && got.Resolved == tc.in.Consumed.Resolved { t.Fatal("NextConsumed shares the input's Resolved") } }) } } // installAt stages ver of km on this box and points host at it, as an // earlier pass would have. func installAt(t *testing.T, s Store, opt, host, ver string) { t.Helper() if _, err := ensureStaged(context.Background(), s, opt, "bab", "km", ver, host); err != nil { t.Fatal(err) } if err := PointLive(opt, host, "bab", ver, "km"); err != nil { t.Fatal(err) } } // R-SETTINGS-UNREADABLE (a): a control name this binary does not know (a // newer console's) is ignored by root; the rest of the column still holds. func TestAnUnknownControlNameKeepsTheHold(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}) if err := reg.SetUpdateSettings("a.example.com", `{"updates":"manual","channel":"canary/x","control":["schedule","pin","schedule"]}`); err != nil { t.Fatal(err) } installAt(t, s, opt, "a.example.com", "v1") passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q: an unknown control name lifted the manual hold", v) } if c, _ := ReadConsumed(opt, "a.example.com"); c == nil || c.Reason != string(ReasonManual) { t.Fatalf("consumed = %+v; want reason manual", c) } u, err := ParseUpdateSettingsLenient(`{"updates":"manual","control":["schedule","pin","pin"]}`) if err != nil || u.Updates != UpdatesManual || !reflect.DeepEqual(u.Control, []string{ControlPin}) { t.Fatalf("lenient parse = %+v, %v; want manual with control [pin]", u, err) } if _, err := ParseUpdateSettings(`{"updates":"manual","control":["schedule"]}`); err == nil { t.Fatal("the strict parse (the console's) accepted an unknown control name") } } // R-SETTINGS-UNREADABLE (b): a column that still does not parse holds the // route where it is, and root records reason unknown for the wake check. func TestAnUnreadableSettingsColumnHolds(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } for host, col := range map[string]string{ "later.example.com": `{"updates":"scheduled","channel":"canary/x"}`, "garbage.example.com": `{not json`, } { settingsRoute(t, reg, host, "km", "canary/x", UpdateSettings{}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}) if err := reg.SetUpdateSettings(host, col); err != nil { t.Fatal(err) } } installAt(t, s, opt, "later.example.com", "v1") var agent bytes.Buffer withCapturedStdout(t, &agent, func() { if err := Pass(ctx, reg, s, pub, opt, nil, WithUpdateSettings(nil, nil, false)); err != nil { t.Fatal(err) } }) if strings.Contains(agent.String(), "update settings") { t.Fatalf("an agent pass printed the settings line:\n%s", agent.String()) } if c, _ := ReadConsumed(opt, "later.example.com"); c != nil { t.Fatalf("an agent pass wrote %+v", c) } out := passWithSettings(t, reg, s, pub, opt) for host, want := range map[string]string{"later.example.com": "v1", "garbage.example.com": ""} { if v, _ := LiveVersion(opt, host); v != want { t.Fatalf("%s live %q, want %q kept", host, v, want) } c, err := ReadConsumed(opt, host) if err != nil || c == nil || c.Reason != string(ReasonUnknown) || !strings.Contains(c.Detail, "update settings") || c.Desired != "" { t.Fatalf("%s consumed = %+v, %v; want reason unknown naming the parse problem", host, c, err) } } if !strings.Contains(out, "update settings unreadable") { t.Fatalf("root printed no line:\n%s", out) } } // R-AGENT-RELAY: an agent pass that stops for root's resolution asks root // for a pass, once, through the stage relay, with the request's sentinel. func TestAnAgentPassRelaysAPendingRequestToRoot(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true}) type relay struct { host, ver string err error } var got []relay var out bytes.Buffer withCapturedStdout(t, &out, func() { err := MaterializeHost(ctx, reg, s, pub, opt, t.TempDir(), "a.example.com", OnStageBlocked(func(h, v string, err error) { got = append(got, relay{h, v, err}) }), WithUpdateSettings(nil, nil, false)) if err == nil { t.Fatal("materialized a request root has not resolved") } }) if len(got) != 1 || got[0].host != "a.example.com" || got[0].ver != "v1" || !errors.Is(got[0].err, ErrInstallRequestNeedsRoot) { t.Fatalf("relays = %+v; want exactly one for a.example.com v1 with ErrInstallRequestNeedsRoot", got) } // Root's pass resolves; the relay's head is then installed. passWithSettings(t, reg, s, pub, opt) if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q after root's pass", v) } } // R-T6-MISC fix round 1, amended by Task 12a: the up-to-date refusal clear // judges both the selected build and the head. Pinned and live on v1: a // refusal of the head v2 that is still current is kept (unpinning would // otherwise retry a refused promotion once); a refusal of v1 itself (its // rollback failed) stays too. func TestTheLiveBuildsRefusalIsJudgedOnTheSelectedBuild(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg, bake := t.TempDir(), testRegistry(t), t.TempDir() for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } head, err := Channel(ctx, s, pub, "bab", "km", "canary/x") if err != nil { t.Fatal(err) } settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true}) installAt(t, s, opt, "a.example.com", "v1") pass := func() { var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, nil, BakeDir(bake), WithUpdateSettings(nil, nil, true)); err != nil { t.Fatal(err) } }) } if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v2", PointerPromotedAt: head.PromotedAt, Reason: "did not come up"}); err != nil { t.Fatal(err) } pass() if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r == nil || r.Version != "v2" { t.Fatalf("refusal = %+v, %v; the pin cleared a refusal of the head that is still current", r, err) } // A refusal of an earlier promotion of v2 blocks neither build: cleared. if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v2", PointerPromotedAt: head.PromotedAt.Add(-time.Hour), Reason: "did not come up"}); err != nil { t.Fatal(err) } pass() if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r != nil { t.Fatalf("refusal = %+v, %v; a refusal that blocks nothing was kept", r, err) } if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v1", Reason: "did not come up"}); err != nil { t.Fatal(err) } pass() if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r == nil || r.Version != "v1" { t.Fatalf("refusal = %+v, %v; the live build's own refusal was cleared", r, err) } if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" { t.Fatalf("live %q", v) } } // failingGets is a Store whose Get of any key in fail returns err, as a // bucket that cannot be read at that moment. type failingGets struct { Store fail map[string]bool err error } func (f failingGets) Get(ctx context.Context, key string) ([]byte, string, error) { if f.fail[key] { return nil, "", f.err } return f.Store.Get(ctx, key) } // Astra round 1 finding 7 (§4.1): this box's staged copy whose bytes still // hash to its sibling manifest is recovered without remote evidence, so a // promotion record GET or a manifest GET that fails does not block it. With // no verified copy the same failures still refuse, and nothing is downloaded: // bytes that are fetched or replaced need the record's digest. func TestAVerifiedLocalCopyNeedsNoRemoteEvidence(t *testing.T) { ctx := context.Background() priv, pub := testKeys(t) base := NewDirStore(t.TempDir()) shipRelease(t, base, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, base, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } blip := errors.New("s3: 503 slow down") for name, fail := range map[string]map[string]bool{ "record GET fails": {PromotionRecordKey("bab", "km", "v1"): true}, "manifest GET fails": {ManifestKey("bab", "km", "v1"): true}, } { t.Run(name, func(t *testing.T) { opt := t.TempDir() s := failingGets{Store: base, fail: fail, err: blip} // No copy on this box: refused, nothing staged. if _, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{}); err == nil { t.Fatal("staged with no verified copy and no readable remote evidence") } if _, err := os.Stat(VersionPath(opt, "bab", "v1", "km")); err == nil { t.Fatal("bytes were downloaded without remote evidence") } // The box's verified copy (staged earlier, sibling beside it). if _, err := ensureStaged(ctx, base, opt, "bab", "km", "v1", "seed"); err != nil { t.Fatal(err) } m, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{}) if err != nil || m == nil || m.Version != "v1" { t.Fatalf("stageAuthorized = %+v, %v; want the verified local copy recovered", m, err) } // A copy whose bytes no longer match its sibling is not recovered. if err := os.WriteFile(VersionPath(opt, "bab", "v1", "km"), []byte("tampered"), 0o755); err != nil { t.Fatal(err) } if _, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{}); err == nil { t.Fatal("recovered a staged copy that no longer matches its sibling") } }) } } // Ruling R-HELD-PURE: root's consumed decision records the settings column // it was made for, so the held wake check can tell a decision for the // current settings from a stale one. A different column has another digest. func TestNextConsumedRecordsTheSettingsItDecidedFor(t *testing.T) { one := 1 u := UpdateSettings{Updates: UpdatesAuto, Channel: "stable", Major: &one} in := TargetInput{Route: registry.Route{Host: "a.example.com", UpdateSettings: u.Column()}, Settings: u, Build: BuildStateCopy{Known: true}} got := NextConsumed(in, TargetDecision{Reason: ReasonMajor}) if got.Settings == "" || got.Settings != SettingsDigest(u.Column()) { t.Fatalf("Settings = %q, want the digest of %s", got.Settings, u.Column()) } two := 2 if other := (UpdateSettings{Updates: UpdatesAuto, Channel: "stable", Major: &two}).Column(); SettingsDigest(other) == got.Settings { t.Fatal("two columns share a digest") } } // Astra r2 N1 (ruling R22): root's consumed file names the tenant it decided // for, at both of the pass's write sites (the decision, and the unreadable // column's reason unknown), and a hostname that moves to another tenant is // rewritten naming the new one, never keeping the old file's tenant. func TestRootsConsumedFileNamesTheRoutesTenant(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) const host = "a.example.com" for _, ta := range [][2]string{{"cdf", "km"}, {"dee", "blog"}} { shipRelease(t, s, ta[0], ta[1], "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, ta[0], ta[1], "canary/x", "v1"); err != nil { t.Fatal(err) } } route := func(account, app string) { t.Helper() if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: account, App: app, Channel: "canary/x", Hibernate: true}); err != nil { t.Fatal(err) } if err := reg.SetUpdateSettings(host, UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}.Column()); err != nil { t.Fatal(err) } if err := reg.SetBuildState(host, BuildStateCopy{Known: true, Replica: true}.Column()); err != nil { t.Fatal(err) } } check := func(when, account, app string) { t.Helper() c, err := ReadConsumed(opt, host) if err != nil || c == nil { t.Fatalf("%s: consumed = %+v, %v", when, c, err) } if c.Account != account || c.App != app || c.Reason != string(ReasonUnknown) { t.Fatalf("%s: consumed names %q/%q (reason %q), want %s/%s", when, c.Account, c.App, c.Reason, account, app) } } route("cdf", "km") passWithSettings(t, reg, s, pub, opt) check("tenant A's decision", "cdf", "km") // Claims off: the row is retired by deletion, and B's arrives on the same // hostname. Root's next pass reads A's file and writes over it. if err := reg.Delete(host); err != nil { t.Fatal(err) } route("dee", "blog") passWithSettings(t, reg, s, pub, opt) check("tenant B's decision over A's file", "dee", "blog") // The other write site: a column that does not parse is recorded as // reason unknown, for the route's tenant too (A's again, over B's file). if err := reg.Delete(host); err != nil { t.Fatal(err) } if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: "cdf", App: "km", Channel: "canary/x", Hibernate: true}); err != nil { t.Fatal(err) } if err := reg.SetUpdateSettings(host, "{not json"); err != nil { t.Fatal(err) } out := passWithSettings(t, reg, s, pub, opt) if !strings.Contains(out, "update settings unreadable") { t.Fatalf("the pass did not take the unreadable-column path:\n%s", out) } check("an unreadable column", "cdf", "km") }