package host import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "testing" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" ) // The Task 9 carries (Plan 5 Task 9a, M16): (b) a foreign database parked by // F's retirement, (c) a rendered config left behind when its ack fails. // parkedLocally lists the base names under the target's .parked/. func parkedLocally(t *testing.T, dir string) map[string][]byte { t.Helper() out := map[string][]byte{} filepath.WalkDir(filepath.Join(dir, ".parked"), func(p string, d os.DirEntry, err error) error { if err == nil && !d.IsDir() && d.Type().IsRegular() { b, _ := os.ReadFile(p) out[filepath.Base(p)] = b } return nil }) return out } // cleanAfterRollback rolls the landing back, recovers the source and runs // the target's cleaning to cleaned. func (h *rollbackHarness) cleanAfterRollback(t *testing.T) { t.Helper() h.rollBack(t) h.rollbackPass(t) h.recovered(t) for i := 0; i < 4 && h.rb(t).TargetPhase != provision.RbTargetCleaned; i++ { h.rollbackPass(t) } if rb := h.rb(t); rb.TargetPhase != provision.RbTargetCleaned { t.Fatalf("the target did not clean: %+v", rb) } } // TestForeignDatabaseSurvivesRollbackRetirement (carry (b)): the landing // registers only what it placed at local, the files and the env; the // database and its marker are registered once the install proves them the // build's. A database that is not the build, standing at DBPath when a // landing stopped at local rolls back, is neither retired with F nor taken // by the moves-owned rename. func TestForeignDatabaseSurvivesRollbackRetirement(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) h.landAt = "after-inventory" h.crashLand(t) if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal { t.Fatalf("fixture: the landing is at %s", m.LandingPhase) } paths := h.claim(t).Inventory["uk-box"].Paths if _, ok := paths[p1Host+".db"]; ok || len(paths) != 2 { t.Fatalf("local registered %v, want the files and the env only", paths) } db := provision.DefaultDBPath(h.tdir, p1Host) foreign := []byte("a database this landing never installed") if err := os.WriteFile(db, foreign, 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(db+".gen", []byte("someone-else\n"), 0o600); err != nil { t.Fatal(err) } h.cleanAfterRollback(t) if got, err := os.ReadFile(db); err != nil || !bytes.Equal(got, foreign) { t.Fatalf("the foreign database: %q %v", got, err) } if _, err := os.Stat(db + ".gen"); err != nil { t.Fatalf("the foreign marker: %v", err) } for name, body := range parkedLocally(t, h.tdir) { if bytes.Equal(body, foreign) { t.Fatalf("the foreign database was parked as %s", name) } } if _, err := os.Lstat(filepath.Join(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the landing's files directory is still live: %v", err) } } // TestInstalledDatabaseIsRegisteredAfterTheInstall (carry (b)): once // configured installs the build, the database and its marker are registered // under F, and a crash between that install and the record's stamp still // leaves them the op's by content: the cleaning parks the build. func TestInstalledDatabaseIsRegisteredAfterTheInstall(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) h.landAt = "after-db-rename" h.crashLand(t) if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") { t.Fatalf("fixture: %+v", m) } if _, ok := h.claim(t).Inventory["uk-box"].Paths[p1Host+".db"]; ok { t.Fatal("the database was registered before the install finished") } db := provision.DefaultDBPath(h.tdir, p1Host) built, err := os.ReadFile(db) if err != nil { t.Fatal(err) } h.cleanAfterRollback(t) if _, err := os.Lstat(db); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the build is still at DBPath: %v", err) } if got, ok := parkedLocally(t, h.tdir)[p1Host+".db"]; !ok || !bytes.Equal(got, built) { t.Fatal("the build was not parked") } h2 := newLandingHarness(t) h2.flip(t) h2.land(t) h2.installed(t) paths := h2.claim(t).Inventory["uk-box"].Paths for _, p := range []string{p1Host + ".db", p1Host + ".db.gen", p1Host, p1Host + ".env"} { if _, ok := paths[p]; !ok { t.Fatalf("after configured the inventory is %v, missing %s", paths, p) } } } // TestFailedRenderAckIsParkedByCleaning (carry (c)): the held render wrote // .config.env and its config ack failed. The journal recorded the // file's state before the render and the rendered sha256, so the target's // cleaning parks the file; a file whose content is not the render's is // left. func TestFailedRenderAckIsParkedByCleaning(t *testing.T) { t.Parallel() for _, tc := range []struct { name string replace []byte // written over the rendered file before the cleaning parked bool }{ {"the rendered file", nil, true}, {"another file at the path", []byte("SOMEONE=else\n"), false}, } { t.Run(tc.name, func(t *testing.T) { h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) path := filepath.Join(h.tdir, p1Host+".config.env") rendered := []byte("SECRET=landing\n") sum := sha256.Sum256(rendered) ackFailed := errors.New("held config ack: the store refused the write") h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) { if err := before(hex.EncodeToString(sum[:])); err != nil { return appconfig.HeldRender{}, err } if err := os.WriteFile(path, rendered, 0o600); err != nil { return appconfig.HeldRender{}, err } in := h.read(t, p1Host).Move.Inputs return appconfig.HeldRender{Path: path, SHA256: hex.EncodeToString(sum[:]), BundleETags: in.BundleETags, OverlayETag: in.OverlayETag}, ackFailed } h.land(t) if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured { t.Fatalf("fixture: the landing is at %s", m.LandingPhase) } j := h.landJournal(t) want := "sha256:" + hex.EncodeToString(sum[:]) if pre := journalledPreRender(t, j); pre.Before != "absent" || len(pre.Intended) != 1 || pre.Intended[0] != want || j.Steps[landStepRendered] != want { t.Fatalf("the render's journal: pre %q rendered %q", j.Steps[landStepPreRender], j.Steps[landStepRendered]) } if tc.replace != nil { if err := os.WriteFile(path, tc.replace, 0o600); err != nil { t.Fatal(err) } } h.cleanAfterRollback(t) _, err := os.Lstat(path) if tc.parked { if !errors.Is(err, os.ErrNotExist) { t.Fatalf("the rendered config is still live: %v", err) } if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) { t.Fatal("the rendered config was not parked") } return } if err != nil { t.Fatalf("a file that is not the render's was taken: %v", err) } }) } } // TestReleaseTimeArchiveRefusalGoesStuck (M16): a target that served stops, // and its files directory holds something the archive can never carry. The // release records the stuck reason at releasing (not a retry forever), the // operator is paged once through moveCfg.Page, and once the cause is // removed a retry runs the release again. func TestReleaseTimeArchiveRefusalGoesStuck(t *testing.T) { t.Parallel() h := newRollbackHarness(t) h.rollBack(t) link := filepath.Join(h.tdir, p1Host, "link") if err := os.Symlink("/etc/passwd", link); err != nil { t.Fatal(err) } h.rollbackPass(t) m := h.read(t, p1Host).Move if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReleaseTargetFiles || m.Rollback.TargetPhase != provision.RbTargetReleasing || m.Rollback.StuckAt != provision.RbSourceWaiting { t.Fatalf("not stuck at releasing: %+v %+v", m, m.Rollback) } pagedOnce(t, h.pages, "uk-box", "target", provision.StuckReleaseTargetFiles) // A stuck record is not the target's to work on, and pages no more. h.rollbackPass(t) if got := h.read(t, p1Host).Move; got.State != provision.MoveStuck || len(h.pages) != 1 { t.Fatalf("a pass over the stuck record: %+v, %d pages", got, len(h.pages)) } if err := os.Remove(link); err != nil { t.Fatal(err) } h.retry(t) h.rollbackPass(t) if rb := h.rb(t); h.read(t, p1Host).Move.State != provision.MoveRollingBack || rb.TargetPhase == provision.RbTargetReleasing { t.Fatalf("after the retry: %+v", rb) } } // journalledPreRender decodes the landing journal's pre_render step. func journalledPreRender(t *testing.T, j *moves.Journal) preRender { t.Helper() var pre preRender if err := json.Unmarshal([]byte(j.Steps[landStepPreRender]), &pre); err != nil { t.Fatalf("pre_render %q: %v", j.Steps[landStepPreRender], err) } return pre } // TestRenderCrashBeforeItsRecordIsParkedByCleaning (fix round 1, I1): the // render journals the hash of the bytes it is about to place before it // writes them, so a crash after the file is written and before either the // ack or the rendered record lands still leaves the file the op's. The // cleaning parks it by that intended hash; other content at the path is // left. func TestRenderCrashBeforeItsRecordIsParkedByCleaning(t *testing.T) { t.Parallel() for _, tc := range []struct { name string replace []byte parked bool }{ {"the file the render was placing", nil, true}, {"other content at the path", []byte("SOMEONE=else\n"), false}, } { t.Run(tc.name, func(t *testing.T) { h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) path := filepath.Join(h.tdir, p1Host+".config.env") rendered := []byte("SECRET=landing\n") sum := sha256.Sum256(rendered) crashed := errors.New("the agent died after the write") h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) { if err := before(hex.EncodeToString(sum[:])); err != nil { return appconfig.HeldRender{}, err } if err := os.WriteFile(path, rendered, 0o600); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{}, crashed // no ack, no answer } h.land(t) j := h.landJournal(t) if pre := journalledPreRender(t, j); len(pre.Intended) != 1 || pre.Intended[0] != "sha256:"+hex.EncodeToString(sum[:]) || pre.Before != "absent" { t.Fatalf("pre_render %+v", pre) } if _, ok := j.Steps[landStepRendered]; ok { t.Fatal("fixture: the rendered record landed") } if tc.replace != nil { if err := os.WriteFile(path, tc.replace, 0o600); err != nil { t.Fatal(err) } } h.cleanAfterRollback(t) _, err := os.Lstat(path) if tc.parked { if !errors.Is(err, os.ErrNotExist) { t.Fatalf("the rendered config is still live: %v", err) } if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) { t.Fatal("the rendered config was not parked") } return } if err != nil { t.Fatalf("a file that is not the render's was taken: %v", err) } }) } } // TestRenderReplayKeepsEveryIntendedHash (fix round 2): render A is written // and the agent dies before the ack or the rendered record; the retry's // inputs have changed, so it renders B, journals B and fails to write it. A // is still the file on disk. The intended hashes are an append-only set // across the op's attempts, so A's stays recorded and the target's cleaning // still parks the file. func TestRenderReplayKeepsEveryIntendedHash(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) path := filepath.Join(h.tdir, p1Host+".config.env") a, b := []byte("SECRET=a\n"), []byte("SECRET=b\n") sumA, sumB := sha256.Sum256(a), sha256.Sum256(b) attempt := 0 h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) { attempt++ if attempt == 1 { if err := before(hex.EncodeToString(sumA[:])); err != nil { return appconfig.HeldRender{}, err } if err := os.WriteFile(path, a, 0o600); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{}, errors.New("the agent died after writing A") } if err := before(hex.EncodeToString(sumB[:])); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{}, errors.New("writing B failed") } h.land(t) h.land(t) if attempt < 2 { t.Fatalf("fixture: %d render attempts", attempt) } pre := journalledPreRender(t, h.landJournal(t)) wantA, wantB := "sha256:"+hex.EncodeToString(sumA[:]), "sha256:"+hex.EncodeToString(sumB[:]) if len(pre.Intended) != 2 || pre.Intended[0] != wantA || pre.Intended[1] != wantB { t.Fatalf("intended %v; want [A B]", pre.Intended) } if got, err := os.ReadFile(path); err != nil || !bytes.Equal(got, a) { t.Fatalf("fixture: the file is %q %v, want A", got, err) } h.cleanAfterRollback(t) if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { t.Fatalf("A is still live: %v", err) } if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, a) { t.Fatal("A was not parked") } } // renderAttempt is one fake held render: it journals sum through the // installer's before-write hook, writes body to path when write is true, // and then dies (no ack, no answer) unless done is true. func renderAttempt(path string, body []byte, write, done bool, held appconfig.HeldRender) func(func(string) error) (appconfig.HeldRender, error) { return func(before func(string) error) (appconfig.HeldRender, error) { sum := sha256.Sum256(body) if err := before(hex.EncodeToString(sum[:])); err != nil { return appconfig.HeldRender{}, err } if !write { return appconfig.HeldRender{}, errors.New("the write failed") } if err := os.WriteFile(path, body, 0o600); err != nil { return appconfig.HeldRender{}, err } if !done { return appconfig.HeldRender{}, errors.New("the agent died after the write") } held.Path, held.SHA256 = path, hex.EncodeToString(sum[:]) return held, nil } } // TestRenderCrashAfterThePrunedJournal (fix round 4): render A is written, // then B is written; render C prunes the set to what is on disk, B, plus C, // journals it, and dies before writing. The file on disk is B, which the set // still names, so the cleaning parks it. func TestRenderCrashAfterThePrunedJournal(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) path := filepath.Join(h.tdir, p1Host+".config.env") a, b, c := []byte("SECRET=a\n"), []byte("SECRET=b\n"), []byte("SECRET=c\n") for _, step := range []func(func(string) error) (appconfig.HeldRender, error){ renderAttempt(path, a, true, false, appconfig.HeldRender{}), renderAttempt(path, b, true, false, appconfig.HeldRender{}), renderAttempt(path, c, false, false, appconfig.HeldRender{}), } { h.renderHook = step h.land(t) } sumB, sumC := sha256.Sum256(b), sha256.Sum256(c) pre := journalledPreRender(t, h.landJournal(t)) if want := []string{"sha256:" + hex.EncodeToString(sumB[:]), "sha256:" + hex.EncodeToString(sumC[:])}; len(pre.Intended) != 2 || pre.Intended[0] != want[0] || pre.Intended[1] != want[1] { t.Fatalf("intended %v; want [B C]", pre.Intended) } if pre.Before != "absent" { t.Fatalf("the earlier-state record changed: %q", pre.Before) } h.cleanAfterRollback(t) if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { t.Fatalf("B is still live: %v", err) } if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, b) { t.Fatal("B was not parked") } } // TestReturnRenderRetriesStayBounded (fix round 4): a dirty return has no // deadline, so its held render may be retried any number of times. Forty // failed or changed attempts, half written and half not, keep the intended // set at most two hashes, always naming what is on disk; the next render // succeeds and the return completes with that content kept as the source's // config, still named. func TestReturnRenderRetriesStayBounded(t *testing.T) { t.Parallel() h := newReturnHarness(t, false) path := filepath.Join(h.sdir, p1Host+".config.env") for i := 0; i < 40; i++ { h.renderHook = renderAttempt(path, []byte(fmt.Sprintf("SECRET=%d\n", i)), i%2 == 0, false, appconfig.HeldRender{}) if _, err := h.install(t); err == nil { t.Fatalf("attempt %d: the render answered", i) } pre := journalledPreRender(t, h.returnJournal(t)) if len(pre.Intended) > 2 { t.Fatalf("attempt %d: %d intended hashes", i, len(pre.Intended)) } if got, err := fileSHA256(path); err == nil && !containsString(pre.Intended, got) { t.Fatalf("attempt %d: the file on disk (%s) is not named by %v", i, got, pre.Intended) } } final := []byte("SECRET=final\n") h.renderHook = renderAttempt(path, final, true, true, h.render) if _, err := h.install(t); err != nil { t.Fatalf("the successful render: %v", err) } if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceRestored { t.Fatalf("the return did not complete: %+v", rb) } got, err := os.ReadFile(path) if err != nil || !bytes.Equal(got, final) { t.Fatalf("the source's config is %q %v, want the final render", got, err) } sum := sha256.Sum256(final) pre := journalledPreRender(t, h.returnJournal(t)) if len(pre.Intended) > 2 || !containsString(pre.Intended, "sha256:"+hex.EncodeToString(sum[:])) { t.Fatalf("intended %v after the final render", pre.Intended) } } // TestRenderTempIsRemovedByCleaning (fix round 5): the renderer writes the // rendered bytes to .config.env.tmp and renames them over the path // (appconfig syncexec.go). A crash between the two leaves the temp holding // the rendered secrets, an unfinished write of this op's. The target's // cleaning removes it (never parks it: it is not live config), and leaves // .config.env as it was before the render: absent, or an earlier file // that is not the render's, kept. func TestRenderTempIsRemovedByCleaning(t *testing.T) { t.Parallel() for _, earlier := range [][]byte{nil, []byte("EARLIER=1\n")} { t.Run(fmt.Sprintf("earlier %v", earlier != nil), func(t *testing.T) { h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.flip(t) path := filepath.Join(h.tdir, p1Host+".config.env") if earlier != nil { if err := os.WriteFile(path, earlier, 0o600); err != nil { t.Fatal(err) } } rendered := []byte("SECRET=landing\n") sum := sha256.Sum256(rendered) h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) { if err := before(hex.EncodeToString(sum[:])); err != nil { return appconfig.HeldRender{}, err } if err := os.WriteFile(path+".tmp", rendered, 0o600); err != nil { return appconfig.HeldRender{}, err } return appconfig.HeldRender{}, errors.New("the agent died before the rename") } h.land(t) if _, err := os.Stat(path + ".tmp"); err != nil { t.Fatalf("fixture: no temp: %v", err) } h.cleanAfterRollback(t) if _, err := os.Lstat(path + ".tmp"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the renderer's temp is still there: %v", err) } got, err := os.ReadFile(path) switch { case earlier == nil && !errors.Is(err, os.ErrNotExist): t.Fatalf("a config appeared: %q %v", got, err) case earlier != nil && (err != nil || !bytes.Equal(got, earlier)): t.Fatalf("the earlier config was not kept: %q %v", got, err) } for name, body := range parkedLocally(t, h.tdir) { if bytes.Equal(body, rendered) { t.Fatalf("the rendered secrets were parked as %s", name) } } }) } }