package host import ( "context" "errors" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" ) // The final whole-branch review's three blockers (final-review.md, // rulings in final-fix.md): the landing never deletes files it did not // create, every local installation runs inside the host guard with the // destination checked there, and the absolute cut deadline bounds the // landing as well as landing_deadline. // sentinels plants files this landing never wrote at the live files paths: // a file in the files directory and .env. No installation record and // no inventory names them, so local retires nothing (ruling G5). func (h *landingHarness) sentinels(t *testing.T) { t.Helper() if err := os.MkdirAll(filepath.Join(h.tdir, p1Host), 0o755); err != nil { t.Fatal(err) } for name, body := range map[string]string{filepath.Join(p1Host, "sentinel.txt"): "not the landing's", p1Host + ".env": "SENTINEL=1\n"} { if err := os.WriteFile(filepath.Join(h.tdir, name), []byte(body), 0o644); err != nil { t.Fatal(err) } } } // sentinelsIntact fails unless both sentinels are exactly as planted. func (h *landingHarness) sentinelsIntact(t *testing.T, when string) { t.Helper() for name, want := range map[string]string{filepath.Join(p1Host, "sentinel.txt"): "not the landing's", p1Host + ".env": "SENTINEL=1\n"} { body, err := os.ReadFile(filepath.Join(h.tdir, name)) if err != nil || string(body) != want { t.Fatalf("%s: %s is %q (%v), want %q", when, name, body, err, want) } } if _, err := os.Stat(filepath.Join(h.tdir, p1Host, "upload.txt")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s: the archive was installed over the sentinels: %v", when, err) } } // retireInstallation retires the installation record's active stamp, as a // successor taking this box over would before installing its own. func (h *landingHarness) retireInstallation(t *testing.T) { t.Helper() g := hostguard.Guard{DataDir: h.tdir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: h.s}} if err := g.RetireInstallation(context.Background(), p1Host, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } } // Finding 1: files at the live paths that no earlier stay registered are // not the landing's. Across two passes (and a fresh worker between them, // so any attempt marker the first pass journalled is what the second // reads), they are preserved, and the landing records why it stops. func TestLandingNeverDeletesFilesItDidNotCreate(t *testing.T) { t.Parallel() t.Run("present before the landing", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) h.sentinels(t) h.land(t) h.sentinelsIntact(t, "after the first pass") h.freshTarget() h.land(t) h.sentinelsIntact(t, "after the second pass") m := h.read(t, p1Host).Move if m.Reason != provision.RollbackFilesOccupied || m.LandingPhase != provision.LandLocal { t.Fatalf("record: %+v", m) } if _, ok := h.filesGen(t); ok { t.Fatal(".files-gen written over files the landing did not write") } }) t.Run("only an env file", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) if err := os.WriteFile(filepath.Join(h.tdir, p1Host+".env"), []byte("SENTINEL=1\n"), 0o644); err != nil { t.Fatal(err) } h.land(t) h.freshTarget() h.land(t) if body, err := os.ReadFile(filepath.Join(h.tdir, p1Host+".env")); err != nil || string(body) != "SENTINEL=1\n" { t.Fatalf("the env file the landing did not write: %q %v", body, err) } if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied { t.Fatalf("record: %+v", m) } }) // The review's failure: the attempt is journalled, then the files // appear (the landing's own crash in between), and the retry reads the // journal. The point is the one after the journal step that begins the // extraction, whatever it is called. t.Run("appearing after the attempt began", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) planted := false h.landFault = func(p string) error { if !planted && (p == "after-extracting" || p == "after-staged") { planted = true h.sentinels(t) return errCrash } return nil } h.land(t) if !planted { t.Fatalf("the extraction never began: %v", h.landHits) } h.sentinelsIntact(t, "after the crash") h.freshTarget() h.land(t) h.sentinelsIntact(t, "after the retry") h.freshTarget() h.land(t) h.sentinelsIntact(t, "after a third pass") if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied { t.Fatalf("record: %+v", m) } }) // The marker vouches for the tree this landing placed: a files // directory swapped after the placement is not written into. t.Run("replaced after the placement", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) dir := filepath.Join(h.tdir, p1Host) h.landFault = func(p string) error { if p == "before-files-gen" { if err := os.Rename(dir, dir+".moved"); err != nil { t.Fatal(err) } if err := os.Mkdir(dir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "sentinel.txt"), []byte("not the landing's"), 0o644); err != nil { t.Fatal(err) } } return nil } h.land(t) if _, ok := h.filesGen(t); ok { t.Fatal(".files-gen written into a files directory the landing did not place") } if body, err := os.ReadFile(filepath.Join(dir, "sentinel.txt")); err != nil || string(body) != "not the landing's" { t.Fatalf("the sentinel: %q %v", body, err) } if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal { t.Fatalf("record: %+v", m) } }) // An empty files directory is the landing's to fill (as Extract has // always allowed), and a retry after a crash inside the install, with // the files moved and the env not, finishes from the staged tree. t.Run("an empty directory and a crash mid-install", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) if err := os.MkdirAll(filepath.Join(h.tdir, p1Host), 0o750); err != nil { t.Fatal(err) } h.landAt = "after-files-rename" h.crashLand(t) if _, err := os.Stat(filepath.Join(h.tdir, p1Host, "upload.txt")); err != nil { t.Fatalf("the files were not moved before the crash: %v", err) } h.freshTarget() h.land(t) h.installed(t) if fi, err := os.Stat(filepath.Join(h.tdir, p1Host)); err != nil || fi.Mode().Perm() != 0o750 { t.Fatalf("the files directory's mode: %v %v", fi, err) } if _, err := os.Stat(filepath.Join(h.tdir, ".moves", p1Host, p1ID, "extract")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the staging was left behind: %v", err) } }) } // Finding 2: the installation is checked and made inside the host guard. // A successor that appears between the landing's observation and its // install is never replaced. func TestLandingInstallRefusesASuccessor(t *testing.T) { t.Parallel() dbPath := func(h *landingHarness) string { return provision.DefaultDBPath(h.tdir, p1Host) } notRestored := func(t *testing.T, h *landingHarness) { t.Helper() m := h.read(t, p1Host).Move if m.LandingPhase == provision.LandRestored || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") { t.Fatalf("the landing completed over a successor: %+v", m) } } t.Run("a successor database", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) h.landFault = func(p string) error { if p == "before-db-verify" { if err := os.WriteFile(dbPath(h), []byte("successor database"), 0o644); err != nil { t.Fatal(err) } if err := hostbox.WriteGenerationMarker(dbPath(h)+".gen", "successor"); err != nil { t.Fatal(err) } } return nil } h.land(t) if !containsString(h.landHits, "before-db-verify") { t.Fatalf("the install never observed: %v", h.landHits) } if body, err := os.ReadFile(dbPath(h)); err != nil || string(body) != "successor database" { t.Fatalf("the successor's database was replaced: %q %v", body, err) } if gen, err := hostbox.ReadGenerationMarker(dbPath(h) + ".gen"); err != nil || gen != "successor" { t.Fatalf("the successor's marker: %q %v", gen, err) } notRestored(t, h) }) t.Run("a successor database without a marker", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) h.landFault = func(p string) error { if p == "before-db-verify" { if err := os.WriteFile(dbPath(h), []byte("successor database"), 0o644); err != nil { t.Fatal(err) } } return nil } h.land(t) if body, err := os.ReadFile(dbPath(h)); err != nil || string(body) != "successor database" { t.Fatalf("the successor's database was replaced: %q %v", body, err) } if _, err := os.Stat(dbPath(h) + ".gen"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the successor's database was labelled as this landing's: %v", err) } notRestored(t, h) }) // A symlink at a destination is never this landing's, dangling or not: // it is not read through (a dangling one would look absent) and never // replaced (final re-review residual). for _, tc := range []struct{ name, suffix string }{{"a dangling marker symlink", ".gen"}, {"a dangling database symlink", ""}, {"a dangling WAL symlink", "-wal"}} { t.Run(tc.name, func(t *testing.T) { h := newLandingHarness(t) h.flip(t) link, target := dbPath(h)+tc.suffix, filepath.Join(h.tdir, "elsewhere", "nothing") h.landFault = func(p string) error { if p == "before-db-verify" { if err := os.Symlink(target, link); err != nil { t.Fatal(err) } } return nil } h.land(t) if !containsString(h.landHits, "before-db-verify") { t.Fatalf("the install never observed: %v", h.landHits) } fi, err := os.Lstat(link) if err != nil || fi.Mode()&os.ModeSymlink == 0 { t.Fatalf("%s is no longer the symlink the landing did not write: %v %v", link, fi, err) } if got, err := os.Readlink(link); err != nil || got != target { t.Fatalf("%s points at %q (%v), want %q", link, got, err, target) } if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) { t.Fatalf("something was written through the symlink: %v", err) } if tc.suffix != "" { if _, err := os.Lstat(dbPath(h)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("installed beside a symlink the landing did not write: %v", err) } } notRestored(t, h) }) } t.Run("a successor marker", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) h.landFault = func(p string) error { if p == "before-db-verify" { if err := hostbox.WriteGenerationMarker(dbPath(h)+".gen", "successor"); err != nil { t.Fatal(err) } } return nil } h.land(t) if _, err := os.Stat(dbPath(h)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("installed beside a successor's marker: %v", err) } if gen, err := hostbox.ReadGenerationMarker(dbPath(h) + ".gen"); err != nil || gen != "successor" { t.Fatalf("the successor's marker: %q %v", gen, err) } notRestored(t, h) }) // The installation retired between the observation and the install: // only the guard can see it, since nothing is at the destination yet. t.Run("the installation retired before the database install", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) h.landFault = func(p string) error { if p == "before-db-verify" { h.retireInstallation(t) } return nil } h.land(t) for _, p := range []string{dbPath(h), dbPath(h) + ".gen"} { if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s installed under a retired installation: %v", p, err) } } notRestored(t, h) }) t.Run("the installation retired before the files install", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) retired := false h.landFault = func(p string) error { if !retired && (p == "after-extracting" || p == "after-staged") { retired = true h.retireInstallation(t) } return nil } h.land(t) if !retired { t.Fatalf("the extraction never began: %v", h.landHits) } for _, p := range []string{filepath.Join(h.tdir, p1Host), filepath.Join(h.tdir, p1Host+".env")} { if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s installed under a retired installation: %v", p, err) } } if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal { t.Fatalf("record: %+v", m) } }) } // Finding 3: the landing is bounded by the earlier of the cut's absolute // deadline and landing_deadline (spec ยง7.4). The cut deadline here falls a // minute after the flip; the landing deadline is fifteen minutes away. func TestLandingCutDeadlineMarksRollback(t *testing.T) { t.Parallel() ctx := context.Background() setCutDeadline := func(t *testing.T, h *landingHarness) time.Time { t.Helper() rec := h.read(t, p1Host) d := rec.Move.FlippedAt.Add(time.Minute) if _, err := provision.SetRecordMove(ctx, h.s, rec.Account, rec.App, rec.Host, func(r *provision.Record) error { r.Move.CutDeadline = &d return nil }); err != nil { t.Fatal(err) } if ld := h.read(t, p1Host).Move.LandingDeadline; ld == nil || !ld.After(d.Add(2*time.Minute)) { t.Fatalf("fixture: landing deadline %v is not well after the cut deadline %v", ld, d) } return d } pastCut := func(h *landingHarness, d time.Time) { h.clock.Advance(d.Sub(h.clock.Now()) + time.Second) } t.Run("at checking", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) d := setCutDeadline(t, h) pastCut(h, d) h.land(t) // The rollback decision reads the earlier deadline too (J11). rolledBack(t, h, provision.RollbackLandingDeadline, provision.LandChecking) }) t.Run("at configured while the release waits", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) d := setCutDeadline(t, h) h.release = moves.Check{Reason: t7NotProven} h.land(t) if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured || m.Reason != "" { t.Fatalf("waiting on the proof: %+v", m) } pastCut(h, d) h.release = moves.Check{OK: true} h.land(t) rolledBack(t, h, provision.RollbackLandingDeadline, provision.LandConfigured) if _, err := os.Stat(provision.DefaultDBPath(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("installed past the cut deadline: %v", err) } }) t.Run("before the install", func(t *testing.T) { h := newLandingHarness(t) h.flip(t) d := setCutDeadline(t, h) h.landFault = func(p string) error { if p == "after-release-ready" { pastCut(h, d) } return nil } h.land(t) m := h.read(t, p1Host).Move if m.Reason != provision.RollbackLandingDeadline || m.LandingPhase != provision.LandConfigured || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") { t.Fatalf("past the cut deadline before the install: %+v", m) } if _, err := os.Stat(provision.DefaultDBPath(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("installed past the cut deadline: %v", err) } }) } // Re-review finding 8: a seed marker left beside the destination by some // earlier seed never survives the landing's database install, so the // installed database is never read as that seed's leftover. func TestLandingInstallRemovesAStaleSeedMarker(t *testing.T) { t.Parallel() h := newLandingHarness(t) h.flip(t) h.route.Replication = "" db := provision.DefaultDBPath(h.tdir, p1Host) if err := os.MkdirAll(filepath.Dir(db), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(db+activator.SeedMarkerSuffix, []byte("sd_aaaaaaaaaaaaaaaaaaaaaaaaaa\n"), 0o600); err != nil { t.Fatal(err) } h.land(t) h.installed(t) if _, err := os.Stat(db); err != nil { t.Fatalf("the database was not installed: %v", err) } if _, err := os.Lstat(db + activator.SeedMarkerSuffix); !errors.Is(err, os.ErrNotExist) { t.Fatalf("the stale seed marker survived the install: %v", err) } }