//go:build linux package host import ( "bytes" "context" "errors" "os" "path/filepath" "strings" "sync/atomic" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // What a tenant can leave at a name in its own instance directory, and the // error main's safe read (instancefs) answers it with. type tenantPlant struct { name string want error } var tenantPlants = []tenantPlant{ {"a symlink to the right bytes", instancefs.ErrSymlink}, {"a FIFO", instancefs.ErrNotRegular}, {"an oversized file", instancefs.ErrTooLarge}, {"a symlinked parent directory", instancefs.ErrSymlink}, } // plantAt replaces the file at path with what p names. body is the file's // legitimate bytes: the symlink points at a copy of them, so a read that // followed it would see nothing wrong; the oversized file is body padded // past limit with what a reader of the file would ignore (pad), so a read // that took it whole would see nothing wrong either. func plantAt(t *testing.T, p tenantPlant, path string, body []byte, limit int64, pad byte) { t.Helper() switch p.name { case "a symlink to the right bytes": real := filepath.Join(t.TempDir(), "real") if err := os.WriteFile(real, body, 0o600); err != nil { t.Fatal(err) } if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } if err := os.Symlink(real, path); err != nil { t.Fatal(err) } case "a FIFO": if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } if err := syscall.Mkfifo(path, 0o600); err != nil { t.Fatal(err) } case "an oversized file": big := append(append([]byte(nil), body...), bytes.Repeat([]byte{pad}, int(limit)+1-len(body))...) if err := os.WriteFile(path, big, 0o600); err != nil { t.Fatal(err) } case "a symlinked parent directory": dir := filepath.Dir(path) moved := dir + ".moved" if err := os.Rename(dir, moved); err != nil { t.Fatal(err) } if err := os.Symlink(moved, dir); err != nil { t.Fatal(err) } default: t.Fatalf("unknown plant %q", p.name) } } // bounded runs fn and fails the test if it does not return in time: a read // that opened a planted FIFO without O_NONBLOCK would block for ever. func bounded(t *testing.T, fn func() error) error { t.Helper() done := make(chan error, 1) go func() { done <- fn() }() select { case err := <-done: return err case <-time.After(20 * time.Second): t.Fatal("the read did not return: it blocked on what the tenant planted") return nil } } // readInstanceFile is main's safe read for tenant-writable ground, and // absence only on positive evidence. func TestReadInstanceFileRefusesWhatTheTenantCanPlant(t *testing.T) { const max = 64 body := []byte("KEY=v\n") for _, p := range tenantPlants { t.Run(p.name, func(t *testing.T) { dir := filepath.Join(t.TempDir(), "inst") if err := os.Mkdir(dir, 0o700); err != nil { t.Fatal(err) } path := filepath.Join(dir, "shop.example.env") if err := os.WriteFile(path, body, 0o600); err != nil { t.Fatal(err) } plantAt(t, p, path, body, max, '\n') var raw []byte var absent bool err := bounded(t, func() (err error) { raw, absent, err = readInstanceFile(dir, "shop.example.env", max) return err }) if !errors.Is(err, p.want) || absent || raw != nil { t.Fatalf("got %q absent=%v %v; want %v", raw, absent, err, p.want) } }) } t.Run("the cap is inclusive", func(t *testing.T) { dir := t.TempDir() full := bytes.Repeat([]byte("x"), max) if err := os.WriteFile(filepath.Join(dir, "f"), full, 0o600); err != nil { t.Fatal(err) } raw, absent, err := readInstanceFile(dir, "f", max) if err != nil || absent || !bytes.Equal(raw, full) { t.Fatalf("a file at the cap: %d bytes absent=%v %v", len(raw), absent, err) } }) t.Run("absent in a directory that opened", func(t *testing.T) { raw, absent, err := readInstanceFile(t.TempDir(), "f", max) if err != nil || !absent || raw != nil { t.Fatalf("got %q absent=%v %v", raw, absent, err) } }) t.Run("a missing directory is not evidence", func(t *testing.T) { raw, absent, err := readInstanceFile(filepath.Join(t.TempDir(), "gone"), "f", max) if err == nil || absent || raw != nil { t.Fatalf("got %q absent=%v %v", raw, absent, err) } }) } // flushFixture is one host's release flush over an env file and the // releasing writer's adoption record in envDir, and files in filesDir. type flushFixture struct { data release.Store f ReleaseFlushFiles obs hostclaim.Observed envDir, filesDir string env, record []byte envPath, recPath string prefix string filesRoot, secret string } func newFlushFixture(t *testing.T) *flushFixture { t.Helper() const host = "shop.example" data := release.NewDirStore(t.TempDir()) reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: "10.0.0.1:3000", Status: "ready", Hibernate: true}); err != nil { t.Fatal(err) } inst := filepath.Join(t.TempDir(), "inst") files := filepath.Join(inst, host) if err := os.MkdirAll(files, 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(files, "a.txt"), []byte("hello"), 0o600); err != nil { t.Fatal(err) } writer := instancekey.Binding{Host: host, Incarnation: "inc-1", Session: 6, Generation: "g6"} x := &flushFixture{ data: data, envDir: inst, filesDir: files, env: []byte("CARLOS_INSTANCE_KEY=k1\n"), record: instancekey.AdoptionRecord(writer, instancekey.SourcePlatform, instancekey.Digest("k1")), envPath: filepath.Join(inst, host+".env"), recPath: filepath.Join(inst, host+instancekey.AdoptionSuffix), prefix: "instances/" + host + "/inc/files/g-g6/", } if err := os.WriteFile(x.envPath, x.env, 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(x.recPath, x.record, 0o600); err != nil { t.Fatal(err) } // A directory the tenant must never get published as its files: the // box's own data, say. x.filesRoot = t.TempDir() x.secret = "bucket-credentials" if err := os.WriteFile(filepath.Join(x.filesRoot, "litestream.env"), []byte(x.secret), 0o600); err != nil { t.Fatal(err) } x.f = ReleaseFlushFiles{Data: data, Registry: reg, FilesDir: func(string) string { return x.filesDir }, EnvPath: func(string) string { return x.envPath }} x.obs = hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Host: host, Holder: &hostclaim.Holder{Location: "instances/" + host + "/inc", Incarnation: "inc-1", Session: 6, Generation: "g6"}}} return x } func (x *flushFixture) flush(t *testing.T) (int64, error) { t.Helper() var seq int64 err := bounded(t, func() (err error) { seq, err = x.f.Flush(context.Background(), x.obs) return err }) return seq, err } func (x *flushFixture) refused(t *testing.T, err, want error) { t.Helper() if !errors.Is(err, ErrClaimStopUnavailable) || !errors.Is(err, want) { t.Fatalf("flush: %v; want %v", err, want) } if _, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix); ok || err != nil { t.Fatalf("a refused flush committed a manifest: %v %v", ok, err) } } // The release flush reads .env and the adoption record through main's // safe read: a symlink (at the name or a parent), a FIFO or an oversized // file planted at either refuses the flush, nothing is committed, and the // flush never blocks. func TestReleaseFlushRefusesWhatTheTenantPlantsAtTheEnvAndAdoptionRecord(t *testing.T) { t.Parallel() for _, target := range []string{"env", "adoption"} { for _, p := range tenantPlants { if target == "adoption" && p.name == "a symlinked parent directory" { continue // the env file's directory: the env case covers it } t.Run(target+"/"+p.name, func(t *testing.T) { x := newFlushFixture(t) path, body, limit, pad := x.envPath, x.env, int64(instancekey.EnvReadLimit), byte('\n') if target == "adoption" { path, body, limit, pad = x.recPath, x.record, instancekey.AdoptionReadLimit, ' ' } plantAt(t, p, path, body, limit, pad) _, err := x.flush(t) x.refused(t, err, p.want) }) } } } // SyncDir follows a symlink at its root, so the flush opens the files // directory (the tenant's to rename) as a real directory with no link at // any component first: a link there would publish the directory it names. func TestReleaseFlushRefusesALinkedFilesDirectory(t *testing.T) { t.Parallel() for _, tc := range []struct { name string plant func(t *testing.T, x *flushFixture) want error }{ {"a symlink at the files directory", func(t *testing.T, x *flushFixture) { if err := os.RemoveAll(x.filesDir); err != nil { t.Fatal(err) } if err := os.Symlink(x.filesRoot, x.filesDir); err != nil { t.Fatal(err) } }, instancefs.ErrSymlink}, {"a symlinked parent directory", func(t *testing.T, x *flushFixture) { // The files directory lives under a linked parent of its own; the // env directory stays real, so only the files root is at issue. parent := filepath.Join(t.TempDir(), "linked") if err := os.Symlink(filepath.Dir(x.filesRoot), parent); err != nil { t.Fatal(err) } x.filesDir = filepath.Join(parent, filepath.Base(x.filesRoot)) }, instancefs.ErrSymlink}, {"a FIFO", func(t *testing.T, x *flushFixture) { if err := os.RemoveAll(x.filesDir); err != nil { t.Fatal(err) } if err := syscall.Mkfifo(x.filesDir, 0o600); err != nil { t.Fatal(err) } }, syscall.ENOTDIR}, } { t.Run(tc.name, func(t *testing.T) { x := newFlushFixture(t) tc.plant(t, x) _, err := x.flush(t) x.refused(t, err, tc.want) }) } } // Absence stays absence only on positive evidence: a missing env file or // record in a directory that opened stages their deletion as before; a // missing directory refuses. func TestReleaseFlushReadsAbsenceOnlyFromADirectoryThatOpened(t *testing.T) { t.Parallel() x := newFlushFixture(t) for _, p := range []string{x.envPath, x.recPath} { if err := os.Remove(p); err != nil { t.Fatal(err) } } seq, err := x.flush(t) if err != nil { t.Fatal(err) } m, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix) if err != nil || !ok || m.Seq != seq { t.Fatalf("manifest: %v %v", ok, err) } for _, item := range []string{hostfiles.EnvItem, instancekey.AdoptionItem} { if _, ok := m.Platform[item]; ok { t.Fatalf("%s published for a file that is not there", item) } } y := newFlushFixture(t) y.envPath = filepath.Join(t.TempDir(), "gone", "shop.example.env") _, err = y.flush(t) y.refused(t, err, os.ErrNotExist) } // The race a check-then-open left: a symlink renamed in at the files // directory after the flush opened it is never followed. The flush // publishes the directory it opened, never the one the link names. func TestReleaseFlushPublishesTheDirectoryItOpenedNotALinkSwappedInAfter(t *testing.T) { x := newFlushFixture(t) swapOnce(t, "opened", func() { if err := os.Rename(x.filesDir, x.filesDir+".moved"); err != nil { t.Fatal(err) } if err := os.Symlink(x.filesRoot, x.filesDir); err != nil { t.Fatal(err) } }) seq, err := x.flush(t) if err != nil { t.Fatal(err) } m, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix) if err != nil || !ok || m.Seq != seq { t.Fatalf("manifest: %v %v", ok, err) } if _, ok := m.Entries["litestream.env"]; ok { t.Fatal("the flush published the directory a swapped-in link names") } if _, ok := m.Entries["a.txt"]; !ok { t.Fatalf("the flush did not publish the directory it opened: %v", m.Entries) } } // Astra P2 / Opus M4: a directory renamed away and back while // readInstanceFile looks is an error or the file, never "absent": a flush // must never stage the deletion of an env file that is there. func TestReadInstanceFileNeverReadsAVanishingDirectoryAsAbsent(t *testing.T) { t.Parallel() dir := filepath.Join(t.TempDir(), "inst") if err := os.Mkdir(dir, 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "shop.example.env"), []byte("K=v\n"), 0o600); err != nil { t.Fatal(err) } var stop atomic.Bool flapped := make(chan struct{}) go func() { defer close(flapped) for !stop.Load() { os.Rename(dir, dir+".away") os.Rename(dir+".away", dir) } }() defer func() { stop.Store(true); <-flapped }() deadline := time.Now().Add(1500 * time.Millisecond) for time.Now().Before(deadline) { if _, absent, _ := readInstanceFile(dir, "shop.example.env", 64); absent { t.Fatal("a directory that vanished between the steps read as an absent file") } } } // Opus I1: a files tree past the walk's caps refuses the flush whole; // nothing is committed. func TestReleaseFlushRefusesATreePastTheWalksCaps(t *testing.T) { t.Parallel() x := newFlushFixture(t) root, err := os.OpenRoot(x.filesDir) if err != nil { t.Fatal(err) } defer root.Close() if err := root.MkdirAll(strings.TrimSuffix(strings.Repeat("d/", hostfiles.MaxPathBytes/2+2), "/"), 0o700); err != nil { t.Fatal(err) } _, err = x.flush(t) if !errors.Is(err, hostfiles.ErrTooDeep) { t.Fatalf("flush of a tree past the caps: %v", err) } if _, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix); ok || err != nil { t.Fatalf("a refused flush committed a manifest: %v %v", ok, err) } }