//go:build linux package operator import ( "bytes" "context" "database/sql" "encoding/json" "errors" "io" "os" "os/exec" "path/filepath" "testing" "time" sharedseal "amadan.net/oficina/home/handoff/seal" "golang.org/x/sys/unix" "modernc.org/sqlite" ) type openProbe struct { Path, Other, Mode string Device, Inode uint64 } func seedOpenDB(t *testing.T, path string) { t.Helper() db, e := sql.Open("sqlite", "file:"+path) if e != nil { t.Fatal(e) } if _, e = db.Exec("PRAGMA journal_mode=WAL; CREATE TABLE changes(value INTEGER)"); e != nil { t.Fatal(e) } if e = db.Close(); e != nil { t.Fatal(e) } } func TestBoundDatabaseSubprocess(t *testing.T) { if os.Getenv("OFICINA_OPEN_PROBE") == "1" { raw, e := io.ReadAll(io.LimitReader(os.Stdin, 16384)) if e != nil { t.Fatal(e) } var p openProbe if json.Unmarshal(raw, &p) != nil { t.Fatal("probe input") } if p.Mode == "contender" { db, err := sql.Open("sqlite", "file:"+p.Path+"?mode=rw") if err != nil { t.Fatal(err) } defer db.Close() _, err = db.Exec("BEGIN IMMEDIATE") var sqliteErr *sqlite.Error if !errors.As(err, &sqliteErr) || sqliteErr.Code() != 5 { t.Fatal("expected SQLITE_BUSY while owner holds transaction", err) } return } saved := p.Path + ".saved" swap := func() { t.Helper() if e := os.Rename(p.Path, saved); e != nil { t.Fatal(e) } if e := os.Rename(p.Other, p.Path); e != nil { t.Fatal(e) } } restore := func() { t.Helper() if e := os.Rename(p.Path, p.Other); e != nil { t.Fatal(e) } if e := os.Rename(saved, p.Path); e != nil { t.Fatal(e) } } if p.Mode == "symlink" { if e = os.Rename(p.Path, saved); e != nil { t.Fatal(e) } if e = os.Symlink(p.Other, p.Path); e != nil { t.Fatal(e) } } if p.Mode == "symlink-ancestor" { link := filepath.Join(filepath.Dir(p.Path), "alias") if e = os.Symlink(filepath.Dir(p.Path), link); e != nil { t.Fatal(e) } p.Path = filepath.Join(link, filepath.Base(p.Path)) } if p.Mode == "missing" { if e = os.Remove(p.Path); e != nil { t.Fatal(e) } } if p.Mode == "hardlink" { if e = os.Link(p.Path, saved); e != nil { t.Fatal(e) } } if p.Mode == "inherited-writeonly" || p.Mode == "inherited-readwrite" { flag := os.O_WRONLY if p.Mode == "inherited-readwrite" { flag = os.O_RDWR } f, e := os.OpenFile(p.Other, flag, 0) if e != nil { t.Fatal(e) } defer f.Close() } deadline := time.Now().Add(time.Minute) if p.Mode == "expired" { deadline = time.Now().Add(-time.Second) } bound, e := newBoundDatabase(p.Path, p.Device, p.Inode, deadline) early := map[string]bool{"symlink": true, "symlink-ancestor": true, "missing": true, "hardlink": true, "expired": true} if early[p.Mode] { if e != ErrRefused { if bound != nil { bound.close() } t.Fatal("pin refusal", e) } return } if e != nil { t.Fatal(e) } defer bound.close() if p.Mode == "replace-restore" { bound.connector.hook = func(stage string) { if stage == "before-driver" { swap() } if stage == "after-open" { restore() } } } if p.Mode == "replace-after" { bound.connector.hook = func(stage string) { if stage == "after-open" { swap() } } } var extra *os.File defer func() { if extra != nil { extra.Close() } }() if p.Mode == "symlink-after" || p.Mode == "ambiguous" { bound.connector.hook = func(stage string) { if stage != "after-open" { return } if p.Mode == "ambiguous" { extra, e = os.OpenFile(p.Path, os.O_RDWR, 0) if e != nil { t.Fatal(e) } } else { if e = os.Rename(p.Path, saved); e != nil { t.Fatal(e) } if e = os.Symlink(saved, p.Path); e != nil { t.Fatal(e) } } } } e = bound.db.PingContext(context.Background()) refused := p.Mode == "symlink-after" || p.Mode == "ambiguous" || p.Mode == "replace-restore" || p.Mode == "replace-after" || p.Mode == "inherited-writeonly" || p.Mode == "inherited-readwrite" if refused { if e != ErrRefused { t.Fatal("connection refusal", e) } return } if e != nil { t.Fatal("ordinary open", e) } if p.Mode == "locks" || p.Mode == "reconnect" { if p.Mode == "reconnect" { bound.db.SetMaxIdleConns(0) if err := bound.db.Ping(); err != nil { t.Fatal("reconnect", err) } bound.db.SetMaxIdleConns(1) } conn, err := bound.db.Conn(context.Background()) if err != nil { t.Fatal(err) } defer conn.Close() if _, err = conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil { t.Fatal(err) } defer conn.ExecContext(context.Background(), "ROLLBACK") if _, err = conn.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)"); err != nil { t.Fatal(err) } p.Mode = "contender" raw, _ := json.Marshal(p) cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseSubprocess$") cmd.Env = append(os.Environ(), "OFICINA_OPEN_PROBE=1") cmd.Stdin = bytes.NewReader(raw) if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("lock contender: %v %s", err, output) } return } if p.Mode == "wal-replaced" || p.Mode == "shm-symlink" || p.Mode == "foreign-descriptor" || p.Mode == "unlinked-temp" || p.Mode == "expired-connected" { conn, err := bound.db.Conn(context.Background()) if err != nil { t.Fatal(err) } defer conn.Close() var count int if err = conn.QueryRowContext(context.Background(), "SELECT count(*) FROM changes").Scan(&count); err != nil { t.Fatal(err) } if p.Mode == "wal-replaced" || p.Mode == "shm-symlink" { suffix := "-wal" if p.Mode == "shm-symlink" { suffix = "-shm" } if err = os.Rename(p.Path+suffix, p.Path+suffix+".saved"); err != nil { t.Fatal(err) } if p.Mode == "shm-symlink" { err = os.Symlink(p.Path+suffix+".saved", p.Path+suffix) } else { err = os.WriteFile(p.Path+suffix, nil, 0600) } if err != nil { t.Fatal(err) } defer func() { os.Remove(p.Path + suffix) os.Rename(p.Path+suffix+".saved", p.Path+suffix) }() } else if p.Mode == "expired-connected" { bound.connector.deadline = time.Now().Add(-time.Second) } else { f, err := os.OpenFile(p.Other, os.O_RDWR, 0) if err != nil { t.Fatal(err) } defer f.Close() if p.Mode == "unlinked-temp" { if err = os.Remove(p.Other); err != nil { t.Fatal(err) } } } if _, err = conn.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)"); err != ErrRefused { t.Fatal("boundary refusal", err) } return } if p.Mode == "post-commit" { bound.connector.hook = func(stage string) { if stage == "after-exec" { swap() } } } _, e = bound.db.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)") if p.Mode == "post-commit" { if e != ErrUnavailable || errors.Is(e, sql.ErrConnDone) { t.Fatal("commit ambiguity", e) } restore() bound.connector.hook = nil return } if e != nil { t.Fatal("ordinary write", e) } return } modes := []string{"symlink-after", "symlink-ancestor", "ambiguous", "wal-replaced", "shm-symlink", "foreign-descriptor", "unlinked-temp", "expired-connected", "reconnect", "locks", "ordinary", "replace-restore", "replace-after", "symlink", "missing", "hardlink", "expired", "inherited-writeonly", "inherited-readwrite", "post-commit"} for _, mode := range modes { t.Run(mode, func(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "home.db") other := filepath.Join(dir, "other.db") seedOpenDB(t, path) seedOpenDB(t, other) var stat unix.Stat_t if e := unix.Stat(path, &stat); e != nil { t.Fatal(e) } raw, e := json.Marshal(openProbe{path, other, mode, uint64(stat.Dev), stat.Ino}) if e != nil { t.Fatal(e) } cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseSubprocess$") cmd.Env = append(os.Environ(), "OFICINA_OPEN_PROBE=1") cmd.Stdin = bytes.NewReader(raw) if output, e := cmd.CombinedOutput(); e != nil { t.Fatalf("probe failed: %v %s", e, output) } if mode == "missing" { if _, e := os.Stat(path); !os.IsNotExist(e) { t.Fatal("missing DB created") } return } readPath := path if mode == "replace-after" || mode == "symlink" || mode == "symlink-after" { readPath = path + ".saved" } db, e := sql.Open("sqlite", "file:"+readPath+"?mode=rw") if e != nil { t.Fatal(e) } defer db.Close() var count int if e = db.QueryRow("SELECT count(*) FROM changes").Scan(&count); e != nil { t.Fatal(e) } expected := 0 if mode == "ordinary" || mode == "post-commit" { expected = 1 } if count != expected { t.Fatal("wrong mutation count", count, expected) } }) } } func TestBoundDatabaseEngineSubprocess(t *testing.T) { if os.Getenv("OFICINA_ENGINE_OPEN_PROBE") == "1" { var p openProbe if json.NewDecoder(io.LimitReader(os.Stdin, 16384)).Decode(&p) != nil { t.Fatal("probe input") } bound, err := newBoundDatabase(p.Path, p.Device, p.Inode, time.Now().Add(time.Minute)) if err != nil { t.Fatal(err) } defer bound.close() k, verifier, auth := fixture(t) sealer, err := sharedseal.New(bytes.Repeat([]byte{6}, 32), "oficina/operator") if err != nil { t.Fatal(err) } now := time.Unix(auth.Issued, 0) engine, err := NewEngine(bound.db, verifier, sealer, bytes.Repeat([]byte{7}, 32), func() time.Time { return now }) if err != nil { t.Fatal(err) } raw := signed(t, k, auth) out, err := engine.Execute(context.Background(), raw, insertOne) if err != nil || !bytes.Equal(out, privateResult) { t.Fatal("bound engine", err) } now = now.Add(301 * time.Second) out, err = engine.Execute(context.Background(), raw, func(context.Context, *sql.Conn, Request) ([]byte, error) { t.Fatal("replay mutated") return nil, nil }) if err != nil || !bytes.Equal(out, privateResult) { t.Fatal("bound recovery", err) } return } f := newEngineFixture(t) if err := f.db.Close(); err != nil { t.Fatal(err) } var stat unix.Stat_t if err := unix.Stat(f.path, &stat); err != nil { t.Fatal(err) } raw, _ := json.Marshal(openProbe{Path: f.path, Device: uint64(stat.Dev), Inode: stat.Ino}) cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseEngineSubprocess$") cmd.Env = append(os.Environ(), "OFICINA_ENGINE_OPEN_PROBE=1") cmd.Stdin = bytes.NewReader(raw) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("engine probe: %v %s", err, out) } db, err := sql.Open("sqlite", "file:"+f.path+"?mode=rw") if err != nil { t.Fatal(err) } defer db.Close() if countRows(t, db, "test_items") != 1 || countRows(t, db, "directory_operator_results") != 1 { t.Fatal("bound atomic recovery") } }