// Command seed ingests text files straight into a memoria database, // without going through HTTP. // // It exists so a fresh instance can be opened on a corpus that is already // worth searching, and it is the honest ancestor of slice 3's connectors: // same Store, same chunking, same sealing, no network. Nothing here // bypasses the trust model — it needs the same DEK the app runs with, and // what it writes is sealed exactly as the app would seal it. package main import ( "context" "encoding/base64" "flag" "fmt" "io/fs" "log/slog" "os" "path/filepath" "strings" "amadan.net/rastrillo/rastrillo/db" "amadan.net/rastrillo/rastrillo/migrate" memoria "amadan.net/oficina/memoria/internal/memoria" "amadan.net/oficina/memoria/internal/memoria/index" "amadan.net/oficina/memoria/internal/memoria/seal" ) func main() { dbPath := flag.String("db", "", "path to the memoria database") root := flag.String("dir", "", "directory to walk for .md and .txt files") source := flag.String("source", "seed", "the Source recorded on each document") flag.Parse() if *dbPath == "" || *root == "" { fmt.Fprintln(os.Stderr, "usage: seed --db --dir [--source name]") os.Exit(2) } dekRaw := os.Getenv("MEMORIA_DEK") if dekRaw == "" { fmt.Fprintln(os.Stderr, "MEMORIA_DEK is not set; refusing to seed with a key the app cannot derive") os.Exit(1) } dek, err := base64.StdEncoding.DecodeString(dekRaw) if err != nil { fmt.Fprintf(os.Stderr, "MEMORIA_DEK is not valid base64: %v\n", err) os.Exit(1) } sealer, err := seal.New(dek) if err != nil { fmt.Fprintf(os.Stderr, "MEMORIA_DEK rejected: %v\n", err) os.Exit(1) } logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) d, err := db.Open(*dbPath, logger) if err != nil { fmt.Fprintf(os.Stderr, "open database: %v\n", err) os.Exit(1) } defer d.Close() // Apply migrations. Without this, seeding a database the app has // never opened fails with "no such table: documents" once per file // and then reports success with nothing seeded — which is a // plausible empty result, and exactly the shape this codebase treats // as the worst available failure. if _, err := migrate.Apply(context.Background(), d, memoria.BootSchema); err != nil { fmt.Fprintf(os.Stderr, "migrate: %v\n", err) os.Exit(1) } ix, err := index.Open() if err != nil { fmt.Fprintf(os.Stderr, "open index: %v\n", err) os.Exit(1) } defer ix.Close() store := memoria.NewStore(d, sealer, ix, nil) ctx := context.Background() var n, skipped, indexed int err = filepath.WalkDir(*root, func(p string, e fs.DirEntry, err error) error { if err != nil { return err } if e.IsDir() { // Anything under a dot-directory or node_modules is not the // team's writing. base := e.Name() if strings.HasPrefix(base, ".") || base == "node_modules" || base == "releases" { return fs.SkipDir } return nil } switch strings.ToLower(filepath.Ext(p)) { case ".md", ".txt", ".markdown": default: return nil } raw, err := os.ReadFile(p) if err != nil { return err } if len(strings.TrimSpace(string(raw))) == 0 { skipped++ return nil } rel, _ := filepath.Rel(*root, p) title := rel // The relative path is the stable key, so re-seeding after an // edit updates the document rather than adding a second copy. if _, err := store.Ingest(ctx, memoria.IngestSpec{ Source: *source, Key: rel, Title: title, Body: string(raw), }); err != nil { // Fatal, not a skip. An earlier version counted these and // carried on, so a missing table produced 59 skips and a // cheerful "0 documents seeded" summary. return fmt.Errorf("ingest %s: %w", rel, err) } n++ // Report per file only when something actually changed, so a // re-seed of an unchanged tree is quiet rather than looking like // 60 successful writes. if before, _ := store.Count(); before != indexed { indexed = before fmt.Printf("seeded %s\n", rel) } return nil }) if err != nil { fmt.Fprintf(os.Stderr, "walk: %v\n", err) os.Exit(1) } count, _ := store.Count() // count is this process's own index, which holds only what THIS run // chunked. A re-seed of an unchanged tree therefore reports zero, and // that is success, not failure — so say which it is. switch { case count == 0 && n > 0: fmt.Printf("\n%d documents checked, all unchanged — nothing to re-index\n", n) default: fmt.Printf("\n%d documents seeded, %d skipped, %d passages indexed\n", n, skipped, count) } }