package moves import ( "bytes" "encoding/binary" "errors" "fmt" "io" "os" "github.com/superfly/ltx" ) // sqliteHeaderSize is the SQLite database header, the first 100 bytes of // page 1. const sqliteHeaderSize = 100 var sqliteMagic = []byte("SQLite format 3\x00") // ReadDBHeader reads the page size (header bytes 16-17; the value 1 means // 65536) and the in-header database size in pages (bytes 28-31) of the // SQLite database at path. The in-header size is what SQLite keeps current // in WAL mode and what litestream's restore writes; BuildGeneration checks // it against the file size before trusting it. func ReadDBHeader(path string) (pageSize uint32, pageCount uint32, err error) { f, err := os.Open(path) if err != nil { return 0, 0, err } defer f.Close() var hdr [sqliteHeaderSize]byte if _, err := io.ReadFull(f, hdr[:]); err != nil { return 0, 0, fmt.Errorf("read sqlite header of %s: %w", path, err) } if !bytes.Equal(hdr[:len(sqliteMagic)], sqliteMagic) { return 0, 0, fmt.Errorf("%s is not a SQLite database", path) } pageSize = uint32(binary.BigEndian.Uint16(hdr[16:18])) if pageSize == 1 { pageSize = 65536 } if !ltx.IsValidPageSize(pageSize) { return 0, 0, fmt.Errorf("%s has an invalid page size %d", path, pageSize) } pageCount = binary.BigEndian.Uint32(hdr[28:32]) if pageCount == 0 { return 0, 0, fmt.Errorf("%s has no in-header database size", path) } return pageSize, pageCount, nil } // EncodeFull encodes every page of db, pages 1 to pageCount in order, as // one LTX file [min, max] with the NoChecksum flag, and returns the bytes // and the trailer's CRC-64 file checksum. SQLite's lock page // (ltx.LockPgno(pageSize), the page holding PENDING_BYTE) is skipped: the // encoder refuses it, litestream never ships it, and the decoder writes it // back as zeros; Commit keeps the full page count. With min 1 the file is // a snapshot; with min == max == N and walOffset litestream.WALHeaderSize // it is the anchor litestream's open reads its position from (ยง5.1 step // 7). The header timestamp is an input, so the same inputs give the same // bytes, which is what lets a replayed PUT repeat its object. The bytes // are read back and verified before they are returned. func EncodeFull(db io.ReaderAt, pageSize, pageCount uint32, min, max ltx.TXID, timestamp int64, walOffset int64) (data []byte, crc64 uint64, err error) { if pageCount == 0 { return nil, 0, errors.New("moves: cannot encode an empty database") } var buf bytes.Buffer enc, err := ltx.NewEncoder(&buf) if err != nil { return nil, 0, err } if err := enc.EncodeHeader(ltx.Header{ Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: pageSize, Commit: pageCount, MinTXID: min, MaxTXID: max, Timestamp: timestamp, WALOffset: walOffset, }); err != nil { return nil, 0, fmt.Errorf("moves: encode header: %w", err) } lock := ltx.LockPgno(pageSize) page := make([]byte, pageSize) for pgno := uint32(1); pgno <= pageCount; pgno++ { if pgno == lock { continue } if _, err := db.ReadAt(page, int64(pgno-1)*int64(pageSize)); err != nil { return nil, 0, fmt.Errorf("moves: read page %d: %w", pgno, err) } if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, page); err != nil { return nil, 0, fmt.Errorf("moves: encode page %d: %w", pgno, err) } } if err := enc.Close(); err != nil { return nil, 0, fmt.Errorf("moves: close encoder: %w", err) } data = buf.Bytes() dec := ltx.NewDecoder(bytes.NewReader(data)) if err := dec.Verify(); err != nil { return nil, 0, fmt.Errorf("moves: verify encoded %s: %w", ltx.FormatFilename(min, max), err) } if dec.Trailer().FileChecksum != enc.Trailer().FileChecksum { return nil, 0, fmt.Errorf("moves: encoded %s reads back with another checksum", ltx.FormatFilename(min, max)) } return data, uint64(dec.Trailer().FileChecksum), nil }