package layout import ( "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "reflect" "strings" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // Phase is how far a move got. files: sources are being renamed; row: the // files are at their destinations and the row is being swapped; done: the // row is swapped and only the journal remains. type Phase string const ( PhaseFiles Phase = "files" PhaseRow Phase = "row" PhaseDone Phase = "done" ) // Journal is /data/layout/.json (design §8.2 step 6). It records // presence so recovery can tell an optional artifact that was absent from // one that was lost (R2-7). It is NOT an instruction (§3.3): Validate // re-derives both endpoints from the host and the mode and refuses a // journal that names anything else. type Journal struct { Host string `json:"host"` Mode Mode `json:"mode"` From Endpoint `json:"from"` To Endpoint `json:"to"` Artifacts []JournalArtifact `json:"artifacts"` // Extra is the app-made entries moved with the host (extras.go), by // base name. Absent in journals written before #256, which moved // none. Each is re-derived from the host and the name on recovery // and must pass extraNameOK, so the journal cannot direct a move // PlanFor and ScanExtras would not have chosen. Extra []JournalArtifact `json:"extra,omitempty"` Phase Phase `json:"phase"` WrittenAt string `json:"written_at"` } // extraNames is the recorded extra entries' names, in journal order. func (j *Journal) extraNames() []string { var names []string for _, e := range j.Extra { names = append(names, e.Name) } return names } // JournalArtifact records one artifact's presence at the time the journal // was written. The path itself is not stored: it is re-derived by // PlanFor from the host and mode on every read, so a journal can never be // used to redirect a rename anywhere PlanFor itself would not have chosen. type JournalArtifact struct { Name string `json:"name"` Present bool `json:"present"` } // ErrJournalExists is WriteJournal's answer when a journal is already on // disk for the host. var ErrJournalExists = errors.New("layout: a journal already exists for this host") // journalMax bounds how much of a journal file ReadJournal will read. A // journal is a handful of paths and booleans; anything past this is either // corrupt or something else entirely. const journalMax = 64 << 10 func journalDir(dataDir string) string { return filepath.Join(dataDir, "layout") } // syncDir fsyncs a directory. A file's own Sync makes its bytes durable, // but the directory entry that gives those bytes a name (a hard link, a // rename) is separate metadata: on ext4 with delayed allocation, a crash // between an otherwise-successful Link or Rename and this call can still // lose the name while the bytes it would have pointed to survive under // their old one. func syncDir(dir string) error { d, err := os.Open(dir) if err != nil { return err } defer d.Close() return d.Sync() } // WriteJournal creates the journal exactly once. An existing file, whoever // wrote it, is a refusal: the pass never overwrites a record of a move it // did not make. // // It writes the content to a temp file, syncs that file, then links it to // the final name: os.Link fails with EEXIST exactly as O_EXCL would, // without ever making a partly written journal visible at the final path. // Without the temp+sync+link, an O_EXCL create-and-write can leave an // empty file at the final name durable while its content is not (a power // loss, or a kill between OpenFile and Write). Recovery then finds a // journal with no content, refuses it, and the host stays fenced with // nothing to read to explain why. func WriteJournal(dataDir string, j *Journal) error { dir := journalDir(dataDir) if err := os.MkdirAll(dir, 0o700); err != nil { return err } if j.WrittenAt == "" { j.WrittenAt = time.Now().UTC().Format(time.RFC3339) } tmpName, err := writeTemp(dir, j) if err != nil { return err } linkErr := os.Link(tmpName, provision.LayoutJournalPath(dataDir, j.Host)) // The temp file's only job was to become the final name; whether that // succeeded or not, its own name must not linger (nothing reads a // ".tmp-" name, but a stray one left after a failed Link would look // like leftover corruption to anyone auditing the directory by hand). os.Remove(tmpName) if linkErr != nil { if errors.Is(linkErr, os.ErrExist) { return ErrJournalExists } return linkErr } return syncDir(dir) } // ReadJournal reads the host's journal without following a symlink and // without blocking on a special file: /data/layout is agent-owned, but a // flat tenant shares the agent's uid during the transition and can plant // things there. func ReadJournal(dataDir, host string) (*Journal, bool, error) { path := provision.LayoutJournalPath(dataDir, host) f, err := os.OpenFile(path, os.O_RDONLY|syscallNofollow|syscallNonblock, 0) if errors.Is(err, os.ErrNotExist) { return nil, false, nil } if err != nil { return nil, false, err } defer f.Close() fi, err := f.Stat() if err != nil { return nil, false, err } if !fi.Mode().IsRegular() { return nil, false, fmt.Errorf("layout: %s is not a regular file", path) } raw, err := io.ReadAll(io.LimitReader(f, journalMax+1)) if err != nil { return nil, false, err } if len(raw) > journalMax { return nil, false, fmt.Errorf("layout: %s is larger than %d bytes", path, journalMax) } var j Journal if err := json.Unmarshal(raw, &j); err != nil { return nil, false, fmt.Errorf("layout: %s: %w", path, err) } if j.Host != host { return nil, false, fmt.Errorf("layout: %s: content names host %q, path names %q", path, j.Host, host) } return &j, true, nil } // UpdatePhase rewrites the journal with a new phase through a temp file and // rename in the same directory, so a crash leaves either the old phase or // the new one, never a torn file. It marshals a copy carrying the new // phase, not j itself, so a caller that inspects j after an error sees the // phase the journal on disk actually has, not one that was never made // durable. func UpdatePhase(dataDir string, j *Journal, p Phase) error { next := *j next.Phase = p dir := journalDir(dataDir) name, err := writeTemp(dir, &next) if err != nil { return err } if err := os.Rename(name, provision.LayoutJournalPath(dataDir, j.Host)); err != nil { os.Remove(name) return err } // Only now, with the rename itself having succeeded, does the phase // this call promised become true of both the in-memory journal and // the one on disk. j.Phase = p return syncDir(dir) } // writeTemp writes j to a fresh temp file in dir and syncs it, returning // its name. The sync comes before the caller links or renames the file to // the journal's canonical name: that makes these bytes visible there, so // they must already be durable, or a crash just after could leave the name // pointing at content that never landed. func writeTemp(dir string, j *Journal) (string, error) { raw, err := json.MarshalIndent(j, "", " ") if err != nil { return "", err } tmp, err := os.CreateTemp(dir, "."+j.Host+".tmp-*") if err != nil { return "", err } name := tmp.Name() if _, err := tmp.Write(append(raw, '\n')); err != nil { tmp.Close() os.Remove(name) return "", err } if err := tmp.Sync(); err != nil { tmp.Close() os.Remove(name) return "", err } if err := tmp.Close(); err != nil { os.Remove(name) return "", err } return name, nil } // ReplaceJournal hands a host over from the journal the caller holds to a // new one in a single rename, so the host is never without a journal: a // crash, or a failure anywhere here, leaves either the whole old journal // or the whole new one at the canonical name. Removing the old one and // then creating the new one would leave a gap in which nothing fences the // host, and a same-uid tenant that can make /layout unwritable at // the right moment could hold that gap open. // // The file replaced must be exactly held, as ReadJournal reads it back; // anything else there is refused and left alone. No exclusive create is // needed on the final name: the caller holds both the host's fence and // the section config sync is locked out of, and it has just checked what // it is replacing. func ReplaceJournal(dataDir string, held, next *Journal) error { if next.Host != held.Host { return fmt.Errorf("layout: replacing %s's journal with one for %s", held.Host, next.Host) } cur, found, err := ReadJournal(dataDir, held.Host) if err != nil { return err } if !found { return fmt.Errorf("layout: %s: no journal to replace", held.Host) } if !reflect.DeepEqual(cur, held) { return fmt.Errorf("layout: %s: the journal on disk is not the one this pass holds; not replacing it", held.Host) } if next.WrittenAt == "" { next.WrittenAt = time.Now().UTC().Format(time.RFC3339) } dir := journalDir(dataDir) name, err := writeTemp(dir, next) if err != nil { return err } if err := os.Rename(name, provision.LayoutJournalPath(dataDir, next.Host)); err != nil { os.Remove(name) return err } return syncDir(dir) } // RemoveJournal deletes the host's journal. Removing an absent journal is // not an error: callers reach this after a move is already complete, and a // second call (a retried recovery, say) must be idempotent. func RemoveJournal(dataDir, host string) error { err := os.Remove(provision.LayoutJournalPath(dataDir, host)) if errors.Is(err, os.ErrNotExist) { return nil } return err } // JournalHosts names every host with a journal in /layout, in the // directory's order. Dot names are WriteJournal's and UpdatePhase's temp // files, and are not journals. A directory that is not there holds none. func JournalHosts(dataDir string) ([]string, error) { entries, err := os.ReadDir(journalDir(dataDir)) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } var hosts []string for _, e := range entries { if n := e.Name(); !strings.HasPrefix(n, ".") && strings.HasSuffix(n, ".json") { hosts = append(hosts, strings.TrimSuffix(n, ".json")) } } return hosts, nil } // JournalPresent reports whether a journal exists for host, without // reading it. Later tasks use this everywhere a caller needs to know a // host is "fenced" by an in-flight move but has no reason to read the // journal's contents (design §8.1's LayoutFenced, SyncExec's skip, the // litestream fenced hook). An Lstat error other than "not exist" (a // permission problem, say) is treated as present: silently proceeding // past a journal this process cannot even stat is worse than refusing. func JournalPresent(dataDir, host string) bool { return provision.LayoutJournalExists(dataDir, host) } // Validate is the recovery check of design §8.2: the journal's endpoints // must be exactly what PlanFor derives for its host and mode, and the row // must sit at one of them. It returns whether the row is at `from` (the // CAS has not run) or at `to` (it has). func Validate(j *Journal, dataDir, socketDir string, row registry.Route) (atFrom bool, err error) { if j.Host != row.Host { return false, fmt.Errorf("layout: journal is for %s, row is %s", j.Host, row.Host) } switch j.Phase { case PhaseFiles, PhaseRow, PhaseDone: default: return false, fmt.Errorf("layout: %s: journal phase %q is not files, row or done; not acting", j.Host, j.Phase) } p, err := PlanFor(dataDir, socketDir, j.Host, j.Mode) if err != nil { return false, err } if j.From != p.From || j.To != p.To { return false, fmt.Errorf("layout: %s: journal endpoints %+v -> %+v are not the derived %+v -> %+v; not acting", j.Host, j.From, j.To, p.From, p.To) } if len(j.Artifacts) != len(artifactNames) { return false, fmt.Errorf("layout: %s: journal lists %d artifacts, expected %d", j.Host, len(j.Artifacts), len(artifactNames)) } for i, a := range j.Artifacts { if a.Name != artifactNames[i] { return false, fmt.Errorf("layout: %s: journal artifact %d is %q, expected %q", j.Host, i, a.Name, artifactNames[i]) } } // The db is the one artifact recovery cannot proceed without: every // other artifact missing is a smaller, optional file the pass would // simply skip, but a journal claiming its db was never even there // describes a move that cannot have happened. if !j.Artifacts[0].Present { return false, fmt.Errorf("layout: %s: journal's db artifact is not marked present; not acting", j.Host) } // The extra names pass the same rule a scan applies, and an extra is // never absent: the plan carries one because it was seen. if err := p.AddExtras(j.extraNames()); err != nil { return false, fmt.Errorf("%w; not acting", err) } for _, e := range j.Extra { if !e.Present { return false, fmt.Errorf("layout: %s: journal records extra %q absent; not acting", j.Host, e.Name) } } switch (Endpoint{DB: row.DBPath, Socket: row.Socket}) { case j.From: return true, nil case j.To: return false, nil } return false, fmt.Errorf("layout: %s: row holds db %q socket %q, at neither journal endpoint; not acting", j.Host, row.DBPath, row.Socket) }