package host import ( "context" "encoding/json" "errors" "fmt" "log" "os" "path/filepath" "strings" "sync" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" ) // relocationOpJournals are the files under relocate/ops// that name // their host inside: the install and attempt journals and the probe stop's // fence owner. var relocationOpJournals = []string{"install.json", "attempts.json", "probe-fence.json"} // relocationStraySaid throttles the sweep's line about an entry under // relocate/ that is no relocation journal to once per path per process. var relocationStraySaid sync.Map func sayRelocationStray(path string) { if _, said := relocationStraySaid.LoadOrStore(path, true); !said { log.Printf("relocation sweep: %s is not a relocation journal; left in place for an operator", path) } } // sweepRelocationJournals is the lifecycle half of N9 (ruling 61). Every // region HA journal under /relocate names its op in its path and // its host inside (the build scratch names its host in its file name): the // install, attempt and probe-fence journals under ops//, the release // positions at release/.json and the build's scratch under build//. // Each is removed once the host's claim no longer runs that op: replaced // by a successor, the claim gone, or completed and marked discharged (a // completed op's journals stay until that mark, ruling 76). While the op // runs, every one of them stays. // // A journal that cannot be read is left in place and reported, and so is // a claim whose completed relocation cannot be read: both are // census-opaque, so they block nothing, and absence is never evidence. // What no committed journal can be is removed: an ops directory holding // no host-bearing journal, an atomic write's leftover temporary under // release/, a build directory left empty. An entry the sweep does not know // is left in place and logged once. // // It runs in two halves (Task 10 review M6). The first reads the // directories and the claims with no lock held, and decides. The second // holds relocationJournals exclusively (carry 3), so it never runs beside // a relocation journal write in this process (the pass's own steps write // in turn, the wake's probe attempts write attempts.json whenever a probe // runs), and removes only what it finds, read again under the lock, as it // was decided: the same host's journals for an op that is over, or still // no committed journal at all. A claim's "the op is over" never reverts // (op ids are never reused), so a decision read before the lock stands; // the local files are what a write can change, and they are read again. // // The build's scratch (relocationbuild.go, MkdirAll then BuildGeneration) // is written outside the lock, by Participant.Step on the pass's own // goroutine, so it is serialized with the sweep by program order. A build // moved off that goroutine would need the lock too. func sweepRelocationJournals(ctx context.Context, dataDir string, claims release.Store) error { root := filepath.Join(dataDir, "relocate") var errs []error gone := func(host, op string) (bool, error) { obs, err := hostclaim.Read(ctx, claims, host) if err != nil { return false, err } if obs.Absent || obs.Claim == nil { return true, nil } // Ruling 76: a completed relocation's journals, the probe stop's // fence owner among them, stay until the box it completed on has // discharged, released its hold, lifted its fence and marked it. // A record that cannot be read keeps every journal: it may be this // op's, and absence is never evidence. done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) if err != nil { return false, fmt.Errorf("%s: %w", host, err) } if ok && done.Op == op && !done.Discharged { return false, nil } return obs.Claim.Op.ID != op, nil } // A removal decided unlocked: check reads the entry again under the // lock and answers whether it is still as decided. type removal struct { path string all bool // RemoveAll, not Remove check func() bool } var removals []removal // ops//{install,attempts,probe-fence}.json ops, err := os.ReadDir(filepath.Join(root, "ops")) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } for _, e := range ops { dir := filepath.Join(root, "ops", e.Name()) if !e.IsDir() { sayRelocationStray(dir) continue } host, found, readErr := opJournalHost(dir) switch { case host != "": if done, err := gone(host, e.Name()); err != nil { errs = append(errs, err) } else if done { removals = append(removals, removal{path: dir, all: true, check: func() bool { h, _, err := opJournalHost(dir) return err == nil && (h == host || h == "") }}) } case found: errs = append(errs, readErr) // unreadable: kept and reported default: // No host-bearing journal was ever committed here: what is // left is a write's temporary, which no op can need, unless a // write committed one since (read again under the lock). removals = append(removals, removal{path: dir, all: true, check: func() bool { _, found, _ := opJournalHost(dir) return !found }}) } } // release/.json, and an atomic write's leftover temporaries. positions, err := os.ReadDir(filepath.Join(root, "release")) if err != nil && !errors.Is(err, os.ErrNotExist) { return errors.Join(append(errs, err)...) } for _, e := range positions { path := filepath.Join(root, "release", e.Name()) op, isJSON := strings.CutSuffix(e.Name(), ".json") switch { case e.IsDir(): sayRelocationStray(path) case isJSON: host, err := relocationJournalHost(path) if err != nil { errs = append(errs, err) continue } if done, err := gone(host, op); err != nil { errs = append(errs, err) } else if done { removals = append(removals, removal{path: path, check: func() bool { h, err := relocationJournalHost(path) return err == nil && h == host }}) } case relocationWriteLeftover(e.Name()): // Under the lock no write is in flight, so a temporary still // there is a leftover, never a write's. removals = append(removals, removal{path: path, check: func() bool { return true }}) default: sayRelocationStray(path) } } // build//.db and its SQLite sidecars: the scratch of a build // whose op is over. builds, err := os.ReadDir(filepath.Join(root, "build")) if err != nil && !errors.Is(err, os.ErrNotExist) { return errors.Join(append(errs, err)...) } for _, e := range builds { dir := filepath.Join(root, "build", e.Name()) if !e.IsDir() { sayRelocationStray(dir) continue } host, empty, err := buildScratchDirHost(dir) switch { case err != nil: errs = append(errs, err) case host != "": if done, err := gone(host, e.Name()); err != nil { errs = append(errs, err) } else if done { removals = append(removals, removal{path: dir, all: true, check: func() bool { h, _, err := buildScratchDirHost(dir) return err == nil && (h == host || h == "") }}) } case empty: removals = append(removals, removal{path: dir, check: func() bool { _, empty, err := buildScratchDirHost(dir) return err == nil && empty }}) default: sayRelocationStray(dir) } } if len(removals) == 0 { return errors.Join(errs...) } relocationJournals.Lock() defer relocationJournals.Unlock() for _, r := range removals { if _, err := os.Lstat(r.path); errors.Is(err, os.ErrNotExist) { continue } if !r.check() { continue // a write landed since the decision: the next pass decides again } if r.all { errs = append(errs, os.RemoveAll(r.path)) } else { errs = append(errs, os.Remove(r.path)) } } return errors.Join(errs...) } // relocationJournalHost is the host a relocation journal names inside: // "host", or a stop request's "Request.Host" (release positions). func relocationJournalHost(path string) (string, error) { raw, err := os.ReadFile(path) if err != nil { return "", err } var probe struct { Host string `json:"host"` Request struct{ Host string } } if err := json.Unmarshal(raw, &probe); err != nil { return "", fmt.Errorf("%s: %w", path, err) } if probe.Host != "" { return probe.Host, nil } if probe.Request.Host != "" { return probe.Request.Host, nil } return "", fmt.Errorf("%s names no host", path) } // opJournalHost is the host an ops/ directory's journals name: the // first readable one, found when any journal is present at all, and the // read errors of the unreadable ones when none is readable. func opJournalHost(dir string) (host string, found bool, err error) { for _, name := range relocationOpJournals { h, rerr := relocationJournalHost(filepath.Join(dir, name)) if errors.Is(rerr, os.ErrNotExist) { continue } found = true if rerr == nil { return h, true, nil } err = errors.Join(err, rerr) } return "", found, err } // buildScratchDirHost is the host a build/ directory's scratch names, // and whether the directory is empty. func buildScratchDirHost(dir string) (string, bool, error) { files, err := os.ReadDir(dir) if err != nil { return "", false, err } for _, f := range files { if h, ok := buildScratchHost(f.Name()); ok { return h, false, nil } } return "", len(files) == 0, nil } // relocationJournalHosts is every host this box's op journals name // (ops/*/{probe-fence,attempts}.json): the completion-only pass's positive // local evidence of a relocation this box took part in, written before any // completion exists and kept until its mark (ruling 76; Task 10 review // M1). A journal that cannot be read is returned as an error beside the // hosts that could. func relocationJournalHosts(dataDir string) ([]string, error) { ops, err := os.ReadDir(filepath.Join(dataDir, "relocate", "ops")) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } seen := map[string]bool{} var out []string var errs []error for _, e := range ops { if !e.IsDir() { continue } for _, name := range []string{"probe-fence.json", "attempts.json"} { host, err := relocationJournalHost(filepath.Join(dataDir, "relocate", "ops", e.Name(), name)) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { errs = append(errs, err) continue } if !seen[host] { seen[host] = true out = append(out, host) } } } return out, errors.Join(errs...) } // relocationJournals serializes the journal sweep against every relocation // journal write in this process (carry 3). A writer holds it shared from // creating the journal's directory until its atomic rename has landed; the // sweep holds it exclusively. Without it the sweep could find an ops // directory between the writer's MkdirAll and its rename, holding no // committed journal, and remove it under the write. var relocationJournals sync.RWMutex // writeRelocationJournal is every relocation journal write under // /relocate: the directory, then moves' writeFileAtomic, under // relocationJournals shared. func writeRelocationJournal(path string, raw []byte) error { relocationJournals.RLock() defer relocationJournals.RUnlock() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return err } return writeFileAtomic(path, raw, 0o600) } // relocationWriteLeftover is a temporary an interrupted atomic write left // under release/: writeFileAtomic's "..json.", or a fixed // ".json.tmp". func relocationWriteLeftover(name string) bool { return strings.HasSuffix(name, ".json.tmp") || (strings.HasPrefix(name, ".") && strings.Contains(name, ".json.")) } // buildScratchHost is the host a build scratch file names: .db, or // one of SQLite's sidecars beside it (-wal, -shm, -journal). func buildScratchHost(name string) (string, bool) { for _, sidecar := range []string{"-wal", "-shm", "-journal"} { if base, ok := strings.CutSuffix(name, sidecar); ok { name = base break } } host, ok := strings.CutSuffix(name, ".db") return host, ok && host != "" }