package hostfiles import ( "context" "errors" "fmt" "log" "path" "strings" "time" ) type Options struct { Deadline time.Duration // first unpublished modification to publish Scan time.Duration // repair scan while every directory is watched Poll time.Duration // full-scan period while watch coverage is missing Skip func(rel string, isDir bool) bool OnStats func(SyncStats) // called after each full scan; may be nil } // DefaultOptions are region HA spec 1 ยง5.6's values. func DefaultOptions() Options { return Options{Deadline: time.Second, Scan: 10 * time.Minute, Poll: time.Second} } type event struct { rel string rescan bool } // watcher reports changes under one directory. Its event channel closes // when it can report nothing more. type watcher interface { Events() <-chan event Degraded() bool // some directory may have no working watch Repair() error // restore full coverage, or say why not Close() error } // Run publishes dir's changes until ctx ends (it returns ctx.Err()), // another writer supersedes the manifest (ErrSuperseded), or the manifest // holds a path past the cap so no commit can succeed (ErrTooDeep). Where // there is no watcher, or the watcher fails, it polls. func Run(ctx context.Context, p *Publisher, dir string, o Options) error { w, err := newWatcher(dir, o.Skip) if err != nil { log.Printf("hostfiles: no watcher for %s (%v): polling every %s", dir, err, o.Poll) return run(ctx, p, dir, o, nil) } defer w.Close() return run(ctx, p, dir, o, w) } func run(ctx context.Context, p *Publisher, dir string, o Options, w watcher) error { def := DefaultOptions() if o.Deadline <= 0 { o.Deadline = def.Deadline } if o.Scan <= 0 { o.Scan = def.Scan } if o.Poll <= 0 { o.Poll = def.Poll } root, err := openTree(dir) if err != nil { return fmt.Errorf("hostfiles: %s: %w", dir, err) } defer root.Close() var events <-chan event if w != nil { events = w.Events() } q := newQueue(o.Deadline) tick := time.NewTicker(o.Deadline / 4) defer tick.Stop() needScan := true // the first pass is a full scan: changes made while nothing watched var lastScan, lastScanTry, lastRepair time.Time coverageLost, scanFailing := false, false // for logging on transitions only loggedTooDeep := false // publishOne's over-cap skip, logged once for { select { case <-ctx.Done(): return ctx.Err() case ev, open := <-events: if !open { // The watcher has stopped for good. A nil channel never // fires, and with no watcher every pass below polls. log.Printf("hostfiles: watcher for %s stopped: polling every %s", dir, o.Poll) events, w = nil, nil continue } if ev.rescan { needScan = true } else { q.Touch(ev.rel, time.Now()) } continue case <-tick.C: } now := time.Now() if w != nil && w.Degraded() { if !coverageLost { coverageLost = true log.Printf("hostfiles: %s: watch coverage is missing: polling every %s until it is restored", dir, o.Poll) } // A repair is a full tree walk: try it once per polling period. if now.Sub(lastRepair) >= o.Poll { lastRepair = now if err := w.Repair(); err == nil { needScan = true // whatever changed while coverage was missing } } } polling := w == nil || w.Degraded() if coverageLost && !polling { coverageLost = false log.Printf("hostfiles: %s: watch coverage restored", dir) } switch { case polling && now.Sub(lastScan) >= o.Poll: needScan = true case !polling && now.Sub(lastScan) >= o.Scan: needScan = true } if needScan && (!scanFailing || now.Sub(lastScanTry) >= o.Poll) { lastScanTry = now st, err := syncRoot(ctx, p, root, o.Skip) if err != nil { if ctx.Err() != nil { return ctx.Err() } if !scanFailing { scanFailing = true log.Printf("hostfiles: %s: scan failing, retrying every %s: %v", dir, o.Poll, err) } } else { if scanFailing { scanFailing = false log.Printf("hostfiles: %s: scan recovered", dir) } needScan, lastScan = false, now if o.OnStats != nil { o.OnStats(st) } } } for _, rel := range q.Due(now) { again, err := publishOne(ctx, p, root, rel, o.Skip) if errors.Is(err, ErrTooDeep) { // Past the path cap: never staged, so never in a manifest a // reader would refuse, and not retried (it cannot shrink). // Logged once per run, so neither the log nor a set of // paths grows with a tenant's supply of them. if !loggedTooDeep { loggedTooDeep = true log.Printf("hostfiles: %s: a path past the %d-byte cap is not published (further ones are skipped without a line): %v", dir, MaxPathBytes, err) } continue } if err != nil { log.Printf("hostfiles: %s: %v", rel, err) q.Touch(rel, now) // try again at the next deadline continue } if again { q.Touch(rel, now) } } if p.Dirty() { if _, err := p.Commit(ctx); errors.Is(err, ErrSuperseded) { return err } else if errors.Is(err, ErrTooDeep) { // The manifest holds a path past the cap: no commit can // ever succeed, so retrying would only spin. One line, and // stop. log.Printf("hostfiles: %s: cannot commit, stopping: %v", dir, err) return err } else if err != nil { log.Printf("hostfiles: commit: %v", err) // staged changes stay; the next tick retries } } } } // publishOne stages one path's current state. again is true when the // caller should read the path again at the next deadline: the file changed // while it was read (its image is published, flagged), or it was replaced // mid-look and nothing was read. // // A tombstone is staged only on proof: the name is gone, or something // that is not a regular file now stands there or above it. A file that // exists but could not be read is an error, and its manifest entry stays. func publishOne(ctx context.Context, p *Publisher, root *dirHandle, rel string, skip func(string, bool) bool) (again bool, err error) { clean, err := CleanPath(rel) if err != nil { return false, nil // a path the watcher should never have produced: skip it } if err := withinTreeCap(clean); err != nil { return false, err // the caller skips it for good: no reader would accept it } if skipped(skip, clean, false) { return false, nil } parent, name := path.Split(clean) dir, gone, err := root.descend(strings.TrimSuffix(parent, "/")) if err != nil { return false, err } if gone { return false, p.StageDelete(clean) } if dir != root { defer dir.Close() } body, mode, state, err := readStable(dir, name) if err != nil { return false, err } switch state { case fileGone, fileNotRegular: // StageDelete is a no-op for a path the manifest never had, which // is what a directory's own event is. return false, p.StageDelete(clean) case fileUnreadable: return false, fmt.Errorf("%s exists but could not be read; keeping what is published", clean) case fileRetry: return true, nil case fileTooLarge: return false, p.StageSkipped(clean, "too-large") } return state == fileTorn, p.Stage(ctx, clean, body, mode, state == fileTorn) }