package layout import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/controlname" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // Fence is what the pass needs from the activator (design §8.2 step 2): // stop the host and refuse wakes for it, but only when no other operation // already holds the host's one fence slot (FenceAndStopIfUnfenced); let it // wake again (LiftFence, at exactly the generation this pass installed); // and the two reads that decide whether an unproven stop can be trusted. type Fence interface { FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*activator.StopEvidence, error) LiftFence(host string, generation uint64) bool FenceGeneration(host string) uint64 Quiesced(host string) bool StopUncertain(host string) bool } // IntentUnreadable is the intent Recover is given when the agent could not // read the fleet's layout intent at all, as distinct from "" (no intent was // ever recorded). A revert completed under it is held fenced; see // recoverHost. const IntentUnreadable Mode = "unreadable" // Pass is the layout pass for one box. It runs one Converge or Recover at // a time; the agent's single-flight is what guarantees that. type Pass struct { Registry *registry.Registry // Fence is the activator. Nil means there is none (tests): the pass // still journals and moves. Fence Fence DataDir string SocketDir string // "" means provision.SocketDir SocketsGID int // the carlos-sockets group; -1 leaves the group alone // Reload asks routing to pick up a flipped row now rather than on the // next 2s poll. Nil relies on the poll. Reload func() Now func() time.Time Logf func(string, ...any) Generation func() uint64 // monotonic fence generations; default time.Now().UnixNano() // SweptAtStart reports that the agent's startup unit sweep succeeded, // so no instance from an earlier agent can still be running. Without // it an unproven stop is never trusted. SweptAtStart bool // NestedOK is the isolation probe's gate (design §8.5): a box that // cannot establish the namespace does not start nested tenants, so it // does not make any either. Nil means ungated. NestedOK func() error // Serialize, when set, runs the span from journal write to journal // removal under the lock config sync holds, so sync never renders a // host's bundle into the directory the pass is moving it out of. Serialize func(func()) // DeferAwake leaves a host that is awake for a later pass, before any // fence is asked for, rather than stopping it to move it. The design // migrates a row that becomes eligible "when it is asleep", and the // agent's serial-move leg runs while tenants are serving; recovery is // not affected, because a host with a journal cannot be woken. // // The deferral is bounded by AwakeGrace. A flat tenant that never // idles would otherwise defer its own migration for as long as it // liked, and while it runs flat it has the agent's uid and so writes // to every nested sibling's raw directory under /data/.instances: // the box is not isolated until no flat tenant remains on it. Once a // host has been seen awake for the grace period, the pass stops it // the way it stops any host (the fence, §8.2 step 2) and moves it; it // wakes nested on its next request. DeferAwake bool // AwakeGrace is how long an awake host is deferred before the pass // stops it anyway; zero means DefaultAwakeGrace. AwakeGrace time.Duration // awakeSince is when each deferred host was first seen awake under // the current direction. In memory only: an agent restart stops every // tenant, so afterwards they are asleep and migrate on the next pass. awakeMu sync.Mutex awakeSince map[string]time.Time // kept is the generation of each fence this pass kept installed after // a failed handover, so a later pass knows that fence as its own // rather than another operation's. Cleared when the fence is lifted. keptMu sync.Mutex kept map[string]uint64 // held is the reason each host whose nested move was refused after // it had been stopped is kept fenced and down (see migrateHost). It // is cleared when the move goes through, or when the intent turns // flat and the fence is lifted. held map[string]string // notes is the last thing worth telling an operator about each host // that was not a refusal: what a move quarantined or kept aside. It // is an event for status, in memory only; the log carries it too. notes map[string]string // abortAt is a test seam: the step name after which a move returns as // if the agent had died there, leaving its journal for Recover. abortAt string } // Result is what the pass says about one host. Outcome is one of // migrated, reverted, recovered, unchanged, skipped, deferred, refused or // held (refused after the host was stopped for a nested move, and kept // fenced and down; see migrateHost). A Result with an empty Host is about // the whole pass. type Result struct { Host string Outcome string Reason string } // quiesceTimeout bounds one host's stop. const quiesceTimeout = 60 * time.Second // DefaultAwakeGrace is how long an awake host may defer its own move. An // hour is longer than any idle window the activator hibernates on, so a // tenant that is ever quiet in that time moves while asleep as the design // prefers, and one that is never quiet is stopped once, briefly, rather // than left flat beside nested siblings for good. const DefaultAwakeGrace = time.Hour // deferAwake reports whether host, found awake, is still within its // grace, and how much of the grace is left. func (p *Pass) deferAwake(host string) (bool, time.Duration) { grace := p.AwakeGrace if grace <= 0 { grace = DefaultAwakeGrace } p.awakeMu.Lock() defer p.awakeMu.Unlock() if p.awakeSince == nil { p.awakeSince = map[string]time.Time{} } now := p.now() since, seen := p.awakeSince[host] if !seen { p.awakeSince[host] = now return true, grace } if left := grace - now.Sub(since); left > 0 { return true, left } return false, 0 } // forgetAwake clears a host's grace clock once the pass has dealt with it // in some other way than deferring it. func (p *Pass) forgetAwake(host string) { p.awakeMu.Lock() defer p.awakeMu.Unlock() delete(p.awakeSince, host) } func (p *Pass) now() time.Time { if p.Now != nil { return p.Now() } return time.Now() } func (p *Pass) logf(format string, a ...any) { if p.Logf != nil { p.Logf(format, a...) } } func (p *Pass) generation() uint64 { if p.Generation != nil { return p.Generation() } return uint64(time.Now().UnixNano()) } func (p *Pass) socketDir() string { if p.SocketDir != "" { return p.SocketDir } return provision.SocketDir } func (p *Pass) reload() { if p.Reload != nil { p.Reload() } } func (p *Pass) serialize(f func()) { if p.Serialize != nil { p.Serialize(f) return } f() } func (p *Pass) layoutInputs(routes []registry.Route) activator.LayoutInputs { return activator.LayoutInputs{ DataDir: p.DataDir, SocketDir: p.socketDir(), Journal: func(host string) bool { return JournalPresent(p.DataDir, host) }, Peers: func() ([]registry.Route, error) { return routes, nil }, } } // Classify is the box's one classification, shared with the status // snapshot so both say the same thing about a row. func (p *Pass) Classify(rt registry.Route, peers []registry.Route) (activator.Layout, string) { return activator.Classify(rt, p.layoutInputs(peers)) } func deadline(ctx context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(ctx, quiesceTimeout) } // ready is the status a row must have for the pass to move it. Any other // status belongs to an operation in flight on the row, the residence // move's "moving" among them, and a row about to leave the box is not // worth moving. func ready(rt registry.Route) bool { return rt.Status == "" || rt.Status == "ready" } // Converge runs Recover, then moves every managed row that is not already // in mode (design §8.2 steps 1 to 9). Rows are visited in host order so a // log reads the same way twice. A host Recover reported is not visited // again: recovery already converged it, or refused it. // // Recovery runs first and ungated, so a failing isolation probe never // leaves a crashed move fenced; its legs toward nested ask the gate. The // fresh moves to nested wait for the gate as a whole. The gate is asked at // most once per pass, and both share the answer. func (p *Pass) Converge(ctx context.Context, mode Mode) []Result { if mode != ModeNested && mode != ModeFlat { return []Result{{Outcome: "refused", Reason: fmt.Sprintf("layout mode %q is not nested or flat", mode)}} } gate := p.nestedGate() out := p.recover(ctx, mode, gate) if mode == ModeNested { if err := gate(); err != nil { return append(out, Result{Outcome: "refused", Reason: "isolation not ready: " + err.Error()}) } } seen := map[string]bool{} for _, r := range out { seen[r.Host] = true } routes, err := p.Registry.List() if err != nil { p.logf("layout: converge: list routes: %v", err) return append(out, Result{Outcome: "refused", Reason: "list routes: " + err.Error()}) } sort.Slice(routes, func(i, j int) bool { return routes[i].Host < routes[j].Host }) migrated := false for _, rt := range routes { if !activator.ActivatorManaged(rt) || seen[rt.Host] { continue } if ctx.Err() != nil { break } r := p.convergeHost(ctx, rt, routes, mode) migrated = migrated || r.Outcome == "migrated" out = append(out, r) } if migrated { // What no host's move could take (extras.go): a flat app with cwd // / wrote relative paths straight into the data directory, and a // name no row owns cannot be given to one host. Said once per pass // that moved something, for the operator who has to place them. if left := UnattributedEntries(p.DataDir, hostsOf(routes)); len(left) > 0 { p.logf("layout: left in %s, owned by no host on this box: %s. a nested instance cannot see them; move what an app wrote into that app's instance directory by hand", p.DataDir, strings.Join(left, ", ")) } } return out } func (p *Pass) convergeHost(ctx context.Context, rt registry.Route, routes []registry.Route, mode Mode) Result { if !ready(rt) { return Result{rt.Host, "deferred", "status " + rt.Status} } layout, reason := p.Classify(rt, routes) switch layout { case activator.LayoutFenced, activator.LayoutConflict: return Result{rt.Host, "refused", reason} case activator.LayoutForeign: return Result{rt.Host, "skipped", "foreign: " + reason} case activator.LayoutNested: if mode == ModeNested { p.forgetAwake(rt.Host) // A host moved before the pass took the app's own entries // with it (#256) has them still in the data directory, // where its namespace cannot see them. Nothing moves here: // the host is running, and what the app has minted since // may be the copy it wants. Said on every pass, so an // operator reading status or the log sees it while it lasts. if left := FlatLeftovers(p.DataDir, rt.Host, hostsOf(routes)); len(left) > 0 { text := fmt.Sprintf("still in %s from before its move, where the instance cannot see them: %s. move what the app needs into %s by hand", p.DataDir, strings.Join(left, ", "), rawInstanceDir(p.DataDir, rt.Host)) p.logf("layout: %s: %s", rt.Host, text) p.note(rt.Host, text) } return Result{rt.Host, "unchanged", ""} } case activator.LayoutFlat: if mode == ModeFlat { p.forgetAwake(rt.Host) p.releaseHeld(rt.Host) return Result{rt.Host, "unchanged", ""} } } if p.DeferAwake && p.Fence != nil && !p.Fence.Quiesced(rt.Host) { if within, left := p.deferAwake(rt.Host); within { return Result{rt.Host, "deferred", fmt.Sprintf("awake (stopped for the move in %s unless it sleeps first)", left.Round(time.Second))} } p.logf("layout: %s: awake past its %s grace -> stopping it for the move", rt.Host, p.grace()) } p.forgetAwake(rt.Host) return p.migrateHost(ctx, rt.Host, routes, mode) } func (p *Pass) grace() time.Duration { if p.AwakeGrace > 0 { return p.AwakeGrace } return DefaultAwakeGrace } // migrateHost is steps 2 to 9 for one host. It checks what it can before // stopping anything, fences and stops the host, and releases it only after // the move is committed and routing has reloaded, or has been refused. func (p *Pass) migrateHost(ctx context.Context, host string, routes []registry.Route, mode Mode) Result { plan, err := PlanFor(p.DataDir, p.socketDir(), host, mode) if err != nil { return Result{host, "refused", err.Error()} } plan.Logf = p.logf if err := Preflight(plan, routes); err != nil { return Result{host, "refused", err.Error()} } // A host whose database is not on this box cannot move. Finding that // out must not cost it a stop, and the pass directories and a // destination check, on every pass. The same check under the fence // stays the authority. if db := plan.Artifacts[0]; db.Required { if ok, err := lstatPresent(db.Src); err != nil || !ok { return Result{host, "refused", fmt.Sprintf("db is missing at %s", db.Src)} } } r, fence, gen := p.quiesce(ctx, host, p.generation()) if fence != fenceOurs { return r } var hold bool r, _, hold = p.shift(host, mode, nil) // Fail closed on the way to nested. A flat host that has been stopped // for its move does not run flat again because a tenant kept the move // refused: a second name for its database, a file in its own future // directory, a symlink at it, each one syscall from inside, and a host // released here would wake flat on its next request and keep its // raw-path reach over its nested siblings for as long as it kept the // refusal alive. So the fence stays, the host stays down, the reason // is kept for status, and every later pass retries from the kept // fence: when the reason is gone the host moves and is released; when // the fleet's intent turns flat, convergeHost lifts the fence and the // host runs flat as asked. Refusals that come from the row (an // operation in flight, an unflag) are not a tenant's doing and lift as // before, so a residence move is never deadlocked on this fence. A // reverse move is never held: a nested host refused its revert is // already isolated, and leaving it running is the safe side. if hold && mode == ModeNested { p.keep(host, gen) p.hold(host, r.Reason) p.logf("layout: %s: held -> %s", host, r.Reason) return Result{host, "held", r.Reason + "; held fenced and down until the reason is cleared or the layout intent is flat"} } p.release(host, gen) return r } // hold records why host is kept fenced and down after a refused nested // move; Held is how status reads it. The reason is also written to a hold // marker in the layout directory, so an agent restart (every carlos // system update) can restore the hold before anything wakes // (RestoreHolds) instead of handing the tenant a fresh grace. // // The marker is uid carlos's file like everything under /data, so a flat // tenant can unlink it. That buys the tenant nothing durable: the marker // is a hint to re-check, not the authority. At start the pass re-derives // the hold from the files themselves (the planted link or symlink is the // reason, and it is the thing the tenant would have to remove to be rid // of the hold, which is the outcome wanted), so a deleted marker costs // one awake grace after one restart, after which the next pass meets the // same refusal and holds again. A marker whose reason is gone is dropped // at start rather than honoured, for the same reason in reverse. func (p *Pass) hold(host, reason string) { p.keptMu.Lock() if p.held == nil { p.held = map[string]string{} } p.held[host] = reason p.keptMu.Unlock() if err := writeHoldMarker(p.DataDir, host, reason, p.now()); err != nil { p.logf("layout: %s: hold marker: %v (the hold stands in memory until the next restart)", host, err) } } // note records an event for status; Note reads it. func (p *Pass) note(host, text string) { p.keptMu.Lock() defer p.keptMu.Unlock() if p.notes == nil { p.notes = map[string]string{} } p.notes[host] = text } // Note is the last event the pass recorded for host that was not a // refusal: what a move quarantined or kept aside, and when. func (p *Pass) Note(host string) string { p.keptMu.Lock() defer p.keptMu.Unlock() return p.notes[host] } // holdMarker is the on-disk record of a hold: /layout/.hold. // The suffix is not .json, so JournalHosts and the downgrade guard, which // list journals by that suffix, never take a marker for a journal. type holdMarker struct { Host string `json:"host"` Reason string `json:"reason"` Since time.Time `json:"since"` } func holdMarkerName(host string) string { return host + ".hold" } func writeHoldMarker(dataDir, host, reason string, at time.Time) error { if _, err := instancefs.EnsureDirBeneath(dataDir, filepath.Base(journalDir(dataDir)), 0o700); err != nil { return err } raw, err := json.Marshal(holdMarker{Host: host, Reason: reason, Since: at.UTC()}) if err != nil { return err } return instancefs.WriteAtomic(journalDir(dataDir), holdMarkerName(host), raw, 0o600) } func removeHoldMarker(dataDir, host string) { if err := instancefs.RemoveBeneath(journalDir(dataDir), holdMarkerName(host)); err != nil && !errors.Is(err, os.ErrNotExist) { // Best effort: a marker that outlives its hold is dropped at the // next start when its reason does not reproduce. _ = err } } // HoldMarkerPresent reports whether host has a hold marker, without reading // it. Lstat, like JournalPresent, and an error other than "not exist" // counts as present: a marker this process cannot stat is not proof there // is none. func HoldMarkerPresent(dataDir, host string) bool { _, err := os.Lstat(filepath.Join(journalDir(dataDir), holdMarkerName(host))) return !errors.Is(err, os.ErrNotExist) } // HoldMarkers lists the hosts with a hold marker in the layout directory. func HoldMarkers(dataDir string) ([]string, error) { entries, err := os.ReadDir(journalDir(dataDir)) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } var hosts []string for _, e := range entries { if n := e.Name(); !strings.HasPrefix(n, ".") && strings.HasSuffix(n, ".hold") { hosts = append(hosts, strings.TrimSuffix(n, ".hold")) } } return hosts, nil } // RestoreHolds runs at agent start, after journal recovery and before the // activator can wake anything. For each hold marker it re-derives the // hold from the files: a flat, managed host whose pre-journal checks // still refuse for a reason a tenant can plant is fenced again and held, // so a restart does not hand it a fresh grace to run flat in. A marker // whose reason is gone is removed, and under a flat intent, or none, every // marker is removed: the fleet does not want the host nested. An intent // that could not be read is treated as nested, the fail-closed side. func (p *Pass) RestoreHolds(ctx context.Context, intent Mode) []Result { hosts, err := HoldMarkers(p.DataDir) if err != nil { p.logf("layout: restore holds: %v", err) return []Result{{Outcome: "refused", Reason: "read hold markers: " + err.Error()}} } if len(hosts) == 0 { return nil } sort.Strings(hosts) var out []Result for _, host := range hosts { if intent != ModeNested && intent != IntentUnreadable { removeHoldMarker(p.DataDir, host) out = append(out, Result{host, "released", "hold marker dropped: the layout intent is not nested"}) continue } out = append(out, p.restoreHold(ctx, host)) } return out } func (p *Pass) restoreHold(ctx context.Context, host string) Result { rt, err := p.Registry.Get(host) if err != nil { return Result{host, "refused", "row: " + err.Error()} } routes, err := p.Registry.List() if err != nil { return Result{host, "refused", "list routes: " + err.Error()} } if rt == nil || !activator.ActivatorManaged(*rt) { removeHoldMarker(p.DataDir, host) return Result{host, "released", "hold marker dropped: the row is gone or no longer managed"} } if layout, _ := p.Classify(*rt, routes); layout != activator.LayoutFlat { removeHoldMarker(p.DataDir, host) return Result{host, "released", "hold marker dropped: the row is " + string(layout)} } reason, plantable := p.recheck(host, routes) if !plantable { removeHoldMarker(p.DataDir, host) if reason == "" { return Result{host, "released", "hold marker dropped: its reason is gone; the next pass moves the host"} } return Result{host, "released", "hold marker dropped: " + reason} } _, fence, gen := p.quiesce(ctx, host, p.generation()) if fence != fenceOurs { return Result{host, "deferred", "hold not restored: " + reason} } p.keep(host, gen) p.hold(host, reason) p.logf("layout: %s: held -> %s (restored at start)", host, reason) return Result{host, "held", reason + "; restored at start"} } // recheck runs the pre-journal checks a nested move would run for host // and reports the first refusal and whether it is one a tenant can plant // (the same classes shift holds on). An empty reason means the move // would go ahead. func (p *Pass) recheck(host string, routes []registry.Route) (reason string, plantable bool) { plan, err := PlanFor(p.DataDir, p.socketDir(), host, ModeNested) if err != nil { return err.Error(), false } plan.Logf = p.logf if err := Preflight(plan, routes); err != nil { return err.Error(), false } if err := EnsureDirs(p.DataDir, p.socketDir(), host, p.SocketsGID); err != nil { return err.Error(), true } if _, err := presence(plan, hostsOf(routes)); err != nil { return err.Error(), true } return "", false } // hostsOf is every host with a row on the box, whatever its kind: the set // an entry's owner is chosen from (extras.go). func hostsOf(routes []registry.Route) []string { hosts := make([]string, 0, len(routes)) for _, rt := range routes { hosts = append(hosts, rt.Host) } return hosts } // Held reports whether the pass keeps host fenced and down after a // refused nested move, and why. The fence must still be the pass's own: // a record the activator no longer matches is dropped. func (p *Pass) Held(host string) (string, bool) { if _, ok := p.keptFence(host); !ok { p.keptMu.Lock() delete(p.held, host) p.keptMu.Unlock() return "", false } p.keptMu.Lock() defer p.keptMu.Unlock() reason, ok := p.held[host] return reason, ok } // releaseHeld lifts a fence the pass kept for host after a refused nested // move, because the fleet now wants the host flat. func (p *Pass) releaseHeld(host string) { if gen, ok := p.keptFence(host); ok { p.logf("layout: %s: released -> the layout intent is flat; the held fence is lifted", host) p.release(host, gen) } } // fenceState is who holds a host's fence once quiesce returns. type fenceState int const ( // fenceOurs: this pass installed the fence and the stop is proven or // trusted, or it is the fence an earlier pass kept after a failed // handover, which has held the host stopped ever since. fenceOurs fenceState = iota // fenceHeld: another operation's fence holds the host; the pass // installed nothing and must lift nothing. fenceHeld // fenceNone: the pass installed nothing, or lifted what it installed, // and the host is not to be touched. fenceNone ) // quiesce installs this pass's fence, unless another operation's fence // already holds the host, and decides whether the host is proven stopped. // The residence move shares the activator's one fence slot per host, so // installing over its fence and later lifting ours would reopen admission // the move closed; the exclusive install refuses instead, atomically. An unproven stop is trusted only when nothing can be // running: no live entry, no stop on record that failed to finish, and a // startup sweep that stopped anything an earlier agent left. After an // agent restart no asleep host has a stop on record, which is why the // entry-less case needs this rather than a proof. Every refusal lifts the // fence it installed: a fence left behind would keep a host down with // nothing on its way to release it. // // It returns the generation the host is fenced at when the fence is the // pass's: gen for a fence it just installed, or the generation an earlier // pass kept after a failed handover, which the caller lifts in the end. func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, fenceState, uint64) { if p.Fence == nil { return Result{}, fenceOurs, gen } fctx, cancel := deadline(ctx) defer cancel() _, err := p.Fence.FenceAndStopIfUnfenced(fctx, host, gen) if err == nil { return Result{}, fenceOurs, gen } if errors.Is(err, activator.ErrFenceHeld) { if kept, ok := p.keptFence(host); ok { return Result{}, fenceOurs, kept } p.logf("layout: %s: deferred -> fenced by another operation", host) return Result{host, "deferred", "fenced by another operation"}, fenceHeld, 0 } unproven := errors.Is(err, activator.ErrStopUnproven) if unproven && p.SweptAtStart && p.Fence.Quiesced(host) && !p.Fence.StopUncertain(host) { return Result{}, fenceOurs, gen } p.Fence.LiftFence(host, gen) reason := "stop unproven" if !unproven { reason += ": " + err.Error() } p.logf("layout: %s: deferred -> %s", host, reason) return Result{host, "deferred", reason}, fenceNone, 0 } // keep records a fence this pass leaves installed after a failed handover. func (p *Pass) keep(host string, gen uint64) { p.keptMu.Lock() defer p.keptMu.Unlock() if p.kept == nil { p.kept = map[string]uint64{} } p.kept[host] = gen } // keptFence reports the generation of a fence this pass kept for host, // provided the fence installed now is still exactly that one. A record // the activator no longer matches is dropped, so a later fence of another // operation is never taken for the pass's own. func (p *Pass) keptFence(host string) (uint64, bool) { p.keptMu.Lock() defer p.keptMu.Unlock() gen, ok := p.kept[host] if !ok { return 0, false } if p.Fence == nil || p.Fence.FenceGeneration(host) != gen { delete(p.kept, host) return 0, false } return gen, true } func (p *Pass) release(host string, gen uint64) { if p.Fence != nil { p.Fence.LiftFence(host, gen) } p.keptMu.Lock() defer p.keptMu.Unlock() if p.kept[host] == gen { delete(p.kept, host) if _, held := p.held[host]; held { delete(p.held, host) removeHoldMarker(p.DataDir, host) } } } // shift is steps 3 to 9 for a host this pass already holds fenced and // stopped. It re-reads the row under the fence, because the copy the pass // classified can be stale: a move cut can have started, or the row been // unflagged, re-added or migrated. Every return before the journal is // written leaves nothing changed. // // prior is a completed journal recovery is still holding the host with, // or nil. That journal is this pass's own, so it does not count as the // host being fenced here. It is replaced by the new journal in one rename // (ReplaceJournal), so the host holds one journal or the other at every // instant, and a refusal before the handover leaves it held by prior. The // handover result says which of those happened. // // hold is set on a refusal a tenant could have brought about itself from // inside the box, with a syscall on files the agent's uid owns: a planted // symlink or entry at the instance directory, a planted file at a // destination, a second name for a file about to move. A refusal that // comes from the row (an operation in flight, an unflag, a route-name // collision) or from the pass's own seams is not one, and is not held. func (p *Pass) shift(host string, mode Mode, prior *Journal) (res Result, h handover, hold bool) { fresh, err := p.Registry.Get(host) if err != nil { return Result{host, "refused", "row: " + err.Error()}, handoverNone, false } if fresh == nil { return Result{host, "refused", "row vanished under the fence"}, handoverNone, false } if !ready(*fresh) { return Result{host, "refused", "status " + fresh.Status + " under the fence"}, handoverNone, false } if !activator.ActivatorManaged(*fresh) { return Result{host, "refused", "no longer activator-managed under the fence"}, handoverNone, false } routes, err := p.Registry.List() if err != nil { return Result{host, "refused", "list routes: " + err.Error()}, handoverNone, false } want, from := activator.LayoutNested, activator.LayoutFlat if mode == ModeFlat { want, from = activator.LayoutFlat, activator.LayoutNested } in := p.layoutInputs(routes) if prior != nil { in.Journal = func(h string) bool { return h != host && JournalPresent(p.DataDir, h) } } switch layout, reason := activator.Classify(*fresh, in); layout { case want: return Result{host, "unchanged", ""}, handoverNone, false case from: default: return Result{host, "refused", fmt.Sprintf("%s under the fence: %s", layout, reason)}, handoverNone, false } plan, err := PlanFor(p.DataDir, p.socketDir(), host, mode) if err != nil { return Result{host, "refused", err.Error()}, handoverNone, false } plan.Logf = p.logf if err := Preflight(plan, routes); err != nil { return Result{host, "refused", err.Error()}, handoverNone, false } // Both directions: a revert renames out of the raw directory, so it // must be a real directory too, and it is cheaper to find a planted // link here than after the journal has fenced the host. if err := EnsureDirs(p.DataDir, p.socketDir(), host, p.SocketsGID); err != nil { return Result{host, "refused", err.Error()}, handoverNone, true } aside, err := DestinationCheck(plan, p.now) if err != nil { return Result{host, "refused", err.Error()}, handoverNone, true } // The presence scan runs inside the section config sync is locked out // of, with the journal write: a bundle rendered between a scan outside // it and the journal would be recorded absent and left at the old path. p.serialize(func() { present, err := presence(plan, hostsOf(routes)) if err != nil { res, h, hold = Result{host, "refused", err.Error()}, handoverNone, true return } res, h = p.commit(plan, plan.Journal(present), aside, prior) }) if res.Outcome == "migrated" || res.Outcome == "reverted" { p.reload() } return res, h, hold } // presence records which artifacts are at their sources, in artifactNames // order, and refuses a plan whose database is not. It then adds to the // plan every entry the app made for the host (ScanExtras; hosts is the // box's rows, which decide ownership), so the journal written from the // plan names them and the move carries them. func presence(plan *Plan, hosts []string) ([]bool, error) { present := make([]bool, len(plan.Artifacts)) for i, a := range plan.Artifacts { // Nothing is ever renamed for the socket: the listener is gone // with the stopped host, and the next wake binds the new path. if a.Name == "socket" { continue } ok, err := lstatPresent(a.Src) if err != nil { return nil, err } if a.Required && !ok { return nil, fmt.Errorf("%s is missing at %s", a.Name, a.Src) } present[i] = ok plan.Artifacts[i].Present = ok } extras, err := ScanExtras(plan, hosts) if err != nil { return nil, err } if err := plan.AddExtras(extras); err != nil { return nil, err } // With presence known, every file about to move must have one name. // This runs under the same lock as the journal write, so nothing the // agent itself does can add a name between the check and the journal. if err := LinkCheck(plan, func(a Artifact) string { return a.Src }); err != nil { return nil, err } return present, nil } // applyJournal gives a plan rebuilt from the host name what the journal // recorded about the move: which fixed artifacts were present, and the // extra entries by name. Validate has already checked the names' shape; // this checks their ownership against the box's rows as they stand now, // because a journal is not an instruction (design §3.3): a forged journal // for H naming H-x, where H-x is a host on the box, would otherwise move // that host's own directory into H's tree on recovery. func applyJournal(plan *Plan, j *Journal, hosts []string) error { for i := range plan.Artifacts { if i < len(j.Artifacts) { plan.Artifacts[i].Present = j.Artifacts[i].Present } } all := append([]string{plan.Host}, hosts...) for _, name := range j.extraNames() { if owner := ownerOf(name, all); owner != plan.Host { return fmt.Errorf("layout: %s: journal names extra %q, which belongs to %s", plan.Host, name, owner) } } return plan.AddExtras(j.extraNames()) } // commit is steps 6 to 9 up to the journal's removal. The caller reloads // routing afterwards, outside Serialize's lock and still under the fence. // Every return after the journal is written leaves it for Recover, and the // journal keeps the host fenced until then. func (p *Pass) commit(plan *Plan, j *Journal, aside []string, prior *Journal) (Result, handover) { host := plan.Host if prior == nil { if err := WriteJournal(p.DataDir, j); err != nil { return Result{host, "refused", err.Error()}, handoverNone } return p.commitWritten(plan, j, aside), handoverNone } if err := ReplaceJournal(p.DataDir, prior, j); err != nil { // Whatever is on disk now, the pass cannot vouch that it fences // the host, so the caller keeps its own fence instead. return Result{host, "refused", err.Error()}, handoverFailed } return p.commitWritten(plan, j, aside), handoverDone } // handover is what became of the journal recovery held a host with when // the converge leg ran. type handover int const ( // handoverNone: the leg stopped before the handover; the held journal // is untouched and still fences the host. handoverNone handover = iota // handoverDone: the new journal replaced it; the host is fenced by the // new journal until that move completes. handoverDone // handoverFailed: the replacement itself failed, and what is on disk // is not known to fence the host. The in-memory fence must stay. handoverFailed ) // commitWritten is commit after the journal is on disk. func (p *Pass) commitWritten(plan *Plan, j *Journal, aside []string) Result { host := plan.Host if p.abortAt == "after-journal" { return Result{host, "refused", "aborted"} } if err := p.moveFiles(plan); err != nil { return Result{host, "refused", err.Error()} } moved, err := VerifyMoved(plan) aside = append(aside, moved...) if err != nil { return Result{host, "refused", notActing(err)} } if p.abortAt == "after-files" { return Result{host, "refused", "aborted"} } if err := UpdatePhase(p.DataDir, j, PhaseRow); err != nil { return Result{host, "refused", err.Error()} } if err := p.Registry.SetInstancePaths(host, plan.From.DB, plan.From.Socket, plan.To.DB, plan.To.Socket); err != nil { return Result{host, "refused", "row: " + err.Error()} } if p.abortAt == "after-cas" { return Result{host, "refused", "aborted"} } if err := UpdatePhase(p.DataDir, j, PhaseDone); err != nil { return Result{host, "refused", err.Error()} } if p.abortAt == "before-remove" { return Result{host, "refused", "aborted"} } if err := RemoveJournal(p.DataDir, host); err != nil { return Result{host, "refused", err.Error()} } outcome := "migrated" if plan.Mode == ModeFlat { outcome = "reverted" } var events []string if extras := plan.ExtraNames(); len(extras) > 0 { events = append(events, "moved with it: "+strings.Join(extras, ", ")) } if len(aside) > 0 { events = append(events, "kept aside: "+strings.Join(aside, ", ")) } if len(events) > 0 { p.logf("layout: %s: %s -> %s (%s)", host, plan.From.DB, plan.To.DB, strings.Join(events, "; ")) p.note(host, fmt.Sprintf("%s at %s: %s", outcome, p.now().UTC().Format(time.RFC3339), strings.Join(events, "; "))) } else { p.logf("layout: %s: %s -> %s", host, plan.From.DB, plan.To.DB) } return Result{host, outcome, ""} } var errAborted = errors.New("layout: aborted by test seam") // moveFiles renames every artifact the plan records present (from the // scan on a first attempt, from the journal on recovery), the app's own // entries included. One already at its destination is done; one still at // its source is renamed; one at neither is lost, and one at both was not // left that way by a crash (the destination check cleared every // destination before the journal was written, and a rename leaves a file // in exactly one place). Either of the last two is an error that keeps // the journal and so the fence. func (p *Pass) moveFiles(plan *Plan) error { for _, a := range plan.Artifacts { if a.Name == "socket" || !a.Present { continue } atDst, err := lstatPresent(a.Dst) if err != nil { return err } atSrc, err := lstatPresent(a.Src) if err != nil { return err } switch { case atDst && atSrc: return fmt.Errorf("layout: %s: %s is at both %s and %s; refusing to choose, not acting", plan.Host, a.Name, a.Src, a.Dst) case atDst: continue case !atSrc: return fmt.Errorf("layout: %s: %s is at neither %s nor %s; not acting", plan.Host, a.Name, a.Src, a.Dst) } if err := instancefs.RenameBetween(filepath.Dir(a.Src), filepath.Base(a.Src), filepath.Dir(a.Dst), filepath.Base(a.Dst)); err != nil { return fmt.Errorf("layout: %s: move %s: %w", plan.Host, a.Name, err) } if p.abortAt == "mid-files" { return errAborted } } return nil } // Recover completes or refuses every journal in /layout (design // §8.2, "Recovery is the journal"). It runs at agent start and at the top // of every Converge. intent is the fleet's recorded layout: "" when none // was ever recorded, IntentUnreadable when the agent could not read it. // Completing a journal is never gated on the isolation probe, since that // is how a host gets out of the fenced state; a leg after it toward a // nested intent is (see recoverHost). func (p *Pass) Recover(ctx context.Context, intent Mode) []Result { return p.recover(ctx, intent, p.nestedGate()) } func (p *Pass) recover(ctx context.Context, intent Mode, gate func() error) []Result { names, err := JournalHosts(p.DataDir) if err != nil { p.logf("layout: recover: read journals: %v", err) return []Result{{Outcome: "refused", Reason: "read journals: " + err.Error()}} } if len(names) == 0 { return nil } routes, err := p.Registry.List() if err != nil { p.logf("layout: recover: list routes: %v", err) return []Result{{Outcome: "refused", Reason: "list routes: " + err.Error()}} } byHost := map[string]registry.Route{} for _, rt := range routes { byHost[rt.Host] = rt } var out []Result for _, host := range names { if ctx.Err() != nil { break } r := p.recoverHost(ctx, host, byHost, intent, gate) if r.Outcome == "refused" { p.logf("layout: %s: recovery refused -> %s", host, r.Reason) } out = append(out, r) } return out } // nestedGate asks NestedOK at most once, on first use, and remembers the // answer for the rest of one pass. func (p *Pass) nestedGate() func() error { asked := false var answer error return func() error { if p.NestedOK == nil { return nil } if !asked { asked, answer = true, p.NestedOK() } return answer } } // notActing makes a recovery refusal say, in the words design §8.2 uses, // that the pass left the host alone and fenced. func notActing(err error) string { if s := err.Error(); strings.HasSuffix(s, "not acting") { return s } return err.Error() + "; not acting" } // recoverHost checks one journal against the row and both derived // endpoints, completes it to its own end under a fence, and then, still // under that fence, converges the host to intent. The host is released // only once it stands at the intent (design §3.3): while the converge leg // runs, the completed journal stays on disk at done and keeps the host // fenced, and if the leg refuses, for any reason, it stays there. A flat // tenant can forge a valid revert journal for a nested sibling; completing // it costs that sibling a round trip, and if the way back is refused the // sibling waits fenced for an operator rather than running flat. // // Completing a journal is never gated on the isolation probe; the leg // after it toward a nested intent is, exactly as Converge(ModeNested) is. // A gate that refuses the leg holds the host like any other refused leg. // // When another operation's fence already holds the host (the residence // move's cut), the pass completes the journal under that fence, installing // and lifting nothing, provided the host has no live entry: the move's own // fence proved or is proving the stop, and the move refuses a host while a // journal exists, so the journal has to be finished for the move to go on. // Without a fence of its own the pass cannot converge the host, so unless // the completed endpoint is already the intent it keeps the journal at // done. The journal then keeps the host fenced if the move aborts and // lifts its fence, and the next recovery converges it. func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]registry.Route, intent Mode, gate func() error) Result { if !controlname.ValidHostShape(host) { return Result{host, "refused", "journal file name is not a host; not acting"} } j, found, err := ReadJournal(p.DataDir, host) if err != nil { return Result{host, "refused", notActing(err)} } if !found { return Result{host, "unchanged", ""} } rt, ok := byHost[host] if !ok { return Result{host, "refused", "journal names a host with no row; not acting"} } // Checked before the stop as well as after it, so a journal that // cannot be honoured never costs its host a stop. if _, err := Validate(j, p.DataDir, p.socketDir(), rt); err != nil { return Result{host, "refused", notActing(err)} } plan, err := PlanFor(p.DataDir, p.socketDir(), host, j.Mode) if err != nil { return Result{host, "refused", notActing(err)} } plan.Logf = p.logf hosts := make([]string, 0, len(byHost)) for h := range byHost { hosts = append(hosts, h) } if err := applyJournal(plan, j, hosts); err != nil { return Result{host, "refused", notActing(err)} } keepFence := false r, fence, gen := p.quiesce(ctx, host, p.generation()) switch fence { case fenceNone: return r case fenceHeld: if !p.Fence.Quiesced(host) { return Result{host, "deferred", "fenced by another operation and not yet stopped"} } case fenceOurs: defer func() { if !keepFence { p.release(host, gen) } }() } // The row was read before the host stopped; check the journal against // the row as it stands now that nothing else can wake it. fresh, err := p.Registry.Get(host) if err != nil { return Result{host, "refused", notActing(fmt.Errorf("row: %w", err))} } if fresh == nil { return Result{host, "refused", "row vanished under the fence; not acting"} } atFrom, err := Validate(j, p.DataDir, p.socketDir(), *fresh) if err != nil { return Result{host, "refused", notActing(err)} } converge := fence == fenceOurs && (intent == ModeNested || intent == ModeFlat) && intent != j.Mode var holdReason string switch { case fence == fenceHeld && intent != j.Mode: // Only an instance move shares the fence slot, and its cut waits // for this journal to go while the pass waits for the move's // fence to lift. Neither goes on alone, so the reason names both // ways out for whoever reads the agent log. holdReason = fmt.Sprintf("completed to %s under an instance move's fence; journal kept at done. the move waits for this journal and the journal waits for the move. abort the move, or remove %s now that the row stands at its destination, and the move can finish", j.Mode, provision.LayoutJournalPath(p.DataDir, host)) case intent == IntentUnreadable && j.Mode == ModeFlat: // Releasing a completed revert flat before the intent is known is // the forged-revert window again. holdReason = "completed to flat; journal kept at done until the layout intent can be read" } remove := holdReason == "" && !converge var res Result p.serialize(func() { res = p.complete(plan, j, atFrom, remove) }) if res.Outcome != "recovered" { return res } // The row flipped (or was found flipped). A host still held by its // journal is refused wakes by that, but routing should say where it is. p.reload() if holdReason != "" { p.logf("layout: %s: held -> %s", host, holdReason) return Result{host, "deferred", holdReason} } if !converge { return res } if intent == ModeNested { if gerr := gate(); gerr != nil { // Held, not released flat: there is no degraded mode (design // §8.5), and a host released at a forged revert's endpoint is // a nested tenant running flat. The journal at done keeps it // fenced, so the in-memory fence is lifted as usual, and the // first recovery after the probe passes converges it. reason := fmt.Sprintf("completed to %s; journal kept at done: not converging to nested: isolation not ready: %v", j.Mode, gerr) p.logf("layout: %s: held -> %s", host, reason) return Result{host, "deferred", reason} } } next, h, _ := p.shift(host, intent, j) switch { case next.Outcome == "migrated" || next.Outcome == "reverted": return Result{host, "recovered", fmt.Sprintf("completed to %s, then converged to %s", j.Mode, intent)} case h == handoverFailed: // Nothing on disk is known to fence the host, so the pass's own // fence stays and the host stays down. Releasing it here would run // it at the completed endpoint, which after a forged revert is a // nested host running flat. The generation is recorded so the next // pass knows the fence as its own and runs the leg again. keepFence = true p.keep(host, gen) reason := fmt.Sprintf("completed to %s; handing over to the move to %s failed: %s. the fence is held by this pass after a failed handover; the next pass retries it", j.Mode, intent, next.Reason) p.logf("layout: %s: refused -> %s", host, reason) return Result{host, "refused", reason} case h == handoverNone: reason := fmt.Sprintf("completed to %s; journal kept at done: converging to %s: %s %s", j.Mode, intent, next.Outcome, next.Reason) p.logf("layout: %s: held -> %s", host, reason) return Result{host, "deferred", reason} } return Result{host, next.Outcome, fmt.Sprintf("completed to %s; converging to %s: %s", j.Mode, intent, next.Reason)} } // complete runs the journal's remaining phases (design §8.2's recovery // list), and removes the journal only when remove is set; otherwise it // stops at done and the journal keeps holding the host. // // Two states a crash cannot produce are refused rather than acted on. An // artifact the journal records absent that exists at either end: the host // has been fenced since the plan was made, so nothing recorded absent can // have appeared, and moving the rest without it would leave it behind (a // forged journal can use exactly that to strand a database file). And a // done journal whose row is still at from: done is written only after the // row's swap committed, and releasing it would leave a row that names the // files' old place. A journal at row is also checked for every artifact // it records present being at its destination before the row is swapped. func (p *Pass) complete(plan *Plan, j *Journal, atFrom, remove bool) Result { host, phase := plan.Host, j.Phase for _, a := range plan.Artifacts { if a.Name == "socket" || a.Present { continue } for _, path := range []string{a.Src, a.Dst} { present, err := lstatPresent(path) if err != nil { return Result{host, "refused", notActing(err)} } if present { return Result{host, "refused", fmt.Sprintf("journal records %s absent, but it exists at %s; not acting", a.Name, path)} } } } switch j.Phase { case PhaseFiles: if err := p.moveFiles(plan); err != nil { return Result{host, "refused", notActing(err)} } // The same check a first attempt makes before its row flips; the // plan carries the journal's presence (applyJournal) so the check // knows which files it moved. moved, err := VerifyMoved(plan) if len(moved) > 0 { p.logf("layout: %s: quarantined on recovery -> %s", host, strings.Join(moved, ", ")) p.note(host, "quarantined "+strings.Join(moved, ", ")) } if err != nil { return Result{host, "refused", notActing(err)} } if err := UpdatePhase(p.DataDir, j, PhaseRow); err != nil { return Result{host, "refused", err.Error()} } fallthrough case PhaseRow: // The row is about to name the destinations, so every file the // journal records present must be there: a row pointed at a // missing database has the next wake restore a second one. for _, a := range plan.Artifacts { if a.Name == "socket" || !a.Present { continue } present, err := lstatPresent(a.Dst) if err != nil { return Result{host, "refused", notActing(err)} } if !present { return Result{host, "refused", fmt.Sprintf("journal records %s present, but it is missing at %s; not acting", a.Name, a.Dst)} } } if atFrom { if err := p.Registry.SetInstancePaths(host, j.From.DB, j.From.Socket, j.To.DB, j.To.Socket); err != nil { return Result{host, "refused", notActing(fmt.Errorf("row: %w", err))} } } if err := UpdatePhase(p.DataDir, j, PhaseDone); err != nil { return Result{host, "refused", err.Error()} } case PhaseDone: if atFrom { return Result{host, "refused", "journal is done but the row is still at its source; not acting"} } default: return Result{host, "refused", fmt.Sprintf("journal phase %q unknown; not acting", j.Phase)} } if remove { if err := RemoveJournal(p.DataDir, host); err != nil { return Result{host, "refused", err.Error()} } } p.logf("layout: %s: recovered -> %s (journal phase was %s)", host, j.Mode, phase) return Result{host, "recovered", string(j.Mode)} }