package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "io" "log" "os" "path/filepath" "strings" "sync" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) type rootRestartChecks struct { unitShape func(context.Context, string, *registry.Route) error preserveOutcome bool history func() (*release.RootHistoryPass, error) readLive func(string) (string, error) legacy func() (bool, error) legacyLog *sync.Once legacyReady func(registry.Route) func() error legacyStart func(string) error targetDigest string targetFile func(string) (*os.File, error) dispatch func(context.Context, registry.Route) (bool, error) resolve proofBindingResolver started *sync.Map attempts func() (rootstate.Store, error) rollback func(context.Context, proofBinding) (proofBinding, error) prepareRollback func(context.Context, proofBinding) (proofBinding, error) recovered func(context.Context, proofBinding) (proofBinding, error) identity hostwrite.Identity effect func(func() error) error verifyOnly bool host func(context.Context, string) (proofBinding, error) ready func(registry.Route) func() error now func() time.Time sleep func(time.Duration) } func productionRootRestartChecks(sys unitSystem, optDir string) rootRestartChecks { var sourceMu sync.Mutex var cached *proofBindingSource var sourceErr error var opened bool source := func(ctx context.Context) (*proofBindingSource, error) { sourceMu.Lock() defer sourceMu.Unlock() if opened { return cached, sourceErr } opened = true s, err := openDeployStore(ctx) sourceErr = err if err != nil { return nil, err } cached = productionProofBindingSource(s, sys, optDir) return cached, nil } return rootRestartChecks{unitShape: rootClaimRestartShape, legacy: legacyRootRestart, legacyLog: &sync.Once{}, targetFile: rootstate.OpenTrustedFile, started: &sync.Map{}, readLive: rootstate.ReadTrustedLink, recovered: func(ctx context.Context, b proofBinding) (proofBinding, error) { s, err := source(ctx) if err != nil { return proofBinding{}, err } return s.recovered(ctx, b) }, history: func() (*release.RootHistoryPass, error) { s, err := openDeployStore(context.Background()) if err != nil { return nil, err } return openRootHistory(s, optDir) }, dispatch: func(ctx context.Context, rt registry.Route) (bool, error) { s, err := source(ctx) if err != nil { return false, err } return s.needsRestart(ctx, rt) }, attempts: func() (rootstate.Store, error) { if err := rootstate.EnsurePublicDir(release.RootHealthyDir); err != nil { return nil, err } return rootstate.Open(rootRestartState, true, false) }, prepareRollback: func(ctx context.Context, b proofBinding) (proofBinding, error) { s, err := source(ctx) if err != nil { return proofBinding{}, err } return s.prepareRollback(ctx, b) }, rollback: func(ctx context.Context, b proofBinding) (proofBinding, error) { s, err := source(ctx) if err != nil { return proofBinding{}, err } return s.rollback(ctx, b) }, resolve: func(ctx context.Context, rt registry.Route) (proofBinding, error) { s, err := source(ctx) if err != nil { return proofBinding{}, err } return s.resolve(ctx, rt) }, host: func(ctx context.Context, host string) (proofBinding, error) { s, err := source(ctx) if err != nil { return proofBinding{}, err } return s.forHost(ctx, host) }, ready: func(rt registry.Route) func() error { return proofRouteReady(rt, 2*time.Second) }, now: time.Now, sleep: time.Sleep, } } func (c rootRestartChecks) runningVersion(ctx context.Context, rt registry.Route) (string, error) { if rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { return "", nil } // Merge ruling 6: without a census the binding never chooses an // effect, and the rollback target is one. Main's cur stands. legacy, err := c.allowLegacy() if err != nil { return "", err } if legacy { return "", nil } b, err := c.resolve(ctx, rt) if err != nil { return "", err } if b.PID > 0 { return b.Version, nil } return "", nil } func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) string, settled func(registry.Route) error, checks ...rootRestartChecks) func(registry.Route) error { c := productionRootRestartChecks(sys, optDir) if len(checks) == 1 { c = checks[0] } return func(candidate registry.Route) error { c := c legacy, err := c.allowLegacy() if err != nil { return err } if legacy { c.noteLegacy() return legacyCheckedUnitRestart(sys, optDir, getenv, candidate, settled, c) } if err := c.checkUnitShape(context.Background(), candidate.Host, &candidate); err != nil { return err } if c.attempts == nil { return os.ErrPermission } s, err := c.attempts() if err != nil { return err } defer s.Close() unlock, err := s.Lock(context.Background(), rollbackHostKey(candidate.Host)+".lock") if err != nil { return err } defer unlock() binding, err := c.resolve(context.Background(), candidate) if err != nil { return err } before, err := sys.Show(binding.Route.Unit) if err != nil { return err } if before.ActiveState == "inactive" { return errUnitStopped } if !pointerBoundUnit(before) && (before.ActiveState != "active" || before.MainPID <= 0 || before.MainPID != binding.PID || before.InvocationID != binding.Invocation) { return os.ErrPermission } target, err := c.live(optDir, candidate.Host) if err != nil { return err } rel, err := filepath.Rel(filepath.Join(optDir, "versions"), target) if err != nil { return err } version := strings.Split(rel, string(filepath.Separator))[0] if !release.ValidVersion(version) || target != release.VersionPath(optDir, binding.Route.Account, version, binding.Route.App) || c.targetFile == nil { return os.ErrPermission } f, err := c.targetFile(target) if err != nil { return err } hash := sha256.New() _, err = io.Copy(hash, f) f.Close() if err != nil { return err } targetDigest := hex.EncodeToString(hash.Sum(nil)) c.targetDigest = targetDigest captured := binding if pointerBoundUnit(before) { previous, e := filepath.EvalSymlinks(filepath.Join(optDir, "previous", candidate.Host)) if e == nil && previous != target { rel, e := filepath.Rel(filepath.Join(optDir, "versions"), previous) if e != nil { return e } version := strings.Split(rel, string(filepath.Separator))[0] if !release.ValidVersion(version) || previous != release.VersionPath(optDir, binding.Route.Account, version, binding.Route.App) { return os.ErrPermission } f, e := c.targetFile(previous) if e != nil { return e } h := sha256.New() _, e = io.Copy(h, f) f.Close() if e != nil { return e } captured.Version, captured.Digest = version, hex.EncodeToString(h.Sum(nil)) captured, e = c.prepareRollback(context.Background(), captured) if e != nil { return e } } else if e != nil && !errors.Is(e, os.ErrNotExist) { return e } } if err := writeRollbackState(s, rollbackHostKey(candidate.Host)+".previous", captured); err != nil { return err } attempt, err := historyRestartAttempt(s, captured, c, target, getenv) if err != nil { return err } if c.started != nil { c.started.Store(candidate.Host, attempt.ID) } err = restartBoundUnit(sys, optDir, getenv, binding, false, settled, c) if err == nil { return settleRollbackAttempt(s, attempt) } if errors.Is(err, errUnitStopped) { return errors.Join(err, settleRollbackAttempt(s, attempt)) } if errors.Is(err, errRestartBinding) && !restartEffectFailed(err) { return errors.Join(err, settleRollbackAttempt(s, attempt)) } attempt.Failed = true var notUp *release.UnitNotUpError if !errors.As(err, ¬Up) { err = errors.Join(&release.UnitNotUpError{Unit: binding.Route.Unit, Reason: err.Error()}, err) } return errors.Join(err, writeRollbackState(s, rollbackJournalKey(candidate.Host), attempt)) } } func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, binding proofBinding, rollback bool, settled func(registry.Route) error, c rootRestartChecks) error { rt := binding.Route if binding.Source == "" || !binding.Network || c.ready == nil { return errRestartBinding } ready := c.ready(rt) if ready == nil { return errRestartBinding } before, err := sys.Show(rt.Unit) if err != nil { return errors.Join(errRestartBinding, err) } if before.ExecPath != proofExecPath(binding, optDir) { return errRestartBinding } if before.ActiveState == "inactive" { return errUnitStopped } want, err := c.live(optDir, rt.Host) if err != nil { return err } effect := c.effect if effect == nil { effect = func(fn func() error) error { return fn() } } if c.verifyOnly { before.InvocationID = "" } else if err := effect(func() error { if rollback { current, err := c.rollback(context.Background(), binding) if err != nil || current.Source != binding.Source { return errRestartBinding } if err := sys.ResetFailed(rt.Unit); err != nil { return err } if err := sys.Restart(rt.Unit); err != nil { return err } } else { if !pointerBoundUnit(before) && (before.ActiveState != "active" || before.MainPID != binding.PID || before.InvocationID != binding.Invocation) { return errRestartBinding } if before.MainPID > 0 { start, err := sys.StartTime(before.MainPID) if err != nil || start != binding.Start { return errRestartBinding } } current, err := c.host(context.Background(), rt.Host) if err != nil || current.Source != binding.Source || current.Version != binding.Version || current.Digest != binding.Digest || current.PID != binding.PID || current.Invocation != binding.Invocation { return errRestartBinding } if pointerBoundUnit(before) { if err := sys.ResetFailed(rt.Unit); err != nil { return err } return sys.Restart(rt.Unit) } if err := sys.TryRestart(rt.Unit); err != nil { return err } } return nil }); err != nil { return err } var observed proofBinding observe := func() (proofBinding, error) { if rollback && c.recovered != nil { return c.recovered(context.Background(), binding) } return c.host(context.Background(), rt.Host) } var bindingErr error checkedReady := func() error { first, err := observe() if err != nil || first.Source != binding.Source || want != release.VersionPath(optDir, first.Route.Account, first.Version, first.Route.App) { bindingErr = errors.Join(errRestartBinding, err) return bindingErr } if c.targetDigest != "" && first.Digest != c.targetDigest { bindingErr = errRestartBinding return bindingErr } if rollback && (first.Version != binding.Version || first.Digest != binding.Digest) { bindingErr = errRestartBinding return bindingErr } if err := ready(); err != nil { return err } last, err := observe() if err != nil || last.Source != first.Source || last.Version != first.Version || last.Digest != first.Digest || last.Invocation != first.Invocation || last.PID != first.PID || last.Start != first.Start { bindingErr = errors.Join(errRestartBinding, err) return bindingErr } observed = last return nil } err = settleUnit(sys, rt.Unit, before.InvocationID, settleOpts{ Timeout: unitSettleTimeout(getenv), Poll: 500 * time.Millisecond, Hold: 5 * time.Second, WantDir: filepath.Dir(want), Ready: checkedReady, Now: c.now, Sleep: c.sleep, RequireExe: true, }) if err != nil { if bindingErr != nil && !restartEffectFailed(err) { if c.preserveOutcome { return errors.Join(err, bindingErr) } return bindingErr } return err } current, err := observe() if err != nil { return errors.Join(errRestartBinding, err) } if current.Invocation != observed.Invocation || current.PID != observed.PID || current.Start != observed.Start || current.Source != binding.Source || want != release.VersionPath(optDir, current.Route.Account, current.Version, current.Route.App) || (rollback && (current.Version != binding.Version || current.Digest != binding.Digest)) { return errRestartBinding } if settled != nil { if err := effect(func() error { return settled(binding.Route) }); err != nil { log.Print(err) } } return nil } func (c rootRestartChecks) live(optDir, host string) (string, error) { if c.readLive != nil { return c.readLive(release.LiveLink(optDir, host)) } return filepath.EvalSymlinks(release.LiveLink(optDir, host)) } func rollbackUnitRestart(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string, checks ...rootRestartChecks) func(string) error { c := productionRootRestartChecks(sys, optDir) if len(checks) == 1 { c = checks[0] } return func(host string) (result error) { c := c rt, routeErr := reg.Get(host) if err := c.checkUnitShape(context.Background(), host, rt); err != nil { return err } if rt != nil && (rt.Unit == "" || (c.legacyStart != nil && rt.Hibernate)) { return nil } if rt != nil && rt.Unit == activator.UnitName(host) { if c.legacyStart != nil { return c.legacyStart(rt.Unit) } return legacyRollbackRestart(sys, optDir, getenv, *rt, c) } legacy, err := c.allowLegacy() if err != nil { return err } if legacy { c.noteLegacy() if routeErr != nil || rt == nil { return nil } if c.legacyStart != nil { return c.legacyStart(rt.Unit) } return legacyRollbackRestart(sys, optDir, getenv, *rt, c) } if c.attempts == nil || c.rollback == nil { return os.ErrPermission } s, err := c.attempts() if err != nil { return err } defer s.Close() unlock, err := s.Lock(context.Background(), rollbackHostKey(host)+".lock") if err != nil { return err } defer unlock() attempt, binding, err := loadRollback(s, host, c.now()) if attempt.ID != "" { defer func() { result = errors.Join(result, settleRollbackAttempt(s, attempt)) }() } if err != nil { if !errors.Is(err, os.ErrNotExist) && !attempt.Settled && (attempt.ID == "" || !c.now().After(attempt.Deadline.Add(rollbackMargin))) { return err } raw, readErr := s.Read(rollbackHostKey(host) + ".previous") if errors.Is(readErr, os.ErrNotExist) && c.host != nil { binding, readErr = c.host(context.Background(), host) } else if readErr == nil && json.Unmarshal(raw, &binding) != nil { readErr = os.ErrPermission } if readErr != nil || binding.Route.Host != host { return errors.Join(err, readErr) } before, showErr := sys.Show(binding.Route.Unit) if showErr != nil || before.ActiveState != "failed" || before.MainPID != 0 { return errors.Join(err, showErr) } } c.targetDigest = binding.Digest binding, err = c.rollback(context.Background(), binding) if err != nil { return err } before, err := sys.Show(binding.Route.Unit) if err != nil { return err } if before.ActiveState == "inactive" { return errUnitStopped } if before.ActiveState != "failed" && before.MainPID != 0 && !attempt.Failed { return os.ErrPermission } if attempt.Recovering && before.ActiveState == "active" && before.MainPID > 0 { c.verifyOnly = true } attempt.Recovering = true if err := writeRollbackState(s, rollbackJournalKey(host), attempt); err != nil { return err } if err := restartBoundUnit(sys, optDir, getenv, binding, true, nil, c); err != nil { return err } attempt.Verified = true return c.settleHistory(context.Background(), attempt, true) } } func rootStartRoute(reg *registry.Registry, optDir string, checks rootRestartChecks) func(string) error { checks.legacyStart = systemctlStartUnit return rollbackUnitRestart(reg, systemdUnits{}, optDir, os.Getenv, checks) } func rootSelectedRestart(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string, settled func(registry.Route) error, checks rootRestartChecks, rollback bool) hostwrite.Callback { return func(ctx context.Context, selected hostwrite.Selection) error { if selected.Work == nil { return hostwrite.ErrUnavailable } c := checks c.identity = selected.Identity c.effect = func(fn func() error) error { return selected.Work.With(ctx, selected.Identity, fn) } if rollback { return rollbackUnitRestart(reg, sys, optDir, getenv, c)(selected.Host) } return rootRestartRouteWith(reg, sys, optDir, getenv, settled, c)(selected.Host) } } func rootRestartRouteWith(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string, settled func(registry.Route) error, checks rootRestartChecks) func(string) error { restart := checkedUnitRestart(sys, optDir, getenv, settled, checks) return func(host string) error { rt, err := reg.Get(host) if shapeErr := checks.checkUnitShape(context.Background(), host, rt); shapeErr != nil { return shapeErr } if err != nil || rt == nil || rt.Unit == "" { return restartOrdinaryUnit(sys, host) } if rt.Unit == activator.UnitName(host) { return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) } legacy, err := checks.allowLegacy() if err != nil { return err } if legacy { checks.noteLegacy() return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) } if checks.dispatch != nil { need, err := checks.dispatch(context.Background(), *rt) if err != nil { return err } if !need { return restartVerifiedOrdinaryUnit(sys, optDir, host, checks.targetFile) } } err = restart(*rt) if errors.Is(err, errUnitStopped) { return nil } return err } } func restartOrdinaryUnit(sys unitSystem, host string) error { if err := sys.TryRestart(activator.UnitName(host)); err != nil { log.Print(err) } return nil } func restartVerifiedOrdinaryUnit(sys unitSystem, optDir, host string, open func(string) (*os.File, error)) error { unit := activator.UnitName(host) p, err := sys.Show(unit) if err != nil || p.Name != unit || p.ExecPath != release.LiveLink(optDir, host) || p.ActiveState != "active" || open == nil { return nil } file, err := open(p.FragmentPath) if err != nil { return err } file.Close() if err := sys.TryRestart(unit); err != nil { log.Print(err) } return nil }