package host import ( "context" "crypto/ed25519" "encoding/json" "errors" "os" "path/filepath" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rollout" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" "github.com/carlosframework/platform/internal/status" ) type rootLegacyRetry struct { Build, Book string Epoch uint64 } type rootRetryStatusRecord struct { status.HostRollout Mirror *rootLegacyRetry `json:"root_retry,omitempty"` } const rootRetryProjection = "/var/lib/carlos-swap-status" func (p rootUpdateSource) beginRuntime(saved *rootUpdateRecord, save func() error, version, digest, attempt string) (func(string, error, func() error) error, error) { rt := saved.Route if version == "" || digest == "" || attempt == "" { return nil, os.ErrPermission } start := time.Now().UTC() snap := p.book.Snapshot() // Claim launches currently use the serial, hold-mode path. Workers and // overlapping generations remain refused by the existing launch boundary. if err := p.book.Begin(rt.Host, rollout.Intent{Account: rt.Account, App: rt.App, Build: version, ID: attempt, Mode: "hold", Start: start}); err != nil { return nil, err } return func(invocation string, failure error, verify func() error) error { if failure == nil && invocation == "" { return os.ErrPermission } var readinessErr error if failure == nil && p.ready != nil { readinessErr = p.ready(rt) failure = readinessErr } if failure == nil && verify != nil { readinessErr = verify() failure = readinessErr } build := &release.BuildState{Schema: 1, Host: rt.Host} if saved.Build.State != nil { *build = *saved.Build.State } if failure != nil { if _, err := p.book.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: version, Mode: "hold", At: start, HeldDown: []string{rt.Host}}); err != nil { return err } if err := p.book.MarkHoldRan(rt.Host, rollout.HoldRan{Account: rt.Account, App: rt.App, Build: version, Exec: attempt, At: start}); err != nil { return err } } else { if _, err := p.book.ClearHoldRanBefore(rt.Host, snap); err != nil { return err } if saved.Served == nil { saved.Served = map[string]rootServedBuild{} } saved.Served[version] = rootServedBuild{Digest: digest, Invocation: invocation, Source: saved.Installation + "/" + attempt, Epoch: p.book.RestrictionEpoch(rt.Account, rt.App, version, rt.Host)} build.LastExecuted = version } if saved.Consumed != nil && saved.Consumed.Resolved != nil && saved.Consumed.Resolved.Target == version { build.Resolved = saved.Consumed.Resolved build.Completed = build.Resolved.ID build.CompletedFailed = failure != nil } saved.Build = release.BuildStateCopy{Known: true, State: build} if err := save(); err != nil { return err } if err := p.book.End(rt.Host, attempt); err != nil { return err } deliveryErr := p.publishRetry(*saved, version) if !p.enforceFailures { deliveryErr = nil } return errors.Join(readinessErr, deliveryErr) }, nil } func (p rootUpdateSource) retryStatus(saved rootUpdateRecord, version string) status.HostRollout { rt := saved.Route out := status.HostRollout{Account: rt.Account, App: rt.App, Installed: version, RetryV: status.HostRetryV} if saved.Build.State != nil { out.Good = saved.Build.State.LastExecuted } held := version if h, ok := p.book.HoldRanOn(rt.Host); ok && h.Account == rt.Account && h.App == rt.App { held = h.Build } served, ok := saved.Served[held] ok = ok && served.Epoch == p.book.RestrictionEpoch(rt.Account, rt.App, held, rt.Host) if held != "" && (p.book.Blocks(rt.Account, rt.App, held, rt.Host, ok) || p.book.HeldDown(rt.Account, rt.App, held, rt.Host)) { out.HeldBy, out.AttemptBuild = held, held out.HeldBook = p.book.ID() out.HeldEpoch = p.book.PublishedEpoch(rt.Account, rt.App, held, rt.Host) out.Attempt = status.AttemptFailed if p.book.HeldDown(rt.Account, rt.App, held, rt.Host) { out.Attempt = status.AttemptDown } } return out } func (p rootUpdateSource) publishRetry(saved rootUpdateRecord, version string) error { if p.publish == nil { return nil } record := rootRetryStatusRecord{HostRollout: p.retryStatus(saved, version)} if ack := saved.RetryMirror; ack != nil && !p.book.SaveOwed() && !p.book.Blocks(saved.Route.Account, saved.Route.App, ack.Build, saved.Route.Host, false) && !p.book.HeldDown(saved.Route.Account, saved.Route.App, ack.Build, saved.Route.Host) { record.Mirror = ack } return p.publish(saved.Route.Host, record) } func publishRootRetry(host string, report rootRetryStatusRecord) error { s, err := rootstate.Open(rootRetryProjection, true, true) if err != nil { return err } defer s.Close() raw, err := json.Marshal(report) if err != nil { return err } return s.Write(host+".json", raw) } func rootRetryStatus(legacy *status.RolloutReport) *status.RolloutReport { if _, err := os.Lstat(rootRetryProjection); errors.Is(err, os.ErrNotExist) { return legacy } s, err := rootstate.Open(rootRetryProjection, false, true) if errors.Is(err, os.ErrNotExist) { return legacy } if err != nil { return &status.RolloutReport{Unavailable: rolloutBookUnreadable} } defer s.Close() names, err := s.Names() if err != nil { return &status.RolloutReport{Unavailable: rolloutBookUnreadable} } if legacy == nil { legacy = &status.RolloutReport{} } out := *legacy out.Hosts = map[string]status.HostRollout{} for h, r := range legacy.Hosts { out.Hosts[h] = r } for _, name := range names { raw, e := s.Read(name) var r rootRetryStatusRecord if e != nil || json.Unmarshal(raw, &r) != nil || len(name) < 6 || name[len(name)-5:] != ".json" { return &status.RolloutReport{Unavailable: rolloutBookUnreadable} } host := name[:len(name)-5] prior := out.Hosts[host] if ack := r.Mirror; ack != nil && ack.Book != "" && ack.Epoch != 0 && prior.HeldBook == ack.Book && prior.HeldEpoch == ack.Epoch && prior.HeldBy == ack.Build { if m := armedRollout(); m != nil { m.RetryHost(host, ack.Build, ack.Book, ack.Epoch) } else if h := heldHolds(); h != nil { h.retryHost(host, ack.Build, ack.Book, ack.Epoch) } } prior.Account, prior.App = r.Account, r.App prior.HeldBy, prior.HeldBook, prior.HeldEpoch, prior.RetryV = r.HeldBy, r.HeldBook, r.HeldEpoch, r.RetryV if r.HeldBy != "" { prior.Attempt, prior.AttemptBuild = r.Attempt, r.AttemptBuild } out.Hosts[host] = prior } return &out } func beginRootGenerationUpdate(ctx context.Context, s *rootGenerationSource, b release.GenerationBinding) (func(string, error, func() error) error, func() error, error) { pub, err := s.pub() if err != nil { return nil, nil, err } in, err := productionRootUpdatesHeld(dataDir(), &b)(ctx, s.store, pub) if err != nil { return nil, nil, err } if in == nil { return nil, nil, os.ErrPermission } fail := func(err error) (func(string, error, func() error) error, func() error, error) { in.Close() return nil, nil, err } row, err := in.Resolve(ctx, b.Binding.Host) if err != nil { return fail(err) } if row.Begin == nil || row.Failed(b.Version) { return fail(os.ErrPermission) } finish, err := row.Begin(b.Version, b.Digest, b.Attempt) if err != nil { return fail(err) } return finish, in.Close, nil } func rootUpdateReady(ctx context.Context, rt registry.Route) error { probe := proofRouteReady(rt, time.Second) if probe == nil { return os.ErrPermission } deadline := time.NewTimer(5 * time.Second) defer deadline.Stop() for { if err := probe(); err == nil { return nil } select { case <-ctx.Done(): return ctx.Err() case <-deadline.C: return os.ErrPermission case <-time.After(25 * time.Millisecond): } } } func relayRootRetry(pub ed25519.PublicKey, rq retryRequest) bool { state, err := rootstate.Open(rootRetryProjection, false, true) if err != nil { return false } defer state.Close() raw, err := state.Read(rq.host + ".json") var report rootRetryStatusRecord if err != nil || json.Unmarshal(raw, &report) != nil { return false } grant, why := verifyRetry(pub, rq, time.Now()) if why != "" || report.Account != rq.acct || report.App != rq.app || report.HeldBy != grant.Build || report.HeldBook != grant.Book || report.HeldEpoch == 0 || report.HeldEpoch != grant.Epoch { return false } return os.WriteFile(filepath.Join(dataDir(), stageRequestMarker), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0644) == nil } func prepareRootRetryUnitDirectory() (bool, error) { if _, err := os.Lstat(filepath.Join(rootparticipation.Namespace, "census.json")); errors.Is(err, os.ErrNotExist) { return false, nil } else if err != nil { return false, err } ledger, required, err := rootparticipation.OpenForUpdate() if ledger.State != nil { defer ledger.State.Close() } if err != nil { return false, err } if !required { return false, nil } census, err := ledger.Census() if err != nil { return false, err } if !census.Complete || census.NeverEnrolled { return false, os.ErrPermission } // install-control runs from the window's root shell: make the agent's // read directory 0755 whatever that shell's umask (#317's class). if err := mkdirAllExact(rootRetryProjection, 0755); err != nil { return false, err } state, err := rootstate.Open(rootRetryProjection, true, true) if err != nil { return false, err } return true, state.Close() }