package console import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "log" "net/http" "os" "strings" "time" ) // activationHeldError names the executable activation reservation that // blocks a channel (I-4), so the operator learns what to wait for or // cancel. It is still hostactors.ErrRoster to every errors.Is check. type activationHeldError struct{ id, component, channel string } func (e activationHeldError) Error() string { return "activation " + e.id + " is held for " + e.component + "/" + e.channel + ". It is released when every reserved box reports completion or is excluded by a termination fence or quarantine. If it was never published, cancel it with carlos system cancel-activation" } func (e activationHeldError) Unwrap() error { return hostactors.ErrRoster } func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelease, channel, mode string, options release.SystemPromoteOptions, expectedETag ...string) (string, error) { pub := release.RootSigner(s.cfg.Key).RootPublicKey() floor, err := hostfloor.Read(ctx, s.cfg.Releases, pub, s.cfg.Bucket) if errors.Is(err, os.ErrNotExist) { // Legacy mode only when BOTH the floor and its authorization are // absent; any incomplete pair is an unfinished publication, never // the old deployment mode (review round 4). for _, key := range []string{hostfloor.Key, hostfloor.AuthKey} { if _, _, e := s.cfg.Releases.Get(ctx, key); !errors.Is(e, os.ErrNotExist) { return "", hostfloor.ErrFloor } } return "", nil } if err != nil { return "", err } tag := strings.TrimPrefix(rel.Source, "release:") if rel.ReleaseTag != "" { tag = rel.ReleaseTag } if rel.Source == "file" { keys, e := s.cfg.Releases.List(ctx, "control/system/router/releases/") if e != nil { return "", e } tag = "" for _, key := range keys { raw, _, e := s.cfg.Releases.Get(ctx, key) if e != nil { return "", e } identity, e := release.VerifySystemIdentity(pub, raw) if e != nil { return "", e } if identity.Current.Version != rel.Version { continue } matches := true for platform, artifact := range rel.Artifacts { if identity.Current.Artifacts[platform] != artifact { matches = false } } if matches { if tag != "" && tag != identity.Current.ReleaseTag { return "", hostfloor.ErrFloor } tag = identity.Current.ReleaseTag } } } approved, err := release.GetSystemIdentity(ctx, s.cfg.Releases, pub, tag) if err != nil { return "", err } if approved.Current.Version != rel.Version { return "", hostfloor.ErrFloor } for platform, artifact := range rel.Artifacts { if approved.Current.Artifacts[platform] != artifact { return "", hostfloor.ErrFloor } } rel.ReleaseTag = approved.Current.ReleaseTag rel.MovesProtocol = approved.Current.MovesProtocol rel.ConsoleArtifacts = approved.Current.ConsoleArtifacts d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: pub, Bucket: s.cfg.Bucket} o, err := d.Read(ctx) if err != nil { return "", err } // M-8: a floor binds host updates only through a roster published at // exactly its digest. A preparing, unpublished or differently bound // roster is an unfinished publication, never permission to reserve. if o.Roster.Publication != "published" || o.Roster.FloorSHA256 != floor.Digest { return "", hostfloor.ErrFloor } requestRelease := *rel requestRelease.ReleasedAt = time.Time{} request, err := json.Marshal(struct { Channel string Mode string Release release.SystemRelease Spread time.Duration }{channel, mode, requestRelease, options.Spread}) if err != nil { return "", err } digest := sha256.Sum256(request) requestID := hex.EncodeToString(digest[:]) for id, existing := range o.Roster.Activations { if existing.Publication == nil || existing.Publication.Channel != channel { continue } if existing.RequestSHA256 != requestID || existing.FloorSHA256 != floor.Digest { return "", activationHeldError{id: id, component: existing.Publication.Component, channel: channel} } if _, err := existing.Publication.Verify(pub); err != nil { return "", err } return id, nil } var nonce [16]byte if _, err = rand.Read(nonce[:]); err != nil { return "", err } id := hex.EncodeToString(nonce[:]) var publication release.SystemPublication if mode == "rollback" { publication, err = release.PrepareSystemRollback(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, options.Actor, id, expectedETag...) } else { publication, err = release.PrepareSystemPromotion(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, *rel, options, id) } if err != nil { return "", err } h := hostactors.Reservation{Publication: &publication, RequestSHA256: requestID, UpdateMode: mode, ID: id, FloorSHA256: floor.Digest, Actors: map[string]string{}, Releases: map[string]hostactors.Report{}, Hashes: map[string][]string{}} e := hostactors.ExactEvidence{PublicKey: d.PublicKey, Store: s.cfg.Releases, Boxes: d.PrivateBoxReport} for ref, b := range o.Roster.Boxes { if b.State != "active" { continue } report, err := d.PrivateBoxReport(ctx, ref, b) if err != nil { return "", err } if err = hostactors.VerifyReport(ctx, e, pub, floor, report, b.Enrollment, b.Cloud.InstanceID, time.Now()); err != nil { return "", err } if report.SystemChannel == "" { return "", hostfloor.ErrFloor } if report.SystemChannel != channel { continue } target, ok := rel.Artifacts[report.Platform] if !ok { return "", hostfloor.ErrFloor } identityKey, _ := release.IdentityKey(rel.ReleaseTag) identity, _, err := s.cfg.Releases.Get(ctx, identityKey) if err != nil { return "", err } if err = hostfloor.CheckImage(pub, floor, identity, rel.Version, report.Platform, target.SHA256); err != nil { return "", err } if target.SHA256 == report.BinarySHA256 { continue } key := "box:" + ref h.Actors[key] = b.Enrollment h.Releases[key] = report // This transaction refreshes recovery from the authenticated current // image. A box unable to do that retains the reservation and refuses. h.Hashes[key] = []string{report.BinarySHA256, target.SHA256, report.BinarySHA256} } if len(h.Actors) == 0 { return "", nil } _, err = d.Reserve(ctx, release.RootSigner(s.cfg.Key), o, h, true) return id, err } func (s *Server) promoteHostUpdate(ctx context.Context, rel release.SystemRelease, component, channel string, options release.SystemPromoteOptions) (*release.SystemPointer, error) { id, err := s.reserveHostUpdate(ctx, &rel, channel, "promote", options) if err != nil { return nil, err } if id == "" { return release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, rel, options) } d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} return d.PublishActivation(ctx, id) } func (s *Server) rollbackHostUpdate(ctx context.Context, component, channel, actor string) (*release.SystemPointer, error) { d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} floor, err := hostfloor.Read(ctx, s.cfg.Releases, d.PublicKey, s.cfg.Bucket) if err == nil { roster, err := d.Read(ctx) if err != nil { return nil, err } // A successful publication with a lost response must not pop history again. for id, h := range roster.Roster.Activations { if h.Publication == nil || h.Publication.Channel != channel { continue } if h.UpdateMode != "rollback" || h.FloorSHA256 != floor.Digest { return nil, activationHeldError{id: id, component: component, channel: channel} } return d.PublishActivation(ctx, id) } } else if !errors.Is(err, os.ErrNotExist) { return nil, err } current, etag, err := release.GetSystemPointer(ctx, s.cfg.Releases, d.PublicKey, component, channel) if err != nil { return nil, err } if len(current.History) == 0 { return nil, release.ErrNoSystemHistory } target := current.History[0] id, err := s.reserveHostUpdate(ctx, &target, channel, "rollback", release.SystemPromoteOptions{Actor: actor}, etag) if err != nil { return nil, err } if id == "" { return release.RollbackSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, actor, etag) } return d.PublishActivation(ctx, id) } // handleAPISystemActivationCancel is the operator's way out of an executable // activation reservation that was never published (I-4). Such a reservation // blocks rollback and every other promote of its channel, and nothing else // releases it: no box ever saw it, so no box can report completing it. // // It takes the update doors' gate (apiSystemRoller: a person who owns the // ops account or is on CARLOS_SYSTEM_OPERATORS) and the signing key. The // channel names the reservation, since at most one is held per channel; an // activation ID, when given, must be the one held. CancelActivation fences // the pointer at the publication's expected ETag before it removes the // reservation, so a delayed publication can never land afterwards, and a // publication that already landed answers 409: a published activation is // released only by completion or the D7 exclusions, never by this door. func (s *Server) handleAPISystemActivationCancel(w http.ResponseWriter, r *http.Request) { addr, _, ok := s.apiSystemRoller(w, r) if !ok { return } if !s.systemBucket(w) { return } if s.cfg.Key == nil { jsonError(w, http.StatusServiceUnavailable, "no signing key on this host, so the console is running read-only") return } var in struct { Component string `json:"component"` Channel string `json:"channel"` Activation string `json:"activation"` } if !readJSONStrictLimit(w, r, &in, 1<<10) { return } if in.Component == "" { in.Component = release.SystemComponentRouter } if in.Component != release.SystemComponentRouter { jsonError(w, http.StatusBadRequest, "unknown component "+in.Component) return } if in.Channel != release.RungCanary && in.Channel != release.RungStable { jsonError(w, http.StatusBadRequest, "channel must be canary or stable") return } target := in.Component + "/" + in.Channel s.systemMu.Lock() running := s.systemRunning[target] s.systemMu.Unlock() if running != "" { writeJSON(w, http.StatusConflict, map[string]string{ "error": "a promote of " + target + " is running on this console; watch it or wait for it to finish", "job": running, }) return } ctx := r.Context() sg := release.RootSigner(s.cfg.Key) d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: sg.RootPublicKey(), Bucket: s.cfg.Bucket} o, err := d.Read(ctx) if errors.Is(err, os.ErrNotExist) { jsonError(w, http.StatusNotFound, "no activation reservation is held for "+target) return } if err != nil { log.Printf("console: cancel activation %s: read roster: %v", target, err) jsonError(w, http.StatusInternalServerError, "could not read the actor roster; see the console log") return } id := "" for key, h := range o.Roster.Activations { if h.Publication != nil && h.Publication.Component == in.Component && h.Publication.Channel == in.Channel { id = key } } if id == "" { jsonError(w, http.StatusNotFound, "no activation reservation is held for "+target) return } if in.Activation != "" && in.Activation != id { jsonError(w, http.StatusConflict, "the reservation held for "+target+" is "+id+", not "+in.Activation+"; nothing was cancelled") return } err = d.CancelActivation(ctx, sg, id) switch { case errors.Is(err, release.ErrCASConflict): jsonError(w, http.StatusConflict, "activation "+id+" was already published, or its channel pointer has moved, so it cannot be cancelled. It is released when every reserved box reports completion or is excluded by a termination fence or quarantine") return case errors.Is(err, hostactors.ErrRoster): jsonError(w, http.StatusConflict, "activation "+id+" changed while it was being cancelled; nothing was released. Check carlos system status and try again") return case err != nil: log.Printf("console: cancel activation %s on %s: %v", id, target, err) jsonError(w, http.StatusInternalServerError, "could not cancel activation "+id+"; see the console log") return } s.auditSystem(addr, "system-activation-cancel", "cancelled activation "+id+" on "+target) writeJSON(w, http.StatusOK, map[string]string{"component": in.Component, "channel": in.Channel, "activation": id, "state": "cancelled"}) }