package console import ( "context" "crypto/rand" "database/sql" "encoding/json" "errors" "fmt" "github.com/carlosframework/platform/internal/park" "log" "net/http" "os" "slices" "strings" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostresolve" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/svcrole" ) // The residence-move doors (issue #331, instance-move spec): the console // writes REQUESTS onto the record and bumps the serial; everything that // touches data happens on the source box's move worker. Three verbs: // // POST .../instances/{host}/move {"region":"uk"} → preparing // POST .../instances/{host}/move/cut → cutting // DELETE .../instances/{host}/move → aborted // // State is read through the ordinary instance GET — the record carries // `move` verbatim while one is in flight. // moveGate is the shared front half: the instance family's gate, the // byte-identical 404 for a host that names no record, and the class rail — // only an exec-backed instance route's state is the platform's to move // (spec §Out of scope), and fleet placements keep their deliberate // state-does-not-follow semantics. func (s *Server) moveGate(w http.ResponseWriter, r *http.Request) (acct *Account, app, host string, rec *provision.Record, ok bool) { c := mainMoveCopy() if s.cfg.MovesEnabled { c = phase1MoveCopy() } _, acct, app, gok := s.instanceGateSaying(w, r, "instance", c.gate) if !gok { return nil, "", "", nil, false } host = r.PathValue("host") if !provision.ValidHostShape(host) { http.NotFound(w, r) return nil, "", "", nil, false } ctx := r.Context() rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) if errors.Is(err, os.ErrNotExist) { http.NotFound(w, r) return nil, "", "", nil, false } if err != nil { log.Printf("console: api instance move %s/%s/%s: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the instance") return nil, "", "", nil, false } if k := rec.Kind; k != "" && k != "instance" { jsonError(w, http.StatusBadRequest, c.kind(k)) return nil, "", "", nil, false } if rec.Addr != "" || len(rec.Addrs) > 0 { jsonError(w, http.StatusBadRequest, c.addr) return nil, "", "", nil, false } if _, fleet := registry.FleetPlacement(rec.Region); fleet { jsonError(w, http.StatusBadRequest, c.fleet) return nil, "", "", nil, false } if rec.Pool != "" { jsonError(w, http.StatusBadRequest, c.pool) return nil, "", "", nil, false } if !rec.HibernateOrDefault() { jsonError(w, http.StatusBadRequest, c.hibernate) return nil, "", "", nil, false } return acct, app, host, rec, true } // moveCopy is what moveGate says: its class rails and the shared gates' // sentences. With the switch off the move doors speak main's bytes // (mainMoveCopy()); with it on, the Phase 1 sentences (phase1MoveCopy(), copy // record MV-N47 to MV-N54), which no other door ever speaks. type moveCopy struct { gate gateCopy kind func(kind string) string addr, fleet, pool, hibernate string } // mainMoveCopy returns main's sentences, byte for byte (the dashes are escaped // only in the source). func mainMoveCopy() moveCopy { return moveCopy{ gate: mainGateCopy(), kind: func(k string) string { return "only exec-backed instances move \u2014 this route is kind " + k }, addr: "an addr-backed route's process is not platform-managed \u2014 nothing to move", fleet: "fleet-placed instances keep state-does-not-follow semantics \u2014 move is for platform regions", pool: "pooled instances do not move in v1 \u2014 their data lives on the pool's own box", hibernate: "a non-hibernating instance runs outside the activator \u2014 nothing manages its data, nothing can move it", } } func phase1MoveCopy() moveCopy { return moveCopy{ gate: gateCopy{ ambiguous: func(seg, alts string) string { return fmt.Sprintf("account %q is ambiguous. pass --account-id with one of: %s", seg, alts) }, // moveGate asks the family's gate for the noun "instance" and no // other, so the recorded sentence (MV-N52) is written whole. tooMany: func(string) string { return "too many instance requests. wait a minute and try again" }, noBucket: "this console has no deployment bucket, so it cannot answer this", }, kind: func(k string) string { return "only exec-backed instances move. this route is kind " + k }, addr: "an addr-backed route's process is not platform-managed. there is nothing to move", fleet: "fleet-placed instances keep state-does-not-follow semantics. moves are for platform regions", pool: "pooled instances do not move yet. their data lives on the pool's own box", hibernate: "a non-hibernating instance runs outside the activator. nothing manages its data, so nothing can move it", } } // moveScopedHolder is MV-N167's sentence, the one the cut's admission // records (provision.RefusedReplicaLayout); the door speaks it for a holder // it will not move (ruling T5-lease). A literal, because the copy gate // reads this file's sentences; TestMoveDoorRefusesAScopedHolder pins it // equal to the recorded constant. const moveScopedHolder = "the instance keeps its data under a per-incarnation path, which moves cannot cut yet" // moveView is the flat answer every move door speaks — titogo's consumer // contract (teamtito/titogo#1848, region-doors design): state, target, one // freshness stamp. "none" is a real state, not an error: a host with no // move in flight answers 200 {"state":"none"} so a poller needs no 404 // branch. Target carries the same NAME the caller used — the home code is // un-normalized on the way out (see homeRegionOut). // // With CARLOS_MOVES_PHASE1 on (Plan 6 Task 2, preflight L2 and L4) the view // also carries the in-flight move's id, the outcome the last close left on // the record (D11's last), and the Phase 1 marker, so a CLI can bind its // wait to the move it requested and can tell a Phase 1 console from a // legacy one before it sends anything. None of the three reveals anything // the state did not, so every caller of the flat view sees them, the // Instance-role token included (M18). With the switch off they are absent // and the bytes are main's. type moveView struct { State string `json:"state"` Target string `json:"target,omitempty"` UpdatedAt int64 `json:"updated_at,omitempty"` ID string `json:"id,omitempty"` Last *provision.MoveOutcome `json:"last,omitempty"` Protocol string `json:"protocol,omitempty"` // Phase is the step within the state (movePhase), so a poller reads // detail=1 only when the state or the phase changes (preflight L19). Phase string `json:"phase,omitempty"` } // movePhase is a move's phase as the flat view shows it: the cut's phase // while cutting, the landing's while landing, and the rollback's target and // source phases while rolling back. Other states have none. func movePhase(m *provision.MoveState) string { switch m.State { case provision.MoveCutting: return m.CutPhase case provision.MoveLanding: return m.LandingPhase case provision.MoveRollingBack: if m.Rollback != nil { return m.Rollback.TargetPhase + "/" + m.Rollback.SourcePhase } } return "" } // moveProtocolPhase1 is the Phase 1 marker every move answer carries with // the switch on (preflight L4). The CLI refuses to send a mutation to a // console whose answer lacks it. const moveProtocolPhase1 = "phase1" func (s *Server) moveViewOf(rec *provision.Record) moveView { v := moveView{State: "none"} if rec != nil && rec.Move != nil { v = moveView{ State: rec.Move.State, Target: s.homeRegionOut(rec.Move.To), UpdatedAt: rec.Move.UpdatedAt.Unix(), } } if s.cfg.MovesEnabled { v.Protocol = moveProtocolPhase1 if rec != nil { if rec.Move != nil { v.ID, v.Phase = rec.Move.ID, movePhase(rec.Move) } v.Last = rec.LastMove } } return v } // armedCodes renders the armed half of an armedPlacements answer as the // sorted comma list every refusal names — one spelling for the create door // and the move door alike. func armedCodes(set map[string]bool) string { codes := make([]string, 0, len(set)) for c, armed := range set { if armed { codes = append(codes, c) } } slices.Sort(codes) return strings.Join(codes, ", ") } // stagingCodes is armedCodes for the move door's stricter bar: the regions // that publish a staging bucket, which is the set a move can actually name. // A separate renderer rather than a flag on armedCodes, because a refusal // that listed "armed" regions a move still could not use would send the // caller round the same loop again. func stagingCodes(caps map[string]placementCapability) string { codes := make([]string, 0, len(caps)) for c, cap := range caps { if cap.Reachable && cap.Staging { codes = append(codes, c) } } slices.Sort(codes) if len(codes) == 0 { return "(none)" } return strings.Join(codes, ", ") } // stagingNames is stagingCodes as the Phase 1 door says it (MV-N20): each // code under the name a caller passes (homeRegionOut), so movePlacements' // folded home is listed by its name. An unnamed home has no name a caller // could pass and is left out, and a home that is both folded and published // under its name is listed once. func (s *Server) stagingNames(caps map[string]placementCapability) string { names := make([]string, 0, len(caps)) for c, cap := range caps { if name := s.homeRegionOut(c); name != "" && cap.Reachable && cap.Staging { names = append(names, name) } } slices.Sort(names) names = slices.Compact(names) if len(names) == 0 { return "(none)" } return strings.Join(names, ", ") } // moveHomeName is the home region's configured name as the move door uses // it: the claim's home (dbClaimHomeRegion) on a host-claim console, // CONSOLE_HOME_REGION otherwise. It is the name homeRegionIn and // homeRegionOut translate with (they spell the rule out inline, which keeps // the copy gate's derivation of their sentences shallow), and box selection // reads the home with it, so the door and the selection agree when the // console's home is the claim-mode fallback. func (s *Server) moveHomeName() string { if s.cfg.HostClaim { return s.cfg.dbClaimHomeRegion() } return s.cfg.HomeRegion } // moveRegionAllowed is the manifest's allow-list for a move target. A // named target must be listed as it is. The home target ("") is allowed // when some entry names the home, read through homeRegionIn as the door // reads the caller's region. An empty list allows any target. func (s *Server) moveRegionAllowed(allowed []string, to string) bool { if len(allowed) == 0 { return true } if to != "" { return slices.Contains(allowed, to) } for _, r := range allowed { if s.homeRegionIn(r) == "" { return true } } return false } // homeRegionIn/Out translate between the home region's NAME (Config's // CONSOLE_HOME_REGION, what pickers and callers speak — issue #346) and the // record's "" (the unconstrained value the fleet's reconcile has always // scoped by). The record NEVER stores the code; the API never shows "". func (s *Server) homeRegionIn(region string) string { home := s.cfg.HomeRegion if s.cfg.HostClaim { home = s.cfg.dbClaimHomeRegion() } return homePlacementCode(home, region) } // homePlacementCode is homeRegionIn's rule for a given home name: the // home's name is the record's "", any other region is itself, and with no // home named nothing is the home. claimMoveRosterPlaces reads roster // regions through it too, so the door and the API translate home one way. func homePlacementCode(home, region string) string { return park.HomePlacementCode(home, region) } func (s *Server) homeRegionOut(region string) string { if region == "" { if s.cfg.HostClaim { return s.cfg.dbClaimHomeRegion() } return s.cfg.HomeRegion } return region } // moveTransition applies one guarded transition and maps its outcomes: // ErrMoveConflict → 409 with the transition's own sentence, success → the // flat move view + a serial bump so the source box notices in seconds. func (s *Server) moveTransition(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int) { s.moveTransitionSaying(w, r, acct, app, host, mut, okStatus, error.Error, "could not write the move \u2014 try again") // main's bytes } // moveTransitionP1 is moveTransition for the Phase 1 door: a refusal is its // recorded sentence (phase1Conflict), never err.Error(), and the failure // sentence has no em dash. The legacy door keeps main's bytes. func (s *Server) moveTransitionP1(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int) { s.moveTransitionSaying(w, r, acct, app, host, mut, okStatus, phase1Conflict, "could not write the move. try again") } func (s *Server) moveTransitionSaying(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int, conflict func(error) string, failed string) { ctx := r.Context() // Host-claim consoles: request, cut and abort all rewrite the record's // move journal, which only the (unbuilt) Moves protocol may do under // its own operation. Refused before any write. if s.cfg.HostClaim { jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) return } rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, acct.Sqid, app, host, mut) if errors.Is(err, provision.ErrMoveConflict) { jsonError(w, http.StatusConflict, conflict(err)) return } if err != nil { log.Printf("console: api instance move %s/%s/%s: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, failed) return } s.bumpSerial(ctx) writeJSON(w, okStatus, s.moveViewOf(rec)) } // handleAPIInstanceMoveStatus answers the poll: the flat view, "none" when // quiet — the tenant's freeze loop lives on this door. // // With the switch on (Plan 6 Task 2), an operator's GET with detail=1 gets // the detail answer (moveDetail); anyone else's gets the flat view, with no // error (M18). A committed landing whose claim has been converted to a // relocation (Paul's J3) answers state relocating in both, so the flat view // reads the claim while the record says committed, and only then. If that // read fails the flat view says what the record says; the poll goes on. func (s *Server) handleAPIInstanceMoveStatus(w http.ResponseWriter, r *http.Request) { acct, app, host, rec, ok := s.moveGate(w, r) if !ok { return } view := s.moveViewOf(rec) if !s.cfg.MovesEnabled { writeJSON(w, http.StatusOK, view) return } if r.URL.Query().Get("detail") == "1" { if _, isOp := s.moveOperator(r); isOp { s.writeMoveDetail(w, r, acct, app, host, rec, view) return } } if rec.Move != nil && rec.Move.State == provision.MoveCommitted { obs, err := hostclaim.Read(r.Context(), s.cfg.Releases, host) if err != nil { log.Printf("console: api instance move %s/%s/%s: status: claim read: %v", acct.Sqid, app, host, err) } else if moveConvertedAfterCommit(rec, obs) { view.State = provision.MoveRelocating } } writeJSON(w, http.StatusOK, view) } func (s *Server) legacyHandleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) { acct, app, host, rec, ok := s.moveGate(w, r) if !ok { return } if s.refuseSeedMove(w, rec) { return } // A replica is refused at the request only (G14): moveGate also fronts // the status and abort doors, and a replica that got a move before this // rail existed must still be able to read and abort it. if rec.Role != "" { jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region") return } var req struct { Region string `json:"region"` } if !readJSON(w, r, &req) { return } region := strings.TrimSpace(req.Region) to := s.homeRegionIn(region) if region == "" || !provision.ValidRegion(to) { jsonError(w, http.StatusBadRequest, "invalid region") return } // A move to the HOME region ("" after normalization) has no armed-edges // row to validate against and no staging lane wired for the home's // bucket yet — refused plainly rather than wedging in preparing. if to == "" { jsonError(w, http.StatusBadRequest, "moves to the home region are not supported yet — the home publishes no staging lane") return } // The armed gate is MANDATORY for moves — unlike create there is no // dormant path, because a move needs a live target box to restore on; // a deployment with no armed placements cannot be a move target at all. // // And the bar here is STAGING, not merely hosting: the requirement is // unconditional and is the target's published data-bucket coordinates, // because that is precisely what the worker on the other side demands. // internal/host's moveTargets skips any target whose edges file carries // no data_bucket, and runMoveRound then answers "target region %s // publishes no data bucket yet — waiting" on every pass, forever. Before // this the door accepted such a target on reachability alone and the // move wedged in preparing with no way out but a manual cancel — the // door and the worker disagreeing about the same fact, which is the // failure this whole family of gates exists to prevent. Unknown is not // an acceptable answer for a move the way it is for a create: a create // into an unconfigured box strands one instance, while a move stakes a // LIVE instance's residence on a lane that does not exist. caps, published := placements(r.Context(), s.cfg.Releases) if !published { jsonError(w, http.StatusBadRequest, "this deployment publishes no armed regions — nothing to move to") return } if !caps[to].Reachable || !caps[to].Staging { jsonError(w, http.StatusBadRequest, "region "+to+" cannot take a move in this deployment — it publishes no staging bucket. Regions that can: "+stagingCodes(caps)) return } s.moveTransition(w, r, acct, app, host, provision.MoveRequest(to, time.Now()), http.StatusAccepted) } func (s *Server) legacyHandleAPIInstanceMoveCut(w http.ResponseWriter, r *http.Request) { acct, app, host, _, ok := s.moveGate(w, r) if !ok { return } s.moveTransition(w, r, acct, app, host, provision.MoveMarkCutting(), http.StatusAccepted) } // legacyHandleAPIInstanceMoveAbort: abort is two-phase (review round 1): // this door marks MoveAborting and the // SOURCE box deletes the staged copy before clearing the field — a bare // clear would leave staging behind to win a later landing on the target // (staging-wins). 202, like its siblings: the caller polls the record. func (s *Server) legacyHandleAPIInstanceMoveAbort(w http.ResponseWriter, r *http.Request) { acct, app, host, _, ok := s.moveGate(w, r) if !ok { return } s.moveTransition(w, r, acct, app, host, provision.MoveAbort(), http.StatusAccepted) } // ---- Phase 1 (CARLOS_MOVES_PHASE1, index D13) ---- // // With the switch off each handler below hands the request to its legacy // twin above, unchanged, and the resolve door answers the family's 404: // the console behaves exactly as before Phase 1. With it on, every // mutation is operator-only, the request door CASes the host claim from // serving to moving before it journals the move (§7.1, K2), and cut and // abort carry the journal's id and incarnation (MoveGuard). // moveOperatorGate is moveGate for the mutations (§12): the caller must be // an operator presenting their own CLI token, else the family's // byte-identical 404. A member's token and the Instance-role service token // keep the status GET only (M18), so the member-minted instance token can // no longer move anything (codex 11). The routes keep // svcrole.Allow(svcrole.Instance) so a service token meets this 404 rather // than the role gate's 403. func (s *Server) moveOperatorGate(w http.ResponseWriter, r *http.Request) (acct *Account, app, host string, rec *provision.Record, addr string, ok bool) { acct, app, host, rec, gok := s.moveGate(w, r) if !gok { return nil, "", "", nil, "", false } addr, ok = s.moveOperator(r) if !ok { http.NotFound(w, r) return nil, "", "", nil, "", false } return acct, app, host, rec, addr, true } // moveOperator reports whether the request presents an operator's own CLI // token, and whose. A service token, a member's token and an error reading // the operator flag are all "no" (the error is logged). func (s *Server) moveOperator(r *http.Request) (addr string, ok bool) { addr, ok = s.store.CLITokenAddress(bearer(r), time.Now()) if !ok { return "", false } isOp, err := s.store.IsOperator(addr) if err != nil { log.Printf("console: api instance move: is-operator %s: %v", addr, err) } if err != nil || !isOp { return "", false } return addr, true } // moveGuardOf is the guard a door derives from the record it read. A record // with no move yields the zero guard, which every guarded transition // refuses with its own 409 ("no journaled move in flight", MV-N12). func moveGuardOf(rec *provision.Record) provision.MoveGuard { if rec.Move == nil { return provision.MoveGuard{} } return provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation} } // handleAPIInstanceMoveRequest is the request door (§7.1). Its order: the // class rails and the operator (moveOperatorGate), the record's own rails, // the body, the region, the app's regions, the instance's storage binding, // the armed placements, the fleet (selectMoveBoxes: one eligible box on // each side, the floor), then the claim (present, this incarnation, // serving, executed by the source's eligible box). Every refusal comes // before the claim CAS, so a refused request writes nothing. // // If the record write fails after the claim CAS, the door answers 502 and // logs: the claim is then moving with no journaled move, and the console // pass of Plan 5 Task 7 releases it. func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) { if !s.cfg.MovesEnabled { s.legacyHandleAPIInstanceMoveRequest(w, r) return } acct, app, host, rec, addr, ok := s.moveOperatorGate(w, r) if !ok { return } if s.refuseSeedMove(w, rec) { return } // Host-claim consoles start the move under the app's admission: the // claim runtime's authority registers it (RegisterMove), and the start // is hostops.BeginClaimMove then hostops.StartClaimMoveRecord, never a // plain BeginMove (whose claim C7 cannot convert). A console with the // flag on but no runtime cannot admit anything, so it refuses before any // write, like cut and abort (moveTransitionSaying). if s.cfg.HostClaim && s.cfg.ClaimAuthority == nil { jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) return } ctx := r.Context() if rec.Role != "" { jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region") return } if len(rec.Replicas) > 0 { jsonError(w, http.StatusBadRequest, "a home with replicas does not move. remove its replicas first") return } var req struct { Region string `json:"region"` Cut string `json:"cut"` } if !readJSON(w, r, &req) { return } cut := req.Cut if cut == "" { cut = "auto" } if cut != "auto" && cut != "manual" { jsonError(w, http.StatusBadRequest, `cut must be "auto" or "manual"`) return } region := strings.TrimSpace(req.Region) to := s.homeRegionIn(region) if region == "" || !provision.ValidRegion(to) { jsonError(w, http.StatusBadRequest, "invalid region") return } // Ruling R1 (home part 2): a move into the home region is admitted only // on a host-claim console, whose claim pins the home's engine region // (@home) under the roster's designation. The plain path refuses it // before any read or write. if to == "" && !s.cfg.HostClaim { jsonError(w, http.StatusBadRequest, "moves into the home region need host claims. this console runs without them") return } // toName is the target as the caller named it, which is // homeRegionOut(to): homeRegionIn maps only the home's own name to "", // so the round trip gives region back (Ruling R3: the caller names the // home by its configured name). Every sentence below names the home so, // never by the record's "". toName := region if to == rec.Region { jsonError(w, http.StatusConflict, "the instance is already in "+toName) return } // Both policy reads fail closed (round-4 finding 13): an unreadable // manifest or storage binding is a 503, never a pass. m, err := provision.GetManifest(ctx, s.cfg.Releases, acct.Sqid, app) if err != nil && !errors.Is(err, os.ErrNotExist) { log.Printf("console: api instance move %s/%s/%s: manifest: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusServiceUnavailable, "could not read the app's instances manifest. try again") return } if m != nil && !s.moveRegionAllowed(m.Regions, to) { jsonError(w, http.StatusBadRequest, "region "+toName+" is not one of this app's regions: "+strings.Join(m.Regions, ", ")) return } // Private storage is the instance's BINDING, not the app's create-time // policy (which is mutable and says nothing about an instance that // already exists): the create door stamps a store per instance under // its environment, or its slug when it has none (InstanceStoreEnv), and // turning the policy off later removes no store. A name that cannot be // an environment can hold no store, so it is not read. if env := provision.InstanceStoreEnv(rec); env == "" || appconfig.ValidEnvironmentName(env) { store, _, err := objstore.GetIn(ctx, s.cfg.Releases, acct.Sqid, app, env) if err != nil { log.Printf("console: api instance move %s/%s/%s: storage binding: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusServiceUnavailable, "could not read this instance's storage binding. try again") return } if store != nil { jsonError(w, http.StatusBadRequest, host+" uses private storage. Instances with private storage cannot move yet.") return } } // Region HA (spec 2 §7, H27): a destination with no box that has room // is the no-room refusal, before the placement checks below answer it // as a 400. It only refuses: nothing is chosen. An incomplete read is // the temporarily unavailable 503, never no room. if err := s.roomIn(ctx, to, ""); err != nil { writeClaimResponseError(w, err) return } caps, homeNamed, published := movePlacements(ctx, s.cfg.Releases, s.moveHomeName()) // Review Focus 5: a box that sets CARLOS_REGION to the home's name is // home to the console but not to itself, so it cannot take a move into // home. It is refused here, before the staging check: the home's fold // takes placementless files only, so that check would answer "publishes // no staging bucket", which hides what the operator has to fix. The // files are not liveness-joined, so a stale file of a box that has gone // refuses too (fail closed): the sentence names the file's box. if to == "" && len(homeNamed) > 0 { jsonError(w, http.StatusConflict, homeNamed[0]+" sets CARLOS_REGION, so it cannot take a move into the home region") return } if !published { jsonError(w, http.StatusBadRequest, "this deployment publishes no armed regions. nothing to move to") return } if !caps[to].Reachable || !caps[to].Staging { jsonError(w, http.StatusBadRequest, "region "+toName+" cannot take a move in this deployment. it publishes no staging bucket. Regions that can: "+s.stagingNames(caps)) return } from := s.homeRegionOut(rec.Region) sel, err := s.selectMoveBoxes(ctx, host, to, rec.Region, acct.Sqid+"/"+app) // No box in the destination has room (region HA spec 2 §7, H27), or the // read that would tell could not be completed: the claim doors' answers, // not the protocol floor's below. var ( full *noCapacityError check *roomCheckError ) if errors.As(err, &full) || errors.As(err, &check) { writeClaimResponseError(w, err) return } if errors.Is(err, errEdgesIncomplete) { jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again") return } if err != nil { log.Printf("console: api instance move %s/%s/%s: protocol floor: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusServiceUnavailable, "could not read the moves protocol floor. try again") return } // Phase 1 is for one-box regions (spec rev 7.20 §7.1, index D15): a // region with more than one eligible box, on either side, is refused. if sel.Many { jsonError(w, http.StatusBadRequest, sel.ManyRegion+" has more than one box. Moves between regions with several boxes are not supported yet.") return } if sel.Target == "" { jsonError(w, http.StatusBadRequest, "no box in "+toName+" publishes an edges file yet") return } if sel.Source == "" { jsonError(w, http.StatusBadRequest, "no eligible box in "+from+" publishes an edges file") return } if sel.ConsoleBelowFloor { jsonError(w, http.StatusConflict, "this console is below the moves protocol floor. roll the console first") return } if !sel.FloorOK { jsonError(w, http.StatusConflict, "a box in "+from+" or "+toName+" is below the moves protocol floor. roll the platform first") return } inc, err := hostclaim.Incarnation(acct.Sqid, app, host, rec.CreatedAt) if err != nil { jsonError(w, http.StatusBadRequest, "the record's identity is malformed") return } obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) if err != nil { log.Printf("console: api instance move %s/%s/%s: claim read: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the host claim") return } if obs.Absent || obs.Claim.Holder == nil { jsonError(w, http.StatusConflict, "the instance has no host claim yet. it is adopted on its next reconcile pass") return } if obs.Claim.Holder.Incarnation != inc { jsonError(w, http.StatusConflict, "the host claim names another incarnation of this instance") return } if obs.Claim.Op.Kind != hostclaim.KindServing { jsonError(w, http.StatusConflict, "the instance is under a "+obs.Claim.Op.Kind+" operation. wait for it to finish") return } // Ruling T5-lease (Task 5 fix round 2): Phase 1 cuts only a holder // whose location is the activator's own lease prefix. A scoped holder // (instances//) is refused here, before any claim // write, with the sentence the cut's admission would record (MV-N167); // when this console knows the activator's prefix, so is any other // location than the activator's. Moving such a host waits for a later // plan. if loc := strings.TrimSuffix(obs.Claim.Holder.Location, "/"); loc == "instances/"+host+"/"+inc || (s.cfg.HibernatePrefix != "" && loc != activator.HibernatePrefix(s.cfg.HibernatePrefix, host)) { jsonError(w, http.StatusConflict, moveScopedHolder) return } // §7.1 binds the source to the claim's executor (round-5 finding 7). if obs.Claim.Op.Exec != sel.Source { jsonError(w, http.StatusConflict, "the source box "+obs.Claim.Op.Exec+" is not the eligible box of "+from) return } // Ruling E2, kept as the early refusal: from ONE roster observation, // the source must be an active roster entry with an enrollment and must // carry no termination fence. Anything else is not a box to move from // (fail closed). A roster or fence that cannot be read or trusted is a // 503, with the error logged and never sent. On a host-claim console // BeginClaimMove checks both boxes again from the roster and records // both enrollments on the op, which is what the claim's conversion (C7) // is pinned to. dir := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} roster, err := dir.Read(ctx) if err != nil { log.Printf("console: api instance move %s/%s/%s: host roster: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") return } if _, eligible := sourceEnrollment(roster.Roster, obs.Claim.Op.Exec); !eligible { jsonError(w, http.StatusConflict, "the source box is not eligible to move from") return } fenced, err := dir.TerminationFence(ctx, roster, obs.Claim.Op.Exec, time.Now()) if err != nil { log.Printf("console: api instance move %s/%s/%s: termination fence of %s: %v", acct.Sqid, app, host, obs.Claim.Op.Exec, err) jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") return } if fenced { jsonError(w, http.StatusConflict, "the source box is not eligible to move from") return } id, err := moves.NewMoveID(rand.Reader) if err != nil { log.Printf("console: api instance move %s/%s/%s: move id: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not mint a move id") return } now := time.Now() ms := provision.MoveStart{ID: id, Incarnation: inc, From: rec.Region, To: to, Cut: cut, By: addr, Now: now} start := provision.MoveStartTransition(ms) // The record's own refusals (a move already journaled, a legacy move // in flight) are tried on a copy first, so they too come before the // claim CAS. The transition only replaces Move, so a shallow copy // leaves rec as read. probe := *rec if err := start(&probe); err != nil { jsonError(w, http.StatusConflict, phase1Conflict(err)) return } exec := obs.Claim.Op.Exec if s.cfg.HostClaim { s.startClaimModeMove(w, r, acct, app, host, obs, ms, sel.Target, from, rec.Region, dir, roster) return } moving, err := hostclaim.Update(ctx, s.cfg.Releases, obs, hostclaim.BeginMove(id, addr, exec, sel.Target, to, now)) if err != nil { moveClaimCASFailed(w, acct, app, host, toName, err) return } got, err := provision.SetRecordMove(ctx, s.cfg.Releases, acct.Sqid, app, host, start) if err != nil { log.Printf("console: api instance move %s/%s/%s: the claim is moving under %s but the record write failed: %v", acct.Sqid, app, host, id, err) got = s.settleFailedMoveWrite(w, r, ctx, acct, app, host, id, exec, moving, false) if got == nil { return } } s.bumpSerial(ctx) writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) } // moveClaimCASFailed answers a refused or failed start CAS, plain or // admitted, with its recorded sentence. The error itself is only logged. // toName is the target as the caller named it (homeRegionOut), never the // record's "" for the home. func moveClaimCASFailed(w http.ResponseWriter, acct *Account, app, host, toName string, err error) { switch { case errors.Is(err, hostclaim.ErrRelocationFinishing): jsonError(w, http.StatusConflict, "the instance's last relocation is still finishing. try again in a few minutes") case errors.Is(err, hostclaim.ErrPendingChore): jsonError(w, http.StatusConflict, "the instance has unfinished cleanup in "+toName+". wait for it to finish") case errors.Is(err, hostclaim.ErrConflict), errors.Is(err, hostclaim.ErrWrongState): jsonError(w, http.StatusConflict, "the host claim changed under this request. try again") default: log.Printf("console: api instance move %s/%s/%s: claim CAS: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not write the host claim") } } // startClaimModeMove is the request door's start on a host-claim console // (claim Plan 5 Task 7): the claim runtime's authority registers the move // under the app's admission (the registration id is the move id), then // BeginClaimMove CASes the same claim observation from serving to moving // with the app id, the registration and both roster enrollments on the op, // and StartClaimMoveRecord journals the move on the record as one effect // of that registration. The door never builds an authority of its own. // // Before anything is registered, the door's own roster observation (the // one E2 read) must place the move as BeginClaimMove will require: both // boxes active and enrolled, the source in the record's region, the target // in to, and no termination fence on the target. A roster that does not is // MV-N177. BeginClaimMove rereads the roster and the record; if it still // refuses (ErrIdentity, ErrRoster), something changed under the request // and the door does not guess what (MV-N178). // // Every refusal after RegisterMove and before the claim CAS writes nothing // but the registration, which the admission sweep settles once its TTL // passes. A failed record start after the CAS is settled as on the plain // path, with the release written under the registration's lifecycle grant. func (s *Server) startClaimModeMove(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, obs hostclaim.Observed, ms provision.MoveStart, target, from, recRegion string, dir hostactors.Directory, roster hostactors.Observed) { ctx := r.Context() a := s.cfg.ClaimAuthority // to is the record's code ("" for the home), which the claim calls // receive; toName is what the sentences say (home part 2). to := ms.To toName := s.homeRegionOut(to) if !roster.Roster.MatchesHome(s.cfg.dbClaimHomeRegion()) || !claimMoveRosterPlaces(roster.Roster, obs.Claim.Op.Exec, target, recRegion, to, s.cfg.dbClaimHomeRegion()) { jsonError(w, http.StatusConflict, "the host roster does not allow this move. the source and target must be active boxes in "+from+" and "+toName) return } fenced, err := dir.TerminationFence(ctx, roster, target, time.Now()) if err != nil { log.Printf("console: api instance move %s/%s/%s: termination fence of %s: %v", acct.Sqid, app, host, target, err) jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") return } if fenced { jsonError(w, http.StatusConflict, "the host roster does not allow this move. the source and target must be active boxes in "+from+" and "+toName) return } appID, err := s.store.ClaimAppID(acct.ID, app) if errors.Is(err, sql.ErrNoRows) || errors.Is(err, hostops.ErrIdentity) { jsonError(w, http.StatusConflict, "this app has no host-claim admission yet. try again shortly") return } if err != nil { log.Printf("console: api instance move %s/%s/%s: app id: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusServiceUnavailable, "could not read this app's admission. try again") return } h, err := a.RegisterMove(ctx, obs, appID, ms.ID) if err != nil { switch { case errors.Is(err, os.ErrNotExist): jsonError(w, http.StatusConflict, "this app has no host-claim admission yet. try again shortly") case errors.Is(err, hostops.ErrPending): jsonError(w, http.StatusConflict, "another operation holds this app. try again when it finishes") default: log.Printf("console: api instance move %s/%s/%s: register move %s: %v", acct.Sqid, app, host, ms.ID, err) jsonError(w, http.StatusBadGateway, "could not register the move. try again") } return } moving, err := hostops.BeginClaimMove(ctx, s.cfg.Releases, a.Directory, h, obs, ms.ID, ms.By, target, to, ms.Now, s.cfg.dbClaimHomeRegion()) if err != nil { switch { case errors.Is(err, hostops.ErrIdentity), errors.Is(err, hostactors.ErrRoster): log.Printf("console: api instance move %s/%s/%s: begin move %s: %v", acct.Sqid, app, host, ms.ID, err) jsonError(w, http.StatusConflict, "the instance or its boxes changed under this request. read its state and try again") case errors.Is(err, hostops.ErrPending): jsonError(w, http.StatusConflict, "another operation holds this app. try again when it finishes") default: moveClaimCASFailed(w, acct, app, host, toName, err) } return } got, err := hostops.StartClaimMoveRecord(ctx, s.cfg.Releases, h, moving, ms) if err != nil { log.Printf("console: api instance move %s/%s/%s: the claim is moving under %s but the record start failed: %v", acct.Sqid, app, host, ms.ID, err) // No retry is promised (MV-N179): the record-start effect is left // uncertain and refuses the next start on this record until the // claim code resolves it (task E-switch fix round 2). got = s.settleFailedMoveWrite(w, r, h.Lifecycle(ctx), acct, app, host, ms.ID, obs.Claim.Op.Exec, moving, true) if got == nil { return } } s.bumpSerial(ctx) writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) } // settleFailedMoveWrite decides a record write that failed after the claim // CAS succeeded. A failed write may still have landed, so the record is // read again first: // // - it carries this move's id: the write landed and only its answer was // lost. The move is returned and the door answers success; // - it definitely does not: the claim goes back to serving under exec // (ReleaseMove, guarded by RequireMoveOp so it can only undo this // move, before its seal) and the caller is told to try again; // - the read fails, or the release does: the outcome is unknown, so the // claim is left as it is and the answer promises nothing. The host and // move id are logged. Plan 5's recovery pass, which must land before // the switch is turned on anywhere, finds a moving claim with no // journaled move and releases it. // // claimCtx carries the authority the release's claim writes run under: the // request's own context on the plain path, the move registration's // lifecycle grant on a host-claim console (whose data boundary refuses an // unbound claim CAS). held picks the 502's sentence once the claim is // released: the plain path's invites a retry; the host-claim path's // (MV-N179, held) does not, because its record-start effect stays uncertain // and blocks the next one until the claim code can resolve it. // // It answers w itself and returns nil in every case but the first. func (s *Server) settleFailedMoveWrite(w http.ResponseWriter, r *http.Request, claimCtx context.Context, acct *Account, app, host, id, exec string, moving hostclaim.Observed, held bool) *provision.Record { ctx := r.Context() unknown := func(why string, err error) { log.Printf("console: api instance move %s/%s/%s: move %s: %s: %v; the claim is left moving", acct.Sqid, app, host, id, why, err) jsonError(w, http.StatusBadGateway, "could not confirm whether the move was recorded. read its state before trying again") } rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) if err != nil { unknown("record reread", err) return nil } if rec.Move != nil && rec.Move.ID == id { return rec } release := func(c *hostclaim.Claim) error { if err := hostclaim.RequireMoveOp(id, hostclaim.KindMoving, hostclaim.PhaseMoving)(c); err != nil { return err } return hostclaim.ReleaseMove(id, exec)(c) } _, err = hostclaim.Update(claimCtx, s.cfg.Releases, moving, release) if errors.Is(err, hostclaim.ErrConflict) { // Something else wrote the claim since the CAS: read it and try // once more. The guard refuses unless it is still this move. var cur hostclaim.Observed if cur, err = hostclaim.Read(ctx, s.cfg.Releases, host); err == nil { _, err = hostclaim.Update(claimCtx, s.cfg.Releases, cur, release) } } if err != nil { unknown("claim release", err) return nil } if held { jsonError(w, http.StatusBadGateway, "the move could not be recorded and is held until an operator checks the record") return nil } jsonError(w, http.StatusBadGateway, "could not record the move. try again") return nil } // handleAPIInstanceMoveCut is ready → cutting/requested under the guard. A // move journaled before ids existed (Legacy) has no guard to carry; it can // only be aborted (MV-N7 says so). func (s *Server) handleAPIInstanceMoveCut(w http.ResponseWriter, r *http.Request) { if !s.cfg.MovesEnabled { s.legacyHandleAPIInstanceMoveCut(w, r) return } acct, app, host, rec, _, ok := s.moveOperatorGate(w, r) if !ok { return } if rec.Move.Legacy() { jsonError(w, http.StatusConflict, "this move was started by an older console and has no id. abort it and request it again") return } if s.cfg.HostClaim { s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveCut) return } s.moveTransitionP1(w, r, acct, app, host, provision.MoveGuardedCut(moveGuardOf(rec)), http.StatusAccepted) } // handleAPIInstanceMoveAbort marks the move aborting under the guard // (legal in preparing, ready and cutting/requested, §7.11). It has no // claim-side effect: the claim stays moving until the source's interim // abort deletes staging and releases it (hostclaim.ReleaseMove, Task 14). // A Legacy() record takes the legacy MoveAbort (F7(a)), the one transition // that accepts it. Once the cut has started the answer is M-C13 (preflight // L5) on either console: without the host claim abortBeforeTheCut decides // it under the record's CAS; a host-claim console decides it from the record // the operator gate read, before claim Plan 10's typed grant // (claimConsoleMove), which after the seal would refuse with "the move // changed" (final review I5). A stuck move keeps the grant's own answer. func (s *Server) handleAPIInstanceMoveAbort(w http.ResponseWriter, r *http.Request) { if !s.cfg.MovesEnabled { s.legacyHandleAPIInstanceMoveAbort(w, r) return } acct, app, host, rec, _, ok := s.moveOperatorGate(w, r) if !ok { return } if rec.Move.Legacy() { s.moveTransitionP1(w, r, acct, app, host, legacyAbortOf(*rec.Move), http.StatusAccepted) return } if s.cfg.HostClaim { // A console without its claim runtime keeps HC5-N1 for every move // (claimConsoleMove), and a record with no move keeps its refusal. if a := s.cfg.ClaimAuthority; a != nil && a.Control != nil && rec.Move != nil && moveCutStarted(rec.Move) { jsonError(w, http.StatusConflict, moveCutStartedSentence) return } s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveAbort) return } s.moveTransitionP1(w, r, acct, app, host, abortBeforeTheCut(moveGuardOf(rec)), http.StatusAccepted) } // moveCutStartedSentence is M-C13, the abort door's answer once the cut has // started (preflight L5): the move then finishes forward or moves back by // itself, and an abort could only confuse the operator about which. const moveCutStartedSentence = "This move has started its cut. It will finish or move back on its own." // moveCutStarted is "past cutting/requested and not stuck": a cut past its // request (MoveGuardedAbort's own bar), a landing, a committed move, a // rollback or a revert. A stuck move keeps provision's refusal, and // relocating and aborting keep theirs. func moveCutStarted(m *provision.MoveState) bool { switch m.State { case provision.MoveCutting: return m.CutPhase != provision.CutRequested case provision.MoveLanding, provision.MoveCommitted, provision.MoveRollingBack, provision.MoveReverting: return true } return false } // abortBeforeTheCut is MoveGuardedAbort that refuses with M-C13 when the // journal it is about to abort, this guard's move, has started its cut. It // decides on the record SetRecordMove reads under its CAS, not on the // door's earlier read, so a cut that advances in between is still M-C13. // Anything else is MoveGuardedAbort's own answer. func abortBeforeTheCut(g provision.MoveGuard) func(*provision.Record) error { abort := provision.MoveGuardedAbort(g) return func(rec *provision.Record) error { if m := rec.Move; m != nil && m.ID == g.ID && m.Incarnation == g.Incarnation && moveCutStarted(m) { return fmt.Errorf("%w: %s", provision.ErrMoveConflict, moveCutStartedSentence) } return abort(rec) } } // handleAPIInstanceMoveResolve is the operator's door out of a stuck move // (M-C11). Its body is {"retry": true} (Plan 4 Task 7, §7.8, §7.10): the // record write is provision.MoveRetry under the guard from the record, which // answers 409 unless the move is stuck. The retry clears the stuck reason // only; the box then resumes at the phase the stuck rollback recorded // (a verification runs again and is never bypassed, a probe gets its nine // attempts again, a revert resumes its rev_phase). The answer is the flat // view: the stuck reason is the CLI's to say (M-C11), read from the record. // Any other body is "nothing to resolve yet" until Plan 4 Task 10 and Plan 6 // Task 2 give the door the rest. With the switch off it is the 404 the // route answered before it existed. func (s *Server) handleAPIInstanceMoveResolve(w http.ResponseWriter, r *http.Request) { if !s.cfg.MovesEnabled { http.NotFound(w, r) return } acct, app, host, rec, addr, ok := s.moveOperatorGate(w, r) if !ok { return } if s.cfg.HostClaim { s.claimMoveResolve(w, r, acct, app, host, rec, addr) return } var req struct { Retry bool `json:"retry"` DeclareLost string `json:"declare_lost"` } if !readJSON(w, r, &req) { return } // A console without the host claim cannot run the claim's loss protocol // (claim Plan 10's inspect and declare): MV-N11, nothing written // (preflight L7). if req.DeclareLost != "" { jsonError(w, http.StatusConflict, "declare-lost is not enabled in this deployment yet") return } if !req.Retry { jsonError(w, http.StatusConflict, "nothing to resolve yet") return } s.moveTransitionP1(w, r, acct, app, host, provision.MoveRetry(moveGuardOf(rec)), http.StatusAccepted) } // claimMoveResolve is the resolve door on a host-claim console (claim Plan // 10, rulings J17 and J4). Its body is exactly one of // // {"retry": true} the typed retry (claimConsoleMove) // {"declare_lost": ""} the signed challenge for the target's loss // {"declare_lost": "", "submission": {…}} the declaration of that challenge // // decoded strictly: an unknown field, or retry beside a declaration, is // refused before anything is read. An empty body is still "nothing to // resolve yet". Without its claim runtime the console keeps HC5-N1. func (s *Server) claimMoveResolve(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, actor string) { var req struct { Retry bool `json:"retry"` DeclareLost string `json:"declare_lost"` Submission *hostfence.Submission `json:"submission"` } if !readJSONStrictLimit(w, r, &req, 1<<20) { return } switch { case req.Retry && (req.DeclareLost != "" || req.Submission != nil), req.DeclareLost == "" && req.Submission != nil: jsonError(w, http.StatusBadRequest, "invalid resolution request") case req.Retry: s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveRetry) case req.DeclareLost != "": s.claimMoveLoss(w, r, acct, app, host, actor, req.DeclareLost, req.Submission) default: jsonError(w, http.StatusConflict, "nothing to resolve yet") } } // claimMoveLoss inspects (no submission) or declares (a submission) the // loss of a move's target through the configured ClaimAuthority, never an // authority, writer or store of its own. A declaration answers the move's // flat view, like retry. func (s *Server) claimMoveLoss(w http.ResponseWriter, r *http.Request, acct *Account, app, host, actor, box string, sub *hostfence.Submission) { a := s.cfg.ClaimAuthority if a == nil || a.Control == nil { jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) return } ctx := r.Context() if sub == nil { in, err := a.InspectMoveLoss(ctx, actor, acct.Sqid, app, host, box) if err != nil { claimMoveLossFailed(w, acct, app, host, err) return } writeJSON(w, http.StatusOK, in) return } if err := s.declareMoveLossLogged(ctx, acct.Sqid, app, host, *sub, func() error { return a.DeclareMoveLoss(ctx, actor, acct.Sqid, app, host, box, *sub) }); err != nil { claimMoveLossFailed(w, acct, app, host, err) return } s.bumpSerial(ctx) got, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) if err != nil { log.Printf("console: api instance move %s/%s/%s: declared loss: record reread: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the instance") return } writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) } // claimMoveLossFailed answers a refused inspection or declaration: a // recorded refusal is its sentence, a stale or refused binding is // moveChangedSentence, anything else the fixed failure. Errors are logged. func claimMoveLossFailed(w http.ResponseWriter, acct *Account, app, host string, err error) { switch { case errors.Is(err, provision.ErrMoveConflict): jsonError(w, http.StatusConflict, phase1Conflict(err)) case errors.Is(err, hostresolve.ErrResolve), errors.Is(err, hostfence.ErrChallenge), errors.Is(err, hostfence.ErrObservation), errors.Is(err, hostactors.ErrRoster), errors.Is(err, hostclaim.ErrLossRecord), errors.Is(err, hostclaim.ErrStaleOp), errors.Is(err, hostclaim.ErrLossOwnerRequired): log.Printf("console: api instance move %s/%s/%s: declared loss refused: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusConflict, moveChangedSentence) default: claimConsoleMoveFailed(w, acct, app, host, err) } } // claimConsoleMove is the cut, abort and retry doors on a host-claim console // (claim Plan 10, ruling J17): the configured ControlWriter's typed // transition, bound to the claim and record observed here. It never builds // an authority, a writer or a store. A console without its claim runtime, // and a move journal with no id, keep HC5-N1. Every refusal is a recorded // sentence; anything else is logged and answers the fixed failure. func (s *Server) claimConsoleMove(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, action hostops.ConsoleMoveAction) { a := s.cfg.ClaimAuthority if a == nil || a.Control == nil || rec.Move.Legacy() { jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) return } if rec.Move == nil { jsonError(w, http.StatusConflict, "no journaled move in flight") return } ctx := r.Context() o, err := hostops.ObserveConsoleMove(ctx, s.cfg.Releases, acct.Sqid, app, host) var got *provision.Record if err == nil && (o.OpID != rec.Move.ID || o.Incarnation != rec.Move.Incarnation || o.RecordSeq != rec.RecordSeq) { // The gate read a different move, incarnation or record than the // claim and record observed now: something replaced it in between. err = hostops.ErrIdentity } if err == nil { got, err = a.Control.ConsoleMove(ctx, o, action) } if err != nil { claimConsoleMoveFailed(w, acct, app, host, err) return } s.bumpSerial(ctx) writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) } // claimConsoleMoveFailed answers a refused typed move write. A provision // refusal is its recorded sentence; a changed claim, record or registration // is moveChangedSentence; the error itself is only logged. func claimConsoleMoveFailed(w http.ResponseWriter, acct *Account, app, host string, err error) { switch { case errors.Is(err, provision.ErrMoveConflict): jsonError(w, http.StatusConflict, phase1Conflict(err)) case errors.Is(err, hostops.ErrIdentity), errors.Is(err, hostops.ErrPending), errors.Is(err, hostclaim.ErrConflict), errors.Is(err, hostclaim.ErrWrongState), errors.Is(err, release.ErrCASConflict), errors.Is(err, os.ErrNotExist): log.Printf("console: api instance move %s/%s/%s: typed move write refused: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusConflict, moveChangedSentence) default: log.Printf("console: api instance move %s/%s/%s: typed move write: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not write the move. try again") } } // registerMovesPhase1Routes adds the routes that exist only with the switch // on. The Instance role stays on resolve only so a service token meets the // family's 404 rather than the role gate's 403. func (s *Server) registerMovesPhase1Routes() { s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/resolve", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceMoveResolve) } // errMoveChanged refuses a write whose move is no longer the one the door // read. It wraps ErrMoveConflict so moveTransition answers 409. var errMoveChanged = fmt.Errorf("%w: %s", provision.ErrMoveConflict, moveChangedSentence) const moveChangedSentence = "the move changed since this request read it. read its state and try again" // legacyAbortOf is the legacy MoveAbort bound to the legacy move the door // read (F7(a)). SetRecordMove reads the record again before it writes, and // MoveAbort alone would abort whatever preparing or ready move it found // there, so a request delayed past that move's end could abort its // replacement. A legacy journal has no id; its target and request time // identify it. func legacyAbortOf(seen provision.MoveState) func(*provision.Record) error { return func(rec *provision.Record) error { m := rec.Move if m == nil || !m.Legacy() || m.To != seen.To || !m.RequestedAt.Equal(seen.RequestedAt) { return errMoveChanged } return provision.MoveAbort()(rec) } } // phase1Conflict maps a provision transition refusal to its recorded // sentence (copy record MV-N12 to MV-N16, MV-N38 to MV-N46, and Plan 4's // MV-N300 to MV-N309: the rollback, retry and declare-lost refusals and the // rollback and stuck reasons): never the // "provision: move state conflict:" prefix, never an em dash. A refusal // this table does not know answers a fixed sentence, so a new provision // sentence cannot reach an operator unreviewed. func phase1Conflict(err error) string { msg := strings.TrimPrefix(err.Error(), provision.ErrMoveConflict.Error()+": ") switch { case msg == "past the point of no return \u2014 the cut completes forward": // The legacy MoveAbort's sentence, reworded. return "past the point of no return. the cut completes forward" case strings.Contains(msg, "\u2014"): case msg == "no journaled move in flight", msg == "malformed move start", msg == "past the point of no return. the cut completes forward or moves back on its own", msg == "cut is only legal from ready", msg == "already aborting", msg == "no move in flight", msg == moveChangedSentence, msg == moveCutStartedSentence, // provision.RefusedCommitted, RefusedNotStuck and // RefusedNothingToDeclare (TestPhase1ConflictPassesPlan4Refusals // holds the literals to the constants). msg == "the move has committed. it cannot roll back", msg == "the move is not stuck. there is nothing to retry", msg == "the target holds nothing the move still needs. there is nothing to declare lost", // The reasons Plan 4 records (ruling J9): provision.RollbackDeclaredLost, // StuckBaselineMismatch, StuckRecoveryProbeAttempts, StuckRevertProbeBudget, // and the shared installer's StuckReturnFilesOccupied, // StuckReturnInstallMismatch and StuckReturnUnknownStamp (Task 3), and // the target's StuckReturnTargetDatabase, StuckReturnTargetChanged // and StuckReturnTargetFiles (Task 4). msg == "the target box was declared lost", msg == "the source no longer matches what it recorded when it stopped", msg == "the recovery probe ran out of attempts", msg == "the source did not answer within 15 minutes of moving back", msg == "the files directory holds files the return did not write", msg == "the returned database does not match the build", msg == "the source holds files of an installation the return cannot account for", msg == "the target's stopped database is missing", msg == "the target changed after it stopped", msg == "the target's files cannot be archived", strings.HasPrefix(msg, "the record's move is ") && strings.Contains(msg, " for incarnation "), strings.HasPrefix(msg, "a move to ") && strings.Contains(msg, " is already "), strings.HasPrefix(msg, "the instance is already in "), strings.HasPrefix(msg, "a move in state ") && strings.HasSuffix(msg, " cannot be aborted"), strings.HasPrefix(msg, "a ") && strings.HasSuffix(msg, " does not move. delete it and create one in the new region"): return msg } return "this move's state does not allow that. read its state and try again" } // claimMoveRosterPlaces is BeginClaimMove's roster precondition, read from // the door's own observation: both boxes have active, enrolled entries // under their box references, they differ, the source is in the record's // region and the target in to, which is not the source's region. // // The roster and the record name regions differently for one region, the // home: the roster refuses an empty region, so a home box is enrolled under // the home's name (CONSOLE_HOME_REGION, as boxchoice reads it), while the // record keeps "". Both boxes' roster regions are read as record codes // through homePlacementCode with home, the deployment's home name (the rule // homeRegionIn applies), so a move out of home is placed and a named region // is compared as it is. With no home named nothing translates, and a home // source is refused as before (home-region parks, part 1). func claimMoveRosterPlaces(r hostactors.Roster, source, target, recRegion, to string, home ...string) bool { return hostops.ClaimMoveRosterPlaces(r, source, target, recRegion, to, home...) } // sourceEnrollment is the enrollment of the box that executes the claim, and // whether the roster lets a move start from it: an entry under that box // reference (never a label), in state "active", with a non-empty // enrollment. The termination fence is a separate check on the same // observation. func sourceEnrollment(r hostactors.Roster, box string) (string, bool) { b, ok := r.Boxes[box] if !ok || b.State != "active" || b.Enrollment == "" { return "", false } return b.Enrollment, true } // ---- GET …/move?detail=1 (Plan 6 Task 2, spec §12) ---- // moveDetail is the operator's detail answer: the flat view, unchanged, and // beside it the detail object. Everything in it is a record field, a claim // field, a report the boxes published for the CLI, or an object key: never // an evidence or log body, and never a launch environment value. The // console cannot tell an HMAC from a raw value of the same shape in a // malformed report, so neither report's env section passes any value: the // source's environment is names and kinds (moveSourceReport), and the // target's env input is blanked. type moveDetail struct { moveView Detail moveDetailBody `json:"detail"` } type moveDetailBody struct { // Move is the record's journal as read; nil when no move is in flight. Move *provision.MoveState `json:"move,omitempty"` // Claim is the host claim's operation; nil when the host has no claim. Claim *moveClaimView `json:"claim,omitempty"` // TargetReport and SourceReport are the reports under the move's report // prefix, nil when absent or unreadable as a report. TargetReport *moves.TargetReport `json:"target_report,omitempty"` SourceReport *moveSourceReport `json:"source_report,omitempty"` // Evidence names the stop evidence the record points at, by key. Evidence moveEvidenceKeys `json:"evidence"` // Log is every key under the move's log/ prefix, sorted (preflight L26: // a move writes a few dozen; the CLI reads bodies only on demand). Log []string `json:"log"` // Timings are a closed move's timings against their budgets (Plan 6 // Task 4, moves.Timings), in seconds: the timing fields of its done or // rolled_back log entry and nothing else of it, as the entry's writer // worked them out from the record's journal. Absent until that entry // is logged. Timings map[string]float64 `json:"timings,omitempty"` // WaitingForBox is the region a relocating move waits on for a live // enrolled box (M-C22), empty otherwise and when it cannot be told. WaitingForBox string `json:"waiting_for_box,omitempty"` // From is the move's origin and Region the record's region now, each // named as the caller names it (homeRegionOut), so the CLI's sentences // (M-C1, M-C7, M-C9) never print the record's "" for the home. From string `json:"from,omitempty"` Region string `json:"region"` } // moveSourceReport is the source report with its environment replaced by // one entry per variable: the name and its T3 kind (moves.Class). The // report's Env (per-name HMACs and their digest) is shadowed by this Env // and never encoded. type moveSourceReport struct { moves.SourceReport Env []moveEnvEntry `json:"env"` } // moveEnvEntry is a launch environment variable as the detail shows it. The // reports compare values only by HMAC and record no per-name result, so // the kind is all there is to say beside the name. type moveEnvEntry struct { Name string `json:"name"` Kind string `json:"kind"` } // sourceReportWithoutValues is the source report as the detail shows it. func sourceReportWithoutValues(r moves.SourceReport) *moveSourceReport { names := make([]string, 0, len(r.Env.Vars)) for name := range r.Env.Vars { names = append(names, name) } slices.Sort(names) out := &moveSourceReport{Env: make([]moveEnvEntry, 0, len(names))} for _, name := range names { out.Env = append(out.Env, moveEnvEntry{Name: name, Kind: moves.Class(name)}) } r.Env = moves.EnvReport{} out.SourceReport = r return out } // moveClaimView is the claim operation's shape and nothing else (no journal, // no holder). Target is the move's target box, Intent["target"], which M-C10 // names (preflight L28). type moveClaimView struct { Kind string `json:"kind"` Phase string `json:"phase,omitempty"` Exec string `json:"exec,omitempty"` Participants []string `json:"participants,omitempty"` Target string `json:"target,omitempty"` } // moveEvidenceKeys are the stop evidence keys the record carries: the // source's (Move.Source.Evidence, written at stopped) and the target's // (Move.Rollback.Evidence, written when a rollback released it). type moveEvidenceKeys struct { Source string `json:"source,omitempty"` Target string `json:"target,omitempty"` } // moveRelocationRegion is the region of a relocating claim operation that // belongs to this record's move (the same holder incarnation), and whether // there is one. func moveRelocationRegion(rec *provision.Record, obs hostclaim.Observed) (string, bool) { if rec == nil || rec.Move == nil || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil { return "", false } c := obs.Claim if c.Op.Kind != hostclaim.KindRelocating || c.Holder.Incarnation != rec.Move.Incarnation || c.Op.Intent["region"] == "" { return "", false } return c.Op.Intent["region"], true } // moveConvertedAfterCommit is Paul's J3 as the status door sees it: the // record says committed and the claim is relocating this incarnation in the // move's target region, because the target was terminated after the commit. // The door answers relocating for it. func moveConvertedAfterCommit(rec *provision.Record, obs hostclaim.Observed) bool { region, ok := moveRelocationRegion(rec, obs) return ok && rec.Move.State == provision.MoveCommitted && park.PlacementMatches(obs.Claim.Op.Intent["home_region"], rec.Move.To, region) } // writeMoveDetail answers an operator's detail=1. The claim read and the // report and log reads fail closed with a fixed sentence; a report that does // not parse is left out, as the boxes treat it (no report). The region // check behind waiting_for_box never fails the answer: when it cannot tell, // the field is left out and the state still says relocating. func (s *Server) writeMoveDetail(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, view moveView) { ctx := r.Context() obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) if err != nil { log.Printf("console: api instance move %s/%s/%s: detail: claim read: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the host claim") return } out := moveDetail{moveView: view, Detail: moveDetailBody{Move: rec.Move, Log: []string{}, Region: s.homeRegionOut(rec.Region)}} if rec.Move != nil { out.Detail.From = s.homeRegionOut(rec.Move.From) } if !obs.Absent && obs.Claim != nil { op := obs.Claim.Op out.Detail.Claim = &moveClaimView{Kind: op.Kind, Phase: op.Phase, Exec: op.Exec, Participants: op.Participants, Target: op.Intent["target"]} } if m := rec.Move; m != nil { if m.Source != nil { out.Detail.Evidence.Source = m.Source.Evidence } if m.Rollback != nil { out.Detail.Evidence.Target = m.Rollback.Evidence } } // The reports and the log of the move in flight, or of the last one to // close. A legacy journal has no id and so no report prefix. opID := "" if rec.Move != nil { opID = rec.Move.ID } else if rec.LastMove != nil { opID = rec.LastMove.ID } if provision.ValidMoveID(opID) { prefix := moves.ReportPrefix(rec.Account, rec.App, rec.Host, opID) var target moves.TargetReport var source moves.SourceReport tok, terr := s.readMoveReport(ctx, moves.TargetReportKey(prefix), &target) sok, serr := s.readMoveReport(ctx, moves.SourceReportKey(prefix), &source) keys, lerr := s.cfg.Releases.List(ctx, prefix+"log/") if err := errors.Join(terr, serr, lerr); err != nil { log.Printf("console: api instance move %s/%s/%s: detail: reports of %s: %v", acct.Sqid, app, host, opID, err) jsonError(w, http.StatusBadGateway, "could not read the move's reports. try again") return } if tok { target.Inputs.EnvReport = "" out.Detail.TargetReport = &target } if sok { out.Detail.SourceReport = sourceReportWithoutValues(source) } slices.Sort(keys) out.Detail.Log = append(out.Detail.Log, keys...) if rec.Move == nil && rec.LastMove != nil { timings, err := s.readMoveTimings(ctx, prefix, rec.LastMove.Outcome) if err != nil { log.Printf("console: api instance move %s/%s/%s: detail: timings of %s: %v", acct.Sqid, app, host, opID, err) jsonError(w, http.StatusBadGateway, "could not read the move's reports. try again") return } out.Detail.Timings = timings } } if (rec.Move != nil && rec.Move.State == provision.MoveRelocating) || moveConvertedAfterCommit(rec, obs) { out.State = provision.MoveRelocating // Any phase that runs on no box (starting, releasing and pinning // today, hostclaim's validateRelocating) is waiting for one. if region, ok := moveRelocationRegion(rec, obs); ok && obs.Claim.Op.Exec == "none" { code, valid := park.EnginePlacement(obs.Claim.Op.Intent["home_region"], region) live, known := false, false if valid { live, known = s.regionHasLiveBox(ctx, code) } if known && !live { out.Detail.WaitingForBox = s.homeRegionOut(code) } } } writeJSON(w, http.StatusOK, out) } // readMoveTimings is the timings a closed move's terminal log entry carries // (moves.TimingsOf over its Fields): nil for an outcome that carries none // (aborted), for an entry not logged yet, and for one that does not parse // (logged); an error when the read failed. func (s *Server) readMoveTimings(ctx context.Context, prefix, outcome string) (map[string]float64, error) { if outcome != "done" && outcome != "rolled_back" { return nil, nil } var ev moves.LogEvent ok, err := s.readMoveReport(ctx, moves.LogKeyFor(prefix, outcome), &ev) if !ok || err != nil { return nil, err } if t := moves.TimingsOf(ev.Fields); len(t) > 0 { return t, nil } return nil, nil } // readMoveReport reads one report: (false, nil) when it is absent or does // not parse (logged), an error when the read failed. func (s *Server) readMoveReport(ctx context.Context, key string, into any) (bool, error) { body, _, err := s.cfg.Releases.Get(ctx, key) if errors.Is(err, os.ErrNotExist) { return false, nil } if err != nil { return false, fmt.Errorf("read %s: %w", key, err) } if err := json.Unmarshal(body, into); err != nil { log.Printf("console: move report %s does not parse: %v", key, err) return false, nil } return true, nil } // regionHasLiveBox reports whether the region (record vocabulary) has a live // enrolled box, through the region HA contracts (Task 2 rulings, L18): // boxchoice.Eligible (eligibleBoxes) over the live edges snapshot, with the // roster read through hostactors.Directory, never directly. A roster entry // counts when it is active, has an enrollment, is in the region, and has no // termination fence; its placement and cloud are the roster's. The AWS // region is the one the code's eligible boxes publish (regionCands), as the // move door selects. known is false when the snapshot is incomplete or the // roster or a fence cannot be read or trusted: then nothing is concluded. func (s *Server) regionHasLiveBox(ctx context.Context, region string) (live, known bool) { snap, err := s.liveEdges(ctx) if err != nil || !snap.EvidenceComplete() { log.Printf("console: move detail: the edges snapshot of %s is incomplete (%v)", region, err) return false, false } dir := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} o, err := dir.Read(ctx) if err != nil { log.Printf("console: move detail: host roster: %v", err) return false, false } now := time.Now() roster := map[string]rosterBox{} for ref, b := range o.Roster.Boxes { if b.State != "active" || b.Enrollment == "" || s.homeRegionIn(b.Region) != region { continue } fenced, err := dir.TerminationFence(ctx, o, ref, now) if err != nil { log.Printf("console: move detail: termination fence of %s: %v", ref, err) return false, false } if !fenced { roster[ref] = rosterBox{Placement: b.Region, Cloud: b.Cloud} } } aws := "" for _, c := range s.regionCands(snap) { if c.Code == region { aws = c.AWSRegion } } return len(eligibleBoxes(snap, roster, region, aws, s.cfg.dbClaimHomeRegion(), s.expectedSecretsFP(), nil)) > 0, true } // ---- the delete door during a move (Plan 6 Task 2b, preflight L6) ---- // moveDeleteRefusal is M-C19, what the delete door says for a host that is // being moved. func moveDeleteRefusal(host string) string { return host + " is being moved. Delete it after the move finishes, or abort the move first." } // moveClaimKinds are the claim operations a move runs under. A relocating // operation is a move's only when the record still journals the move (a // region HA relocation of its own is not one). var moveClaimKinds = map[string]bool{hostclaim.KindMoving: true, hostclaim.KindLanding: true, hostclaim.KindReverting: true, hostclaim.KindRestoring: true} // refuseDeleteDuringMove is the delete door's move check, with the switch // on: before any retire effect, a host that is being moved answers 409 // M-C19 and nothing is written. On a console without the host claim that is // a record carrying a move (any state, legacy included); on a host-claim // console it is a claim whose operation is a move's. The claim would refuse // the retire anyway; this check only says why, before the retire registers // anything. A record that is not there is main's idempotent delete. A read // that fails is a 502 (HC5-N3), never a pass. It reports whether it // answered. func (s *Server) refuseDeleteDuringMove(ctx context.Context, w http.ResponseWriter, acct, app, host string) bool { if !s.cfg.MovesEnabled { return false } rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct, app, host) if errors.Is(err, os.ErrNotExist) { return false } if err != nil { log.Printf("console: api instances delete %s/%s/%s: move check: record: %v", acct, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the instance") return true } moving := rec.Move != nil if s.cfg.HostClaim { obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) if err != nil { log.Printf("console: api instances delete %s/%s/%s: move check: claim: %v", acct, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the instance") return true } kind := "" if !obs.Absent && obs.Claim != nil { kind = obs.Claim.Op.Kind } moving = moveClaimKinds[kind] || (kind == hostclaim.KindRelocating && rec.Move != nil) } if !moving { return false } jsonError(w, http.StatusConflict, moveDeleteRefusal(host)) return true }