package instancectl import ( "context" "fmt" "net/http" "strings" "sync" "time" "github.com/carlosframework/platform/internal/provision" ) func (s *Server) cmdPoll() time.Duration { if s.cmdPollEvery > 0 { return s.cmdPollEvery } return 250 * time.Millisecond } func (s *Server) cmdWait() time.Duration { if s.cmdWaitBound > 0 { return s.cmdWaitBound } return 30 * time.Second } func (s *Server) cmdStartWait() time.Duration { if s.cmdStartBound > 0 { return s.cmdStartBound } return 10 * time.Second } // handleCmd starts host's cmd unit and answers at once with the new // invocation (run spec §5.4); /seed blocks, this never does. The agent // stops a cmd unit after every run, so a unit still loaded (active, // activating, deactivating) belongs to a run this request must not end: it // is refused, never restarted. A failed one is reset first, or start would // keep its old result. func (s *Server) handleCmd(w http.ResponseWriter, r *http.Request) { req, err := parseRequest(r, verbShape{tier: true, exe: true, cmd: true}) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // One start per host at a time: a second request waits here, then reads // the unit loaded and is refused, instead of racing the first one's // invocation check and stopping its command. lock, _ := s.cmdLocks.LoadOrStore(req.host, new(sync.Mutex)) lock.(*sync.Mutex).Lock() defer lock.(*sync.Mutex).Unlock() ctx := r.Context() unit := UnitName(RoleCmd, req.host) switch state := s.showValue(ctx, "ActiveState", unit); state { case "active", "activating", "deactivating", "reloading", "refreshing": http.Error(w, "cmd: "+unit+" is "+state, http.StatusConflict) return case "failed": if out, err := s.Systemctl(ctx, "reset-failed", unit); err != nil { http.Error(w, "reset-failed: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) return } } // With generations on, the agent names the serving generation's staged // build, which root validates before it lays out the mount, as for the // legacy sidecar (zero-downtime spec §7.5). Without one, the live link, // as the seed. if s.refuseStagedExe(w, req, unit) { return } // The command's executable is laid out at its own source (merge ruling // M4), never at the host's app, which an instance or sidecar start may // be rewriting for a different build at this moment. if s.refuseStart(w, req.host, unit, func() error { return s.ensureCmdExeAt(req.host, req.exe) }) { return } // A command runs beside an awake instance, whose own start set the // slice. With generations on the slice is the generation parent, and a // swap may have raised it to two tiers (zero-downtime spec §7.3); its // restore is the swap owner's. So a populated slice is left alone, as // startGen and the sidecar's start leave it, and only a cold one gets // the tier. if s.slicePopulated(req.host) { s.logf("instance-control: %s: slice populated, leaving its limit to the instance's owner", req.host) } else if err := s.setTier(ctx, req.host, req.tier); err != nil { http.Error(w, "set-property: "+err.Error(), http.StatusBadGateway) return } before := s.showValue(ctx, "InvocationID", unit) if out, err := s.Systemctl(ctx, "start", "--no-block", unit); err != nil { http.Error(w, "start: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) return } // --no-block returns once the job is queued; the invocation exists // once the job runs. deadline := time.Now().Add(s.cmdStartWait()) for { if inv := s.showValue(ctx, "InvocationID", unit); inv != "" && inv != before { s.logf("instance-control: %s: started %s memory.max=%d", req.host, unit, req.tier.MemoryMax) writeJSON(w, http.StatusOK, StartResponse{InvocationID: inv}) return } if time.Now().After(deadline) || ctx.Err() != nil { dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) s.stopUnit(dctx, unit) cancel() http.Error(w, "cmd: "+unit+" showed no new invocation", http.StatusBadGateway) return } time.Sleep(s.cmdPoll()) } } // handleCmdStatus is the bounded long-poll: it answers as soon as the named // invocation is done or gone, and running at the bound. func (s *Server) handleCmdStatus(w http.ResponseWriter, r *http.Request) { host := r.PathValue("host") if !provision.ValidHostShape(host) { http.Error(w, fmt.Sprintf("host %q is not a host", host), http.StatusBadRequest) return } want := r.URL.Query().Get("invocation") if want == "" { http.Error(w, "invocation is required", http.StatusBadRequest) return } ctx := r.Context() unit := UnitName(RoleCmd, host) deadline := time.Now().Add(s.cmdWait()) for { out, err := s.Systemctl(ctx, "show", "-p", "InvocationID,ActiveState,SubState,Result,ExecMainStatus,ExecMainCode", unit) if err != nil { http.Error(w, "show: "+err.Error(), http.StatusBadGateway) return } st := cmdStatusOf(parseShow(out), want) if st.State != CmdRunning || !time.Now().Before(deadline) { writeJSON(w, http.StatusOK, st) return } select { case <-ctx.Done(): return case <-time.After(s.cmdPoll()): } } } // cmdStatusOf reads one show for the named invocation. func cmdStatusOf(m map[string]string, want string) CmdStatus { st := CmdStatus{InvocationID: m["InvocationID"], Result: m["Result"], Status: m["ExecMainStatus"], Code: m["ExecMainCode"]} active, sub := m["ActiveState"], m["SubState"] switch { case st.InvocationID != want: st.State = CmdGone case active == "failed", active == "inactive", active == "active" && sub == "exited": st.State = CmdDone default: st.State = CmdRunning } return st } // cleanCmdExe removes host's cmd executable sources after its cmd unit has // stopped. It holds the host's cmd lock, so a start cannot be laying them // out meanwhile, and leaves them for a unit that is not inactive or failed // (a start that has since begun). func (s *Server) cleanCmdExe(ctx context.Context, host string) { lock, _ := s.cmdLocks.LoadOrStore(host, new(sync.Mutex)) lock.(*sync.Mutex).Lock() defer lock.(*sync.Mutex).Unlock() switch s.showValue(ctx, "ActiveState", UnitName(RoleCmd, host)) { case "inactive", "failed": s.removeCmdExe(provision.InstanceID(host)) } }