package host import ( "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "log" "net/http" "os" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/release" ) // regionHAEnabled is CARLOS_REGION_HA: on only for a value // strconv.ParseBool reads as true ("1", "t", "true" in any case); "", // "0", "false", "off", "no" and anything unparseable are off. Every region // HA behaviour on a box is behind it (spec 1 §8 step 2). func regionHAEnabled(getenv func(string) string) bool { on, err := strconv.ParseBool(strings.TrimSpace(getenv("CARLOS_REGION_HA"))) return err == nil && on } const ( imdsBase = "http://169.254.169.254" imdsIdentityDocument = "/latest/dynamic/instance-identity/document" imdsIdentitySignature = "/latest/dynamic/instance-identity/signature" boxRecordRetry = time.Minute ) // imdsGetRaw reads one IMDSv2 path and returns its body untouched. The // identity document is signed byte for byte, so unlike imdsLeaf nothing is // trimmed. func imdsGetRaw(client *http.Client, base, token, path string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, base+path, nil) if err != nil { return nil, err } req.Header.Set("X-aws-ec2-metadata-token", token) resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("IMDS %s answered %d", path, resp.StatusCode) } return body, nil } // readIdentityRecord builds this box's record (spec §5.1) from IMDS: the // identity document exactly as served, and its base64 RSA-SHA256 // signature with any line breaks removed (base64 ignores them; Go's // decoder does not). func readIdentityRecord(client *http.Client, base, box string, now time.Time) (boxid.Record, error) { req, err := http.NewRequest(http.MethodPut, base+"/latest/api/token", nil) if err != nil { return boxid.Record{}, err } req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60") resp, err := client.Do(req) if err != nil { return boxid.Record{}, fmt.Errorf("IMDS token: %w", err) } tok, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) resp.Body.Close() if resp.StatusCode != http.StatusOK || len(strings.TrimSpace(string(tok))) == 0 { return boxid.Record{}, fmt.Errorf("IMDS token answered %d", resp.StatusCode) } token := strings.TrimSpace(string(tok)) doc, err := imdsGetRaw(client, base, token, imdsIdentityDocument) if err != nil { return boxid.Record{}, err } sig, err := imdsGetRaw(client, base, token, imdsIdentitySignature) if err != nil { return boxid.Record{}, err } cleaned := strings.Join(strings.Fields(string(sig)), "") if _, err := base64.StdEncoding.DecodeString(cleaned); err != nil || cleaned == "" { return boxid.Record{}, errors.New("IMDS signature is not base64") } return boxid.Record{V: 1, Box: box, IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: cleaned, WrittenAt: now.UTC().Format(time.RFC3339)}, nil } // recordInstanceID is the instanceId in a record's identity document. It // does not check the signature: it only tells whether a record names this // box's instance. func recordInstanceID(rec boxid.Record) (string, error) { doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) if err != nil { return "", errors.New("identity document is not base64") } var d struct { InstanceID string `json:"instanceId"` } if err := json.Unmarshal(doc, &d); err != nil || d.InstanceID == "" { return "", errors.New("identity document names no instance") } return d.InstanceID, nil } // publishBoxRecord makes one attempt to publish this box's enrollment // evidence. done is true once there is nothing left to retry: the record // is in place, or a different one already holds this reference, which // only an operator can sort out and which is logged loudly. A record // already under this reference is never rewritten, but it is checked // against this box's own identity: one signed for another instance, or // one that cannot be read, is reported the same loud way. func publishBoxRecord(ctx context.Context, s release.Store, box string, read func() (boxid.Record, error)) (done bool, err error) { if !boxid.ValidBox(box) { return true, fmt.Errorf("box reference %q is not a label; no enrollment evidence is published", box) } existing, err := boxid.Load(ctx, s, box) if err != nil && !errors.Is(err, os.ErrNotExist) { return false, err } rec, rerr := read() if rerr != nil { return false, rerr } if err == nil { var held boxid.Record if jerr := json.Unmarshal(existing, &held); jerr != nil { return true, fmt.Errorf("the record already under %s cannot be read: %v", box, jerr) } theirs, ierr := recordInstanceID(held) if ierr != nil { return true, fmt.Errorf("the record already under %s: %v", box, ierr) } mine, ierr := recordInstanceID(rec) if ierr != nil { return false, fmt.Errorf("this box's own identity: %v", ierr) } if theirs != mine { return true, fmt.Errorf("the record already under %s is signed for instance %s, this box is %s", box, theirs, mine) } return true, nil // create-only: an existing record is never rewritten } err = boxid.Publish(ctx, s, rec) if errors.Is(err, release.ErrCASConflict) { return true, fmt.Errorf("another record already holds %s: %w", box, err) } return err == nil, err } // boxRecordLoop publishes this box's enrollment evidence once, at start, // when CARLOS_REGION_HA is on, retrying each minute until it is in place. // The record grants nothing: enrollment (an operator, or for Auto Scaling // boxes the console with a membership confirmation) reads it later. // Platform-box job: skipped on fleet boxes. func boxRecordLoop() { if !regionHAEnabled(os.Getenv) || shouldSkipPublisherElection(os.Getenv) { return } ctx := context.Background() store, err := openDeployStore(ctx) if err != nil { log.Printf("agent: box record not published: %v", err) return } client := &http.Client{Timeout: 2 * time.Second} for attempt := 1; ; attempt++ { // A group box's reference comes from its identity document, which // IMDS may not answer yet: that is a retry, never a refusal // (boxreference.go). box := metricsBoxLabel(os.Getenv, readMachineID()) read := func() (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) } done, err := false, errors.New("this box's group reference could not be read") if box != "" { done, err = publishBoxRecord(ctx, store, box, read) } name := box if name == "" { name = "this group box" } if done { if err != nil { log.Printf("agent: box record for %s: %v (region HA cannot enroll this box until an operator resolves it)", name, err) } return } if attempt == 1 || attempt%60 == 0 { log.Printf("agent: box record for %s not published yet (#%d): %v", name, attempt, err) } time.Sleep(boxRecordRetry) } }