package host import ( "bytes" "context" "crypto/x509" "encoding/base64" "encoding/json" "errors" "flag" "fmt" "io" "net/http" "os" "strings" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/cliflag" "github.com/carlosframework/platform/internal/release" ) // carlos system box publish-record: one publishBoxRecord attempt, by hand, // without the CARLOS_REGION_HA gate boxRecordLoop has (census ruling // box-publish-record, R1-R4). // // The host claim's enrollment needs a box's signed identity record at // control/boxes/.json, and the agent writes it only when region HA is // on. This verb lets an operator publish it once, with the box's own AWS // credentials and IMDS: AWS signs the identity document, not the box and // not the operator. // // It runs as the agent user and refuses root before any store, IMDS or // host read (R1); the one supported way to run it is systemd-run with // User=carlos and host.env as its EnvironmentFile (R2). It refuses on a // fleet box, as the loop skips one, and anywhere without an S3 deployment // bucket: only S3's If-None-Match makes the write create-only, and a // deployment directory's check-then-rename does not. // // It reads the record already under the box's key first, and keeps one // that enrollment's own verifier accepts as evidence for this box // (heldEvidence: boxid.Verifier.Enroll, with the account, AWS region and // instance id taken from this box's current IMDS identity). Its bytes are // never touched, however old: an instance's identity document changes // with its lifecycle (pendingTime, instanceType) while the instance stays // the same, and the record cannot be replaced. Anything the verifier // refuses is refused and left alone. Keeping a record says only that // valid evidence is in place; enrollment still makes its roster and // operator checks. With nothing there, it calls publishBoxRecord through // boxRecordStore, which can read and create this box's one record key and // nothing else (R4), so it can never delete or overwrite: the dev bucket's // box_records policy (BoxRecordsAreCreateOnly, BoxRecordsAreNeverDeleted) // denies both anyway. const boxPublishRecordCommand = "systemd-run --wait --pipe -p EnvironmentFile=/etc/carlos/host.env -p User=carlos /opt/carlos/carlos system box publish-record" const boxPublishRecordUsage = "usage: carlos system box publish-record\n\n" + "Publish this box's AWS-signed identity record to the deployment bucket, once. It never replaces or deletes a record.\n" + "Run it on the box as the agent user, with host.env loaded:\n\n" + " " + boxPublishRecordCommand + "\n" // errBoxRecordStore is boxRecordStore refusing anything but reading and // creating this box's own record. publishBoxRecord never asks for more, so // a user only sees it if that changes. var errBoxRecordStore = errors.New("publish-record only reads and creates this box's own record") // boxRecordStore is the only store publish-record gives publishBoxRecord // (R4): Get on this box's record key, and Put on it create-only (ifMatch // "", which S3Store sends as If-None-Match: *). Everything else is // refused before it reaches the bucket. attempted is whether a create // reached the bucket, and wrote whether one was answered as done. type boxRecordStore struct { inner release.Store key string attempted bool wrote bool } func (s *boxRecordStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key != s.key { return nil, "", errBoxRecordStore } return s.inner.Get(ctx, key) } func (s *boxRecordStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { if key != s.key || ifMatch != "" { return "", errBoxRecordStore } s.attempted = true etag, err := s.inner.Put(ctx, key, body, "") if err == nil { s.wrote = true } return etag, err } func (s *boxRecordStore) List(context.Context, string) ([]string, error) { return nil, errBoxRecordStore } func (s *boxRecordStore) ListDirs(context.Context, string) ([]string, error) { return nil, errBoxRecordStore } func (s *boxRecordStore) Delete(context.Context, string) error { return errBoxRecordStore } // The lines publish-record prints. Every one is in the copy review deck. func boxRecordPublishedLine(box, instance string) string { return fmt.Sprintf("published the identity record for %s (instance %s) to %s", box, instance, boxid.Key(box)) } func boxRecordInPlaceLine(box, instance string) string { return fmt.Sprintf("the record at %s is valid AWS-signed evidence for %s (instance %s), so nothing was written. enrollment still makes its own checks before it accepts the box", boxid.Key(box), box, instance) } const ( boxRecordRootRefusal = "carlos system box publish-record runs as the agent user, not root. run it with systemd-run -p User=carlos." boxRecordFleetRefusal = "this is a fleet box. fleet boxes publish no identity record, so nothing was written" boxRecordNoBucket = "this box has no deployment bucket (CARLOS_DEPLOYMENT_BUCKET is not set). the record is written only to an S3 bucket, which refuses to overwrite it, so nothing was written" boxRecordNoNameRefusal = "this box's name could not be read, so nothing was written" ) // errOwnIdentityIncomplete is the IMDS identity document naming no account, // region or instance: there is nothing to check a stored record against, // and nothing worth publishing. var errOwnIdentityIncomplete = errors.New("the identity document AWS served this box names no account, region or instance") func boxRecordBadName(box string) error { return fmt.Errorf("this box's name %q is not a valid box reference, so nothing was written", box) } func boxRecordStoreFailure(err error) error { return fmt.Errorf("the deployment bucket could not be opened: %v. nothing was written", err) } func boxRecordRefusal(err error) error { return fmt.Errorf("refused: %v. nothing was changed", err) } func boxRecordMismatch(box, what string) error { return fmt.Errorf("refused: the record already under %s is not valid evidence for this box: %s. nothing was changed", box, what) } // boxRecordRetryable is a failure before any write reached the bucket. func boxRecordRetryable(box string, err error) error { return fmt.Errorf("the identity record for %s was not published: %v. nothing was written, so it is safe to run again", box, err) } // boxRecordUncertain is a failure once a create reached the bucket: S3 may // have committed it. A second run only ever creates, and reports what it // finds. func boxRecordUncertain(box string, err error) error { return fmt.Errorf("the identity record for %s may or may not have been written: %v. it is safe to run again, which only ever creates the record and says what is there", box, err) } // awsIdentity is the part of an identity document enrollment checks. type awsIdentity struct { Account string `json:"accountId"` Region string `json:"region"` Instance string `json:"instanceId"` } // identityOf reads rec's identity document. It does not check the // signature. func identityOf(rec boxid.Record) (awsIdentity, bool) { doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) if err != nil { return awsIdentity{}, false } var id awsIdentity if json.Unmarshal(doc, &id) != nil { return awsIdentity{}, false } return id, true } // heldEvidence says why raw, the record already under the box's key, is // not evidence for this box, or "" when it is. The verdict is // boxid.Verifier.Enroll's, with the account, AWS region and instance id // (TofuInstanceID) taken from own, this box's current IMDS identity, and // certs as enrollment's: a strict v1 envelope naming this box, no other // fields and nothing after it, and AWS's signature under the certificate // for the document's region. There is no freshness check: the document's // mutable fields (pendingTime, instanceType) and written_at may differ // from today's. The rest only names the refusal, in Enroll's own order. func heldEvidence(raw []byte, box string, own awsIdentity, certs map[string]*x509.Certificate) string { v := boxid.Verifier{Account: own.Account, Certs: certs} _, err := v.Enroll(raw, boxid.Expect{Box: box, AWSRegion: own.Region, TofuInstanceID: own.Instance}) if err == nil { return "" } const unreadable = "it cannot be read as an identity record" dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() var held boxid.Record if dec.Decode(&held) != nil { return unreadable } if _, terr := dec.Token(); !errors.Is(terr, io.EOF) { return unreadable } switch { case held.V != 1: return fmt.Sprintf("its version is %d, not 1", held.V) case held.Box != box: return fmt.Sprintf("it is for box %q", held.Box) } id, ok := identityOf(held) switch { case !ok: return unreadable case id.Instance != own.Instance: return fmt.Sprintf("it is signed for instance %s, and this box is %s", id.Instance, own.Instance) case id.Account != own.Account: return fmt.Sprintf("it is signed for account %s, and this box is in account %s", id.Account, own.Account) case id.Region != own.Region: return fmt.Sprintf("it is signed in region %s, and this box is in %s", id.Region, own.Region) } reason := strings.TrimPrefix(err.Error(), boxid.ErrUnverified.Error()+": ") return fmt.Sprintf("it does not verify as AWS-signed evidence: %s", reason) } // boxPublishRun is one run of the verb, with its seams: the environment, // the effective uid (R1), the machine id (a host read, made only after the // uid check), the deployment store, the IMDS read and the AWS signing // certificates enrollment verifies with. type boxPublishRun struct { getenv func(string) string geteuid func() int machineID func() string open func(context.Context, func(string) string) (release.Store, error) read func(box string) (boxid.Record, error) certs func() (map[string]*x509.Certificate, error) } // publish is the verb. It returns the line to print, or the refusal. func (r boxPublishRun) publish(ctx context.Context) (string, error) { if r.geteuid() == 0 { return "", errors.New(boxRecordRootRefusal) } if shouldSkipPublisherElection(r.getenv) { return "", errors.New(boxRecordFleetRefusal) } if r.getenv("CARLOS_DEPLOYMENT_BUCKET") == "" { return "", errors.New(boxRecordNoBucket) } box := metricsBoxLabel(r.getenv, r.machineID()) if box == "" { return "", errors.New(boxRecordNoNameRefusal) } if !boxid.ValidBox(box) { return "", boxRecordBadName(box) } opened, err := r.open(ctx, r.getenv) if err != nil { return "", boxRecordStoreFailure(err) } if _, ok := opened.(*release.S3Store); !ok { return "", errors.New(boxRecordNoBucket) } store := &boxRecordStore{inner: opened, key: boxid.Key(box)} existing, _, gerr := store.Get(ctx, store.key) if gerr != nil && !errors.Is(gerr, os.ErrNotExist) { return "", boxRecordRetryable(box, gerr) } mine, err := r.read(box) if err != nil { return "", boxRecordRetryable(box, err) } own, ok := identityOf(mine) if !ok || own.Account == "" || own.Region == "" || own.Instance == "" { return "", boxRecordRetryable(box, errOwnIdentityIncomplete) } // keep decides a record that is already there: kept as it is when the // verifier accepts it, refused otherwise. keep := func(raw []byte) (string, error) { certs, err := r.certs() if err != nil { return "", boxRecordRetryable(box, err) } if what := heldEvidence(raw, box, own, certs); what != "" { return "", boxRecordMismatch(box, what) } return boxRecordInPlaceLine(box, own.Instance), nil } if gerr == nil { return keep(existing) } done, err := publishBoxRecord(ctx, store, box, func() (boxid.Record, error) { return mine, nil }) switch { case err != nil && done: return "", boxRecordRefusal(err) case err != nil && store.attempted: return "", boxRecordUncertain(box, err) case err != nil: return "", boxRecordRetryable(box, err) case store.attempted: // Either this create landed, or a retry of it was answered 412 // and boxid.Publish found exactly these bytes, written_at // included: this run's record either way. return boxRecordPublishedLine(box, own.Instance), nil } // No create was attempted: a record arrived after the read above and // publishBoxRecord took it as this instance's. It is held to the same // verifier as one that was there from the start. raw, _, err := store.Get(ctx, store.key) if err != nil { return "", boxRecordRetryable(box, err) } return keep(raw) } // RunSystemBoxPublishRecord is `carlos system box publish-record`. func RunSystemBoxPublishRecord(args []string) { fs := flag.NewFlagSet("system box publish-record", flag.ExitOnError) fs.Usage = cliflag.Usage(fs, boxPublishRecordUsage) _ = fs.Parse(args) if fs.NArg() != 0 { fs.Usage() os.Exit(2) } client := &http.Client{Timeout: 2 * time.Second} run := boxPublishRun{getenv: os.Getenv, geteuid: os.Geteuid, machineID: readMachineID, open: openDeployStoreEnv, read: func(box string) (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) }, certs: boxid.EmbeddedCerts} line, err := run.publish(context.Background()) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } fmt.Println(line) }