// The app-declared private-storage policy (issue #412, spec // 2026-09-22-instance-private-storage-design.md): an app may say once, on // its instances manifest, that every instance it creates gets a config // environment of its own and an object store bound to that environment. // // The whole point is the SPLIT. Declaring the policy is // `handleAPIInstanceEnable` — person-only, zero Set, no service credential // reaches it. Spending it is `handleAPIInstanceCreate`, which an // `instance`-role credential can already reach. So a public signup path // provisions a tenant's own bucket without ever holding the power to create // one, which is what makes this a better answer than widening the role // (#410, which this makes unnecessary). // // Nothing here is a second implementation of provisioning. The walk this // file drives is the one `handleAPIStoreProvision` drives, with the same // lock, the same seal key, the same timeout and the same // GrantSourceAccountApproval — factored so the automatic path and the // member's own `carlos store create --environment` cannot drift. package console import ( "context" "crypto/ecdh" "errors" "fmt" "log" "net/http" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" ) // storeMaxPerAppDefault is the ceiling on ENVIRONMENT stores under one app // when the deployment names none (spec §8, issue #412's point 3). // // A bucket plus an IAM user per instance is a bigger footprint than an // instance, and the binding limit is IAM's: 5,000 users per cloud account, // an adjustable quota // (https://docs.aws.amazon.com/general/latest/gr/iam-service.html). So a // signup path that can mint them needs a bar it stops at — but the bar has // to be sized against that number rather than picked for looking tidy. An // earlier draft said 100, which was a fifth of one app's fair share of the // quota and would have refused a perfectly ordinary managed product at its // hundredth customer (James, 2026-09-23). // // Per APP rather than per account, and that is a cost decision stated as // such: the count is one prefix listing of the app, which the create door // can afford on every request. A deployment-wide count would mean listing // every app's stores on every create. // // WHAT THIS DOES NOT BOUND, said plainly because the per-app shape invites // the wrong reading: five apps at this default reach 5,000 between them, and // the deployment's own IAM principals are not in that sum. This is a brake // on ONE runaway signup path, not a budget for the cloud account. A real // account-wide bound needs a counter kept on the side rather than one // derived by listing, and it is not here. const storeMaxPerAppDefault = 1000 // storeMaxPerApp is the ceiling in force. Zero in the config means "not // configured", not "no stores allowed": a cap of zero would turn an // unset variable into an app that can have no environment store at all. func (s *Server) storeMaxPerApp() int { if s.cfg.StoreMaxPerApp > 0 { return s.cfg.StoreMaxPerApp } return storeMaxPerAppDefault } // storeCeiling refuses an app that has reached its environment-store cap. // // env is the environment ABOUT to be given a store, and a name that already // has one is never refused: the cap is on the FOOTPRINT, and re-declaring // an existing store adds no bucket and no principal. Without that carve-out // a retry of a create that failed just after provisioning would be turned // away by the very ceiling it had reached, with no way forward at all. // // A listing failure is a refusal, not a pass. This is a spend gate; reading // "how many are there already" as "none" on a bad round trip is the one // error direction that costs money. func (s *Server) storeCeiling(ctx context.Context, acct *Account, app, env string) *doorRefusal { if env == "" { return nil // the app's own store is not one of the capped collection } envs, err := objstore.StoreEnvironments(ctx, s.cfg.Releases, acct.Sqid, app) if err != nil { log.Printf("console: store ceiling %s/%s: %v", acct.Sqid, app, err) return &doorRefusal{Status: http.StatusBadGateway, Msg: "could not count this app's stores — try again"} } for _, e := range envs { if e == env { return nil } } max := s.storeMaxPerApp() if len(envs) < max { return nil } return &doorRefusal{Status: http.StatusConflict, Msg: fmt.Sprintf("%s already has %d environment stores, which is this deployment's limit of %d — destroy one in the console, or ask an operator to raise CARLOS_STORE_MAX_PER_APP", app, len(envs), max)} } // instanceStoreNames refuses an environment that cannot carry a store, for // an app under the policy (spec §6). // // derived says where the name came from — the host's first label, or the // caller's own `environment` — and it changes only the sentence, never the // rails. The first label is the derived name because it is the one // predictable from the host, already unique within the app (the record key // IS the host), and readable in `store status` at a hundred of them, which // is #412's point 1. // // Three refusals, all BEFORE anything is written, because each of them is // otherwise discovered somewhere worse: // // - a DNS label is not an environment name. Labels run to 63 characters // and environment names to 32, and "default" is a perfectly legal label // that names the app's own bundle rather than a named overlay. // - the bucket name is S3's 63. The walk checks this itself, but reaching // it there means an instance record already written over an estate that // cannot exist — and the walk's own refusal arrives as a 502 the caller // cannot act on, not a 400 naming the part to shorten. // - the principal name is IAM's 64, for the same reason. An environment // store's name carries three components, so a long enough (account, // app, environment) is simply unnameable. // // Each sentence names the part the caller can shorten, because from a // signup path the host label is usually the only one they control. func (s *Server) instanceStoreNames(acct *Account, app, env string, derived bool) *doorRefusal { source := "the environment you named" fix := "choose a shorter environment name" if derived { source = "the host's first label" fix = "choose a different host label" } if !appconfig.ValidEnvironmentName(env) { return &doorRefusal{Status: http.StatusBadRequest, Msg: fmt.Sprintf("%s gives every instance its own storage, which needs %s to be a usable environment name — %q is not (want ^[a-z0-9-]{1,32}$, and not \"default\"). Please %s", app, source, env, fix)} } bucket := objstore.MintedBucketIn(s.cfg.StoreDeployment, acct.Sqid, app, env) if !objstore.ValidBucketName(bucket) { return &doorRefusal{Status: http.StatusBadRequest, Msg: fmt.Sprintf("this instance's bucket would be named %q, which is %d characters and not a legal S3 name — %s", bucket, len(bucket), fix)} } user := objstore.UserNameIn(acct.Sqid, app, env) if !objstore.ValidUserName(user) { return &doorRefusal{Status: http.StatusBadRequest, Msg: fmt.Sprintf("this instance's IAM user would be named %q, which is %d characters, over AWS's %d — %s, or shorten the app name", user, len(user), objstore.MaxIAMUserName, fix)} } return nil } // instanceStoreProvision declares and provisions one environment's store // under the account's standing entitlement, and answers w itself on every // refusal. The caller has already proved membership and spent its write // budget; what is left is the store plane's own gates, in the order // handleAPIStoreProvision applies them. // // actor is the create door's principal — a person's address or the service // credential's `app:/` audit actor — and it lands in the grant // block, so a bucket minted by a signup path names the signup path. // // The env prefix is INHERITED from the app's own store when it has one. An // app that declared `carlos store create --env-prefix JELLY_S3` reads those // names in its code, and an instance store delivering CARLOS_STORE_BUCKET // into the same app would be a second vocabulary for one thing. With no app // store declared, the default prefix is taken — what a fresh declaration // would have taken anyway. // // ctx must be the REQUEST context, not a short-deadline derivative of it: // the walk below takes its own storeGrantTimeout, which is longer than the // create door's own deadline, and a walk that inherited a fifteen-second // budget would abandon a bucket it had already paid for. func (s *Server) instanceStoreProvision(ctx context.Context, w http.ResponseWriter, acct *Account, app, env, actor string) (*objstore.Record, bool) { out, ok, _ := s.instanceStoreProvisionFor(ctx, w, acct, app, env, actor, "") return out, ok } // errSeedStoreTaken is instanceStoreProvisionFor's one refusal it does not // answer itself: the environment already has a store that holds estate and // was not made by this seed (seeded-instances spec §4 Storage). The seed // door speaks that refusal, naming the store it got back. var errSeedStoreTaken = errors.New("console: the environment's store is not this seed's") // instanceStoreProvisionFor is instanceStoreProvision for a seeded create // when seedID is set. With seedID "" it is instanceStoreProvision exactly, // and its error is always nil. // // With a seed, the store must be new and empty, or this seed's own from an // earlier attempt. That is re-checked under the store lock, on the record // read there, before anything is declared or granted: a check made before // the lock (the seed door's own) can be overtaken by a concurrent `carlos // store create --environment`, and GrantIn would then reuse that grant. // On that refusal it answers nothing on w and returns the record it found, // false and errSeedStoreTaken. The environment must also be one no other // instance names (final review F6): errSeedStoreTaken again, or // errSeedEnvironmentUnread when that cannot be read, also unanswered. // Every other refusal is answered on w and returns a nil error, as // instanceStoreProvision's do. // // A seed's declaration carries its id (objstore.Declared.Seed). A record // that is declared and holds no estate is declared again for this seed. // When the grant then fails before writing its block, the declaration is // rolled back under the lock: deleted if this call created the record, // else restored to its previous bytes. func (s *Server) instanceStoreProvisionFor(ctx context.Context, w http.ResponseWriter, acct *Account, app, env, actor, seedID string) (*objstore.Record, bool, error) { pub, ok := s.instanceStorePreflight(ctx, w, acct, app) if !ok { return nil, false, nil } effectStore, effectProvisioner := claimStoreAdapters(ctx, s.cfg.Releases, s.cfg.StoreProvisioner) // The same lock handleAPIStoreProvision takes, keyed the same way, so a // signup creating an instance and a member running `carlos store create // --environment` against the same name serialise against each other. defer s.storeWalks.lock(acct.Sqid, app, env)() // The walk's deadline starts AFTER the lock, for the reason that door // states: a request that queued behind a full-length walk would // otherwise reach the backend holding a spent context. walkCtx, cancel := context.WithTimeout(ctx, storeGrantTimeout) defer cancel() // Re-read under the lock: a queued request may have waited while an // operator declined the account. e, ok := s.storeProvisionApproval(walkCtx, w, acct) if !ok { return nil, false, nil } // The declaration the walk requires. GrantIn refuses a store nobody // declared (ErrNotDeclared) because the env prefix is the member's // choice and a store with nowhere to be delivered is not one to invent // — under this policy the APP made that choice, once, on the manifest, // so the declaration is written here rather than demanded of a caller // who has no verb to write it with. prefix, ok := s.storeInheritedPrefix(walkCtx, w, acct, app) if !ok { return nil, false, nil } rec, etag, err := objstore.GetIn(walkCtx, effectStore, acct.Sqid, app, env) if err != nil { log.Printf("console: instance store read %s/%s/%s: %v", acct.Sqid, app, env, err) jsonError(w, http.StatusBadGateway, "could not read this instance's store record — try again") return nil, false, nil } if seedID != "" && !seedOwnsStore(rec, seedID) { return rec, false, errSeedStoreTaken } // A seed's environment must also be unused (final review F6), checked // here under the lock an environment binding takes (seedenvbind.go): a // binding either landed before this read, or finds the store marked // and takes it from the seed. if seedID != "" { user, err := s.seedEnvironmentUser(walkCtx, acct.Sqid, app, env, seedID) if err != nil { log.Printf("console: instance store %s/%s/%s: seed %s: who names the environment: %v", acct.Sqid, app, env, seedID, err) return nil, false, errSeedEnvironmentUnread } if user != "" { log.Printf("console: instance store %s/%s/%s: seed %s: instance %s already names the environment", acct.Sqid, app, env, seedID, user) return rec, false, errSeedStoreTaken } } existed := rec != nil if rec == nil { rec = &objstore.Record{} } // Only when absent. A re-run over an existing declaration must not // re-stamp its author and timestamp, and must not move a prefix the app // is already reading — that is the one rewrite handleAPIStoreCreate // refuses with a 409, and this path has no member to show it to. // // A seed is the exception: a declared-only record holds no estate, and // one not marked with this seed is declared again, for this seed, so // the store this seed makes is marked as its own. That replaces the // other writer's EnvPrefix, By and At: a member's chosen --env-prefix // on a store with no estate yet is lost, and the app's inherited prefix // is delivered instead. redeclare := rec.Declared == nil if seedID != "" && rec.Declared != nil && rec.Grant == nil && rec.Provisioned == nil && rec.Declared.Seed != seedID { redeclare = true } // What a seed's failed grant rolls back to (below): the record's exact // bytes before this call declared it, or nil when there was no record. var prevRaw []byte if seedID != "" && redeclare && existed { raw, rawTag, err := effectStore.Get(walkCtx, objstore.KeyIn(acct.Sqid, app, env)) if err != nil { log.Printf("console: instance store read %s/%s/%s: %v", acct.Sqid, app, env, err) jsonError(w, http.StatusBadGateway, "could not read this instance's store record — try again") return nil, false, nil } if rawTag != etag { jsonError(w, http.StatusConflict, "this instance's store record changed while it was being created — try again") return nil, false, nil } prevRaw = raw } declaredTag := "" if redeclare { rec.Declared = &objstore.Declared{EnvPrefix: prefix, At: time.Now().UTC(), By: actor, Seed: seedID} if declaredTag, err = objstore.PutIn(walkCtx, effectStore, acct.Sqid, app, env, rec, etag); err != nil { if errors.Is(err, release.ErrCASConflict) { jsonError(w, http.StatusConflict, "this instance's store record changed while it was being created — try again") return nil, false, nil } log.Printf("console: instance store declare %s/%s/%s: %v", acct.Sqid, app, env, err) jsonError(w, http.StatusBadGateway, "could not declare this instance's store — nothing was created; try again") return nil, false, nil } } out, err := objstore.GrantInFor(walkCtx, effectStore, effectProvisioner, s.cfg.StoreDeployment, pub, acct.Sqid, app, env, e.Decision.By, objstore.GrantSourceAccountApproval, "", seedID, time.Now()) if err != nil && seedID != "" && redeclare { // A seed's declaration must not outlive a grant that wrote nothing: // another path could grant that record later and keep the mark, and // the seed's own finish would then tear down a store it never made // (spec §4 Storage). Still under the lock, the record goes back to // what it was, conditionally on declaredTag. A grant that did write // its block has moved the record past declaredTag, and that estate // is this seed's to finish. rbCtx, rbCancel := context.WithTimeout(context.WithoutCancel(ctx), storeTimeout) rerr := objstore.RevertIn(rbCtx, effectStore, acct.Sqid, app, env, prevRaw, declaredTag) rbCancel() if errors.Is(rerr, release.ErrCASConflict) { log.Printf("console: instance store %s/%s/%s: seed %s declaration not rolled back: the record was written since (its grant block or another writer), left as it is", acct.Sqid, app, env, seedID) } else if rerr != nil { log.Printf("console: instance store %s/%s/%s: roll back seed %s declaration: %v", acct.Sqid, app, env, seedID, rerr) } } if err == nil && seedID != "" && (out.Declared == nil || out.Declared.Seed != seedID) { // The declare door takes no store lock, so a `carlos store create // --environment` between this declaration and the grant can rewrite // it without the mark. The store is still new, but this seed's // abort can no longer tear it down: said in the log, for an operator. log.Printf("console: instance store %s/%s/%s: granted for seed %s but its declaration no longer names the seed", acct.Sqid, app, env, seedID) } if errors.Is(err, objstore.ErrStoreNameTaken) { log.Printf("console: instance store provision %s/%s/%s: %v", acct.Sqid, app, env, err) jsonError(w, http.StatusConflict, "another store already uses this instance's storage names (app and environment names are joined with \"-\"), so the instance was not created. use a different host name") return nil, false, nil } if err != nil { // Logged, not returned: a walk error carries bucket names, IAM ARNs // and request ids. The sentence the caller gets is the one they can // act on, and it is honest about what is now true — the declaration // may stand, the instance does not, and a retry of the SAME host // resumes this exact store because its name came from the host. log.Printf("console: instance store provision %s/%s/%s: %v", acct.Sqid, app, env, err) jsonError(w, http.StatusBadGateway, "could not provision this instance's storage, so the instance was not created — run the same command again to resume it") return nil, false, nil } // Audited from here as well as from the store doors, because this is the // one path where a bucket is minted by a caller that is not a person: // an account reading its own rows has to be able to see that its signup // path spent money, and by whom it was authorised. s.audit(acct, app, actor, "store-provision", "provisioned "+storeBucketName(out)+" for instance environment "+env+" (instance create)") return out, true, nil } // instanceStorePreflight is the store plane's gates that write nothing and // take no lock: a temporary app, no backend, no standing approval, no // sealing key. It answers w itself on a refusal. instanceStoreProvisionFor // runs it first; a seed door runs it before reserving its record, so these // refusals never leave an aborted record behind. func (s *Server) instanceStorePreflight(ctx context.Context, w http.ResponseWriter, acct *Account, app string) (*ecdh.PublicKey, bool) { if s.refuseTemporary(w, acct, app, "give it object storage") { return nil, false } if s.cfg.StoreProvisioner == nil { // The manifest carries a policy this deployment cannot honour. The // enable door refuses to write one in this state (§5), so reaching // here means the backend was unwired afterwards — a configuration // fact, named as one. jsonError(w, http.StatusServiceUnavailable, "object storage is not configured for this deployment (CARLOS_STORE_BACKEND), and "+app+" gives every instance its own storage — no instance can be created until one is wired or the policy is turned off (carlos instances enable --private-storage=false)") return nil, false } preCtx, preCancel := context.WithTimeout(ctx, storeTimeout) defer preCancel() if _, ok := s.storeProvisionApproval(preCtx, w, acct); !ok { return nil, false } return s.storeSealKey(w) } // seedOwnsStore reports whether a seeded create may use rec (spec §4 // Storage): there is no store, the record holds no estate yet, or the // estate was made by this very seed. func seedOwnsStore(rec *objstore.Record, seedID string) bool { if rec == nil || (rec.Grant == nil && rec.Provisioned == nil) { return true } return rec.Declared != nil && rec.Declared.Seed == seedID } // storeInheritedPrefix reads the app's own store for the variable-name // prefix a new environment store should deliver under. Absent is not an // error — an app with no store of its own is the ordinary shape for a // private-storage app, and takes the default. func (s *Server) storeInheritedPrefix(ctx context.Context, w http.ResponseWriter, acct *Account, app string) (string, bool) { rec, _, err := objstore.Get(ctx, s.cfg.Releases, acct.Sqid, app) if err != nil { log.Printf("console: instance store prefix %s/%s: %v", acct.Sqid, app, err) jsonError(w, http.StatusBadGateway, "could not read this app's store settings — try again") return "", false } return storeEffectivePrefix(rec), true } // instancePrivateStorageAllowed is the enable door's half: the two states in // which declaring the policy would be a trap rather than a policy (spec §5). // // A manifest that says "every instance gets a bucket" on a deployment with // no object storage, or in an account nobody approved, does not fail when it // is written. It fails once per signup, afterwards, in a door whose caller // is a machine — and the two sentences below are the only two that name the // fix. So they are said here, to the person, at the moment they declare it. // // Turning the policy OFF is never refused, and that asymmetry is the point: // an account whose entitlement was later declined must be able to get its // create door working again without an operator. func (s *Server) instancePrivateStorageAllowed(ctx context.Context, w http.ResponseWriter, acct *Account) bool { if s.cfg.StoreProvisioner == nil { jsonError(w, http.StatusServiceUnavailable, "object storage is not configured for this deployment (CARLOS_STORE_BACKEND) — an app cannot be given per-instance storage here") return false } e, _, err := objstore.GetEntitlement(ctx, s.cfg.Releases, acct.Sqid) if err != nil { log.Printf("console: instances enable entitlement %s: %v", acct.Sqid, err) jsonError(w, http.StatusBadGateway, "could not read this account's storage approval — try again") return false } if !e.Approved() { jsonError(w, http.StatusForbidden, "this account is not approved for object storage, so its instances cannot be given storage of their own — run `carlos store apply` and ask an operator to approve it first") return false } return true }