// Package-file doc: control/changes.json is the change index (S3 change-index // design ยง3): a map from change scope to a generation number that every // successful bucket mutation inside that scope bumps. A box that has read the // index can compare one small object's etag, and the generations behind it, // instead of re-walking the deployment bucket. Like control/serial it is // UNSIGNED and carries no authority: it can only say "something may have // changed", so a reader that cannot trust it walks as before. package release import ( "context" "encoding/json" "errors" "fmt" "log" "math/rand/v2" "os" "strings" "time" ) // ChangeIndexKey sits under control/ next to SerialKey, outside every prefix // that app or account listings walk. const ChangeIndexKey = "control/changes.json" // ErrChangeIndexSchema is an index written by a newer schema than this build // understands. Readers treat it as unknown (walk as today); writers refuse to // overwrite it. var ErrChangeIndexSchema = errors.New("release: change index schema unsupported") const changeIndexAttempts = 4 // ChangeIndex is the body of ChangeIndexKey. Generations only increase and // entries are never removed in v1. type ChangeIndex struct { Schema int `json:"schema"` Entries map[string]uint64 `json:"entries"` } // dataPrefixes are the per-app prefixes that hold high-volume data rather // than state a box loop consumes; writes there never bump a scope. // schedule-status is each box's report of its timers, written on every // scheduled fire and read only by the console's schedule door, never by a // box loop: bumping on it would run a scoped pass on every armed box per fire. var dataPrefixes = map[string]bool{ "logs": true, "errors": true, "analytics": true, "runs": true, "metrics": true, "schedule-status": true, } // ChangeScope maps a bucket key to the scope whose generation a write to it // bumps. The scopes are apps// (except the data prefixes), accounts/, // control/certs and control/edges; every other key reports ok=false. func ChangeScope(key string) (scope string, ok bool) { parts := strings.Split(key, "/") switch parts[0] { case "apps": // apps///: all three segments non-empty. if len(parts) < 4 || parts[1] == "" || parts[2] == "" || parts[3] == "" || dataPrefixes[parts[3]] { return "", false } return "apps/" + parts[1] + "/" + parts[2], true case "accounts": if len(parts) < 3 || parts[1] == "" || parts[2] == "" { return "", false } return "accounts/" + parts[1], true case "control": if len(parts) < 3 || parts[2] == "" { return "", false } if parts[1] == "certs" || parts[1] == "edges" { return "control/" + parts[1], true } } return "", false } // ReadChangeIndex returns the index and its etag for CAS. An absent object is // the ordinary pre-first-bump state: an empty schema-1 index and etag "". A // schema other than 1 is ErrChangeIndexSchema and an unparseable body is an // error too; neither is ever reported as empty, because "empty" would read as // "nothing changed". func ReadChangeIndex(ctx context.Context, s Store) (ChangeIndex, string, error) { return readChangeIndex(ctx, s.Get) } func readChangeIndex(ctx context.Context, get func(context.Context, string) ([]byte, string, error)) (ChangeIndex, string, error) { body, etag, err := get(ctx, ChangeIndexKey) if errors.Is(err, os.ErrNotExist) { return ChangeIndex{Schema: 1, Entries: map[string]uint64{}}, "", nil } if err != nil { return ChangeIndex{}, "", err } var idx ChangeIndex if err := json.Unmarshal(body, &idx); err != nil { return ChangeIndex{}, "", fmt.Errorf("release: change index malformed: %w", err) } if idx.Schema != 1 { return ChangeIndex{}, "", fmt.Errorf("%w: schema %d", ErrChangeIndexSchema, idx.Schema) } if idx.Entries == nil { idx.Entries = map[string]uint64{} } return idx, etag, nil } // bumpChangeIndex increments every distinct scope's generation in one read-modify-CAS, // retrying a lost race up to four times with 20-120ms of jitter between // attempts. It takes raw get/put funcs rather than a Store so the stores' // own mutation methods can call it with their unhooked internals. It makes no // context of its own: the caller supplies the (detached, time-boxed) one. // Callers treat failure as log-only; a lost bump costs latency, never // correctness, and must never fail the write that triggered it. func bumpChangeIndex(ctx context.Context, get func(context.Context, string) ([]byte, string, error), put func(context.Context, string, []byte, string) (string, error), scopes ...string) error { var lastErr error for attempt := 0; attempt < changeIndexAttempts; attempt++ { if attempt > 0 { jitter := 20*time.Millisecond + rand.N(100*time.Millisecond) t := time.NewTimer(jitter) select { case <-ctx.Done(): t.Stop() return ctx.Err() case <-t.C: } } idx, etag, err := readChangeIndex(ctx, get) if err != nil { return err } seen := make(map[string]bool, len(scopes)) for _, sc := range scopes { if !seen[sc] { // one write is one generation per scope, however many keys it touched seen[sc] = true idx.Entries[sc]++ } } body, err := json.Marshal(idx) if err != nil { return err } _, err = put(ctx, ChangeIndexKey, body, etag) if err == nil { return nil } if !errors.Is(err, ErrCASConflict) { return err } lastErr = err } return fmt.Errorf("bump %s: lost CAS race %d times: %w", ChangeIndexKey, changeIndexAttempts, lastErr) } // changeIndexBumpBudget bounds one post-mutation bump, retries included. A // var so tests can shrink it. var changeIndexBumpBudget = 2 * time.Second // bumpAfterMutation is the hook both concrete stores call after a successful // mutation of key. It does nothing unless the store enabled the index and key // falls in a change scope. It runs under a context detached from the caller's // cancellation (a request that returns right after its write must not drop the // bump) but bounded by changeIndexBumpBudget, and it takes the stores' raw, // unhooked get/put so writing the index never bumps again. A failure is logged // and swallowed: the write already happened. func bumpAfterMutation(ctx context.Context, enabled bool, key string, get func(context.Context, string) ([]byte, string, error), put func(context.Context, string, []byte, string) (string, error)) { if !enabled { return } scope, ok := ChangeScope(key) if !ok { return } bctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), changeIndexBumpBudget) defer cancel() if err := bumpChangeIndex(bctx, get, put, scope); err != nil { log.Printf("release: change index bump %s: %v", scope, err) } }