package relay import ( "context" "encoding/json" "errors" "os" "time" "github.com/carlosframework/platform/internal/release" ) const ( AuthorityKey = "control/relay.json" LeaseMaxAge = 15 * time.Minute ) type AuthorityApp struct { Account string `json:"account"` App string `json:"app"` Incarnation string `json:"incarnation"` Regions []string `json:"regions"` } type Authority struct { Apps []AuthorityApp `json:"apps"` WrittenAt time.Time `json:"written_at"` } func (a *Authority) Lookup(account, app string) (AuthorityApp, bool) { if a == nil { return AuthorityApp{}, false } for _, x := range a.Apps { if x.Account == account && x.App == app { return x, true } } return AuthorityApp{}, false } func ReadAuthority(ctx context.Context, s release.Store) (*Authority, string, error) { body, etag, err := s.Get(ctx, AuthorityKey) if errors.Is(err, os.ErrNotExist) { return nil, "", nil } if err != nil { return nil, "", err } var a Authority if err := json.Unmarshal(body, &a); err != nil { return nil, etag, err } return &a, etag, nil } // Lease is the relay's serving lease: live while the last SUCCESSFUL // authority read is younger than LeaseMaxAge. A failed read renews nothing. type Lease struct{ last time.Time } func (l *Lease) Renew(now time.Time) { l.last = now } func (l *Lease) Live(now time.Time) bool { return !l.last.IsZero() && now.Sub(l.last) <= LeaseMaxAge } // Last is the time of the last successful read, zero if there has never // been one. It is what the heartbeat reports as read_at: the wall clock // would say "just now" on a relay whose every read has failed for twenty // minutes, which is the one thing a console watching this record must not // be told. func (l *Lease) Last() time.Time { return l.last } func HeartbeatKey(box string) string { return "control/relay-ack/" + box + ".json" } type Heartbeat struct { Box string `json:"box"` Region string `json:"region"` WrittenAt time.Time `json:"written_at"` // of the authority record held; zero if none // ReadAt is the last SUCCESSFUL authority read — the lease clock, not // the wall clock — so the gap between it and the heartbeat's own // arrival is how far the relay has drifted from the record it serves. ReadAt time.Time `json:"read_at"` Listeners []string `json:"listeners"` Kids []string `json:"kids"` // Retiring is kid -> retire_at for the loaded masters that carry one, // so `carlos relay status` can tell an operator mid-rotation when the // old key may leave the box's masters file (discussion #120). Additive: // a relay older than this field simply reports none. Retiring map[string]time.Time `json:"retiring,omitempty"` // The two "why is this record stale" fields: the last masters-file // reload error and the last authority read error, either of which // explains a ReadAt that has stopped moving. MastersError string `json:"masters_error,omitempty"` AuthorityError string `json:"authority_error,omitempty"` } // WriteHeartbeat writes a box's heartbeat by read-then-CAS: Get the current // etag (absent means create with ifMatch ""), then Put with it. A box owns // its own heartbeat key, so an ErrCASConflict can only be our own previous // write racing a restart — retry the whole sequence once more and return // whatever the second attempt reports. func WriteHeartbeat(ctx context.Context, s release.Store, hb Heartbeat) error { body, err := json.Marshal(hb) if err != nil { return err } key := HeartbeatKey(hb.Box) var lastErr error for attempt := 0; attempt < 2; attempt++ { etag := "" _, curEtag, err := s.Get(ctx, key) switch { case errors.Is(err, os.ErrNotExist): // no existing record: create-only Put below. case err != nil: return err default: etag = curEtag } if _, err := s.Put(ctx, key, body, etag); err != nil { lastErr = err if errors.Is(err, release.ErrCASConflict) { continue } return err } return nil } return lastErr }