// relay.go is `carlos relay`: the platform STUN/TURN relay's own verbs // (relay spec §3-§5). Three audiences, one command: // // enable a MEMBER turning the relay on for their app, and having // adopted CARLOS_RELAY delivered to its config. status is the only // status sight they have of delivery per route and health per region // disable without box access; adopted is their cutover ack for a route // this platform cannot restart for them. // rotate an OPERATOR moving a region onto a newly staged master. // keygen an OPERATOR minting one. Its output is a secret. // serve what carlos-relay.service runs. Not typed by hand. // // The five member/operator verbs are API mode only, on `carlos store`'s // model (store.go): the record lives in the console's deployment bucket, and // the key is derived from a master only the console holds and sealed with a // key only the app's boxes can open — there is no bucket-direct equivalent of // any of that. See requireRelayAPISession. // // The output-honesty rule is restart.go's, and it bites hardest here: enable // prints what was ISSUED and written, never that a route has picked it up. // Only status can say that, and even status says only what the console can // observe — a route's own acknowledgement, not a guess from elapsed time. package main import ( "context" "crypto/rand" "encoding/base64" "encoding/hex" "flag" "fmt" "io" "log" "os" "os/signal" "sort" "strings" "syscall" "github.com/carlosframework/platform/internal/host" "github.com/carlosframework/platform/internal/relay" ) const relayUsage = `usage: carlos relay [flags] enable --app [--regions ] turn the relay on for one app and deliver CARLOS_RELAY to its config. Regions default to every one this deployment arms, in the console's order — the order every browser tries them in. status --app what was issued per region, what each route has done about it, and each region relay's heartbeat, lease and loaded masters. The only sight of any of that without box access. disable --app remove the key. Every allocation the app holds ends on the relays' next authority read, within a minute. adopted --app --host tell the platform that a TCP-address route you restarted by hand now runs the issued key. The one route shape this platform cannot restart for you, so the acknowledgement has to come from you. rotate --region operator: move the region's issuer pointer onto the newly staged master and re-issue every app enabled there. Needs an address on CARLOS_RELAY_OPERATORS; anyone else meets the same 404 as an unknown console door, never a 403. keygen mint a region master: one " -" line for /etc/carlos/relay-masters and the same pair for the console's CARLOS_RELAY_MASTERS_ secret, which holds every master the console may need as :[,:]. serve run the relay (what carlos-relay.service execs; not typed by hand) --account account the app belongs to (default: your only account, or the configured/chosen one) --account-id account sqid, matched as a sqid ONLY. --console which logged-in console to use (only needed with several) enable, status, disable, adopted and rotate are signed-in only (carlos auth login): the relay's keys are derived from a master the console holds and sealed with the deployment's secrets key, so there is no direct-bucket equivalent.` func runRelay(args []string) { if len(args) == 0 { fmt.Fprintln(os.Stderr, relayUsage) os.Exit(2) } switch args[0] { case "enable": runRelayEnable(args[1:]) case "status": runRelayStatus(args[1:]) case "disable": runRelayDisable(args[1:]) case "adopted": runRelayAdopted(args[1:]) case "rotate": runRelayRotate(args[1:]) case "serve": runRelayServe(args[1:]) case "keygen": runRelayKeygen(args[1:]) case "-h", "--help", "help": fmt.Println(relayUsage) default: fmt.Fprintln(os.Stderr, relayUsage) os.Exit(2) } } // runRelayServe is the unit's entry point. SIGTERM is systemd's stop, and // it must reach relay.Serve rather than killing the process, so the relay // closes its allocations and flushes its last minute of metering — the // bytes it forwarded in that minute are billable, and a hard kill loses // them. func runRelayServe(_ []string) { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() // A store that will not open is waited out, not died on: the unit is // Restart=always and systemd's start limit would park it in `failed` // within ten seconds, taking STUN — which needs no store — down with // it. Configuration errors below are fatal, because waiting fixes // nothing about a missing public IP or a malformed CIDR. store, err := relay.OpenStoreWithRetry(ctx, host.OpenDeployStore, log.Printf) if err != nil { return // ctx cancelled: systemd asked us to stop before we ever started } cfg, err := relay.ConfigFromEnv(ctx, os.Getenv, host.ReadPublicIP, host.ReadPlacementRegion, store) if err != nil { log.Fatalf("relay serve: %v", err) } // The box label is the metrics loop's, so the relay's heartbeat and its // hourly metering objects land under the same name every other row this // box publishes uses. It lives in internal/host, which internal/relay // does not import, so it is filled in here. if cfg.Box == "" { cfg.Box = host.BoxLabel() } if err := relay.Serve(ctx, cfg, log.Printf); err != nil { log.Fatalf("relay serve: %v", err) } } // runRelayKeygen prints one masters-file line: a short key id, the key, and // "-" for "no retirement date". The key is printed once and never stored by // this process — the operator stages it on the region's boxes and puts the // same pair in the console's secret (spec §4). func runRelayKeygen(_ []string) { key := make([]byte, relay.MasterKeyLen) if _, err := rand.Read(key); err != nil { log.Fatalf("relay keygen: %v", err) } kid := make([]byte, 4) if _, err := rand.Read(kid); err != nil { log.Fatalf("relay keygen: %v", err) } fmt.Printf("%s %s -\n", hex.EncodeToString(kid), base64.StdEncoding.EncodeToString(key)) } // ---- the app-facing family (spec §5) ---------------------------------------- // requireRelayAPISession resolves signed-in mode for every relay verb below, // refusing direct-bucket mode with relay's OWN message — requireStoreAPISession's // pattern (store.go), and for the same reason: a refusal that names another // verb's advice sends someone to a fix that cannot help them. // // There is no headless fallback, and the reason is stronger than the store's. // Enabling a relay does not merely write an object: it derives an app key from // a region master held in the CONSOLE's own environment and seals it with the // deployment's secrets key. A shell in bucket mode has neither, so there is // nothing here it could drive on its own — and a key minted anywhere else // would be one no relay in the region would accept. func requireRelayAPISession(consoleFlag string) *apiSession { sess, err := resolveMode(consoleFlag) if err != nil { log.Fatal(err) } if sess == nil { log.Fatal("carlos relay works through a console, and this shell is in direct bucket mode\n" + "(CARLOS_DEPLOYMENT_BUCKET/CARLOS_DEPLOYMENT_DIR is set). The relay's keys are derived from a\n" + "region master the CONSOLE holds and sealed with the deployment's secrets key, neither of which\n" + "exists in this shell — there is no direct-bucket equivalent. Unset the store env and run:\n" + "carlos auth login --console .") } return sess } func runRelayEnable(args []string) { fs := flag.NewFlagSet("relay enable", flag.ExitOnError) app := appFlag(fs, "app to enable the relay for") regions := fs.String("regions", "", "comma-separated regions to enable (default: every region this deployment arms)") acctRaw, acctIDRaw, _ := accountFlags(fs) consoleFlag := consoleOption(fs, "console origin to enable through (API mode only; default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) name := strings.TrimSpace(*app) usage := "usage: carlos relay enable --app [--regions ] [--account ] [--account-id ] [--console ]" if name != "" { if err := validAppName(name); err != nil { log.Fatalf("relay enable: %v", err) } } sess := requireRelayAPISession(*consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) name = resolvedApp(sess, acct, name, usage) res, err := sess.relayEnable(context.Background(), acct, name, splitList(*regions)) if err != nil { log.Fatalf("relay enable: %v", err) } printRelayStatus(stdout, res) // The output-honesty rule this whole CLI follows (restart.go's): the // console's own walk finished before this process got its 200, so the // record and the aggregate ARE written — but a route picking the key up // is a converge a box does on its own schedule, and only `status` can // say whether it has. mutedf(stdout, "delivery per route is reported by: carlos relay status --app %s", name) } func runRelayStatus(args []string) { fs := flag.NewFlagSet("relay status", flag.ExitOnError) app := appFlag(fs, "app whose relay to show") acctRaw, acctIDRaw, _ := accountFlags(fs) consoleFlag := consoleOption(fs, "console origin to read through (API mode only; default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) name := strings.TrimSpace(*app) usage := "usage: carlos relay status --app [--account ] [--account-id ] [--console ]" if name != "" { if err := validAppName(name); err != nil { log.Fatalf("relay status: %v", err) } } sess := requireRelayAPISession(*consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) name = resolvedApp(sess, acct, name, usage) res, err := sess.relayGet(context.Background(), acct, name) if err != nil { log.Fatalf("relay status: %v", err) } printRelayStatus(stdout, res) } func runRelayDisable(args []string) { fs := flag.NewFlagSet("relay disable", flag.ExitOnError) app := appFlag(fs, "app to disable the relay for") acctRaw, acctIDRaw, _ := accountFlags(fs) consoleFlag := consoleOption(fs, "console origin to disable through (API mode only; default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) name := strings.TrimSpace(*app) usage := "usage: carlos relay disable --app [--account ] [--account-id ] [--console ]" if name != "" { if err := validAppName(name); err != nil { log.Fatalf("relay disable: %v", err) } } sess := requireRelayAPISession(*consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) name = resolvedApp(sess, acct, name, usage) res, err := sess.relayDisable(context.Background(), acct, name) if err != nil { log.Fatalf("relay disable: %v", err) } printRelayStatus(stdout, res) } func runRelayAdopted(args []string) { fs := flag.NewFlagSet("relay adopted", flag.ExitOnError) app := appFlag(fs, "app the route belongs to") host := fs.String("host", "", "the route that now runs the issued relay key") acctRaw, acctIDRaw, _ := accountFlags(fs) consoleFlag := consoleOption(fs, "console origin to acknowledge through (API mode only; default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) name := strings.TrimSpace(*app) usage := "usage: carlos relay adopted --app --host [--account ] [--account-id ] [--console ]" if name != "" { if err := validAppName(name); err != nil { log.Fatalf("relay adopted: %v", err) } } if strings.TrimSpace(*host) == "" { log.Fatalf("relay adopted: --host is required\n%s", usage) } sess := requireRelayAPISession(*consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) name = resolvedApp(sess, acct, name, usage) res, err := sess.relayAdopted(context.Background(), acct, name, strings.TrimSpace(*host)) if err != nil { log.Fatalf("relay adopted: %v", err) } printRelayStatus(stdout, res) } // runRelayRotate is the operator's verb and the one here with no --app: it // moves a REGION's issuer pointer and re-issues every app enabled there. A // caller who is not on CARLOS_RELAY_OPERATORS meets the console's byte- // identical 404, so the error printed here reads as "no such thing" rather // than "not allowed" — deliberately, since a 403 would confirm to any token // holder that this deployment runs a relay at all. func runRelayRotate(args []string) { fs := flag.NewFlagSet("relay rotate", flag.ExitOnError) region := fs.String("region", "", "region whose issuer pointer to move onto the newly staged master") consoleFlag := consoleOption(fs, "console origin to rotate through (API mode only; default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) if strings.TrimSpace(*region) == "" { log.Fatal("relay rotate: --region is required\nusage: carlos relay rotate --region [--console ]") } sess := requireRelayAPISession(*consoleFlag) printUsing(stdout, sess) res, err := sess.relayRotate(context.Background(), strings.TrimSpace(*region)) if err != nil { log.Fatalf("relay rotate: %v", err) } printRelayRotate(stdout, res) } // splitList turns a comma-separated flag into a list, dropping empties so a // trailing comma is not a region named "". func splitList(s string) []string { var out []string for _, p := range strings.Split(s, ",") { if p = strings.TrimSpace(p); p != "" { out = append(out, p) } } return out } // printRelayStatus prints one relay record: what the member ASKED for, what // the console ISSUED, what each route has done about it, and how each region's // own relay looks. The four are kept apart on purpose — a member debugging a // failing call needs to know which of them is the one that has not happened, // and a single "relay: ok" line would hide exactly that. func printRelayStatus(w io.Writer, res apiRelayReply) { c := colorFor(w) fmt.Fprintf(w, "%s %s\n", c.head("relay for"), res.App) switch { case res.Enabled: fmt.Fprintf(w, " enabled in %s, incarnation %s (op %d, %s)\n", strings.Join(res.Regions, ", "), res.Incarnation, res.Op, res.Phase) case res.Incarnation != "": // A record that HAS been enabled and is now off still carries an // operation and a phase, and both matter: a disable that stopped at // phase "config" is one the reconcile has yet to finish, and hiding // that behind "not enabled" would make a half-done disable look // complete. fmt.Fprintf(w, " disabled (op %d, %s)\n", res.Op, res.Phase) default: mutedf(w, "not enabled — carlos relay enable --app %s", res.App) } // The issued identity per region: account, app, incarnation and kid, the // public half of the credential the app mints per call. Printed so a // member can match a rejected call against the kid a region issues from. for _, region := range res.Regions { if suffix := res.Issued[region]; suffix != "" { fmt.Fprintf(w, " issued %s: %s\n", region, suffix) } } for _, rt := range res.Routes { line := fmt.Sprintf(" route %s (%s): %s", rt.Host, rt.Shape, rt.Delivery) // A route the relay key will never reach is the one row here that is // a fault rather than a state: it blocks the next master retirement // until someone fixes it, and only the member can. if rt.Shape == "no-delivery" { warnf(w, "%s", line) continue } fmt.Fprintln(w, line) } if len(res.Routes) == 0 && res.Enabled { mutedf(w, " no routes yet — the key is written and waits for the first one") } // Disable progress before region health, spec §5's own order: when a // member has just disabled, "is it off everywhere yet" is the question // they typed the command to answer. if len(res.DisablePending) > 0 { warnf(w, " disable pending on %s — those relays have not read the record without this app yet; "+ "allocations there end on their next authority read, within a minute", strings.Join(res.DisablePending, ", ")) } else if !res.Enabled && res.Incarnation != "" { fmt.Fprintln(w, " revoked: every relay has read a record without this app") } for _, rg := range res.RegionStatus { line := fmt.Sprintf(" relay %s in %s: heartbeat %s", rg.Box, rg.Region, rg.HeartbeatAge) if !rg.Current { line += ", holding an older record" } if len(rg.Listeners) > 0 { line += ", listening on " + strings.Join(rg.Listeners, " ") } if len(rg.Kids) > 0 { line += ", masters " + strings.Join(rg.Kids, ",") } // A retiring master is the operator's cue, not the member's: it is // the only thing that says when the old key may leave this box's // /etc/carlos/relay-masters. Printed per box, because the region is // done with a key only once every relay there has stopped loading // it (discussion #120). if len(rg.Retiring) > 0 { kids := make([]string, 0, len(rg.Retiring)) for kid := range rg.Retiring { kids = append(kids, kid) } sort.Strings(kids) parts := make([]string, 0, len(kids)) for _, kid := range kids { parts = append(parts, kid+" after "+rg.Retiring[kid]) } line += ", retiring " + strings.Join(parts, ",") } switch { case rg.Stale: // STALE is the important word and it gets the warning colour: a // relay that stopped reading the authority keeps serving until its // lease runs out, so "up" and "current" are different facts. The // relay's own last error, when it reported one, is the whole // difference between a member filing a bug and an operator // fixing a masters file. warnf(w, "%s — STALE (no successful read in over five minutes)%s", line, relayWhyStale(rg)) case !rg.LeaseLive: warnf(w, "%s — lease expired, this relay is no longer serving", line) default: fmt.Fprintln(w, line+", lease live") } } if res.Enabled && len(res.RegionStatus) == 0 { warnf(w, " no relay in %s has heartbeated at all — calls will not connect through TURN", strings.Join(res.Regions, ", ")) } } // relayWhyStale is the reason half of a stale row: the relay's own last // masters-file error, its last authority-read error, or nothing when it // reported neither (a relay that simply stopped writing heartbeats has no // error to tell us about, and inventing one would be worse than the silence). func relayWhyStale(rg apiRelayRegion) string { switch { case rg.MastersError != "" && rg.AuthorityError != "": return ": masters " + rg.MastersError + "; authority " + rg.AuthorityError case rg.MastersError != "": return ": masters " + rg.MastersError case rg.AuthorityError != "": return ": authority " + rg.AuthorityError } return "" } // printRelayRotate is the operator's answer: which master the region issues // from now, how many apps were re-issued under it, and — the line the runbook // turns on — the earliest the OLD master may be taken out of the masters file. // Retiring it before then kills credentials that are still inside their own // four-hour lifetime (spec §5, step 4). func printRelayRotate(w io.Writer, res apiRelayRotateReply) { c := colorFor(w) fmt.Fprintf(w, "%s %s\n", c.head("rotated"), res.Region) fmt.Fprintf(w, " issuing from kid %s; %d app(s) re-issued\n", res.Kid, res.Apps) fmt.Fprintf(w, " retire the previous master no earlier than %s\n", res.RetireNoEarlierThan) mutedf(w, "leave the old line in /etc/carlos/relay-masters and the old pair in the console secret until then") }