package relay import ( "context" "encoding/json" "errors" "net" "os" "path/filepath" "strconv" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/release" ) func TestReadPSI(t *testing.T) { for _, tc := range []struct { name string body string want float64 err bool }{ { name: "quiet box", body: "some avg10=0.00 avg60=0.00 avg300=0.00 total=0\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n", want: 0, }, { // The `full` line is deliberately higher: taking it instead of // `some` would read as pressure the relay must not shed on. name: "under pressure", body: "some avg10=23.47 avg60=11.20 avg300=3.01 total=91234567\nfull avg10=44.00 avg60=9.00 avg300=1.00 total=44\n", want: 23.47, }, {name: "no some line", body: "full avg10=1.00 avg60=0 avg300=0 total=0\n", err: true}, {name: "unparseable", body: "some avg10=lots avg60=0 avg300=0 total=0\n", err: true}, {name: "empty", body: "", err: true}, } { t.Run(tc.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "pressure") if err := os.WriteFile(path, []byte(tc.body), 0o644); err != nil { t.Fatal(err) } got, err := readPSI(path) if tc.err { if err == nil { t.Fatalf("readPSI(%q) = %v, want an error", tc.body, got) } return } if err != nil { t.Fatalf("readPSI: %v", err) } if got != tc.want { t.Fatalf("readPSI = %v, want %v", got, tc.want) } }) } } // A missing pressure file is the ordinary state on a kernel without PSI and // on the platforms this binary cross-compiles for: an error the caller skips // a tick on, never a panic. func TestReadPSIMissingFile(t *testing.T) { if _, err := readPSI(filepath.Join(t.TempDir(), "nope")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("readPSI on a missing file = %v, want ErrNotExist", err) } } func TestConfigFromEnvRefusesNoPublicIP(t *testing.T) { _, err := ConfigFromEnv(context.Background(), func(string) string { return "" }, func() string { return "" }, func() string { return "eu-west-1" }, newMemStore(t)) if err == nil { t.Fatal("ConfigFromEnv with no public IP anywhere: want an error") } if !strings.Contains(err.Error(), "CARLOS_RELAY_PUBLIC_IP") { t.Fatalf("error should name the override: %v", err) } } func TestConfigFromEnv(t *testing.T) { env := map[string]string{ "CARLOS_RELAY_PUBLIC_IP": " 203.0.113.7 ", "CARLOS_AWS_REGION": "us-east-1", "CARLOS_RELAY_DENY_CIDRS": "10.4.0.0/16, 198.51.100.9/32", "CARLOS_RELAY_MASTERS_PATH": "/tmp/masters", "CARLOS_RELAY_BOX": "box-7", "CARLOS_RELAY_LIMIT_APP_LIVE": "9", "CARLOS_RELAY_PRESSURE_PATH": "/tmp/pressure", "CARLOS_DEPLOYMENT_BUCKET_ETC": "ignored", } cfg, err := ConfigFromEnv(context.Background(), func(k string) string { return env[k] }, func() string { return "192.0.2.1" }, func() string { return "eu-west-1" }, newMemStore(t)) if err != nil { t.Fatalf("ConfigFromEnv: %v", err) } // Both overrides must beat IMDS, and the region one must match // edgesFileRegion's rule so the relay and the edges file agree. if got := cfg.PublicIP.String(); got != "203.0.113.7" { t.Errorf("PublicIP = %s, want the override", got) } if cfg.Region != "us-east-1" { t.Errorf("Region = %s, want the override", cfg.Region) } if cfg.Box != "box-7" || cfg.MastersPath != "/tmp/masters" || cfg.PressurePath != "/tmp/pressure" { t.Errorf("config = %+v", cfg) } if len(cfg.DenyCIDRs) != 2 || cfg.DenyCIDRs[0].String() != "10.4.0.0/16" || cfg.DenyCIDRs[1].String() != "198.51.100.9/32" { t.Errorf("DenyCIDRs = %v", cfg.DenyCIDRs) } if cfg.Limits.AppLive != 9 { t.Errorf("AppLive = %d, want the env limit", cfg.Limits.AppLive) } } // A malformed deny list is an operator typo that would otherwise open the // VPC to anything that can reach 3478, so it stops the relay. func TestConfigFromEnvRefusesBadDenyCIDR(t *testing.T) { env := map[string]string{"CARLOS_RELAY_PUBLIC_IP": "203.0.113.7", "CARLOS_RELAY_DENY_CIDRS": "10.4.0.0/16,not-a-cidr"} if _, err := ConfigFromEnv(context.Background(), func(k string) string { return env[k] }, nil, nil, newMemStore(t)); err == nil { t.Fatal("want an error for a malformed CIDR") } } // With no override, IMDS supplies both, and an off-EC2 box with neither // region source is region-"" rather than an error: it serves STUN. func TestConfigFromEnvFallsBackToIMDS(t *testing.T) { cfg, err := ConfigFromEnv(context.Background(), func(string) string { return "" }, func() string { return "192.0.2.5" }, func() string { return "" }, newMemStore(t)) if err != nil { t.Fatalf("ConfigFromEnv: %v", err) } if cfg.PublicIP.String() != "192.0.2.5" || cfg.Region != "" || cfg.MastersPath != defaultMastersPath { t.Fatalf("config = %+v", cfg) } } func TestKidsOfNilMasters(t *testing.T) { if got := kidsOf(nil); got != nil { t.Fatalf("kidsOf(nil) = %v, want nil", got) } m, err := ParseMasters(strings.NewReader("k1 " + b64(testMaster()) + " -\nk2 " + b64(testMaster()) + " -\n")) if err != nil { t.Fatal(err) } if got := kidsOf(m); len(got) != 2 || got[0] != "k1" || got[1] != "k2" { t.Fatalf("kidsOf = %v", got) } } // seedWindows must charge a restart's rebuilt window with in+out, keyed on // the FIRST slash: an app name can never contain one, an account sqid never // can either, and a malformed key is skipped rather than seeding a garbage // identity. func TestSeedWindows(t *testing.T) { srv, _, _, _, _ := newTestServer(t, "203.0.113.7", nil, func(cfg *ServerConfig) { cfg.Limits = Limits{AppLive: 1, AppPerMinute: 1, AppBytesPerHour: 1000, AccountLive: 1, AccountPerMinute: 1, AccountBytesPerHour: 1 << 40, BoxLive: 1, BoxPerMinute: 1} }) at := time.Now().Add(-30 * time.Minute) seedWindows(srv, []MeterSample{{ At: at, Apps: map[string]MeterApp{ "abc/meet": {In: 400, Out: 300}, "abc/other": {In: 1, Out: 1}, "nokey": {In: 999999}, }, }}) meet := Identity{Account: "abc", App: "meet"} if srv.adm.OverBudget(meet, time.Now()) { t.Fatal("700 of 1000 bytes seeded should not be over budget") } seedWindows(srv, []MeterSample{{At: at, Apps: map[string]MeterApp{"abc/meet": {In: 400}}}}) if !srv.adm.OverBudget(meet, time.Now()) { t.Fatal("1100 of 1000 bytes seeded should be over budget") } if srv.adm.OverBudget(Identity{Account: "nokey"}, time.Now()) { t.Fatal("a key with no slash must not seed anything") } } // The serve loop, end to end on ephemeral ports: it binds, reads the // authority record, writes the heartbeat the console reads, and shuts down // cleanly when its context is cancelled (systemd's SIGTERM). func TestServeWritesHeartbeatAndStops(t *testing.T) { store := newMemStore(t) dir := t.TempDir() masters := filepath.Join(dir, "relay-masters") // k2 carries a retirement date, so this fixture also covers the hop that // puts it on the heartbeat for `carlos relay status` (discussion #120). if err := os.WriteFile(masters, []byte( "k1 "+b64(testMaster())+" -\n"+ "k2 "+b64(testMaster())+" 2026-09-09T12:00:00Z\n"), 0o640); err != nil { t.Fatal(err) } cfg := Config{ Box: "box-serve", Region: "eu-west-1", MastersPath: masters, PublicIP: net.ParseIP("203.0.113.7"), Store: store, Limits: DefaultLimits(), PressurePath: filepath.Join(dir, "no-pressure-here"), udp: "127.0.0.1:0", tcp: "127.0.0.1:0", } // The masters watcher outlives Serve by up to one poll interval, and // t.Logf from a finished test is a data race, so the log goes quiet // when the test body does. var logMu sync.Mutex quiet := false logf := func(format string, args ...any) { logMu.Lock() defer logMu.Unlock() if !quiet { t.Logf(format, args...) } } t.Cleanup(func() { logMu.Lock(); quiet = true; logMu.Unlock() }) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- Serve(ctx, cfg, logf) }() hb := waitHeartbeat(t, store, "box-serve") if hb.Region != "eu-west-1" || len(hb.Listeners) != 2 { t.Errorf("heartbeat = %+v, want both listeners and the region", hb) } if hb.ReadAt.IsZero() { t.Error("heartbeat carries no read time, so the lease clock is invisible") } // Without this the relay half of the retiring display can be deleted with // the whole suite green, and an operator mid-rotation silently reads // "nothing is retiring". Polled rather than asserted on the heartbeat // above: the masters watcher is a goroutine, so the FIRST heartbeat can // legitimately be written before the file has been read. deadline := time.Now().Add(10 * time.Second) for { hb := waitHeartbeat(t, store, "box-serve") if got, ok := hb.Retiring["k2"]; ok { if want := "2026-09-09T12:00:00Z"; got.UTC().Format(time.RFC3339) != want { t.Errorf("k2 retire_at = %s, want %s", got.UTC().Format(time.RFC3339), want) } if _, ok := hb.Retiring["k1"]; ok { t.Error("k1 has no retirement date and must not be reported as retiring") } break } if time.Now().After(deadline) { t.Fatalf("no heartbeat ever carried k2's retirement (last: %+v)", hb.Retiring) } time.Sleep(50 * time.Millisecond) } cancel() select { case err := <-done: if err != nil { t.Fatalf("Serve returned %v, want nil on a cancelled context", err) } case <-time.After(10 * time.Second): t.Fatal("Serve did not return after its context was cancelled") } } // A failed authority read must not look like a healthy one. read_at is the // lease clock — the last SUCCESSFUL read — so the gap between it and the // heartbeat's arrival is the relay's real drift from the record it serves; // a wall clock there would say "just now" for a relay whose every read has // failed for twenty minutes, and the console would believe it. func TestServeHeartbeatIsHonestAboutAFailedRead(t *testing.T) { inner := newMemStore(t) store := &flakyStore{Store: inner} dir := t.TempDir() masters := filepath.Join(dir, "relay-masters") if err := os.WriteFile(masters, []byte("k1 "+b64(testMaster())+" -\n"), 0o640); err != nil { t.Fatal(err) } cfg := Config{ Box: "box-honest", Region: "eu-west-1", MastersPath: masters, PublicIP: net.ParseIP("203.0.113.7"), Store: store, Limits: DefaultLimits(), PressurePath: filepath.Join(dir, "none"), udp: "127.0.0.1:0", tcp: "127.0.0.1:0", } var logMu sync.Mutex quiet := false logf := func(format string, args ...any) { logMu.Lock() defer logMu.Unlock() if !quiet { t.Logf(format, args...) } } t.Cleanup(func() { logMu.Lock(); quiet = true; logMu.Unlock() }) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- Serve(ctx, cfg, logf) }() // The first read succeeds, so the lease clock is set and honest. good := waitHeartbeat(t, inner, "box-honest") if good.ReadAt.IsZero() || good.AuthorityError != "" { t.Fatalf("first heartbeat = %+v, want a read time and no error", good) } cancel() if err := <-done; err != nil { t.Fatalf("Serve: %v", err) } // Now every authority read fails. Serve is stopped, so drive readAuth's // contract directly through a second run whose store is already broken: // the lease clock must not move and the reason must be recorded. store.failGet(AuthorityKey) ctx2, cancel2 := context.WithCancel(context.Background()) done2 := make(chan error, 1) go func() { done2 <- Serve(ctx2, cfg, logf) }() deadline := time.Now().Add(10 * time.Second) var hb Heartbeat for { hb = waitHeartbeat(t, inner, "box-honest") if hb.AuthorityError != "" || time.Now().After(deadline) { break } time.Sleep(20 * time.Millisecond) } cancel2() <-done2 if hb.AuthorityError == "" { t.Fatalf("heartbeat = %+v, want the read error recorded", hb) } // A relay that has never read successfully has no lease and no read // time to report — the one thing it must not do is invent one. if !hb.ReadAt.IsZero() { t.Errorf("read_at = %v on a relay whose every read failed, want zero", hb.ReadAt) } } // flakyStore is a release.Store that can be told to fail Get for one key, // so a test can watch what the relay says when the bucket stops answering. type flakyStore struct { release.Store mu sync.Mutex fail map[string]bool } func (f *flakyStore) failGet(key string) { f.mu.Lock() if f.fail == nil { f.fail = map[string]bool{} } f.fail[key] = true f.mu.Unlock() } func (f *flakyStore) Get(ctx context.Context, key string) ([]byte, string, error) { f.mu.Lock() bad := f.fail[key] f.mu.Unlock() if bad { return nil, "", errors.New("bucket unreachable") } return f.Store.Get(ctx, key) } // OpenStoreWithRetry waits a store out rather than dying on it: the unit is // Restart=always, systemd's start limit would park it in `failed` within // ten seconds, and STUN — which needs no store at all — would go down with // it. func TestOpenStoreWithRetry(t *testing.T) { defer swapRetryTimings(t)() want := newMemStore(t) calls := 0 got, err := OpenStoreWithRetry(context.Background(), func(context.Context) (release.Store, error) { calls++ if calls < 3 { return nil, errors.New("no credentials yet") } return want, nil }, t.Logf) if err != nil { t.Fatalf("OpenStoreWithRetry: %v", err) } if calls != 3 { t.Errorf("opener called %d times, want 3 (two failures then a success)", calls) } if got != want { t.Errorf("returned a different store than the opener gave") } } // And it gives up the moment systemd asks the process to stop, rather than // holding a shutdown open for the length of the backoff. func TestOpenStoreWithRetryStopsOnCancel(t *testing.T) { defer swapRetryTimings(t)() ctx, cancel := context.WithCancel(context.Background()) calls := 0 _, err := OpenStoreWithRetry(ctx, func(context.Context) (release.Store, error) { calls++ if calls == 2 { cancel() } return nil, errors.New("still down") }, nil) if !errors.Is(err, context.Canceled) { t.Fatalf("err = %v, want context.Canceled", err) } } func swapRetryTimings(t *testing.T) func() { t.Helper() oi, om, ol := storeRetryInitial, storeRetryMax, storeRetryLogEvery storeRetryInitial, storeRetryMax, storeRetryLogEvery = time.Millisecond, 2*time.Millisecond, time.Millisecond return func() { storeRetryInitial, storeRetryMax, storeRetryLogEvery = oi, om, ol } } // Serve refuses to start without the two things it cannot invent: the box // label its objects are keyed by, and a store to write them to. func TestServeRefusesIncompleteConfig(t *testing.T) { base := Config{Box: "b", Store: newMemStore(t), PublicIP: net.ParseIP("203.0.113.7")} for name, mutate := range map[string]func(*Config){ "no box": func(c *Config) { c.Box = "" }, "no store": func(c *Config) { c.Store = nil }, "no ip": func(c *Config) { c.PublicIP = nil }, } { cfg := base mutate(&cfg) if err := Serve(context.Background(), cfg, nil); err == nil { t.Errorf("%s: Serve returned nil, want an error", name) } } } func waitHeartbeat(t *testing.T, store release.Store, box string) Heartbeat { t.Helper() deadline := time.Now().Add(10 * time.Second) for { body, _, err := store.Get(context.Background(), HeartbeatKey(box)) if err == nil { var hb Heartbeat if err := json.Unmarshal(body, &hb); err != nil { t.Fatalf("heartbeat is not JSON: %v", err) } return hb } if time.Now().After(deadline) { t.Fatalf("no heartbeat at %s within the deadline (last error %v)", HeartbeatKey(box), err) } time.Sleep(20 * time.Millisecond) } } // heartbeatUDPPort pulls the udp listener's port out of a heartbeat, which is // the only place a test outside Serve can learn the port it bound. func heartbeatUDPPort(t *testing.T, hb Heartbeat) int { t.Helper() for _, l := range hb.Listeners { if p, ok := strings.CutPrefix(l, "udp:"); ok { n, err := strconv.Atoi(p) if err != nil { t.Fatalf("udp listener %q has no port: %v", l, err) } return n } } t.Fatalf("heartbeat lists no udp listener: %v", hb.Listeners) return 0 } // TestServeEndsAllocationsOnDisable is the end-to-end disable path through // the real Serve loop, which spec §7 asks for and the branch never carried // (discussion #120): a pion client holding a live allocation, an app removed // from the authority record, and the relay acting on it within one authority // read rather than on a direct EndWhere call. // // The serial is what makes it a test and not a minute-long wait: the relay // polls control/serial every three seconds and re-reads the authority record // when it moves, which is the same path a console-mediated disable takes on // a real box. func TestServeEndsAllocationsOnDisable(t *testing.T) { store := newMemStore(t) dir := t.TempDir() mastersPath := filepath.Join(dir, "relay-masters") if err := os.WriteFile(mastersPath, []byte("k1 "+b64(testMaster())+" -\n"), 0o640); err != nil { t.Fatal(err) } ctx := context.Background() putAuthority := func(apps []AuthorityApp) time.Time { t.Helper() writtenAt := time.Now().UTC() body, err := json.Marshal(Authority{Apps: apps, WrittenAt: writtenAt}) if err != nil { t.Fatal(err) } _, etag, _ := store.Get(ctx, AuthorityKey) if _, err := store.Put(ctx, AuthorityKey, body, etag); err != nil { t.Fatal(err) } if err := release.BumpSerial(ctx, store, time.Now()); err != nil { t.Fatal(err) } return writtenAt } putAuthority([]AuthorityApp{{Account: "abc", App: "meet", Incarnation: "0123456789abcdef", Regions: []string{"eu-west-1"}}}) cfg := Config{ Box: "box-disable", Region: "eu-west-1", MastersPath: mastersPath, // The advertised relay address is deliberately NOT loopback, as on a // real box: the peers are on loopback and a relay IP equal to them // does not forward (TestServerRelaysBothLegs makes the same choice). PublicIP: net.ParseIP("203.0.113.7"), Store: store, Limits: DefaultLimits(), PressurePath: filepath.Join(dir, "no-pressure-here"), udp: "127.0.0.1:0", tcp: "127.0.0.1:0", extraAllow: mustCIDRs("127.0.0.0/8"), } var logMu sync.Mutex quiet := false logf := func(format string, args ...any) { logMu.Lock() defer logMu.Unlock() if !quiet { t.Logf(format, args...) } } t.Cleanup(func() { logMu.Lock(); quiet = true; logMu.Unlock() }) runCtx, cancel := context.WithCancel(ctx) defer cancel() done := make(chan error, 1) go func() { done <- Serve(runCtx, cfg, logf) }() port := heartbeatUDPPort(t, waitHeartbeat(t, store, "box-disable")) username, password := creds(time.Now(), "abc", "meet", "0123456789abcdef") client := turnClient(t, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: port}, username, password) relayConn, err := client.Allocate() if err != nil { t.Fatalf("allocate against a relay that has read an enabling record: %v", err) } defer relayConn.Close() // A peer on loopback, permitted through the extraAllow seam, and one // packet through the relay to prove the allocation carries traffic while // the app IS enabled. Without this the "no traffic after" check below // would pass against a relay that never forwarded anything at all. peer, err := net.ListenPacket("udp4", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer peer.Close() peerAddr := peer.LocalAddr().(*net.UDPAddr) if err := client.CreatePermission(peerAddr); err != nil { t.Fatalf("permission for a loopback peer: %v", err) } if _, err := relayConn.WriteTo([]byte("before"), peerAddr); err != nil { t.Fatalf("write through the relay: %v", err) } buf := make([]byte, 64) peer.SetReadDeadline(time.Now().Add(5 * time.Second)) if n, _, err := peer.ReadFrom(buf); err != nil || string(buf[:n]) != "before" { t.Fatalf("the relay did not carry traffic while enabled: n=%d err=%v", n, err) } // The disable: the same record without the app, and a serial bump so the // relay re-reads inside three seconds rather than at the next minute. disabledAt := putAuthority(nil) // Wait on the relay's OWN report that it is holding the new record — // the heartbeat carries the WrittenAt of the authority copy it serves — // rather than on an allocation failing. Retrying Allocate until any // error appeared was the first shape of this test and it was not a test // at all: a refusal for an unrelated reason (a per-app allocation cap, // say) reads exactly like the disable, so the assertion passed with the // app still enabled. deadline := time.Now().Add(20 * time.Second) for { hb := waitHeartbeat(t, store, "box-disable") if !hb.WrittenAt.Before(disabledAt) { break } if time.Now().After(deadline) { t.Fatalf("the relay never picked up the record written at %s (last heartbeat holds %s)", disabledAt, hb.WrittenAt) } time.Sleep(100 * time.Millisecond) } // The allocation taken before the disable must stop carrying traffic. // This is the assertion the test is named for, and the first version of // it did not make: it only re-Allocated, which proves admission refuses // a NEW allocation and says nothing about the live one. Deleting Serve's // EndWhere(..., "authority") call left that version green. if _, err := relayConn.WriteTo([]byte("after"), peerAddr); err == nil { peer.SetReadDeadline(time.Now().Add(time.Second)) if n, _, err := peer.ReadFrom(buf); err == nil { t.Errorf("the relay forwarded %q after the app was disabled: the allocation did not end", buf[:n]) } } // And a credential that was good a moment ago must be refused, which is // what a member sees after `relay disable`. probe := turnClient(t, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: port}, username, password) if _, err := probe.Allocate(); err == nil { t.Error("the relay allocated for an app the authority record no longer carries") } cancel() select { case err := <-done: if err != nil { t.Fatalf("Serve returned %v, want nil on a cancelled context", err) } case <-time.After(10 * time.Second): t.Fatal("Serve did not return after its context was cancelled") } }