package domains import ( "context" "encoding/json" "errors" "strconv" "strings" "testing" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // addInstance upserts a provisioned instance route the way the provision // pass mints one: Slug is the host's first label. func addInstance(t *testing.T, reg *registry.Registry, host, slug, account, app string) { t.Helper() if err := reg.Upsert(registry.Route{ Host: host, Slug: slug, App: app, Account: account, Kind: "instance", Status: "ready", }); err != nil { t.Fatal(err) } if err := reg.SetProvisioned(host, true); err != nil { t.Fatal(err) } } func aliasMap(t *testing.T, reg *registry.Registry) map[string]registry.RouteAlias { t.Helper() aliases, err := reg.ListAliases() if err != nil { t.Fatal(err) } out := map[string]registry.RouteAlias{} for _, a := range aliases { out[a.Alias] = a } return out } // jellyStore is the letsjelly.dev testbed in miniature: account bad holds // the apex through its static site, and dev-managed-jelly holds the // instance catch-all. func jellyStore(t *testing.T) release.Store { t.Helper() s := release.NewDirStore(t.TempDir()) putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) putClaim(t, s, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "dev-managed-jelly", Instances: true}) return s } func reconcile(t *testing.T, s release.Store, reg *registry.Registry, deploymentRoutes map[string]bool) { t.Helper() if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { t.Fatal(err) } } func TestInstanceCatchallDerivesOneAliasPerInstance(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") addInstance(t, reg, "smoketest.bad.carlos.exciting.io", "smoketest", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) got := aliasMap(t, reg) if len(got) != 2 { t.Fatalf("got %d aliases, want 2: %+v", len(got), got) } a := got["acme.letsjelly.dev"] if a.Host != "acme.bad.carlos.exciting.io" || a.Account != "bad" || a.App != "dev-managed-jelly" { t.Fatalf("acme alias = %+v", a) } if got["smoketest.letsjelly.dev"].Host != "smoketest.bad.carlos.exciting.io" { t.Fatalf("smoketest alias = %+v", got["smoketest.letsjelly.dev"]) } } func TestInstanceCatchallFollowsTheInstance(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) if _, ok := aliasMap(t, reg)["acme.letsjelly.dev"]; !ok { t.Fatal("no alias for a live instance") } // The instance leaves the box (deleted, or moved elsewhere): its name // leaves with it on the next pass. if err := reg.Delete("acme.bad.carlos.exciting.io"); err != nil { t.Fatal(err) } reconcile(t, s, reg, nil) if got := aliasMap(t, reg); len(got) != 0 { t.Fatalf("alias outlived its instance: %+v", got) } } func TestInstanceCatchallDetachDropsEveryName(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) deleteClaim(t, s, "*.letsjelly.dev") reconcile(t, s, reg, nil) if got := aliasMap(t, reg); len(got) != 0 { t.Fatalf("aliases outlived the catch-all: %+v", got) } } // An exact claim on a derived name wins, whoever holds it: a same-account // one (the Jelly manager's per-team claims, www, auth) and another // account's alike. func TestInstanceCatchallExactClaimWins(t *testing.T) { s := jellyStore(t) putClaim(t, s, Claim{Host: "acme.letsjelly.dev", Account: "bad", App: "dev-managed-jelly", Route: "acme.bad.carlos.exciting.io"}) putClaim(t, s, Claim{Host: "beta.letsjelly.dev", Account: "zzz", App: "other", Route: "x.zzz.carlos.exciting.io"}) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") addInstance(t, reg, "beta.bad.carlos.exciting.io", "beta", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) got := aliasMap(t, reg) // acme is still served — by its exact claim's row, which says the same. if got["acme.letsjelly.dev"].Host != "acme.bad.carlos.exciting.io" { t.Fatalf("acme = %+v", got["acme.letsjelly.dev"]) } // beta's exact claim belongs to another account whose route is not on // this box: no row here at all, and certainly not the derived one. if a, ok := got["beta.letsjelly.dev"]; ok { t.Fatalf("derived alias shadowed another account's exact claim: %+v", a) } // Removing the exact claim hands the name back to the catch-all. deleteClaim(t, s, "acme.letsjelly.dev") reconcile(t, s, reg, nil) if a := aliasMap(t, reg)["acme.letsjelly.dev"]; a.Host != "acme.bad.carlos.exciting.io" { t.Fatalf("after the exact claim went, acme = %+v", a) } } func TestInstanceCatchallRoutesWin(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") addInstance(t, reg, "beta.bad.carlos.exciting.io", "beta", "bad", "dev-managed-jelly") addRoute(t, reg, "acme.letsjelly.dev", "bad", "something") reconcile(t, s, reg, map[string]bool{"beta.letsjelly.dev": true}) got := aliasMap(t, reg) if len(got) != 0 { t.Fatalf("a derived name shadowed a real route: %+v", got) } } // The catch-all is honoured only while its parent is held by the same // account — released, or held by someone else, it derives nothing. func TestInstanceCatchallNeedsTheParentInItsAccount(t *testing.T) { for name, apex := range map[string]*Claim{ "released": nil, "other account": {Host: "letsjelly.dev", Account: "zzz", App: "squat"}, } { t.Run(name, func(t *testing.T) { s := release.NewDirStore(t.TempDir()) putClaim(t, s, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "dev-managed-jelly", Instances: true}) if apex != nil { putClaim(t, s, *apex) } reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) if got := aliasMap(t, reg); len(got) != 0 { t.Fatalf("derived without the parent: %+v", got) } }) } } // Only the claim's own account and app, and only provisioned instance rows // whose Slug is the host's first label. func TestInstanceCatchallOnlyItsOwnInstances(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") // A sibling app in the same account. addInstance(t, reg, "sib.bad.carlos.exciting.io", "sib", "bad", "dev-jelly-manager") // Another account's app of the same name. addInstance(t, reg, "evil.zzz.carlos.exciting.io", "evil", "zzz", "dev-managed-jelly") // A hand-added route (not provisioned), with a slug. if err := reg.Upsert(registry.Route{Host: "hand.bad.carlos.exciting.io", Slug: "hand", App: "dev-managed-jelly", Account: "bad", Kind: "instance", Status: "ready"}); err != nil { t.Fatal(err) } // A replica row: Slug is the whole host (provision.Replica) ... addInstance(t, reg, "acme-r.bad.carlos.exciting.io", "acme-r.bad.carlos.exciting.io", "bad", "dev-managed-jelly") // ... or a label, when the create door wrote it; the mark excludes it. addInstance(t, reg, "rep.bad.carlos.exciting.io", "rep", "bad", "dev-managed-jelly") if err := reg.SetReplica("rep.bad.carlos.exciting.io", true); err != nil { t.Fatal(err) } // A sidecar of the app. if err := reg.Upsert(registry.Route{Host: "side.sidecar.internal", Slug: "side", App: "dev-managed-jelly", Account: "bad", Kind: "sidecar", Status: "ready"}); err != nil { t.Fatal(err) } // A slug that is not the host's first label. addInstance(t, reg, "odd.bad.carlos.exciting.io", "other", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) got := aliasMap(t, reg) if len(got) != 1 || got["acme.letsjelly.dev"].Host != "acme.bad.carlos.exciting.io" { t.Fatalf("got %+v, want only acme.letsjelly.dev", got) } } // A catch-all carrying fields no door writes on one is not honoured. func TestInstanceCatchallRefusesHandWrittenShapes(t *testing.T) { for name, c := range map[string]Claim{ "route": {Route: "acme.bad.carlos.exciting.io"}, "wildcard": {Wildcard: true}, "paths": {Paths: map[string]string{"x": "acme.bad.carlos.exciting.io"}}, } { t.Run(name, func(t *testing.T) { s := release.NewDirStore(t.TempDir()) putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) c.Host, c.Account, c.App, c.Instances = "*.letsjelly.dev", "bad", "dev-managed-jelly", true putClaim(t, s, c) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) for alias := range aliasMap(t, reg) { if alias == "acme.letsjelly.dev" { t.Fatalf("derived from a hand-written catch-all (%s)", name) } } }) } } // The catch-all sits beside the apex's own one-route catch-all: the exact // derived names win at the edge (an exact key beats "*."), and neither pass // removes the other's row. func TestInstanceCatchallBesideARouteCatchall(t *testing.T) { s := release.NewDirStore(t.TempDir()) putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site", Wildcard: true, Catchall: true}) putClaim(t, s, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "dev-managed-jelly", Instances: true}) reg := aliasRegWith(t) if err := reg.Upsert(registry.Route{Host: "letsjelly.dev", App: "dev-jelly-site", Account: "bad", Kind: "static", Status: "ready"}); err != nil { t.Fatal(err) } addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) got := aliasMap(t, reg) if got["*.letsjelly.dev"].Host != "letsjelly.dev" || got["acme.letsjelly.dev"].Host != "acme.bad.carlos.exciting.io" { t.Fatalf("got %+v", got) } } func TestParseInstanceCatchallHost(t *testing.T) { for raw, want := range map[string]string{ "*.letsjelly.dev": "letsjelly.dev", "*.LetsJelly.dev.": "letsjelly.dev", "*.a.b.example.com": "a.b.example.com", "*.dev": "", "letsjelly.dev": "", "*letsjelly.dev": "", "*.*.letsjelly.dev": "", "x.*.letsjelly.dev": "", "*.": "", } { got, ok := ParseInstanceCatchallHost(raw) if ok != (want != "") || got != want { t.Errorf("ParseInstanceCatchallHost(%q) = %q, %v; want %q", raw, got, ok, want) } } } func TestAttachInstanceCatchall(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if _, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-managed-jelly", "t", "j"); !errors.Is(err, ErrParentNotHeld) { t.Fatalf("unclaimed parent: err = %v, want ErrParentNotHeld", err) } putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "zzz", App: "squat"}) if _, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-managed-jelly", "t", "j"); !errors.Is(err, ErrParentNotHeld) { t.Fatalf("parent held elsewhere: err = %v, want ErrParentNotHeld", err) } deleteClaim(t, s, "letsjelly.dev") putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) created, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-managed-jelly", "t", "j") if err != nil || !created { t.Fatalf("first attach: created=%v err=%v", created, err) } c, err := GetClaim(ctx, s, "*.letsjelly.dev") if err != nil || !c.Instances || c.Account != "bad" || c.App != "dev-managed-jelly" || c.Route != "" || c.Wildcard || c.Catchall { t.Fatalf("stored = %+v, %v", c, err) } created, err = AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-managed-jelly", "t", "j") if err != nil || created { t.Fatalf("re-attach: created=%v err=%v, want idempotent", created, err) } // Another app of the same account does not take it over. if _, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-jelly-site", "t", "j"); !errors.Is(err, ErrHostUnavailable) { t.Fatalf("sibling app: err = %v, want ErrHostUnavailable", err) } // Capability and path writers refuse a catch-all even to its owner. if _, err := SetClaimCapabilities(ctx, s, "*.letsjelly.dev", "bad", "dev-managed-jelly", true, false); !errors.Is(err, ErrHostUnavailable) { t.Fatalf("capabilities on a catch-all: err = %v", err) } if _, err := SetClaimPaths(ctx, s, "*.letsjelly.dev", "bad", "dev-managed-jelly", map[string]string{"x": "a.b.c"}); !errors.Is(err, ErrHostUnavailable) { t.Fatalf("paths on a catch-all: err = %v", err) } // The parent changes hands: the old catch-all is inert, and the new // holder replaces it instead of meeting a squat. deleteClaim(t, s, "letsjelly.dev") putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "zzz", App: "new"}) if created, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "zzz", "teams", "t", "j"); err != nil || !created { t.Fatalf("new parent holder: created=%v err=%v", created, err) } if c, _ := GetClaim(ctx, s, "*.letsjelly.dev"); c.Account != "zzz" || c.App != "teams" || !c.Instances { t.Fatalf("after replacement: %+v", c) } deleteClaim(t, s, "*.letsjelly.dev") deleteClaim(t, s, "letsjelly.dev") putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) if _, err := AttachInstanceCatchall(ctx, s, "letsjelly.dev", "bad", "dev-managed-jelly", "t", "j"); err != nil { t.Fatal(err) } // Detach is the ordinary owner-gated delete. if removed, err := DetachClaim(ctx, s, "*.letsjelly.dev", "bad", "dev-jelly-site"); err != nil || removed { t.Fatalf("sibling detach: removed=%v err=%v", removed, err) } if removed, err := DetachClaim(ctx, s, "*.letsjelly.dev", "bad", "dev-managed-jelly"); err != nil || !removed { t.Fatalf("owner detach: removed=%v err=%v", removed, err) } } func TestForeignInstanceCatchall(t *testing.T) { ctx := context.Background() s := jellyStore(t) for host, want := range map[string]map[string]bool{ "acme.letsjelly.dev": {"bad": false, "zzz": true}, "x.acme.letsjelly.dev": {"bad": false, "zzz": false}, // deeper: not the catch-all's "letsjelly.dev": {"bad": false, "zzz": false}, // the parent itself "acme.example.com": {"bad": false, "zzz": false}, } { for acct, w := range want { c, err := ForeignInstanceCatchall(ctx, s, host, acct) if got := c != nil; err != nil || got != w { t.Errorf("ForeignInstanceCatchall(%s, %s) = %+v, %v; want %v", host, acct, c, err, w) } } } // Released parent: the catch-all serves nothing, so it holds nothing. deleteClaim(t, s, "letsjelly.dev") if c, err := ForeignInstanceCatchall(ctx, s, "acme.letsjelly.dev", "zzz"); err != nil || c != nil { t.Errorf("after the parent went: %+v, %v; want nil", c, err) } } // What a binary that predates Instances makes of a catch-all claim: the // shape it decodes (no Route, no Wildcard, no Paths) is one every pass // already skips. Simulated by clearing the field this binary knows. func TestInstanceCatchallIsInertWithoutTheField(t *testing.T) { s := release.NewDirStore(t.TempDir()) putClaim(t, s, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) putClaim(t, s, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "dev-managed-jelly"}) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) if got := aliasMap(t, reg); len(got) != 0 { t.Fatalf("got %+v", got) } wds, err := reg.ListWildcardDomains() if err != nil || len(wds) != 0 { t.Fatalf("wildcard domains %+v, %v", wds, err) } } // The per-box cap holds, deterministically: the first names in host order // are kept, and nothing past the cap is written. func TestInstanceCatchallCap(t *testing.T) { s := jellyStore(t) reg := aliasRegWith(t) for i := 0; i <= maxInstanceAliases; i++ { label := "t" + strconv.Itoa(10000+i) addInstance(t, reg, label+".bad.carlos.exciting.io", label, "bad", "dev-managed-jelly") } reconcile(t, s, reg, nil) got := aliasMap(t, reg) if len(got) != maxInstanceAliases { t.Fatalf("got %d aliases, want the cap %d", len(got), maxInstanceAliases) } if _, ok := got["t"+strconv.Itoa(10000+maxInstanceAliases)+".letsjelly.dev"]; ok { t.Fatal("the last name in host order was kept past the cap") } } // unreadableKeyStore fails every read of one key, as a spurious store error would, // while the key still shows in listings. type unreadableKeyStore struct { *release.DirStore failKey string } func (f unreadableKeyStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == f.failKey { return nil, "", errors.New("synthetic 403") } return f.DirStore.Get(ctx, key) } // A claim whose body cannot be read this pass still beats the derived // name: precedence is decided from the listing, never from what decoded. func TestInstanceCatchallUnreadableClaimStillWins(t *testing.T) { dir := release.NewDirStore(t.TempDir()) putClaim(t, dir, Claim{Host: "letsjelly.dev", Account: "bad", App: "dev-jelly-site"}) putClaim(t, dir, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "dev-managed-jelly", Instances: true}) putClaim(t, dir, Claim{Host: "acme.letsjelly.dev", Account: "zzz", App: "other", Route: "x.zzz.carlos.exciting.io"}) s := unreadableKeyStore{dir, ClaimKey("acme.letsjelly.dev")} reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") for _, cache := range []*ClaimCache{nil, {}} { if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, cache); err != nil { t.Fatal(err) } if a, ok := aliasMap(t, reg)["acme.letsjelly.dev"]; ok { t.Fatalf("derived over an unreadable claim (cache %v): %+v", cache != nil, a) } } } // replacingStore hands back the claim at key as it stood, then lets another // writer replace it before the caller acts — the old owner's detach racing // the parent's new holder. type replacingStore struct { *release.DirStore key string next []byte done bool } func (r *replacingStore) Get(ctx context.Context, key string) ([]byte, string, error) { body, etag, err := r.DirStore.Get(ctx, key) if key == r.key && !r.done && err == nil { r.done = true if _, perr := r.DirStore.Put(ctx, key, r.next, etag); perr != nil { return nil, "", perr } } return body, etag, err } func TestDetachCatchallNeverDeletesAReplacement(t *testing.T) { ctx := context.Background() dir := release.NewDirStore(t.TempDir()) putClaim(t, dir, Claim{Host: "*.letsjelly.dev", Account: "bad", App: "old", Instances: true}) next, _ := json.Marshal(Claim{Schema: 1, Host: "*.letsjelly.dev", Account: "zzz", App: "new", Instances: true}) s := &replacingStore{DirStore: dir, key: ClaimKey("*.letsjelly.dev"), next: next} removed, err := DetachClaim(ctx, s, "*.letsjelly.dev", "bad", "old") if err != nil || removed { t.Fatalf("detach over a replacement: removed=%v err=%v, want false and no error", removed, err) } c, err := GetClaim(ctx, dir, "*.letsjelly.dev") if err != nil || c.Account != "zzz" || c.App != "new" { t.Fatalf("the replacement was not kept: %+v, %v", c, err) } } // putRecordKey drops an (empty) instance record at its key: the derivation's // duplicate check reads record KEYS only. func putRecordKey(t *testing.T, s release.Store, account, app, host string) { t.Helper() if _, err := s.Put(context.Background(), instanceRecordsPrefix(account, app)+host+".json", []byte(`{}`), ""); err != nil { t.Fatal(err) } } // A label two instances share (after an instance-domain change) derives // nothing, on every box alike — here, the box holding only one of them. func TestInstanceCatchallRefusesASharedLabel(t *testing.T) { s := jellyStore(t) putRecordKey(t, s, "bad", "dev-managed-jelly", "acme.bad.carlos.exciting.io") putRecordKey(t, s, "bad", "dev-managed-jelly", "acme.teams.example.net") putRecordKey(t, s, "bad", "dev-managed-jelly", "beta.bad.carlos.exciting.io") reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") addInstance(t, reg, "beta.bad.carlos.exciting.io", "beta", "bad", "dev-managed-jelly") reconcile(t, s, reg, nil) got := aliasMap(t, reg) if _, ok := got["acme.letsjelly.dev"]; ok { t.Fatalf("a shared label derived a name: %+v", got) } if got["beta.letsjelly.dev"].Host != "beta.bad.carlos.exciting.io" { t.Fatalf("an unshared label lost its name: %+v", got) } } // listFailStore fails listings under one prefix, and nothing else. type listFailStore struct { *release.DirStore prefix string } func (f listFailStore) List(ctx context.Context, prefix string) ([]string, error) { if strings.HasPrefix(prefix, f.prefix) { return nil, errors.New("synthetic list failure") } return f.DirStore.List(ctx, prefix) } // When the app's instances cannot be listed, the catch-all's names stay as // they were for the pass: a blip must not take every team's name down. func TestInstanceCatchallKeepsNamesWhenTheListingFails(t *testing.T) { dir := jellyStore(t).(*release.DirStore) reg := aliasRegWith(t) addInstance(t, reg, "acme.bad.carlos.exciting.io", "acme", "bad", "dev-managed-jelly") reconcile(t, dir, reg, nil) if _, ok := aliasMap(t, reg)["acme.letsjelly.dev"]; !ok { t.Fatal("precondition: the name derives") } failing := listFailStore{dir, instanceRecordsPrefix("bad", "dev-managed-jelly")} if err := ReconcileAliases(context.Background(), failing, reg, aliasNow(), nil, nil); err != nil { t.Fatal(err) } if a, ok := aliasMap(t, reg)["acme.letsjelly.dev"]; !ok || a.Host != "acme.bad.carlos.exciting.io" { t.Fatalf("the name was dropped on a listing failure: %+v", aliasMap(t, reg)) } } func TestInstanceLabelCounts(t *testing.T) { s := release.NewDirStore(t.TempDir()) putRecordKey(t, s, "bad", "app", "acme.bad.carlos.exciting.io") putRecordKey(t, s, "bad", "app", "acme.old.example.net") putRecordKey(t, s, "bad", "app", "beta.bad.carlos.exciting.io") putRecordKey(t, s, "bad", "other", "gamma.bad.carlos.exciting.io") got, err := InstanceLabelCounts(context.Background(), s, "bad", "app") if err != nil || got["acme"] != 2 || got["beta"] != 1 || got["gamma"] != 0 { t.Fatalf("counts = %v, %v", got, err) } }