package console import ( "bytes" "context" "encoding/json" "errors" "io" "log" "os" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) // mustTestSecretsKey is Task 9's (regionconfig_test.go). var testKeyPub, testKeyFP = mustTestSecretsKey() func bp(b bool) *bool { return &b } type edgeOpts struct { hib, shared, inbound *bool // nil = the writer predates the field fp string } var eligibleBox = edgeOpts{hib: bp(true), shared: bp(true), inbound: bp(true), fp: testKeyFP} // edgeJSON writes one box's edges file the way internal/host does. func edgeJSON(box, eip, awsRegion, placement string, o edgeOpts) string { m := map[string]any{"v": 1, "box": box, "eip": eip, "region": awsRegion, "hosts": map[string]any{}} if placement != "" { m["placement"] = placement } if o.hib != nil { m["hibernation_configured"] = *o.hib } if o.shared != nil { m["serves_shared_pool"] = *o.shared } if o.inbound != nil { m["accepts_inbound"] = *o.inbound } if o.fp != "" { m["secrets_key_fp"] = o.fp } b, _ := json.Marshal(m) return string(b) } var ( homeBox = edgeJSON("ie-box", "5.6.7.8", "eu-west-1", "", eligibleBox) ukEligible = edgeJSON("uk-box", "1.2.3.4", "eu-west-2", "uk", eligibleBox) ukNoKey = edgeJSON("uk-box", "1.2.3.4", "eu-west-2", "uk", edgeOpts{hib: bp(true), shared: bp(true), inbound: bp(true)}) ukWrongKey = edgeJSON("uk-box", "1.2.3.4", "eu-west-2", "uk", edgeOpts{hib: bp(true), shared: bp(true), inbound: bp(true), fp: "sha256:" + strings.Repeat("0", 64)}) ukOldWriter = edgeJSON("uk-box", "1.2.3.4", "eu-west-2", "uk", edgeOpts{}) auPoolOnly = edgeJSON("au-box", "9.9.9.9", "ap-southeast-2", "au", edgeOpts{hib: bp(true), shared: bp(false), inbound: bp(true), fp: testKeyFP}) usNoInbound = edgeJSON("us-box", "8.8.4.4", "us-east-1", "us", edgeOpts{hib: bp(true), shared: bp(true), inbound: bp(false), fp: testKeyFP}) nzNoHosting = edgeJSON("nz-box", "6.6.6.6", "ap-southeast-6", "nz", edgeOpts{hib: bp(false), shared: bp(true), inbound: bp(true), fp: testKeyFP}) jpNoEIP = edgeJSON("jp-box", "", "ap-northeast-1", "jp", eligibleBox) fleetBoxFile = edgeJSON("fl-box", "4.4.4.4", "eu-west-2", "fleet/acme", eligibleBox) // ukKeylessSibling is a second uk box reconcile would admit (shared pool, // inbound) that holds no key: the F2 hazard beside ukEligible. ukKeylessSibling = edgeJSON("uk-box-2", "1.2.3.5", "eu-west-2", "uk", edgeOpts{hib: bp(true), shared: bp(true), inbound: bp(true)}) ) // Partial reads (re-review r1): automatic non-home eligibility needs // COMPLETE evidence, the edges snapshot Complete and every listed box's // status read succeeded. Otherwise a failing sibling can simply drop out // of the fold (its edges GET failed) or read as not fresh (its status GET // failed), and the region becomes eligible through the box that was read. // On any incomplete read no non-home region is eligible for that request // (whole-request scope, the stricter choice); the home stays as it is. func TestIncompleteReadsNeverMakeARegionEligible(t *testing.T) { ctx := context.Background() auBox := edgeJSON("au-box", "9.9.9.9", "ap-southeast-2", "au", eligibleBox) s := release.NewDirStore(t.TempDir()) for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible, "uk-box-2": ukKeylessSibling, "au-box": auBox} { if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { t.Fatal(err) } setBoxSnapshot(t, s, key, time.Now()) } check := func(name string, store release.Store, wantAU bool) { t.Helper() snap, err := readLiveEdgesSnapshot(ctx, store, time.Now()) if err != nil { t.Fatalf("%s: %v", name, err) } cands := regionCandidates(snap, "ie", testKeyFP) uk, _ := findCandidate(cands, "uk") au, _ := findCandidate(cands, "au") home, _ := findCandidate(cands, "") if uk.Eligible { t.Errorf("%s: uk = %+v, want ineligible (a keyless sibling was listed)", name, uk) } if au.Eligible != wantAU { t.Errorf("%s: au eligible = %v, want %v", name, au.Eligible, wantAU) } if !home.Eligible { t.Errorf("%s: home = %+v, want eligible regardless", name, home) } } // Control: everything read. uk fails on its keyless sibling, au is clean. check("complete", s, true) // Hole 1: the keyless sibling's EDGES object cannot be read. It drops out // of the fold, and only the keyed box remains to speak for uk. check("edges GET failure", failingGet{Store: s, key: "control/edges/uk-box-2.json"}, false) // Hole 2: the keyless sibling's STATUS snapshot cannot be read. Unknown // liveness is not staleness: it must not be excluded as stale. check("status GET failure", failingGet{Store: s, key: "status/boxes/uk-box-2.json"}, false) // A corrupt status body is an incomplete read too (loadBoxes skips it). corrupt := release.NewDirStore(t.TempDir()) for _, key := range []string{"ie-box", "uk-box", "uk-box-2", "au-box"} { body, _, err := s.Get(ctx, "control/edges/"+key+".json") if err != nil { t.Fatal(err) } if _, err := corrupt.Put(ctx, "control/edges/"+key+".json", body, ""); err != nil { t.Fatal(err) } setBoxSnapshot(t, corrupt, key, time.Now()) } if _, err := corrupt.Put(ctx, "status/boxes/uk-box-2.json", []byte("{not json"), mustEtag(t, corrupt, "status/boxes/uk-box-2.json")); err != nil { t.Fatal(err) } check("corrupt status snapshot", corrupt, false) } // Unmatched boxes, ruling (A) (re-review r2): a FRESH status snapshot // whose box has no edges file is incomplete evidence, whole-request, the // same as a failed read. Reconcile does not need the edges file to admit a // box, so a keyless live box whose edges publication failed would // otherwise never reach the fold while its keyed sibling made the region // eligible. Matched by the status Box.Label against the edges file's own // box field, the pair edgesfile.EdgeIPs joins on. A STALE snapshot with no // edges file is a retired box and is ignored. func TestFreshStatusWithoutAnEdgesFileIsIncompleteEvidence(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible} { if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { t.Fatal(err) } setBoxSnapshot(t, s, key, time.Now()) } check := func(name string, wantUK bool) { t.Helper() snap, err := readLiveEdgesSnapshot(ctx, s, time.Now()) if err != nil { t.Fatalf("%s: %v", name, err) } cands := regionCandidates(snap, "ie", testKeyFP) uk, _ := findCandidate(cands, "uk") home, _ := findCandidate(cands, "") if uk.Eligible != wantUK { t.Errorf("%s: uk = %+v, want eligible %v", name, uk, wantUK) } if !home.Eligible { t.Errorf("%s: home = %+v, want eligible regardless", name, home) } } check("control: every fresh box has an edges file", true) // Box B reports status, fresh, and has published no edges file. setBoxSnapshot(t, s, "uk-box-2", time.Now()) check("fresh status-only box", false) // The same box gone dark: a retired box, ignored. setBoxSnapshot(t, s, "uk-box-2", time.Now().Add(-time.Hour)) check("stale status-only box", true) } // Unmatched boxes, ruling (B) REVERSED (re-review r2, production // evidence): on the flagship every box has both objects, and a retired // box leaves both behind, so it reads as a STALE snapshot, never an absent // one. An edges file whose box has no status object at all is therefore // not a retired box but unknown liveness, and it makes the evidence // incomplete, whole-request, like (A). A stale snapshot beside the file is // the retired box: not fresh, counts for neither side, and the region // stays eligible through its fresh keyed sibling (the F1 case). func TestEdgesFileWithoutAnyStatusSnapshotIsIncompleteEvidence(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible} { if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { t.Fatal(err) } setBoxSnapshot(t, s, key, time.Now()) } if _, err := s.Put(ctx, "control/edges/uk-box-2.json", []byte(ukKeylessSibling), ""); err != nil { t.Fatal(err) } snap, err := readLiveEdgesSnapshot(ctx, s, time.Now()) if err != nil { t.Fatal(err) } if snap.EvidenceComplete() { t.Fatalf("a keyless edges file with NO status object left the evidence complete (edges %v, boxes %v)", snap.Complete, snap.BoxesComplete) } if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); c.Eligible { t.Fatalf("uk = %+v, want ineligible: a box with no status object has unknown liveness", c) } if h, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), ""); !h.Eligible { t.Fatalf("home = %+v, want eligible regardless", h) } // The retired box: the same edges file beside a STALE snapshot. F1's // case, and it must keep working. setBoxSnapshot(t, s, "uk-box-2", time.Now().Add(-time.Hour)) snap, err = readLiveEdgesSnapshot(ctx, s, time.Now()) if err != nil { t.Fatal(err) } if !snap.EvidenceComplete() { t.Fatalf("a stale snapshot beside its edges file made the evidence incomplete (edges %v, boxes %v); a retired box is complete evidence", snap.Complete, snap.BoxesComplete) } if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); !c.Eligible { t.Fatalf("retired keyless sibling: uk = %+v, want eligible through the fresh keyed box", c) } } func mustEtag(t *testing.T, s release.Store, key string) string { t.Helper() _, etag, err := s.Get(context.Background(), key) if err != nil { t.Fatal(err) } return etag } // Incomplete evidence is said once, like the other liveness diagnostics. func TestIncompleteEvidenceIsLoggedOnce(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible} { if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { t.Fatal(err) } setBoxSnapshot(t, s, key, time.Now()) } var buf bytes.Buffer restore := captureLogTo(&buf) defer restore() livenessSaid.Delete("incomplete") for i := 0; i < 2; i++ { snap, err := readLiveEdgesSnapshot(ctx, failingGet{Store: s, key: "status/boxes/uk-box.json"}, time.Now()) if err != nil { t.Fatal(err) } if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); c.Eligible { t.Fatalf("round %d: uk eligible on an incomplete read", i) } } if n := strings.Count(buf.String(), "incomplete"); n != 1 { t.Fatalf("incomplete evidence logged %d times, want once: %s", n, buf.String()) } } // setEdges creates OR replaces edges files, and gives each box a FRESH // status snapshot: automatic eligibility needs one (F1), and the fixture // boxes are all meant to be live unless a test says otherwise through // setBoxSnapshot. putEdges is create-only (an empty ifMatch on DirStore and // S3), so it cannot change a box's file. func setEdges(t *testing.T, fx *consoleFixture, files map[string]string) { t.Helper() ctx := context.Background() for name := range files { setBoxSnapshot(t, fx.rel, name, time.Now()) } for name, body := range files { key := "control/edges/" + name + ".json" etag := "" if _, cur, err := fx.rel.Get(ctx, key); err == nil { etag = cur } else if !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } if _, err := fx.rel.Put(ctx, key, []byte(body), etag); err != nil { t.Fatalf("set %s: %v", key, err) } } } // setBoxSnapshot writes (or rewrites) a box's status/boxes/