package release import ( "go/ast" "go/parser" "go/token" "io/fs" "os" "path/filepath" "sort" "strconv" "strings" "testing" ) // The change index (control/changes.json) only moves if every writer of the // deployment bucket bumps it. A bump lives in the concrete store and is off // until EnableChangeIndex, so the way a deployment-bucket writer goes missing // is a new call site that opens the bucket with the plain constructor. This // test finds every place the module builds a store over a bucket or a // directory (NewS3Store, NewS3StoreIn, an S3Store literal, NewDirStore) and // makes each one say which kind it is for: // // - an S3 deployment bucket (apps/, accounts/, control/): the site calls // NewDeploymentS3Store[In], so it never appears here at all; // - a deployment directory (CARLOS_DEPLOYMENT_DIR) or a hand-built // S3Store literal over the deployment bucket: listed in // deploymentManualSites, and the test checks the same function calls // EnableChangeIndex at least once per site; // - anything else: listed in otherBucketOpeners with the reason. // // Keys are "path/from/module/root.go:enclosingFunc". Every entry carries the // number of sites it covers, and the test fails when the count found under // the key differs. A new open inside a listed function is therefore a // failure, not a silent pass. type openerEntry struct { reason string count int } // otherBucketOpeners are the sites that open a store which is not the // deployment bucket: through NewS3Store, NewS3StoreIn, an S3Store literal or // NewDirStore. var otherBucketOpeners = map[string]openerEntry{ "cmd/console/main.go:serve": {"console seed data store: the database/data bucket (seeds/), not the deployment bucket", 1}, "internal/host/seedwiring.go:seedDataOpener": {"this region's data bucket, where seeds/ lives", 1}, "internal/host/relocatepass.go:productionRelocationRuntimeInputs": {"relocation files store: the data bucket, manifests under moves/ prefixes", 1}, "internal/host/relocatepass.go:registerRelocationSource": {"relocation files store: the data bucket, manifests under moves/ prefixes", 1}, "internal/host/claimfiles.go:newProductionFilesStore": {"claim files manifests: the data bucket, moves/ prefixes", 1}, "internal/hostfence/writer.go:OpenRootFenceStore": {"fence writer: the fence bucket, written under its own scoped role", 1}, "internal/lease/execution_store.go:ExecutionStore": {"execution/build state: the hibernate and claim data bucket, keyed by host prefix", 1}, "internal/status/dryrun.go:DryRun": {"status dry run: a throwaway scratch directory, not a deployment", 1}, } // deploymentManualSites are stores over the deployment (CARLOS_DEPLOYMENT_DIR // directory, or a hand-built S3Store literal over the bucket) that are not // built by NewDeploymentS3Store. Each function must call EnableChangeIndex // once per site. var deploymentManualSites = map[string]openerEntry{ "cli.go:deploymentReadSeam": {"CARLOS_DEPLOYMENT_DIR", 1}, "cli.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, "cmd/console/main.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, "internal/edge/edge.go:enableStaticFromEnv": {"CARLOS_DEPLOYMENT_DIR", 1}, "internal/host/agent.go:openDeployStoreEnv": {"CARLOS_DEPLOYMENT_DIR", 1}, "internal/ops/ops.go:openDeployStore": {"CARLOS_DEPLOYMENT_DIR", 1}, "internal/vet/vet.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, } type openerSite struct { key string // path:func line int what string // what was found, for the failure message kind string // "s3call", "literal" or "dir" // enables is how many EnableChangeIndex calls the enclosing function has. enables int } const releaseImportPath = "github.com/carlosframework/platform/internal/release" // scanOpenerSites lists the store constructions in one parsed file. // inReleasePkg is true for files of package release itself, where the // constructors and the type are referred to without a qualifier. func scanOpenerSites(fset *token.FileSet, f *ast.File, rel string, inReleasePkg bool) []openerSite { qual := "" for _, im := range f.Imports { p, _ := strconv.Unquote(im.Path.Value) if p != releaseImportPath { continue } qual = "release" if im.Name != nil { qual = im.Name.Name } } if qual == "" && !inReleasePkg { return nil } // isRelease reports whether e names the identifier in package release. isRelease := func(e ast.Expr, name string) bool { switch x := e.(type) { case *ast.SelectorExpr: id, ok := x.X.(*ast.Ident) return ok && qual != "" && id.Name == qual && x.Sel.Name == name case *ast.Ident: return inReleasePkg && x.Name == name } return false } var sites []openerSite for _, decl := range f.Decls { fn := "" var root ast.Node = decl if fd, ok := decl.(*ast.FuncDecl); ok { fn = fd.Name.Name if fd.Body == nil { continue } root = fd.Body } enables := 0 var found []openerSite ast.Inspect(root, func(n ast.Node) bool { switch x := n.(type) { case *ast.CallExpr: if sel, ok := x.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "EnableChangeIndex" { enables++ } for _, name := range []string{"NewS3Store", "NewS3StoreIn"} { if isRelease(x.Fun, name) { found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "call to " + name, kind: "s3call"}) } } if isRelease(x.Fun, "NewDirStore") { found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "call to NewDirStore", kind: "dir"}) } case *ast.CompositeLit: if isRelease(x.Type, "S3Store") { found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "S3Store literal", kind: "literal"}) } } return true }) for _, s := range found { s.enables = enables sites = append(sites, s) } } return sites } func moduleRoot(t *testing.T) string { t.Helper() dir, err := os.Getwd() if err != nil { t.Fatal(err) } for { if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { return dir } parent := filepath.Dir(dir) if parent == dir { t.Fatal("no go.mod above the test directory") } dir = parent } } func TestS3StoreOpenersClassified(t *testing.T) { root := moduleRoot(t) fset := token.NewFileSet() var sites []openerSite err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } name := d.Name() if d.IsDir() { if path != root && (strings.HasPrefix(name, ".") || name == "vendor" || name == "testdata" || name == "node_modules") { return filepath.SkipDir } if path != root { if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { return filepath.SkipDir // a nested module } } return nil } if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { return nil } rel, err := filepath.Rel(root, path) if err != nil { return err } rel = filepath.ToSlash(rel) if rel == "internal/release/store.go" { return nil // the constructors themselves } f, err := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) if err != nil { return err } sites = append(sites, scanOpenerSites(fset, f, rel, f.Name.Name == "release" && filepath.ToSlash(filepath.Dir(rel)) == "internal/release")...) return nil }) if err != nil { t.Fatal(err) } byKey := map[string][]openerSite{} for _, s := range sites { byKey[s.key] = append(byKey[s.key], s) } var bad []string for key, ss := range byKey { if e, ok := deploymentManualSites[key]; ok { for _, s := range ss { if s.kind == "s3call" { bad = append(bad, key+" (line "+strconv.Itoa(s.line)+"): "+s.what+" in a deployment site; use NewDeploymentS3Store[In]") } } if len(ss) != e.count { bad = append(bad, key+": "+strconv.Itoa(len(ss))+" store opens found, deploymentManualSites says "+strconv.Itoa(e.count)+" ("+e.reason+"); classify the new one and update the count") } if ss[0].enables < len(ss) { bad = append(bad, key+": "+strconv.Itoa(len(ss))+" deployment store opens but only "+strconv.Itoa(ss[0].enables)+" EnableChangeIndex calls in the function") } continue } if e, ok := otherBucketOpeners[key]; ok { if len(ss) != e.count { bad = append(bad, key+": "+strconv.Itoa(len(ss))+" store opens found, otherBucketOpeners says "+strconv.Itoa(e.count)+" ("+e.reason+"); classify the new one and update the count") } continue } for _, s := range ss { bad = append(bad, key+" (line "+strconv.Itoa(s.line)+"): "+s.what+" is not classified. S3 deployment bucket: use NewDeploymentS3Store[In]. Deployment directory or literal: call EnableChangeIndex and list it in deploymentManualSites. Anything else: add it to otherBucketOpeners with the reason") } } for k := range otherBucketOpeners { if byKey[k] == nil { bad = append(bad, k+": listed in otherBucketOpeners but no longer opens a store (remove the entry)") } } for k := range deploymentManualSites { if byKey[k] == nil { bad = append(bad, k+": listed in deploymentManualSites but no longer opens a store (remove the entry)") } } sort.Strings(bad) for _, b := range bad { t.Error(b) } if len(sites) == 0 { t.Fatal("the scan found no store sites at all; the walk is not reaching the module") } } // TestOpenerScanRecognisesEveryForm keeps the scanner honest: qualified and // aliased constructors, literals with and without &, and the enable check. func TestOpenerScanRecognisesEveryForm(t *testing.T) { const src = `package x import rel "github.com/carlosframework/platform/internal/release" func a() { rel.NewS3Store(nil, "b") } func b() { rel.NewS3StoreIn(nil, "b", "r") } func c() { _ = &rel.S3Store{Bucket: "b"} } func d() { s := rel.S3Store{Bucket: "b"}; _ = s } func e() { s := &rel.S3Store{Bucket: "b"}; s.EnableChangeIndex() } func f() { rel.NewDeploymentS3Store(nil, "b") } func g() { rel.NewDirStore("d") } func h() { d := rel.NewDirStore("d"); d.EnableChangeIndex(); _ = rel.NewDirStore("e") } ` fset := token.NewFileSet() f, err := parser.ParseFile(fset, "x.go", src, 0) if err != nil { t.Fatal(err) } got := map[string][]openerSite{} for _, s := range scanOpenerSites(fset, f, "x.go", false) { got[s.key] = append(got[s.key], s) } for _, want := range []string{"x.go:a", "x.go:b", "x.go:c", "x.go:d", "x.go:e", "x.go:g"} { if len(got[want]) != 1 { t.Errorf("%s: %d sites, want 1", want, len(got[want])) } } if len(got["x.go:f"]) != 0 { t.Error("NewDeploymentS3Store is not a plain opener") } if len(got["x.go:h"]) != 2 || got["x.go:h"][0].enables != 1 { t.Errorf("h: want two dir sites and one enable call, got %+v", got["x.go:h"]) } if got["x.go:c"][0].kind != "literal" || got["x.go:c"][0].enables != 0 { t.Errorf("c: literal without enable, got %+v", got["x.go:c"]) } if got["x.go:g"][0].kind != "dir" || got["x.go:e"][0].enables != 1 { t.Errorf("g/e: kinds wrong, got %+v %+v", got["x.go:g"], got["x.go:e"]) } }