package host import ( "context" "errors" "os" "path/filepath" "testing" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // Harness Run 9, finding 6: a nested instance created with settings has no // instance directory until its first wake, so config sync's write of its // .config.env failed, and only the hourly pass came back for it. The // instance woke and ran without its settings (its store binding included). // // Through the agent's own wiring: the boot or hourly pass (syncExecSerialized) // skips the row and names it, while a sibling whose directory exists gets its // file in the same pass; the generation loop's tick (configGenTick) does // nothing while the directory is absent, and on the first tick after the // first wake has made it, writes the file. func TestConfigGenTickWritesANestedRowsConfigOnceItsDirAppears(t *testing.T) { configDirsPending = pendingDirs{} t.Cleanup(func() { configDirsPending = pendingDirs{} }) ctx := context.Background() s := release.NewDirStore(t.TempDir()) reg := restartTestRegistry(t) dd := t.TempDir() privText, _, err := appconfig.GenerateKey() if err != nil { t.Fatal(err) } priv, err := appconfig.ParsePrivateKey(privText, "test") if err != nil { t.Fatal(err) } if err := appconfig.SetEnv(ctx, s, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil { t.Fatal(err) } fresh, woken := "shop.example.com", "older.example.com" for _, h := range []string{fresh, woken} { if err := reg.Upsert(registry.Route{Host: h, Kind: "instance", App: "shop", Account: "bab", Hibernate: true, Status: "ready", DBPath: provision.NestedDBPath(dd, h), Socket: provision.NestedSocket(h)}); err != nil { t.Fatal(err) } } // The sibling has woken before: the activator's prepare made its // instance directory (os.MkdirAll(dataDir, 0o700), activator/prepare.go). firstWake := func(host string) { t.Helper() if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil { t.Fatal(err) } } firstWake(woken) cfgPath := func(h string) string { return filepath.Join(provision.InstanceDir(dd, h), h+".config.env") } // The boot pass (configSyncLoop's first, a minute after start). if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil { t.Fatalf("a nested row with no directory failed the pass: %v", err) } if b, err := os.ReadFile(cfgPath(woken)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" { t.Fatalf("the sibling's bundle = %q, %v", b, err) } if _, err := os.Lstat(provision.InstanceDir(dd, fresh)); !os.IsNotExist(err) { t.Fatalf("config sync made the instance directory itself (err %v): that is the activator's", err) } // The generation loop: the first tick seeds its cache, and no tick // writes while the directory is absent. cache := newGenCache() tick := func() { t.Helper() routes, err := reg.List() if err != nil { t.Fatal(err) } configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache) } for range 2 { tick() } if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) { t.Fatalf("a bundle with nowhere to go: %v", err) } // The first wake makes the directory; the next tick writes the bundle, // with no config change and no hourly pass. firstWake(fresh) tick() if b, err := os.ReadFile(cfgPath(fresh)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" { t.Fatalf("after the first wake, the bundle = %q, %v", b, err) } if _, ok := configDirsPending.appeared(); ok { t.Fatal("the written host is still waited for") } // Nothing more to wait for: a later tick does not run the pass again. if err := os.Remove(cfgPath(fresh)); err != nil { t.Fatal(err) } tick() if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) { t.Fatalf("a tick with nothing pending ran the pass (err %v)", err) } } // A pass that fails part way does not forget a host an earlier pass was // waiting for; a complete pass that no longer has the host in scope drops it. func TestConfigDirsPendingKeepsWhatAFailedPassDidNotReach(t *testing.T) { var p pendingDirs dir := filepath.Join(t.TempDir(), "instances", "shop.example.com") p.record(map[string]string{"shop.example.com": dir}, nil, true) p.record(map[string]string{}, nil, false) if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatal(err) } if h, ok := p.appeared(); !ok || h != "shop.example.com" { t.Fatalf("appeared = %q %v after a failed pass", h, ok) } p.record(map[string]string{}, map[string]bool{}, true) if h, ok := p.appeared(); ok { t.Fatalf("appeared = %q after a complete pass with the host out of scope", h) } } // blipStore fails reads of one key while fail is set: a bucket blip on // one app's bundle. type blipStore struct { release.Store key string fail bool } func (f *blipStore) Get(ctx context.Context, key string) ([]byte, string, error) { if f.fail && key == f.key { return nil, "", errors.New("transient: connection reset") } return f.Store.Get(ctx, key) } // Fix round 1, Astra's finding 3: the first tick after the first wake makes // the directory meets a transient failure reading the app's bundle. The pass // returns nil, leaving the file unwritten; the host stays waited for, and the // next tick writes its config, with no config change and no hourly pass. A // bundle that keeps failing is retried on a backoff, not every tick. func TestConfigGenTickRetriesAHostAPassLeftUndone(t *testing.T) { configDirsPending = pendingDirs{} t.Cleanup(func() { configDirsPending = pendingDirs{} }) ctx := context.Background() base := release.NewDirStore(t.TempDir()) s := &blipStore{Store: base, key: appconfig.Key("bab", "shop")} reg := restartTestRegistry(t) dd := t.TempDir() privText, _, err := appconfig.GenerateKey() if err != nil { t.Fatal(err) } priv, err := appconfig.ParsePrivateKey(privText, "test") if err != nil { t.Fatal(err) } if err := appconfig.SetEnv(ctx, base, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil { t.Fatal(err) } host := "shop.example.com" if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "shop", Account: "bab", Hibernate: true, Status: "ready", DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host)}); err != nil { t.Fatal(err) } cfgPath := filepath.Join(provision.InstanceDir(dd, host), host+".config.env") cache := newGenCache() passes := 0 tick := func() { t.Helper() routes, err := reg.List() if err != nil { t.Fatal(err) } if _, due := configDirsPending.peek(); due { passes++ } configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache) } if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil { t.Fatal(err) } tick() // seeds the cache if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil { t.Fatal(err) } s.fail = true tick() if _, err := os.Lstat(cfgPath); !os.IsNotExist(err) { t.Fatalf("a bundle that did not read was written (err %v)", err) } if _, ok := configDirsPending.peek(); !ok { t.Fatal("the pass that left the host undone dropped it") } s.fail = false tick() if b, err := os.ReadFile(cfgPath); err != nil || string(b) != "STORE_BUCKET=store-shop\n" { t.Fatalf("the next tick's bundle = %q, %v", b, err) } if _, ok := configDirsPending.peek(); ok { t.Fatal("the written host is still waited for") } // A bundle that keeps failing: retries back off (1, 2, 4 ... ticks apart). if err := os.Remove(cfgPath); err != nil { t.Fatal(err) } configDirsPending.record(map[string]string{host: provision.InstanceDir(dd, host)}, nil, false) s.fail = true passes = 0 for range 20 { tick() } if passes == 0 || passes > 6 { t.Fatalf("%d retries in 20 ticks of a failing bundle, want a backoff (1 to 6)", passes) } } // peek is appeared without the tick: whether a retry is due now. func (p *pendingDirs) peek() (string, bool) { p.mu.Lock() defer p.mu.Unlock() for h, e := range p.dirs { if _, err := os.Lstat(e.dir); err == nil && e.waitTicks == 0 { return h, true } } return "", false } // TestPendingDirsBackoffSaturates: a host that keeps failing backs off to // the cap and stays there; the failure count never grows far enough for the // shift to overflow into a negative or zero delay (Astra fix round 1). func TestPendingDirsBackoffSaturates(t *testing.T) { var p pendingDirs dir := t.TempDir() p.record(map[string]string{"shop.example.com": dir}, nil, false) for i := 0; i < 200; i++ { p.record(nil, map[string]bool{"shop.example.com": false}, false) e := p.dirs["shop.example.com"] if e == nil { t.Fatalf("pass %d: the waiting host was dropped", i) } if e.waitTicks < 0 || e.waitTicks > pendingRetryCap-1 { t.Fatalf("pass %d: waitTicks %d outside [0,%d]", i, e.waitTicks, pendingRetryCap-1) } } if got := p.dirs["shop.example.com"].waitTicks; got != pendingRetryCap-1 { t.Fatalf("after 200 failures waitTicks = %d, want the cap %d", got, pendingRetryCap-1) } p.record(nil, map[string]bool{"shop.example.com": true}, false) if _, ok := p.dirs["shop.example.com"]; ok { t.Fatal("a delivered host stays pending") } }