package console import ( "context" "errors" "strings" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" ) var errTransientList = errors.New("transient listing failure") // flakyListStore fails listings under one prefix while failing is set. It // embeds the concrete *release.DirStore so PageLister and ModLister stay // promoted (see drainFixture). type flakyListStore struct { *release.DirStore failing atomic.Bool // dirsPrefix fails ListDirs(prefix) for exactly this prefix. dirsPrefix string // listPrefix fails List/ListPage for keys under this prefix. listPrefix string } func (f *flakyListStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { if f.failing.Load() && f.dirsPrefix != "" && prefix == f.dirsPrefix { return nil, errTransientList } return f.DirStore.ListDirs(ctx, prefix) } func (f *flakyListStore) List(ctx context.Context, prefix string) ([]string, error) { if f.failing.Load() && f.listPrefix != "" && strings.HasPrefix(prefix, f.listPrefix) { return nil, errTransientList } return f.DirStore.List(ctx, prefix) } func (f *flakyListStore) ListPage(ctx context.Context, prefix, startAfter string, limit int) ([]string, error) { if f.failing.Load() && f.listPrefix != "" && strings.HasPrefix(prefix, f.listPrefix) { return nil, errTransientList } return f.DirStore.ListPage(ctx, prefix, startAfter, limit) } func indexed(fx *consoleFixture, acct, app string) bool { for _, ref := range fx.srv.scanQueue.Apps() { if ref.Account == acct && ref.App == app { return true } } return false } // TestRebuildRetriesAfterPartialFailure: a nested listing failure must make // rebuildScanQueue return an error (so StartScanDrain retries) while the // healthy app is still found, and the retry must find the skipped app. func TestRebuildRetriesAfterPartialFailure(t *testing.T) { const badAcct, goodAcct = "acct-bad", "acct-good" cases := []struct { name string setup func(f *flakyListStore) // acct/app of the skipped app, and a healthy app that must still be found skipAcct, skipApp string goodAcct, goodApp string }{ { name: "account app-list failure", setup: func(f *flakyListStore) { f.dirsPrefix = "apps/" + badAcct + "/" }, skipAcct: badAcct, skipApp: "shop", goodAcct: goodAcct, goodApp: "shop", }, { name: "per-app queue-list failure", setup: func(f *flakyListStore) { f.listPrefix = objstore.QueuePrefix(badAcct, "shop") }, skipAcct: badAcct, skipApp: "shop", goodAcct: badAcct, goodApp: "blog", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() fx, _, _ := drainFixture(t) flaky := &flakyListStore{DirStore: fx.rel.(*release.DirStore)} tc.setup(flaky) fx.srv.cfg.Releases = flaky // Names sort so the failing account/app comes first. enableStore(t, fx, tc.skipAcct, tc.skipApp, true) enableStore(t, fx, tc.goodAcct, tc.goodApp, true) at := time.Now().Add(-time.Minute) enqueueStranded(t, fx, tc.skipAcct, tc.skipApp, "uploads/skipped.pdf", at) enqueueStranded(t, fx, tc.goodAcct, tc.goodApp, "uploads/healthy.pdf", at) flaky.failing.Store(true) err := fx.srv.rebuildScanQueue(ctx) if !errors.Is(err, errTransientList) { t.Fatalf("first rebuild err = %v, want the transient listing failure so the drain retries", err) } if !indexed(fx, tc.goodAcct, tc.goodApp) { t.Errorf("healthy app %s/%s not indexed: one failure cost the others their queue", tc.goodAcct, tc.goodApp) } if indexed(fx, tc.skipAcct, tc.skipApp) { t.Errorf("skipped app %s/%s indexed despite its listing failing", tc.skipAcct, tc.skipApp) } // Nothing was deleted or lost by the failed pass. if n := queueLen(t, fx, tc.skipAcct, tc.skipApp); n != 1 { t.Errorf("skipped app has %d queue item(s) after the failed pass, want 1", n) } if n := queueLen(t, fx, tc.goodAcct, tc.goodApp); n != 1 { t.Errorf("healthy app has %d queue item(s) after the pass, want 1", n) } flaky.failing.Store(false) if err := fx.srv.rebuildScanQueue(ctx); err != nil { t.Fatalf("second rebuild after the failure cleared: %v", err) } if !indexed(fx, tc.skipAcct, tc.skipApp) { t.Errorf("retry did not find the previously skipped app %s/%s", tc.skipAcct, tc.skipApp) } if n := queueLen(t, fx, tc.skipAcct, tc.skipApp); n != 1 { t.Errorf("skipped app has %d queue item(s) after the retry, want 1", n) } if d := fx.srv.ScanQueueDepth(); d != 2 { t.Errorf("index depth %d after the retry, want 2", d) } }) } }