package assetcache import ( "errors" "fmt" "os" "sync" "testing" "time" ) // fakeFS is a settable free-space source. Tests using it are not acceptance: // the real FS is covered by TestStatfsReportsTheTempDir. type fakeFS struct { mu sync.Mutex total, free uint64 err error } func (f *fakeFS) Space(string) (uint64, uint64, error) { f.mu.Lock() defer f.mu.Unlock() return f.total, f.free, f.err } func (f *fakeFS) setFree(v uint64) { f.mu.Lock() f.free = v f.mu.Unlock() } const worstReserve = 2*MaxBody + metaReserve // fillStore opens a Store over a real temporary directory with a fake FS that // has plenty of room, and a budget big enough for many worst-case fills. func fillStore(t *testing.T) (*Store, *fakeFS, string) { t.Helper() dir := t.TempDir() fs := &fakeFS{total: 64 << 30, free: 64 << 30} cfg := testConfig(dir) cfg.Budget = 4 << 30 cfg.FS = fs s, err := Open(cfg) if err != nil { t.Fatal(err) } return s, fs, dir } // TestReserveWorstCaseWithUnknownLength is not acceptance: it uses a fake FS. func TestReserveWorstCaseWithUnknownLength(t *testing.T) { s, _, _ := fillStore(t) f, err := s.Reserve(testKey("shop", "/a.css"), -1) if err != nil { t.Fatal(err) } defer f.Abort() if got, want := s.Stats().Reserved, int64(2*MaxBody+64<<10); got != want { t.Fatalf("Reserved = %d, want %d", got, want) } } // TestReserveUsesContentLength is not acceptance: it uses a fake FS. func TestReserveUsesContentLength(t *testing.T) { s, _, _ := fillStore(t) f, err := s.Reserve(testKey("shop", "/a.css"), 1000) if err != nil { t.Fatal(err) } defer f.Abort() if got, want := s.Stats().Reserved, int64(2*1000+metaReserve); got != want { t.Fatalf("Reserved = %d, want %d", got, want) } } // TestReserveRefusesTooLarge is not acceptance: it uses a fake FS. func TestReserveRefusesTooLarge(t *testing.T) { s, _, dir := fillStore(t) if _, err := s.Reserve(testKey("shop", "/big"), MaxBody+1); !errors.Is(err, ErrTooLarge) { t.Fatalf("err = %v, want ErrTooLarge", err) } f, err := s.Reserve(testKey("shop", "/max"), MaxBody) if err != nil { t.Fatalf("a body of exactly MaxBody: %v", err) } f.Abort() if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after refusals and abort, want 0", got) } if got := listDir(t, dir); len(got) != 0 { t.Fatalf("files left: %v", got) } } // TestReserveRefusesBelowTheFloor is not acceptance: it uses a fake FS. It is // spec section 7's sixteen-fill case reduced to its rule: free space minus // outstanding reservations minus this one must stay at the 15% floor. func TestReserveRefusesBelowTheFloor(t *testing.T) { s, fs, _ := fillStore(t) fs.total = 8 << 30 fs.free = 8<<30*15/100 + worstReserve // exactly one reservation above the floor first, err := s.Reserve(testKey("shop", "/a"), -1) if err != nil { t.Fatalf("first reservation: %v", err) } defer first.Abort() if _, err := s.Reserve(testKey("blog", "/b"), -1); !errors.Is(err, ErrBelowFloor) { t.Fatalf("second concurrent reservation: err = %v, want ErrBelowFloor", err) } if got := s.Stats().Reserved; got != worstReserve { t.Fatalf("Reserved = %d after a refusal, want %d", got, int64(worstReserve)) } // Once the first is released the room is back. first.Abort() again, err := s.Reserve(testKey("blog", "/b"), -1) if err != nil { t.Fatalf("after abort: %v", err) } again.Abort() } // TestReserveRefusesWhenSomethingElseFillsTheDisk is not acceptance: it uses a // fake FS. Free space drops between two calls, as when logs fill the disk. func TestReserveRefusesWhenSomethingElseFillsTheDisk(t *testing.T) { s, fs, _ := fillStore(t) fs.total = 8 << 30 fs.setFree(4 << 30) f, err := s.Reserve(testKey("shop", "/a"), -1) if err != nil { t.Fatal(err) } defer f.Abort() fs.setFree(8 << 30 * 15 / 100) // the floor itself, nothing to spare if _, err := s.Reserve(testKey("shop", "/b"), 10); !errors.Is(err, ErrBelowFloor) { t.Fatalf("err = %v, want ErrBelowFloor", err) } // The fill in flight is untouched and still writes up to its reservation. if _, err := f.Write([]byte("still fine")); err != nil { t.Fatalf("in-flight write: %v", err) } } // TestReserveRefusesWithAnFSError is not acceptance: it uses a fake FS. func TestReserveRefusesWithAnFSError(t *testing.T) { s, fs, _ := fillStore(t) boom := errors.New("statfs failed") fs.err = boom if _, err := s.Reserve(testKey("shop", "/a"), 10); !errors.Is(err, boom) { t.Fatalf("err = %v, want the FS error", err) } if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after an FS error", got) } s.mu.Lock() boxFills := s.fills s.mu.Unlock() if boxFills != 0 { t.Fatalf("fill slots held after an FS error: %d", boxFills) } } // TestReserveRefusesOverBudget is not acceptance: it uses a fake FS. func TestReserveRefusesOverBudget(t *testing.T) { dir := t.TempDir() cfg := testConfig(dir) cfg.Budget = 2*500 + metaReserve + 10 // room for one 500-byte fill, not two cfg.FS = &fakeFS{total: 64 << 30, free: 64 << 30} s, err := Open(cfg) if err != nil { t.Fatal(err) } a, err := s.Reserve(testKey("shop", "/a"), 500) if err != nil { t.Fatal(err) } defer a.Abort() if _, err := s.Reserve(testKey("shop", "/b"), 500); !errors.Is(err, ErrNoRoom) { t.Fatalf("err = %v, want ErrNoRoom", err) } } // TestReserveBoundsFillsPerBoxAndPerApp is not acceptance: it uses a fake FS. func TestReserveBoundsFillsPerBoxAndPerApp(t *testing.T) { s, _, _ := fillStore(t) var fills []*Fill defer func() { for _, f := range fills { f.Abort() } }() reserve := func(app string, i int) (*Fill, error) { f, err := s.Reserve(testKey(app, fmt.Sprintf("/%d", i)), 100) if err == nil { fills = append(fills, f) } return f, err } // Four fills for one app are fine, the fifth is busy. for i := 0; i < maxFillsPerApp; i++ { if _, err := reserve("shop", i); err != nil { t.Fatalf("fill %d for shop: %v", i, err) } } if _, err := reserve("shop", 99); !errors.Is(err, ErrBusy) { t.Fatalf("5th fill for one app: err = %v, want ErrBusy", err) } // Another app is not held up by it. for i := 0; i < maxFillsPerApp; i++ { if _, err := reserve("blog", i); err != nil { t.Fatalf("fill %d for blog: %v", i, err) } } // Fill the box to 16 with further apps, then the 17th is busy. for a := 0; len(fills) < maxFillsPerBox; a++ { if _, err := reserve(fmt.Sprintf("app%d", a), 0); err != nil { t.Fatalf("fill %d: %v", len(fills), err) } } if _, err := reserve("other", 0); !errors.Is(err, ErrBusy) { t.Fatalf("17th fill on the box: err = %v, want ErrBusy", err) } // Releasing one frees a slot for the box and for that app. fills[0].Abort() fills = fills[1:] if _, err := reserve("shop", 100); err != nil { t.Fatalf("after abort: %v", err) } } // TestWritePastTheReservationFails is not acceptance: it uses a fake FS. func TestWritePastTheReservationFails(t *testing.T) { s, _, _ := fillStore(t) f, err := s.Reserve(testKey("shop", "/a"), 10) if err != nil { t.Fatal(err) } defer f.Abort() if n, err := f.Write([]byte("0123456789")); err != nil || n != 10 { t.Fatalf("write of exactly the reservation: n=%d err=%v", n, err) } if n, err := f.Write([]byte("x")); !errors.Is(err, ErrTooLarge) || n != 0 { t.Fatalf("write past the reservation: n=%d err=%v, want 0 and ErrTooLarge", n, err) } if _, err := f.Write(nil); !errors.Is(err, ErrTooLarge) { t.Fatalf("a failed fill keeps failing: err = %v", err) } } // TestWritePastMaxBodyFails is not acceptance: it uses a fake FS. With an // unknown length the identity half is MaxBody, and the fill stops there. func TestWritePastMaxBodyFails(t *testing.T) { s, _, _ := fillStore(t) f, err := s.Reserve(testKey("shop", "/a"), -1) if err != nil { t.Fatal(err) } defer f.Abort() chunk := make([]byte, 1<<20) for i := 0; i < MaxBody>>20; i++ { if _, err := f.Write(chunk); err != nil { t.Fatalf("chunk %d: %v", i, err) } } if _, err := f.Write([]byte("x")); !errors.Is(err, ErrTooLarge) { t.Fatalf("byte past MaxBody: err = %v, want ErrTooLarge", err) } } // TestAbortReleasesEverything is not acceptance: it uses a fake FS. func TestAbortReleasesEverything(t *testing.T) { s, _, dir := fillStore(t) f, err := s.Reserve(testKey("shop", "/a"), 100) if err != nil { t.Fatal(err) } if _, err := f.Write([]byte("some bytes")); err != nil { t.Fatal(err) } if len(listDir(t, dir)) == 0 { t.Fatal("expected a temporary while the fill is open") } f.Abort() if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after Abort", got) } if got := listDir(t, dir); len(got) != 0 { t.Fatalf("files left after Abort: %v", got) } s.mu.Lock() boxFills, appFills := s.fills, len(s.fillsByApp) s.mu.Unlock() if boxFills != 0 || appFills != 0 { t.Fatalf("slots after Abort: box %d, apps %d, want 0 and 0", boxFills, appFills) } if _, err := f.Write([]byte("x")); err == nil { t.Fatal("write after Abort succeeded") } } // TestAbortTwiceIsSafe is not acceptance: it uses a fake FS. A second Abort // must not release another fill's reservation. func TestAbortTwiceIsSafe(t *testing.T) { s, _, _ := fillStore(t) a, err := s.Reserve(testKey("shop", "/a"), 100) if err != nil { t.Fatal(err) } b, err := s.Reserve(testKey("shop", "/b"), 100) if err != nil { t.Fatal(err) } defer b.Abort() a.Abort() a.Abort() if got, want := s.Stats().Reserved, int64(2*100+metaReserve); got != want { t.Fatalf("Reserved = %d, want %d (b's alone)", got, want) } } // TestReserveAndAbortConcurrently is not acceptance: it uses a fake FS. Run // with -race: counters must come back to zero. func TestReserveAndAbortConcurrently(t *testing.T) { s, _, dir := fillStore(t) var wg sync.WaitGroup for g := 0; g < 8; g++ { wg.Add(1) go func(g int) { defer wg.Done() for i := 0; i < 50; i++ { f, err := s.Reserve(testKey([]string{"shop", "blog"}[g%2], fmt.Sprintf("/%d/%d", g, i)), 64) if err != nil { if !errors.Is(err, ErrBusy) { t.Errorf("unexpected error: %v", err) } continue } if _, err := f.Write([]byte("body")); err != nil { t.Errorf("write: %v", err) } f.Abort() } }(g) } wg.Wait() if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after all aborted", got) } if got := listDir(t, dir); len(got) != 0 { t.Fatalf("files left: %v", got) } } // TestAbortKeepsLeftoverBytesCountedAndRetries is not acceptance: it uses a // fake FS. When the temporaries cannot be removed (the directory is not // writable), the bytes still on disk stay counted, the fill slot is released, // and a second Abort removes the files and releases the bytes. func TestAbortKeepsLeftoverBytesCountedAndRetries(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root ignores directory permissions") } s, _, dir := fillStore(t) f, err := s.Reserve(testKey("shop", "/a"), 100) if err != nil { t.Fatal(err) } if _, err := f.Write([]byte("ten bytes!")); err != nil { t.Fatal(err) } if err := os.Chmod(dir, 0o500); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(dir, 0o700) }) f.Abort() if got := s.Stats().Reserved; got != 10 { t.Fatalf("Reserved = %d after a failed cleanup, want the 10 bytes still on disk", got) } if got := listDir(t, dir); len(got) != 1 { t.Fatalf("files after a failed cleanup: %v, want the one body temporary", got) } s.mu.Lock() boxFills, appFills := s.fills, len(s.fillsByApp) s.mu.Unlock() if boxFills != 0 || appFills != 0 { t.Fatalf("slots after a failed cleanup: box %d, apps %d, want 0 and 0", boxFills, appFills) } f.Abort() // still failing: nothing changes, nothing is released twice if got := s.Stats().Reserved; got != 10 { t.Fatalf("Reserved = %d after a second failed cleanup, want 10", got) } if err := os.Chmod(dir, 0o700); err != nil { t.Fatal(err) } f.Abort() if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after the retry, want 0", got) } if got := listDir(t, dir); len(got) != 0 { t.Fatalf("files after the retry: %v", got) } f.Abort() // and a further Abort is a no-op if got := s.Stats().Reserved; got != 0 { t.Fatalf("Reserved = %d after a third Abort", got) } } // TestReserveResamplesWhenACommitLandsAfterItsSample is not acceptance: it // uses a fake FS. A fill commits between another Reserve's free-space sample // and its decision. The committed bytes have left the reservation counter // and are not in the sample either, so a decision that combined the two // would count them nowhere. The numbers make that stale combination pass // the floor and a fresh sample fail it. func TestReserveResamplesWhenACommitLandsAfterItsSample(t *testing.T) { s, fs, _ := fillStore(t) const total = 8 << 30 floor := uint64(total) * floorPercent / 100 body := make([]byte, 1<<20) onDisk := uint64(len(body)) + 4096 // what the fake disk loses when a commits start := floor + worstReserve + onDisk/2 fs.total = total fs.setFree(start) a := fill(t, s, testKey("blog", "/a.css"), body) samples := 0 s.afterSample = func() { samples++ if samples > 1 { return } if err := a.Commit(Meta{Header: cssHeader("1"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, false); err != nil { t.Errorf("Commit: %v", err) } fs.setFree(start - onDisk) } f, err := s.Reserve(testKey("shop", "/b.css"), -1) if err == nil { f.Abort() } if !errors.Is(err, ErrBelowFloor) { t.Fatalf("Reserve after a commit landed behind its sample: err = %v, want ErrBelowFloor", err) } if samples < 2 { t.Fatalf("free space sampled %d times, want a second sample after the commit", samples) } if st := s.Stats(); st.Reserved != 0 || st.Entries != 1 { t.Fatalf("Stats = %+v, want a's entry and nothing reserved", st) } } // TestReserveDecidesUnderTheLockWhenReleasesKeepComing is not acceptance: it // uses a fake FS. Every unlocked sample is overtaken by a release, so after // sampleTries of them Reserve samples with the store locked, where nothing // can move the counter, and decides on that. func TestReserveDecidesUnderTheLockWhenReleasesKeepComing(t *testing.T) { s, _, _ := fillStore(t) var others []*Fill for i := 0; i < sampleTries; i++ { others = append(others, fill(t, s, testKey("blog", fmt.Sprintf("/%d", i)), []byte("x"))) } samples := 0 s.afterSample = func() { if samples < len(others) { others[samples].Abort() } samples++ } f, err := s.Reserve(testKey("shop", "/a.css"), 10) if err != nil { t.Fatalf("Reserve: %v", err) } f.Abort() if samples != sampleTries { t.Fatalf("unlocked samples = %d, want %d and then one under the lock", samples, sampleTries) } if st := s.Stats(); st.Reserved != 0 { t.Fatalf("Reserved = %d after every fill aborted", st.Reserved) } }