package console

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/release"
	"github.com/carlosframework/platform/internal/runs"
	"github.com/carlosframework/platform/internal/svcrole"
)

// startRun admits one run through the door and returns its id.
func (fx *runFx) startRun(t *testing.T) string {
	t.Helper()
	status, body := postRun(t, fx.runsURL(runTestHost), fx.member, okMeta(), nil)
	if status != http.StatusAccepted {
		t.Fatalf("create: %d %s", status, body)
	}
	var c runs.Created
	if err := json.Unmarshal([]byte(body), &c); err != nil {
		t.Fatal(err)
	}
	return c.ID
}

func (fx *runFx) transition(t *testing.T, mut func(*provision.Record) error) {
	t.Helper()
	if _, err := provision.SetRecordRun(context.Background(), fx.rel, release.OpsAccount, "shop", runTestHost, mut); err != nil {
		t.Fatal(err)
	}
}

func (fx *runFx) chunk(t *testing.T, id string, seq int64, stream, data string) {
	t.Helper()
	if _, err := fx.rel.Put(context.Background(), runs.ChunkKey(release.OpsAccount, "shop", id, seq, stream), []byte(data), ""); err != nil {
		t.Fatal(err)
	}
}

func (fx *runFx) page(t *testing.T, id string, after int64) runs.Page {
	t.Helper()
	status, _, body := rawReq(t, http.MethodGet, fmt.Sprintf("%s/%s?after=%d", fx.runsURL(runTestHost), id, after), fx.member, "")
	if status != http.StatusOK {
		t.Fatalf("GET page: %d %s", status, body)
	}
	var p runs.Page
	if err := json.Unmarshal([]byte(body), &p); err != nil {
		t.Fatal(err)
	}
	return p
}

func (fx *runFx) post(t *testing.T, id, verb, body string) (int, string) {
	t.Helper()
	status, _, out := rawReq(t, http.MethodPost, fx.runsURL(runTestHost)+"/"+id+"/"+verb, fx.member, body)
	return status, out
}

func TestAPIRunGetPagesChunksAndTheResult(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	fx.chunk(t, id, 1, runs.StreamOut, "hello ")
	fx.chunk(t, id, 2, runs.StreamErr, "warn")
	fx.chunk(t, id, 3, runs.StreamOut, "world")
	p := fx.page(t, id, 0)
	if p.State != provision.RunPicked || len(p.Chunks) != 3 || p.Result != nil || string(p.Chunks[1].Data) != "warn" || p.Chunks[1].Stream != runs.StreamErr {
		t.Fatalf("page %+v", p)
	}
	if p := fx.page(t, id, 2); len(p.Chunks) != 1 || p.Chunks[0].Seq != 3 {
		t.Fatalf("after 2: %+v", p)
	}
}

func TestAPIRunGetPagesSeveralPagesAfterTheResult(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	for seq := int64(1); seq <= 3*runs.PageChunks; seq++ {
		fx.chunk(t, id, seq, runs.StreamOut, "x")
	}
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0, LastSeq: 3 * runs.PageChunks}); err != nil {
		t.Fatal(err)
	}
	var after int64
	pages := 0
	for {
		p := fx.page(t, id, after)
		pages++
		for _, c := range p.Chunks {
			if c.Seq != after+1 {
				t.Fatalf("seq %d after %d", c.Seq, after)
			}
			after = c.Seq
		}
		if p.Result == nil {
			t.Fatal("every page must carry the result once it exists")
		}
		if !p.More {
			break
		}
	}
	if after != 3*runs.PageChunks || pages != 3 {
		t.Fatalf("read to %d in %d pages", after, pages)
	}
}

func TestAPIRunGetReadsTheResultBeforeTheChunks(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	fx.chunk(t, id, 1, runs.StreamOut, "a")
	// The box writes the last chunk, then the result. A store that shows
	// the result to the console's read must also show every chunk to the
	// listing after it.
	order := []string{}
	fx.srv.cfg.Releases = &orderedStore{Store: fx.rel, seen: &order}
	fx.page(t, id, 0)
	result, list := -1, -1
	for i, call := range order {
		if result < 0 && strings.HasSuffix(call, "/result.json") {
			result = i
		}
		if list < 0 && strings.HasPrefix(call, "list ") && strings.HasSuffix(call, "/out/") {
			list = i
		}
	}
	if result < 0 || list < 0 || result > list {
		t.Fatalf("store calls %q: the result must be read before the chunks are listed", order)
	}
}

// orderedStore records Get and List calls in order.
type orderedStore struct {
	release.Store
	seen *[]string
}

func (o *orderedStore) Get(ctx context.Context, key string) ([]byte, string, error) {
	*o.seen = append(*o.seen, "get "+key)
	return o.Store.Get(ctx, key)
}
func (o *orderedStore) List(ctx context.Context, prefix string) ([]string, error) {
	*o.seen = append(*o.seen, "list "+prefix)
	return o.Store.List(ctx, prefix)
}

func TestAPIRunAckChecksLastSeqThenCleansUp(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	if status, body := fx.post(t, id, "ack", `{"last_seq":0}`); status != http.StatusConflict || errorOf(t, body) != "run "+id+" has not finished yet" {
		t.Fatalf("ack before the result: %d %s", status, body)
	}
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	fx.chunk(t, id, 1, runs.StreamOut, "a")
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0, LastSeq: 1}); err != nil {
		t.Fatal(err)
	}
	fx.transition(t, provision.RunFinish(id))
	if status, body := fx.post(t, id, "ack", `{"last_seq":0}`); status != http.StatusConflict || errorOf(t, body) != "run "+id+" ended at chunk 1, not 0" {
		t.Fatalf("wrong last_seq: %d %s", status, body)
	}
	if left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id)); len(left) == 0 {
		t.Fatal("a refused ack deleted the run")
	}
	if status, body := fx.post(t, id, "ack", `{"last_seq":1}`); status != http.StatusOK {
		t.Fatalf("ack: %d %s", status, body)
	}
	if left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id)); len(left) != 0 {
		t.Fatalf("ack left %q", left)
	}
	if rec, _ := provision.GetRecord(context.Background(), fx.rel, release.OpsAccount, "shop", runTestHost); rec.ActiveRun != nil {
		t.Fatalf("ack left the run on the record: %+v", rec.ActiveRun)
	}
}

func TestAPIRunCancelBeforeAndAfterPickup(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	if status, body := fx.post(t, id, "cancel", ""); status != http.StatusOK || !strings.Contains(body, `"cancelled"`) {
		t.Fatalf("queued cancel: %d %s", status, body)
	}
	if status, body := fx.post(t, id, "cancel", ""); status != http.StatusConflict || errorOf(t, body) != "run "+id+" has already ended" {
		t.Fatalf("cancel a cancelled run: %d %s", status, body)
	}
	id2 := fx.startRun(t)
	fx.transition(t, provision.RunPick(id2, "box-1", "boot-1", time.Now().Add(time.Hour)))
	if status, body := fx.post(t, id2, "cancel", ""); status != http.StatusOK || !strings.Contains(body, `"cancel"`) {
		t.Fatalf("picked cancel: %d %s", status, body)
	}
	events, _ := fx.st.Events(OpsAccountID, "shop", 20)
	ends := 0
	for _, e := range events {
		if e.Action == "run-end" && strings.Contains(e.Detail, id) && strings.Contains(e.Detail, `"cancelled"`) {
			ends++
		}
	}
	if ends != 1 {
		t.Errorf("a run cancelled before pickup must write one run-end row, got %d", ends)
	}
}

func TestAPIRunExpireRacesAPickup(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	if status, body := fx.post(t, id, "expire", ""); status != http.StatusOK || !strings.Contains(body, `"expired"`) {
		t.Fatalf("expire: %d %s", status, body)
	}
	id2 := fx.startRun(t)
	fx.transition(t, provision.RunPick(id2, "box-1", "boot-1", time.Now().Add(time.Hour)))
	if status, body := fx.post(t, id2, "expire", ""); status != http.StatusConflict || errorOf(t, body) != "run "+id2+" has already started" {
		t.Fatalf("expire after pickup: %d %s", status, body)
	}
}

func TestAPIRunRoutesNameAMissingRun(t *testing.T) {
	fx := runFixture(t)
	missing := "run_bbbbbbbbbbbbbbbbbbbbbbbbbb"
	status, _, body := rawReq(t, http.MethodGet, fx.runsURL(runTestHost)+"/"+missing, fx.member, "")
	if status != http.StatusNotFound || errorOf(t, body) != runTestHost+" has no run "+missing {
		t.Fatalf("GET: %d %s", status, body)
	}
	for _, verb := range []string{"cancel", "expire", "ack"} {
		if status, body := fx.post(t, missing, verb, `{"last_seq":0}`); status != http.StatusNotFound || errorOf(t, body) != runTestHost+" has no run "+missing {
			t.Errorf("%s: %d %s", verb, status, body)
		}
	}
}

func TestAPIRunGetWritesRunEndOnce(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 7, Box: "box-1"}); err != nil {
		t.Fatal(err)
	}
	fx.page(t, id, 0)
	fx.page(t, id, 0)
	events, _ := fx.st.Events(OpsAccountID, "shop", 20)
	n := 0
	for _, e := range events {
		if e.Action == "run-end" && strings.Contains(e.Detail, id) {
			n++
			if !strings.Contains(e.Detail, `"exit":7`) || e.Address != "alice@keymail.dev" {
				t.Errorf("run-end %+v", e)
			}
		}
	}
	if n != 1 {
		t.Fatalf("run-end rows %d, want 1", n)
	}
}

func TestAPIRunAckKeepsTheObjectsOfALiveRun(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	fx.chunk(t, id, 1, runs.StreamOut, "a")
	// The box has written the result and not yet moved the run to done.
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0, LastSeq: 1}); err != nil {
		t.Fatal(err)
	}
	if status, body := fx.post(t, id, "ack", `{"last_seq":1}`); status != http.StatusConflict || errorOf(t, body) != "run "+id+" has not finished yet" {
		t.Fatalf("ack of a picked run: %d %s", status, body)
	}
	if left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id)); len(left) == 0 {
		t.Fatal("an ack deleted the objects of a run the record holds as picked")
	}
	if rec, _ := provision.GetRecord(context.Background(), fx.rel, release.OpsAccount, "shop", runTestHost); rec.ActiveRun == nil || rec.ActiveRun.State != provision.RunPicked {
		t.Fatalf("record %+v", rec.ActiveRun)
	}
}

func TestAPIRunAckAfterTheRecordMovedOnDeletesTheObjects(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0}); err != nil {
		t.Fatal(err)
	}
	fx.transition(t, provision.RunFinish(id))
	fx.transition(t, provision.RunClear(id))
	// A repeated ack, the record already clear: the objects go, the answer stands.
	if status, body := fx.post(t, id, "ack", `{"last_seq":0}`); status != http.StatusOK {
		t.Fatalf("ack: %d %s", status, body)
	}
	if left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id)); len(left) != 0 {
		t.Fatalf("left %q", left)
	}
}

func TestAPIRunRoutesRefuseAnotherHostsRun(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	other := "other.example.com"
	putShopRecord(t, fx.consoleFixture, provision.Record{Host: other, Channel: "stable", CreatedAt: time.Now().UTC()})
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0}); err != nil {
		t.Fatal(err)
	}
	want := other + " has no run " + id
	status, _, body := rawReq(t, http.MethodGet, fx.runsURL(other)+"/"+id, fx.member, "")
	if status != http.StatusNotFound || errorOf(t, body) != want {
		t.Errorf("GET: %d %s", status, body)
	}
	for _, verb := range []string{"cancel", "expire", "ack"} {
		_, _, body := rawReq(t, http.MethodPost, fx.runsURL(other)+"/"+id+"/"+verb, fx.member, `{"last_seq":0}`)
		if errorOf(t, body) != want {
			t.Errorf("%s: %s", verb, body)
		}
	}
	if left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id)); len(left) == 0 {
		t.Fatal("another host's ack deleted the run")
	}
}

func TestAPIRunRoutesRefuseAServiceCredentialAndABadID(t *testing.T) {
	fx := runFixture(t)
	svc := serviceToken(t, fx.st, OpsAccountID, "ci", "shop", svcrole.Instance)
	id := "run_aaaaaaaaaaaaaaaaaaaaaaaaaa"
	if status, _, _ := rawReq(t, http.MethodGet, fx.runsURL(runTestHost)+"/"+id, svc, ""); status != http.StatusForbidden {
		t.Errorf("service credential: %d, want 403", status)
	}
	status, _, body := rawReq(t, http.MethodGet, fx.runsURL(runTestHost)+"/not-a-run", fx.member, "")
	if status != http.StatusNotFound || errorOf(t, body) != runTestHost+" has no run not-a-run" {
		t.Errorf("bad id: %d %s", status, body)
	}
}

// faultStore injects the failures a bucket has: a delete that fails, a Put
// whose response is lost (landed or not), a Get that fails.
type faultStore struct {
	release.Store
	delete func(key string) error
	put    func(key string) (landed bool, err error)
	get    func(key string) error
}

func (f *faultStore) Delete(ctx context.Context, key string) error {
	if f.delete != nil {
		if err := f.delete(key); err != nil {
			return err
		}
	}
	return f.Store.Delete(ctx, key)
}

func (f *faultStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
	if f.put != nil {
		if landed, err := f.put(key); err != nil {
			if landed {
				_, _ = f.Store.Put(ctx, key, body, ifMatch)
			}
			return "", err
		}
	}
	return f.Store.Put(ctx, key, body, ifMatch)
}

func (f *faultStore) Get(ctx context.Context, key string) ([]byte, string, error) {
	if f.get != nil {
		if err := f.get(key); err != nil {
			return nil, "", err
		}
	}
	return f.Store.Get(ctx, key)
}

// runEnds is the details of the run-end rows written for id.
func (fx *runFx) runEnds(t *testing.T, id string) []string {
	t.Helper()
	events, err := fx.st.Events(OpsAccountID, "shop", 50)
	if err != nil {
		t.Fatal(err)
	}
	var out []string
	for _, e := range events {
		if e.Action == "run-end" && strings.Contains(e.Detail, id) {
			out = append(out, e.Detail)
		}
	}
	return out
}

func (fx *runFx) left(id string) []string {
	left, _ := fx.rel.List(context.Background(), runs.Prefix(release.OpsAccount, "shop", id))
	return left
}

// finishedRun is a picked run with one chunk and a result, finished on the record.
func (fx *runFx) finishedRun(t *testing.T) string {
	t.Helper()
	id := fx.startRun(t)
	fx.transition(t, provision.RunPick(id, "box-1", "boot-1", time.Now().Add(time.Hour)))
	fx.chunk(t, id, 1, runs.StreamOut, "a")
	if err := runs.WriteResult(context.Background(), fx.rel, release.OpsAccount, "shop", id, runs.Result{Exit: 0, LastSeq: 1}); err != nil {
		t.Fatal(err)
	}
	fx.transition(t, provision.RunFinish(id))
	return id
}

func TestAPIRunAckThatCannotDeleteKeepsTheMarkerSoTheRowIsWrittenOnce(t *testing.T) {
	fx := runFixture(t)
	id := fx.finishedRun(t)
	failing := true
	fx.srv.cfg.Releases = &faultStore{Store: fx.rel, delete: func(key string) error {
		if failing && key == runs.ResultKey(release.OpsAccount, "shop", id) {
			return errors.New("delete refused")
		}
		return nil
	}}
	if status, body := fx.post(t, id, "ack", `{"last_seq":1}`); status != http.StatusOK {
		t.Fatalf("ack: %d %s", status, body)
	}
	left := fx.left(id)
	for _, want := range []string{runs.ResultKey(release.OpsAccount, "shop", id), runs.RequestKey(release.OpsAccount, "shop", id), runs.AuditedKey(release.OpsAccount, "shop", id)} {
		if !slices.Contains(left, want) {
			t.Fatalf("after a failed delete %q is gone; left %q", want, left)
		}
	}
	// A follower and the retried ack both find the result; neither writes a second row.
	failing = false
	if p := fx.page(t, id, 1); p.Result == nil {
		t.Fatalf("page %+v", p)
	}
	if status, body := fx.post(t, id, "ack", `{"last_seq":1}`); status != http.StatusOK {
		t.Fatalf("retried ack: %d %s", status, body)
	}
	if left := fx.left(id); len(left) != 0 {
		t.Fatalf("retried ack left %q", left)
	}
	if rows := fx.runEnds(t, id); len(rows) != 1 {
		t.Fatalf("run-end rows %q, want 1", rows)
	}
}

func TestAPIRunEndIsNotWrittenForARunAnAckHasDeleted(t *testing.T) {
	fx := runFixture(t)
	id := fx.finishedRun(t)
	if status, body := fx.post(t, id, "ack", `{"last_seq":1}`); status != http.StatusOK {
		t.Fatalf("ack: %d %s", status, body)
	}
	// A follower that read the result before the ack's delete reaches the
	// audit after it.
	fx.srv.auditRunEnd(context.Background(), &Account{ID: OpsAccountID, Sqid: release.OpsAccount}, "shop", runTestHost, id, "result", &runs.Result{LastSeq: 1})
	if left := fx.left(id); len(left) != 0 {
		t.Fatalf("a late audit left %q", left)
	}
	if rows := fx.runEnds(t, id); len(rows) != 1 {
		t.Fatalf("run-end rows %q, want 1", rows)
	}
}

func TestAPIRunCancelAndExpireBeforePickupLeaveAResult(t *testing.T) {
	fx := runFixture(t)
	cancelled := fx.startRun(t)
	if status, body := fx.post(t, cancelled, "cancel", ""); status != http.StatusOK {
		t.Fatalf("cancel: %d %s", status, body)
	}
	fx.transition(t, provision.RunClear(cancelled))
	expiredID := fx.startRun(t)
	if status, body := fx.post(t, expiredID, "expire", ""); status != http.StatusOK {
		t.Fatalf("expire: %d %s", status, body)
	}
	fx.transition(t, provision.RunClear(expiredID))
	// The record no longer names either run; a follower still learns how it ended.
	if p := fx.page(t, cancelled, 0); p.State != "" || p.Result == nil || !p.Result.Cancelled || p.More || len(p.Chunks) != 0 {
		t.Errorf("cancelled run page %+v", p)
	}
	if p := fx.page(t, expiredID, 0); p.State != "" || p.Result == nil || p.Result.Refused != runs.RefusedExpired {
		t.Errorf("expired run page %+v", p)
	}
	if status, body := fx.post(t, cancelled, "ack", `{"last_seq":0}`); status != http.StatusOK {
		t.Errorf("ack of a cancelled run: %d %s", status, body)
	}
	if left := fx.left(cancelled); len(left) != 0 {
		t.Errorf("left %q", left)
	}
}

func TestAPIRunExpireWritesTheRunEndRow(t *testing.T) {
	fx := runFixture(t)
	id := fx.startRun(t)
	if status, body := fx.post(t, id, "expire", ""); status != http.StatusOK {
		t.Fatalf("expire: %d %s", status, body)
	}
	rows := fx.runEnds(t, id)
	if len(rows) != 1 || !strings.Contains(rows[0], `"outcome":"expired"`) {
		t.Fatalf("run-end rows %q", rows)
	}
}

// A cancel or expire whose CAS landed but whose response was lost is retried
// and finds its own terminal state; the result and the row are still written.
func TestAPIRunRetriedCancelAndExpireStillEndTheRun(t *testing.T) {
	fx := runFixture(t)
	for _, c := range []struct {
		verb, outcome string
		mut           func(string) func(*provision.Record) error
	}{
		{"cancel", "cancelled", provision.RunCancelTransition},
		{"expire", "expired", provision.RunExpire},
	} {
		id := fx.startRun(t)
		fx.transition(t, c.mut(id))
		if status, body := fx.post(t, id, c.verb, ""); status != http.StatusConflict || errorOf(t, body) != "run "+id+" has already ended" {
			t.Fatalf("%s retry: %d %s", c.verb, status, body)
		}
		if res, err := runs.ReadResult(context.Background(), fx.rel, release.OpsAccount, "shop", id); err != nil || res == nil {
			t.Errorf("%s retry left no result: %+v, %v", c.verb, res, err)
		}
		if rows := fx.runEnds(t, id); len(rows) != 1 || !strings.Contains(rows[0], `"outcome":"`+c.outcome+`"`) {
			t.Errorf("%s retry rows %q", c.verb, rows)
		}
		fx.post(t, id, c.verb, "")
		if rows := fx.runEnds(t, id); len(rows) != 1 {
			t.Errorf("%s retried twice: rows %q", c.verb, rows)
		}
		fx.transition(t, provision.RunClear(id))
	}
}

func TestAPIRunAckKeepsTheObjectsWhenTheClearIsNotSettled(t *testing.T) {
	recordKey := provision.RecordKey(release.OpsAccount, "shop", runTestHost)
	for _, c := range []struct {
		name       string
		setup      func(*faultStore, *bool)
		wantStatus int
		wantKept   bool
	}{
		{"clear refused", func(f *faultStore, _ *bool) {
			f.put = func(key string) (bool, error) {
				if key == recordKey {
					return false, errors.New("put refused")
				}
				return false, nil
			}
		}, http.StatusBadGateway, true},
		{"clear landed, response lost", func(f *faultStore, _ *bool) {
			f.put = func(key string) (bool, error) {
				if key == recordKey {
					return true, errors.New("response lost")
				}
				return false, nil
			}
		}, http.StatusOK, false},
		{"clear landed, re-read fails", func(f *faultStore, afterPut *bool) {
			f.put = func(key string) (bool, error) {
				if key == recordKey {
					*afterPut = true
				}
				return false, nil
			}
			f.get = func(key string) error {
				if key == recordKey && *afterPut {
					return errors.New("get refused")
				}
				return nil
			}
		}, http.StatusBadGateway, true},
	} {
		t.Run(c.name, func(t *testing.T) {
			fx := runFixture(t)
			id := fx.finishedRun(t)
			f := &faultStore{Store: fx.rel}
			var afterPut bool
			c.setup(f, &afterPut)
			fx.srv.cfg.Releases = f
			status, body := fx.post(t, id, "ack", `{"last_seq":1}`)
			if status != c.wantStatus {
				t.Fatalf("ack: %d %s, want %d", status, body, c.wantStatus)
			}
			if kept := len(fx.left(id)) != 0; kept != c.wantKept {
				t.Fatalf("objects kept = %v, want %v (left %q)", kept, c.wantKept, fx.left(id))
			}
		})
	}
}