package host import ( "context" "errors" "io" "os" "path/filepath" "sync" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/layout" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/seeds" ) // seedRecordFixture writes an account, the shop app and one record of // shop-1.example.com carrying a seed in state, and returns the record. func seedRecordFixture(t *testing.T, deploy release.Store, kind, state string) provision.Record { t.Helper() ctx := context.Background() sqid := domains.Sqid(2) manifest := []byte(`{"id":2,"sqid":"` + sqid + `","name":"Shop","created_at":"2026-08-02T09:00:00Z"}`) if _, err := deploy.Put(ctx, "accounts/"+sqid+"/account.json", manifest, ""); err != nil { t.Fatal(err) } if err := provision.PutManifest(ctx, deploy, sqid, "shop", provision.Manifest{ Schema: 1, App: "shop", Domain: "example.com", Seed: []string{"seed"}}); err != nil { t.Fatal(err) } id, err := seeds.NewID() if err != nil { t.Fatal(err) } rec := provision.Record{Schema: 1, Host: "shop-1.example.com", Slug: "shop-1", App: "shop", Account: sqid, Channel: "stable", CreatedAt: time.Now().UTC(), Seed: &provision.SeedSource{ID: id, Kind: kind, State: state, SHA256: "a", Bytes: 2, By: "t", At: time.Now().UTC()}} if err := provision.CreateRecord(ctx, deploy, rec); err != nil { t.Fatal(err) } return rec } func putSnapshot(t *testing.T, data release.Store, host, id string) string { t.Helper() k := seeds.SnapshotKey(host, id) if _, err := data.Put(context.Background(), k, []byte("db"), ""); err != nil { t.Fatal(err) } return k } func snapshotPresent(t *testing.T, data release.Store, key string) bool { t.Helper() _, _, err := data.Get(context.Background(), key) if errors.Is(err, os.ErrNotExist) { return false } if err != nil { t.Fatal(err) } return true } func seedStateOf(t *testing.T, deploy release.Store, rec provision.Record) string { t.Helper() got, err := provision.GetRecord(context.Background(), deploy, rec.Account, rec.App, rec.Host) if err != nil || got.Seed == nil { t.Fatalf("record: %+v, %v", got, err) } return got.Seed.State } func TestSeedPlacedHookCASesThenDeletesOnlyThisSeed(t *testing.T) { ctx := context.Background() deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateReady) id := rec.Seed.ID other, _ := seeds.NewID() mine, theirs := putSnapshot(t, data, rec.Host, id), putSnapshot(t, data, rec.Host, other) hook := seedPlacedHook(deploy, data) if err := hook(ctx, rec.Account, rec.App, rec.Host, id); err != nil { t.Fatal(err) } if st := seedStateOf(t, deploy, rec); st != seeds.StatePlaced { t.Fatalf("state = %s, want placed", st) } if snapshotPresent(t, data, mine) { t.Fatal("this seed's snapshot survived its placement") } if !snapshotPresent(t, data, theirs) { t.Fatal("another seed's snapshot was deleted") } // Another id than the record's: refused, nothing deleted. if err := hook(ctx, rec.Account, rec.App, rec.Host, other); !errors.Is(err, provision.ErrSeedConflict) { t.Fatalf("placing another seed's id = %v, want ErrSeedConflict", err) } if !snapshotPresent(t, data, theirs) { t.Fatal("a refused placement deleted a snapshot") } // An id that is not an id: refused before the record or the bucket. if err := hook(ctx, rec.Account, rec.App, rec.Host, "../x"); err == nil { t.Fatal("a malformed id was accepted") } } // The hook runs twice on one finishing wake (Restore's finishPlacement and // doWake's step 5b), and two boxes can finish one placement. Each call is a // success, and a call that finds the record already placed with this id // still deletes the snapshot: the first call's delete may be the one that // failed. func TestSeedPlacedHookIsIdempotent(t *testing.T) { ctx := context.Background() deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateReady) key := putSnapshot(t, data, rec.Host, rec.Seed.ID) boxA, boxB := seedPlacedHook(deploy, data), seedPlacedHook(deploy, data) if err := boxA(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil { t.Fatal(err) } // The first call's delete lost: the snapshot is back, and the record // already reads placed. The second call deletes it. putSnapshot(t, data, rec.Host, rec.Seed.ID) if err := boxA(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil { t.Fatalf("second call on one box: %v", err) } if snapshotPresent(t, data, key) { t.Fatal("an already-placed record's call left the snapshot") } if err := boxB(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil { t.Fatalf("another box finishing the same placement: %v", err) } if st := seedStateOf(t, deploy, rec); st != seeds.StatePlaced { t.Fatalf("state = %s, want placed", st) } // Two boxes at once, from ready: both succeed, one CAS wins, the // snapshot goes. deploy2, data2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) rec2 := seedRecordFixture(t, deploy2, seeds.KindReplica, seeds.StateReady) key2 := putSnapshot(t, data2, rec2.Host, rec2.Seed.ID) var wg sync.WaitGroup errs := make([]error, 2) for i := range errs { wg.Add(1) go func() { defer wg.Done() errs[i] = seedPlacedHook(deploy2, data2)(ctx, rec2.Account, rec2.App, rec2.Host, rec2.Seed.ID) }() } wg.Wait() for i, err := range errs { if err != nil { t.Fatalf("box %d: %v", i, err) } } if st := seedStateOf(t, deploy2, rec2); st != seeds.StatePlaced { t.Fatalf("state = %s, want placed", st) } if snapshotPresent(t, data2, key2) { t.Fatal("the snapshot survived two finishing boxes") } } // A refused CAS for any reason but "already placed with this id" deletes // nothing: the snapshot may be the only copy of the seed. func TestSeedPlacedHookRefusedDeletesNothing(t *testing.T) { ctx := context.Background() for name, tc := range map[string]struct{ kind, state string }{ "aborted": {seeds.KindReplica, seeds.StateAborted}, "rolled-back": {seeds.KindReplica, seeds.StateRolledBack}, "reserving": {seeds.KindReplica, seeds.StateReserving}, "import, ready": {seeds.KindImport, seeds.StateReady}, } { t.Run(name, func(t *testing.T) { deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) rec := seedRecordFixture(t, deploy, tc.kind, tc.state) key := putSnapshot(t, data, rec.Host, rec.Seed.ID) if err := seedPlacedHook(deploy, data)(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); !errors.Is(err, provision.ErrSeedConflict) { t.Fatalf("err = %v, want ErrSeedConflict", err) } if !snapshotPresent(t, data, key) { t.Fatal("a refused placement deleted the snapshot") } if st := seedStateOf(t, deploy, rec); st != tc.state { t.Fatalf("state = %s, want %s unchanged", st, tc.state) } }) } // No record at all: an error, nothing deleted. deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) id, _ := seeds.NewID() key := putSnapshot(t, data, "shop-9.example.com", id) if err := seedPlacedHook(deploy, data)(ctx, domains.Sqid(2), "shop", "shop-9.example.com", id); err == nil { t.Fatal("placing a seed with no record succeeded") } if !snapshotPresent(t, data, key) { t.Fatal("a placement with no record deleted the snapshot") } // A box that has not wired the stores records nothing. if err := seedPlacedHook(nil, data)(ctx, domains.Sqid(2), "shop", "shop-9.example.com", id); err == nil { t.Fatal("an unwired hook reported the seed placed") } } func TestStoreSeedObjectsOpensTheSnapshotKey(t *testing.T) { ctx := context.Background() data := release.NewDirStore(t.TempDir()) id, _ := seeds.NewID() if _, err := data.Put(ctx, seeds.SnapshotKey("shop-1.example.com", id), []byte("db-bytes"), ""); err != nil { t.Fatal(err) } rc, err := storeSeedObjects{data}.Open(ctx, "shop-1.example.com", id) if err != nil { t.Fatal(err) } b, _ := io.ReadAll(rc) rc.Close() if string(b) != "db-bytes" { t.Fatalf("read %q", b) } if _, err := (storeSeedObjects{data}).Open(ctx, "shop-1.example.com", "../x"); err == nil { t.Fatal("a malformed id opened something") } } func resetArmedSeedHooks(t *testing.T) { t.Helper() old := armedSeedHooks.Load() armedSeedHooks.Store(nil) t.Cleanup(func() { armedSeedHooks.Store(old) }) } // wireSeedPlacement sets the claim flag whatever happens to the stores, and // arms the reader, the recorder and the provision pass only together, with // the data bucket always set on the pass's hooks. func TestWireSeedPlacement(t *testing.T) { data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) ok := func(s release.Store) func() (release.Store, error) { return func() (release.Store, error) { return s, nil } } fail := func() (release.Store, error) { return nil, errors.New("no credentials") } t.Run("armed", func(t *testing.T) { resetArmedSeedHooks(t) eb := &activator.ExecBackend{} wireSeedPlacement(eb, true, ok(data), ok(deploy), seedBox{}) if !eb.ClaimArmed || eb.Seeds == nil || eb.OnSeedPlaced == nil { t.Fatalf("ClaimArmed %v Seeds %v OnSeedPlaced set %v", eb.ClaimArmed, eb.Seeds, eb.OnSeedPlaced != nil) } h := armedSeedHooks.Load() if h == nil || h.Data != release.Store(data) { t.Fatalf("armed hooks = %+v, want the data bucket", h) } }) t.Run("claims off", func(t *testing.T) { resetArmedSeedHooks(t) eb := &activator.ExecBackend{ClaimArmed: true} wireSeedPlacement(eb, false, ok(data), ok(deploy), seedBox{}) if eb.ClaimArmed { t.Fatal("ClaimArmed set with claim control off") } }) for name, opens := range map[string][2]func() (release.Store, error){ "no data bucket": {fail, ok(deploy)}, "no record store": {ok(data), fail}, } { t.Run(name, func(t *testing.T) { resetArmedSeedHooks(t) eb := &activator.ExecBackend{} wireSeedPlacement(eb, true, opens[0], opens[1], seedBox{}) if !eb.ClaimArmed { t.Fatal("ClaimArmed not set when a store failed") } if eb.Seeds != nil || eb.OnSeedPlaced != nil || armedSeedHooks.Load() != nil { t.Fatal("seed placement half-armed") } }) } t.Run("hooks without a data bucket are not armed", func(t *testing.T) { resetArmedSeedHooks(t) armSeedHooks(&provision.SeedHooks{}) if armedSeedHooks.Load() != nil { t.Fatal("armed SeedHooks with a nil Data") } }) } // reconcileSerialized hands the provision pass the armed hooks: with them // an aborted seed is finished (its snapshot deleted, the record rolled // back); without them it is left alone. func TestReconcileSerializedCarriesTheArmedSeedHooks(t *testing.T) { t.Parallel() ctx := context.Background() deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateAborted) key := putSnapshot(t, data, rec.Host, rec.Seed.ID) resetArmedSeedHooks(t) if err := reconcileSerialized(ctx, deploy, restartTestRegistry(t), provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil { t.Fatal(err) } if st := seedStateOf(t, deploy, rec); st != seeds.StateAborted || !snapshotPresent(t, data, key) { t.Fatalf("unarmed pass acted on the seed: state %s", st) } armSeedHooks(&provision.SeedHooks{Data: data}) if err := reconcileSerialized(ctx, deploy, restartTestRegistry(t), provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil { t.Fatal(err) } if st := seedStateOf(t, deploy, rec); st != seeds.StateRolledBack { t.Fatalf("state = %s, want rolled-back", st) } if snapshotPresent(t, data, key) { t.Fatal("the aborted seed's snapshot survived an armed pass") } } // fakeFence is the activator's fence as the import rollback meets it: one // fence per host, an exclusive install that refuses any fence already // held, and a lift that takes only the generation it names. // FenceAndStopIfUnfenced answers err once it would install; nil and // ErrStopUnproven leave the fence installed, as the activator does. type fakeFence struct { gens map[string]uint64 err error quiesced bool uncertain bool calls int deadline time.Duration // the time left on the last install's context lifted []uint64 } func (f *fakeFence) FenceAndStopIfUnfenced(ctx context.Context, host string, gen uint64) (*activator.StopEvidence, error) { f.calls++ if d, ok := ctx.Deadline(); ok { f.deadline = time.Until(d) } if f.gens[host] != 0 { return nil, activator.ErrFenceHeld } if f.err == nil || errors.Is(f.err, activator.ErrStopUnproven) { f.gens[host] = gen } return nil, f.err } func (f *fakeFence) LiftFence(host string, gen uint64) bool { f.lifted = append(f.lifted, gen) if gen == 0 || f.gens[host] != gen { return false } delete(f.gens, host) return true } func (f *fakeFence) FenceGeneration(host string) uint64 { return f.gens[host] } func (f *fakeFence) Quiesced(string) bool { return f.quiesced } func (f *fakeFence) StopUncertain(string) bool { return f.uncertain } func storeOpener(s release.Store) func() (release.Store, error) { return func() (release.Store, error) { return s, nil } } // wiredSeedHooks arms seed placement as the agent does on a box without // claim control, with f as the activator and reg's generation source, and // returns the provision pass's hooks. func wiredSeedHooks(t *testing.T, f layout.Fence, reg *registry.Registry, data, deploy release.Store) *provision.SeedHooks { t.Helper() resetArmedSeedHooks(t) eb := &activator.ExecBackend{NestedOK: func() error { return nil }} wireSeedPlacement(eb, false, storeOpener(data), storeOpener(deploy), seedBox{ Fence: f, Prefix: "instances", Swept: func() bool { return true }, NextFenceGen: fenceGenerations(reg)}) h := armedSeedHooks.Load() if h == nil { t.Fatal("no seed hooks armed") } return h } // The rollback's step 2 (spec ยง5.6) by the layout pass's rule, and the // generation it is known by surviving an agent restart. func TestRollbackFenceAfterRestart(t *testing.T) { ctx := context.Background() const host = "shop-1.example.com" swept := true f := &fakeFence{gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true} fence := fenceForRollback(f, func() bool { return swept }) if ok, _ := fence(ctx, host, 7); !ok { t.Fatal("cold, swept, quiesced: want proceed") } if f.deadline <= 0 || f.deadline > 30*time.Second { t.Fatalf("the install ran with %v left, want a deadline of at most 30s", f.deadline) } // Ours already (an earlier pass installed it): proceed without asking. f.err, f.calls = errors.New("must not be called"), 0 if ok, _ := fence(ctx, host, 7); !ok || f.calls != 0 { t.Fatalf("our own fence: want proceed without an install (calls %d)", f.calls) } // Not swept at start: lift and defer. f = &fakeFence{gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true} swept = false fence = fenceForRollback(f, func() bool { return swept }) if ok, _ := fence(ctx, host, 8); ok || len(f.lifted) != 1 || f.lifted[0] != 8 || f.gens[host] != 0 { t.Fatalf("not swept: proceed %v lifted %v fences %v", ok, f.lifted, f.gens) } // Swept, but not quiesced, or its stop uncertain: lift and defer. for name, f := range map[string]*fakeFence{ "not quiesced": {gens: map[string]uint64{}, err: activator.ErrStopUnproven}, "stop uncertain": {gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true, uncertain: true}, "another error": {gens: map[string]uint64{}, err: activator.ErrFenceNotInstalled, quiesced: true}, } { fence := fenceForRollback(f, func() bool { return true }) if ok, err := fence(ctx, host, 9); ok || err == nil || f.gens[host] != 0 { t.Fatalf("%s: proceed %v, %v, fences %v", name, ok, err, f.gens) } } // Another operation's fence: defer and touch nothing. f = &fakeFence{gens: map[string]uint64{host: 3}} fence = fenceForRollback(f, func() bool { return true }) if ok, err := fence(ctx, host, 9); ok || !errors.Is(err, activator.ErrFenceHeld) || len(f.lifted) != 0 || f.gens[host] != 3 { t.Fatalf("held: proceed %v, %v, lifted %v", ok, err, f.lifted) } // Generation 0 is no fence's: on a host with none, FenceGeneration's 0 // would otherwise read as "ours already" and proceed unfenced. f = &fakeFence{gens: map[string]uint64{}} fence = fenceForRollback(f, func() bool { return true }) if ok, err := fence(ctx, host, 0); ok || err == nil || f.calls != 0 { t.Fatalf("generation 0: proceed %v, %v, installs %d", ok, err, f.calls) } // After an agent restart. The journal kept the generation the rollback // was given before it; the activator's fences died with the process, // and another operation, the layout pass, has fenced the host since. // The resumed rollback must neither take that fence for its own nor // lift it: provision lifts at the journal's generation on every path // that does not proceed, the held-stale-journal lift // (seedrollback.go, rollBackRoute's first check) among them. path := filepath.Join(t.TempDir(), "registry.db") reg, err := registry.Open(path) if err != nil { t.Fatal(err) } // The clock before the restart ran an hour ahead of the clock after it // (a box that booted before NTP synced): the generation the journal // stores is ahead of every wall-clock reading the restarted agent makes. if _, err := reg.NextFenceGeneration(time.Now().Add(time.Hour)); err != nil { t.Fatal(err) } data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) before := wiredSeedHooks(t, &fakeFence{gens: map[string]uint64{}}, reg, data, deploy) stored, err := before.NextFenceGen() if err != nil || stored == 0 { t.Fatalf("generation before the restart: %d, %v", stored, err) } if err := reg.Close(); err != nil { t.Fatal(err) } reg, err = registry.Open(path) if err != nil { t.Fatal(err) } defer reg.Close() other := newLayoutPass(reg, nil, t.TempDir(), nil, true).Generation() if other <= stored { t.Fatalf("the layout pass after the restart drew generation %d, not above the %d the journal holds", other, stored) } f = &fakeFence{gens: map[string]uint64{host: other}, err: activator.ErrStopUnproven, quiesced: true} after := wiredSeedHooks(t, f, reg, data, deploy) if ok, err := after.FenceForRollback(ctx, host, stored); ok || !errors.Is(err, activator.ErrFenceHeld) { t.Fatalf("resumed under another operation's fence: proceed %v, %v", ok, err) } after.LiftFence(host, stored) if f.gens[host] != other { t.Fatalf("the resumed rollback lifted another operation's fence: fences %v", f.gens) } if next, err := after.NextFenceGen(); err != nil || next <= other { t.Fatalf("a generation minted after the restart: %d (%v); before %d, the layout pass's %d", next, err, stored, other) } // Once the other operation lets go, the rollback fences at its own // stored generation and proceeds, as on a cold host. f.LiftFence(host, other) if ok, err := after.FenceForRollback(ctx, host, stored); !ok || f.gens[host] != stored { t.Fatalf("after the other fence lifted: proceed %v, %v, fences %v", ok, err, f.gens) } } // The rollback's seven hooks are all or none (provision skips a rollback // missing any), so production wiring must set every one; with claim control // on, seeds are refused until #250 and none of the swap's or the // rollback's hooks is armed. func TestWireSeedPlacementArmsTheSwapAndTheRollback(t *testing.T) { ctx := context.Background() data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) f := &fakeFence{gens: map[string]uint64{}} reg := restartTestRegistry(t) nestedErr := errors.New("probe") box := seedBox{Fence: f, Prefix: "instances", Swept: func() bool { return true }, NextFenceGen: fenceGenerations(reg)} rollbackHooks := func(h *provision.SeedHooks) map[string]bool { return map[string]bool{ "FenceForRollback": h.FenceForRollback != nil, "LiftFence": h.LiftFence != nil, "RemoveLocal": h.RemoveLocal != nil, "DeleteReplica": h.DeleteReplica != nil, "NextFenceGen": h.NextFenceGen != nil, "LocalSeedID": h.LocalSeedID != nil, "RetireKey": h.RetireKey != nil, } } t.Run("armed", func(t *testing.T) { resetArmedSeedHooks(t) eb := &activator.ExecBackend{NestedOK: func() error { return nestedErr }} wireSeedPlacement(eb, false, storeOpener(data), storeOpener(deploy), box) h := armedSeedHooks.Load() if h == nil || h.Data != release.Store(data) { t.Fatalf("armed hooks = %+v", h) } for name, set := range rollbackHooks(h) { if !set { t.Errorf("the rollback hook %s is not wired: no rollback runs", name) } } if h.NestedOK == nil || h.NestedOK() != nestedErr { t.Error("NestedOK is not the exec backend's probe") } if h.LegacyOwners == nil { t.Fatal("LegacyOwners is not wired") } if h.EnvBefore == nil { t.Fatal("EnvBefore is not wired: no import swaps") } body := `{"v":1,"names":{"old.example.com":{"account":"bbw","app":"shop"}}}` if _, err := deploy.Put(ctx, seeds.LegacyOwnersKey, []byte(body), ""); err != nil { t.Fatal(err) } if o, err := h.LegacyOwners(ctx); err != nil || o == nil || o.Names["old.example.com"].App != "shop" { t.Errorf("LegacyOwners read %+v, %v from the record store", o, err) } if g, err := h.NextFenceGen(); err != nil || g == 0 { t.Errorf("NextFenceGen = %d, %v", g, err) } f.gens["shop-1.example.com"] = 41 h.LiftFence("shop-1.example.com", 41) if f.gens["shop-1.example.com"] != 0 { t.Error("LiftFence did not reach the activator") } }) t.Run("claim control on", func(t *testing.T) { resetArmedSeedHooks(t) eb := &activator.ExecBackend{NestedOK: func() error { return nil }} wireSeedPlacement(eb, true, storeOpener(data), storeOpener(deploy), box) h := armedSeedHooks.Load() if h == nil || h.Data != release.Store(data) { t.Fatalf("armed hooks = %+v, want the data bucket", h) } for name, set := range rollbackHooks(h) { if set { t.Errorf("the rollback hook %s is armed with claim control on", name) } } if h.NestedOK != nil || h.LegacyOwners != nil || h.EnvBefore != nil { t.Error("the swap's hooks are armed with claim control on") } }) t.Run("no activator fence", func(t *testing.T) { resetArmedSeedHooks(t) wireSeedPlacement(&activator.ExecBackend{}, false, storeOpener(data), storeOpener(deploy), seedBox{Prefix: "instances"}) h := armedSeedHooks.Load() if h == nil { t.Fatal("no hooks") } for name, set := range rollbackHooks(h) { if set { t.Errorf("the rollback hook %s is armed without a fence: half the rollback", name) } } }) } // The rollback's step 6 deletes the host's replica and nothing beside it: // not a host whose name only begins with this one, not another host, not // the seeds/ prefix. func TestDeleteReplicaDeletesOnlyTheHostsReplica(t *testing.T) { ctx := context.Background() data := release.NewDirStore(t.TempDir()) const host = "shop-1.example.com" put := func(k string) { if _, err := data.Put(ctx, k, []byte("x"), ""); err != nil { t.Fatal(err) } } mine := []string{"instances/" + host + "/generations/a/wal/1.ltx", "instances/" + host + "/lock.json"} theirs := []string{"instances/" + host + ".au/lock.json", "instances/other.example.com/lock.json", "seeds/" + host + "/sd_aaaaaaaaaaaaaaaaaaaaaaaaaa/snapshot.db"} for _, k := range append(append([]string{}, mine...), theirs...) { put(k) } del := deleteReplica(data, "instances") for _, bad := range []string{"", "..", "a/b", "."} { if err := del(ctx, bad); err == nil { t.Fatalf("host %q accepted", bad) } } if err := del(ctx, host); err != nil { t.Fatal(err) } for _, k := range mine { if snapshotPresent(t, data, k) { t.Errorf("%s survived", k) } } for _, k := range theirs { if !snapshotPresent(t, data, k) { t.Errorf("%s was deleted", k) } } if err := del(ctx, host); err != nil { t.Fatalf("nothing there: %v", err) } }