package host import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "reflect" "slices" "sort" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/layout" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The offline operator move's box side (Task 3 of // docs/superpowers/plans/2026-10-20-offline-operator-move.md), tested // through the functions the agent calls: newMovesWorker, armOffline, // reconcileSerializedWith, the production edges-file writer and movesPass, // over one DirStore, real registries and real activator services (the // package's fake wake backend, which only the fence needs). const ( offHost = "shop.example.com" offID = "mv_zyxwvutsrqponmlkjihgfedcba" offID2 = "mv_abcdefghijklmnopqrstuvwxy2" ) type offWorld struct { t *testing.T s release.Store acct string mu sync.Mutex now time.Time } func (wd *offWorld) clock() time.Time { wd.mu.Lock() defer wd.mu.Unlock() return wd.now } func (wd *offWorld) advance(d time.Duration) { wd.mu.Lock() wd.now = wd.now.Add(d) wd.mu.Unlock() } // newOffWorld is a deployment with one account, one app and one instance // record for offHost at the home placement (""). func newOffWorld(t *testing.T) *offWorld { t.Helper() t.Setenv("CARLOS_HOST_CLAIM", "") // the appconfig and provision guards read the process env ctx := context.Background() wd := &offWorld{t: t, s: release.NewDirStore(t.TempDir()), acct: domains.Sqid(2), now: time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)} manifest, err := json.Marshal(map[string]any{"id": 2, "sqid": wd.acct, "name": "fixture", "created_at": wd.now}) if err != nil { t.Fatal(err) } if _, err := wd.s.Put(ctx, "accounts/"+wd.acct+"/account.json", manifest, ""); err != nil { t.Fatal(err) } if err := provision.PutManifest(ctx, wd.s, wd.acct, "app", provision.Manifest{Schema: 1, App: "app", Domain: "example.com"}); err != nil { t.Fatal(err) } wd.create(offHost) return wd } func (wd *offWorld) create(host string) { wd.t.Helper() rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", Region: "", Channel: "stable", CreatedAt: wd.now} if err := provision.CreateRecord(context.Background(), wd.s, rec); err != nil { wd.t.Fatal(err) } } // move applies one offline transition to offHost's record. func (wd *offWorld) move(mut func(*provision.Record) error) provision.Record { wd.t.Helper() return wd.moveHost(offHost, mut) } func (wd *offWorld) moveHost(host string, mut func(*provision.Record) error) provision.Record { wd.t.Helper() rec, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", host, mut) if err != nil { wd.t.Fatal(err) } return *rec } func (wd *offWorld) record() provision.Record { wd.t.Helper() rec, err := provision.GetRecord(context.Background(), wd.s, wd.acct, "app", offHost) if err != nil { wd.t.Fatal(err) } return *rec } // The legal path to a state, from the record at rest. func (wd *offWorld) start() { wd.move(provision.OfflineStart(offID, "", "uk", "op@example.com", wd.clock())) } func (wd *offWorld) seal() { wd.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"})) } func (wd *offWorld) sealed() { wd.move(provision.OfflineSealedBy(offID, "ie-a", provision.MoveSource{})) } func (wd *offWorld) flip() { wd.move(provision.OfflineFlip(offID)) } func (wd *offWorld) landed() { wd.move(provision.OfflineLandedBy(offID, "uk")) } func (wd *offWorld) open() { wd.move(provision.OfflineOpenTransition(offID, wd.clock())) } func (wd *offWorld) finishing() { wd.move(provision.OfflineFinishingTransition(offID)) } func (wd *offWorld) abort() { wd.move(provision.OfflineAbort(offID)) } // report reads one box's report for offID, or fails. func (wd *offWorld) report(role, box string) moves.OfflineReport { wd.t.Helper() r, ok := wd.reportIf(role, box) if !ok { wd.t.Fatalf("no %s report from %s", role, box) } return r } func (wd *offWorld) reportIf(role, box string) (moves.OfflineReport, bool) { wd.t.Helper() body, _, err := wd.s.Get(context.Background(), moves.OfflineReportKey(offHost, offID, role, box)) if errors.Is(err, os.ErrNotExist) { return moves.OfflineReport{}, false } if err != nil { wd.t.Fatal(err) } var r moves.OfflineReport if err := json.Unmarshal(body, &r); err != nil { wd.t.Fatal(err) } if err := moves.ValidateOfflineReport(r); err != nil { wd.t.Fatal(err) } return r, true } // offBox is one box: its own data directory, registry, activator and moves // worker, armed as armMovesPass arms it (armOffline). type offBox struct { wd *offWorld name, region string dd, opt, cg string env map[string]string reg *registry.Registry svc *activator.Service w *movesWorker nested bool last []byte } func (wd *offWorld) box(name, region string, env map[string]string) *offBox { wd.t.Helper() b := &offBox{wd: wd, name: name, region: region, dd: wd.t.TempDir(), opt: wd.t.TempDir(), cg: wd.t.TempDir(), env: env} reg, err := registry.Open(filepath.Join(b.dd, "registry.db")) if err != nil { wd.t.Fatal(err) } wd.t.Cleanup(func() { reg.Close() }) b.reg = reg b.restart() return b } func (b *offBox) getenv(k string) string { if v, ok := b.env[k]; ok { return v } if k == "CARLOS_HIBERNATE_BUCKET" { return "data" } return "" } // restart is an agent restart: a fresh activator (no fences) and a fresh // worker over the same registry and data directory. func (b *offBox) restart() { b.svc = activator.New(activator.Config{Bucket: "data", Idle: time.Hour}, &fakeWakeBackend{}, b.reg) cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock} armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv) // The stage-requested marker is root's business, and its throttled // timer would outlive the test's directories. cfg.Offline = newOfflineView(cfg, nil) cfg.ReleaseHold = releaseHeld(b.reg, nil) cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") } b.w = newMovesWorker(cfg) } func (b *offBox) reconcile() { b.wd.t.Helper() cfg := provision.ReconcileConfig{Region: b.region, DataDir: b.dd} if b.nested { cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil } } else { cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil } } if err := reconcileSerializedWith(context.Background(), b.wd.s, b.reg, cfg, b.w); err != nil { b.wd.t.Fatal(err) } } func (b *offBox) edges() { b.wd.t.Helper() if err := edgesFileWrite(context.Background(), b.wd.s, b.reg, edgesFile{Box: b.name, Placement: b.region}, "", &b.last, false); err != nil { b.wd.t.Fatal(err) } } func (b *offBox) pass() { b.wd.t.Helper() if err := movesPass(context.Background(), b.wd.s, b.w); err != nil { b.wd.t.Fatal(err) } } // tick is the agent's order on a serial bump: reconcile, the edges file, // the moves pass. func (b *offBox) tick() { b.wd.t.Helper() b.reconcile() b.edges() b.pass() } func (b *offBox) row(host string) *registry.Route { b.wd.t.Helper() rt, err := b.reg.Get(host) if err != nil { b.wd.t.Fatal(err) } return rt } // lists reports whether this box's published edges file lists host. func (b *offBox) lists(host string) bool { b.wd.t.Helper() body, _, err := b.wd.s.Get(context.Background(), edgesFileKey(b.name)) if err != nil { b.wd.t.Fatal(err) } var f edgesFile if err := json.Unmarshal(body, &f); err != nil { b.wd.t.Fatal(err) } _, ok := f.Hosts[host] return ok } func writeFile(t *testing.T, p, body string) { t.Helper() if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(body), 0o644); err != nil { t.Fatal(err) } } // --- the hold --------------------------------------------------------------- // offlineHold's table: a pure function of the record and the box's // placement. The home's "" and a named region; every state. func TestOfflineHoldIsAFunctionOfTheRecord(t *testing.T) { home := moveCfg{Region: ""} uk := moveCfg{Region: "uk"} fr := moveCfg{Region: "fr"} rec := func(region, state string) provision.Record { return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}} } cases := []struct { name string box moveCfg rec provision.Record wantStatus, wantOp string }{ {"source stopping", home, rec("", provision.OfflineStopping), "moving", offID}, {"source sealing", home, rec("", provision.OfflineSealing), "moving", offID}, {"source sealed", home, rec("", provision.OfflineSealed), "moving", offID}, {"source reverting", home, rec("", provision.OfflineReverting), "", ""}, {"source after the flip", home, rec("uk", provision.OfflineLanding), "", ""}, {"target before the flip", uk, rec("", provision.OfflineSealed), "", ""}, {"target landing", uk, rec("uk", provision.OfflineLanding), "landing", offID}, {"target landed", uk, rec("uk", provision.OfflineLanded), "landing", offID}, {"target landing, region not yet the target's", uk, rec("", provision.OfflineLanding), "", ""}, {"target open", uk, rec("uk", provision.OfflineOpen), "", ""}, {"target finishing", uk, rec("uk", provision.OfflineFinishing), "", ""}, {"target reverting", uk, rec("", provision.OfflineReverting), "", ""}, {"a third region", fr, rec("", provision.OfflineStopping), "", ""}, {"a third region after the flip", fr, rec("uk", provision.OfflineLanding), "", ""}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { status, op, ok := offlineHold(c.box)(c.rec) if status != c.wantStatus || op != c.wantOp || ok != (c.wantStatus != "") { t.Fatalf("hold = (%q, %q, %v), want (%q, %q)", status, op, ok, c.wantStatus, c.wantOp) } }) } // A record at rest, a legacy move and a Phase 1 move are never held. for _, m := range []*provision.MoveState{nil, {To: "uk", State: provision.MoveCutting}, {ID: offID, From: "", To: "uk", State: provision.OfflineStopping}} { if _, _, ok := offlineHold(home)(provision.Record{Host: offHost, Move: m}); ok { t.Fatalf("held: %+v", m) } } } // B1, the pinning test: a fresh target worker whose keep view has never // seen the move (built from a list with no offline record in it, as a // box's view is before the flip) mints the row held "landing" on its first // reconcile after the flip. And with the view never built at all, reconcile // through the offline hooks alone does the same. func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) { wd := newOffWorld(t) wd.start() wd.seal() wd.sealed() wd.flip() uk := wd.box("uk-box", "uk", nil) uk.w.cfg.Offline.refresh(nil, nil) // stale: no offline move in it uk.reconcile() rt := uk.row(offHost) if rt == nil || rt.Status != "landing" || rt.HoldOp != offID { t.Fatalf("after the flip the target minted %+v, want held landing by %s", rt, offID) } uk.edges() if uk.lists(offHost) { t.Fatal("the target's edges file lists a held landing row") } // The hooks with a view that was never built, straight into // provision.Reconcile: the hold needs no view. fresh := wd.box("uk-box-2", "uk", nil) if fresh.w.cfg.Offline.built() { t.Fatal("fixture: the view is built") } cfg := withOfflineHolds(provision.ReconcileConfig{Region: "uk", DataDir: fresh.dd}, offlineHold(fresh.w.cfg), fresh.w.cfg.Offline) if err := provision.Reconcile(context.Background(), wd.s, fresh.reg, cfg); err != nil { t.Fatal(err) } if rt := fresh.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID { t.Fatalf("an unbuilt view: minted %+v", rt) } } // --- the source boxes ------------------------------------------------------- // start holds the row on both source boxes (one minting it held, one // holding the row it has, keeping its other columns), both report held, // fenced and quiesced, and neither edges file lists the host. An agent // restart re-fences and re-reports. After the flip the rows stay, kept. func TestOfflineStartHoldsEverySourceRow(t *testing.T) { wd := newOffWorld(t) a := wd.box("ie-a", "", nil) b := wd.box("ie-b", "", nil) uk := wd.box("uk-box", "uk", nil) a.tick() // ie-a serves the host before the move; ie-b has no row yet if rt := a.row(offHost); rt == nil || rt.Status != "ready" || !a.lists(offHost) { t.Fatalf("fixture: ie-a does not serve the host: %+v", rt) } if err := a.reg.SetReplication(offHost, "daily"); err != nil { t.Fatal(err) } if err := a.reg.SetNextDue(offHost, 1234); err != nil { t.Fatal(err) } before := *a.row(offHost) wd.start() a.tick() b.tick() uk.tick() for _, bx := range []*offBox{a, b} { rt := bx.row(offHost) if rt == nil || rt.Status != "moving" || rt.HoldOp != offID { t.Fatalf("%s: row %+v, want held moving by the move", bx.name, rt) } if bx.lists(offHost) { t.Fatalf("%s's edges file lists a held host", bx.name) } if !bx.svc.Fenced(offHost) { t.Fatalf("%s: no wake fence", bx.name) } r := wd.report(moves.OfflineRoleSource, bx.name) if !r.Held || !r.Fenced || !r.Quiesced || !r.RowPresent || r.Procs != 0 || r.LayoutJournal || r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(wd.clock()) { t.Fatalf("%s: report %+v", bx.name, r) } } after := *a.row(offHost) after.Status, after.HoldOp = before.Status, before.HoldOp if !reflect.DeepEqual(after, before) { t.Fatalf("holding the existing row changed its other columns:\n before %+v\n after %+v", before, after) } if uk.row(offHost) != nil { t.Fatal("the target minted a row before the flip") } if _, ok := wd.reportIf(moves.OfflineRoleSource, "uk-box"); ok { t.Fatal("the target wrote a source report") } // An agent restart: the fence is gone with the process, and the next // pass installs it again and rewrites the report. a.restart() if a.svc.Fenced(offHost) { t.Fatal("fixture: a fresh activator is fenced") } wd.advance(time.Minute) a.tick() if !a.svc.Fenced(offHost) { t.Fatal("after a restart the pass did not fence again") } if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.Fenced || !r.At.Equal(wd.clock()) { t.Fatalf("after a restart: %+v", r) } // The flip: the record names uk; both source rows stay, held, and keep // reporting; the target mints its row held landing. wd.seal() wd.sealed() wd.flip() wd.advance(time.Minute) for _, bx := range []*offBox{a, b, uk} { bx.tick() } for _, bx := range []*offBox{a, b} { if rt := bx.row(offHost); rt == nil || rt.Status != "moving" || rt.HoldOp != offID { t.Fatalf("%s after the flip: %+v, want the row kept, held", bx.name, rt) } if r := wd.report(moves.OfflineRoleSource, bx.name); !r.Held || !r.Fenced || !r.At.Equal(wd.clock()) { t.Fatalf("%s after the flip: %+v", bx.name, r) } } if rt := uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID { t.Fatalf("target after the flip: %+v", rt) } if uk.lists(offHost) { t.Fatal("the target lists the landing row") } } // What the report reads from disk: a box with only a meta dir has local // state; a process left in the host's cgroup and a layout journal show. func TestOfflineSourceReportReadsTheBox(t *testing.T) { wd := newOffWorld(t) a := wd.box("ie-a", "", nil) b := wd.box("ie-b", "", nil) a.tick() wd.start() // ie-b has no DB file, only the activator's meta dir with one LTX file. meta := activatorMetaDir(provision.DefaultDBPath(b.dd, offHost)) writeFile(t, filepath.Join(meta, "ltx", "0", "0000000000000001-0000000000000007.ltx"), "x") // ie-a: files, env, a process in the host's exec cgroup, a journal. writeFile(t, filepath.Join(a.dd, offHost, "instance.key"), "k") writeFile(t, filepath.Join(a.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=k\n") writeFile(t, filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs"), "4242\n") writeFile(t, provision.LayoutJournalPath(a.dd, offHost), "{}") a.tick() b.tick() rb := wd.report(moves.OfflineRoleSource, "ie-b") if !rb.LocalDB || rb.LocalTXID != 7 || rb.Files || rb.Env || rb.Sidecar { t.Fatalf("a meta dir alone: %+v", rb) } ra := wd.report(moves.OfflineRoleSource, "ie-a") if ra.LocalDB || !ra.Files || !ra.Env || ra.Procs != 1 || !ra.LayoutJournal { t.Fatalf("ie-a: %+v", ra) } // A box that cannot count its processes reports -1, never 0. a.w.cfg.Procs = func(string) (int, error) { return 0, cgroup.ErrUnknown } a.pass() if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Procs != -1 { t.Fatalf("unknown processes reported as %d", r.Procs) } // A sidecar file shows. writeFile(t, filepath.Join(b.dd, offHost+"-sidecar.env"), "X=1\n") b.pass() if r := wd.report(moves.OfflineRoleSource, "ie-b"); !r.Sidecar { t.Fatalf("sidecar not seen: %+v", r) } } // offline-reverting, before and after the flip: every source row is // released and served again, the fence is lifted, the boxes report // reverted, and the next reconcile does not hold the rows again. func TestOfflineRevertReleasesEverySourceRow(t *testing.T) { for _, afterFlip := range []bool{false, true} { t.Run(map[bool]string{false: "before the flip", true: "after the flip"}[afterFlip], func(t *testing.T) { wd := newOffWorld(t) a := wd.box("ie-a", "", nil) b := wd.box("ie-b", "", nil) uk := wd.box("uk-box", "uk", nil) a.tick() b.tick() wd.start() if afterFlip { wd.seal() wd.sealed() wd.flip() } for _, bx := range []*offBox{a, b, uk} { bx.tick() } wd.abort() for i := 0; i < 2; i++ { for _, bx := range []*offBox{a, b, uk} { bx.tick() } } for _, bx := range []*offBox{a, b} { rt := bx.row(offHost) if rt == nil || rt.Status != "ready" || rt.HoldOp != "" { t.Fatalf("%s: %+v, want ready and unheld", bx.name, rt) } if !bx.lists(offHost) { t.Fatalf("%s does not serve the host again", bx.name) } if bx.svc.Fenced(offHost) { t.Fatalf("%s: the fence is still up", bx.name) } if r := wd.report(moves.OfflineRoleSource, bx.name); r.Phase != moves.OfflinePhaseReverted || r.Held { t.Fatalf("%s: %+v", bx.name, r) } } // The move finishes; nothing holds the rows again. wd.move(provision.OfflineFinish(offID, wd.clock())) for _, bx := range []*offBox{a, b} { bx.tick() if rt := bx.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" { t.Fatalf("%s after finish: %+v", bx.name, rt) } } }) } } // offline-open on a flat source: every path of the source-retire row that // is there is renamed into the park directory byte for byte, nothing is // deleted, the fence is lifted, and the parked report lists every path with // its action. A process left in the host's cgroup stops the park first. // offline-finishing then removes the row and the box reports gone. func TestOfflineOpenParksTheFlatSource(t *testing.T) { wd := newOffWorld(t) a := wd.box("ie-a", "", nil) uk := wd.box("uk-box", "uk", nil) a.tick() db := a.row(offHost).DBPath contents := map[string]string{ db: "sqlite", db + "-wal": "wal", filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx", filepath.Join(a.dd, offHost, "instance.key"): "key", filepath.Join(a.dd, offHost+".env"): "CARLOS_INSTANCE_KEY=k\n", filepath.Join(a.dd, offHost+".config.env"): "SECRET=s\n", filepath.Join(a.dd, "programs", offHost+".json"): `{"programs":[]}`, filepath.Join(a.opt, "live", offHost, "app"): "binary", filepath.Join(a.dd, "neighbour.example.com.db"): "not ours", } for p, body := range contents { writeFile(t, p, body) } wd.start() a.tick() // the source reports before the seal, as the seal door requires wd.seal() wd.sealed() wd.flip() wd.landed() a.tick() uk.tick() wd.open() // A process still in the host's cgroup: nothing is renamed. procs := filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs") writeFile(t, procs, "4242\n") a.tick() if _, err := os.Stat(db); err != nil { t.Fatalf("parked with a process left: %v", err) } if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok { t.Fatal("fixture: no report") } if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase == moves.OfflinePhaseParked { t.Fatalf("reported parked with a process left: %+v", r) } writeFile(t, procs, "") a.tick() park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource) for p, body := range contents { switch { case strings.HasPrefix(p, a.opt), strings.Contains(p, "neighbour"): if got, err := os.ReadFile(p); err != nil || string(got) != body { t.Fatalf("%s was touched: %q %v", p, got, err) } continue } if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { t.Fatalf("%s is still in place: %v", p, err) } } for rel, body := range map[string]string{ filepath.Base(db): "sqlite", filepath.Base(db) + "-wal": "wal", filepath.Join(filepath.Base(activatorMetaDir(db)), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx", filepath.Join(offHost, "instance.key"): "key", offHost + ".env": "CARLOS_INSTANCE_KEY=k\n", offHost + ".config.env": "SECRET=s\n", offHost + ".json": `{"programs":[]}`, } { if got, err := os.ReadFile(filepath.Join(park, rel)); err != nil || string(got) != body { t.Fatalf("park/%s = %q %v, want %q", rel, got, err, body) } } r := wd.report(moves.OfflineRoleSource, "ie-a") if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env || r.Fenced { t.Fatalf("parked report: %+v", r) } if a.svc.Fenced(offHost) { t.Fatal("the fence is still up after the park") } want := map[string]string{ db: moves.RetiredParked, db + "-wal": moves.RetiredParked, db + "-shm": moves.RetiredAbsent, db + "-litestream": moves.RetiredAbsent, activatorMetaDir(db): moves.RetiredParked, filepath.Join(a.dd, offHost): moves.RetiredParked, filepath.Join(a.dd, offHost+".env"): moves.RetiredParked, filepath.Join(a.dd, offHost+".config.env"): moves.RetiredParked, filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredParked, filepath.Join(a.opt, "live", offHost): moves.RetiredLeft, } assertRetired(t, r.Retired, want) // A second pass changes nothing and reports the same inventory. before := r.Retired a.tick() if r := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(r.Retired, before) || r.Phase != moves.OfflinePhaseParked { t.Fatalf("second pass: %+v", r.Retired) } // The row stays held through open. if rt := a.row(offHost); rt == nil || rt.HoldOp != offID { t.Fatalf("row at open: %+v", rt) } // finishing: keep no longer holds the row; reconcile removes it and // the box reports gone. The parked state stays. // The view the reconcile reads is the last moves pass's, so the row // goes on the pass after the one that first sees finishing. wd.finishing() a.tick() a.tick() if rt := a.row(offHost); rt != nil { t.Fatalf("row at finishing: %+v", rt) } a.pass() if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone || r.RowPresent || r.LocalDB { t.Fatalf("gone report: %+v", r) } if got, err := os.ReadFile(filepath.Join(park, filepath.Base(db))); err != nil || string(got) != "sqlite" { t.Fatalf("the parked DB: %q %v", got, err) } } func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string) { t.Helper() seen := map[string]string{} for _, rp := range got { if _, dup := seen[rp.Path]; dup { t.Fatalf("%s listed twice", rp.Path) } seen[rp.Path] = rp.Action if rp.Action == moves.RetiredLeft && rp.Reason == "" { t.Fatalf("%s left with no reason", rp.Path) } } for p, action := range want { if seen[p] != action { t.Fatalf("%s: %q, want %q (inventory %+v)", p, seen[p], action, got) } } if len(seen) != len(want) { var extra []string for p := range seen { if _, ok := want[p]; !ok { extra = append(extra, p+"="+seen[p]) } } sort.Strings(extra) t.Fatalf("unexpected entries: %v", extra) } } // M3: a path the agent can never rename (here a file in a directory it may // not write, as a root-owned /data/programs is) is reported left once, with // its reason, and is not tried again: later passes, an agent restart // included, neither fence nor rename, and the report stays parked. func TestOfflineParkLeavesWhatCannotBeRenamedOnce(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root can rename anything") } wd := newOffWorld(t) a := wd.box("ie-a", "", nil) a.tick() db := a.row(offHost).DBPath writeFile(t, db, "sqlite") programs := filepath.Join(a.dd, "programs") writeFile(t, filepath.Join(programs, offHost+".json"), "{}") if err := os.Chmod(programs, 0o555); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(programs, 0o755) }) fences := 0 count := func() { inner := a.w.cfg.FenceAs a.w.cfg.FenceAs = func(ctx context.Context, host string, gen uint64, owner string) error { fences++ return inner(ctx, host, gen, owner) } } wd.start() a.tick() wd.seal() wd.sealed() wd.flip() wd.landed() a.tick() wd.open() count() before := fences a.tick() if fences != before+1 { t.Fatalf("the first park fenced %d times", fences-before) } r := wd.report(moves.OfflineRoleSource, "ie-a") if r.Phase != moves.OfflinePhaseParked { t.Fatalf("phase %s", r.Phase) } var left moves.RetiredPath for _, rp := range r.Retired { if rp.Path == filepath.Join(programs, offHost+".json") { left = rp } if rp.Path == db && rp.Action != moves.RetiredParked { t.Fatalf("the DB: %+v", rp) } } if left.Action != moves.RetiredLeft || !strings.HasPrefix(left.Reason, leftCannot) { t.Fatalf("the unrenameable path: %+v", left) } for i := 0; i < 2; i++ { before = fences a.tick() if fences != before { t.Fatal("a later pass fenced again for a path it cannot rename") } if again := wd.report(moves.OfflineRoleSource, "ie-a"); again.Phase != moves.OfflinePhaseParked || !reflect.DeepEqual(again.Retired, r.Retired) { t.Fatalf("pass %d: %+v", i, again) } } a.restart() count() before = fences a.tick() if fences != before || a.svc.Fenced(offHost) { t.Fatal("after a restart the pass fenced again for a path it cannot rename") } if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) { t.Fatalf("after a restart: %+v", again.Retired) } if _, err := os.Stat(filepath.Join(programs, offHost+".json")); err != nil { t.Fatalf("the left path: %v", err) } } // offline-open on a nested source: the contents of the raw instance // directory are renamed into the park directory, the directory and its // view stay, empty, and the inventory names each path. func TestOfflineOpenParksTheNestedSource(t *testing.T) { wd := newOffWorld(t) a := wd.box("ie-a", "", nil) a.nested = true raw := layout.RawInstanceDir(a.dd, offHost) if err := os.MkdirAll(raw, 0o700); err != nil { t.Fatal(err) } // The bind mount at