package main import ( "bytes" "context" "crypto/ecdh" "encoding/base64" "encoding/json" "net/http" "net/http/httptest" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/console" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) // offlineE2E is a real console.Server with one hibernating instance of the // app shop in ie, an operator's CLI login, and the ie and uk boxes // published with caps. run is moveMain for `carlos move offline ` // against it; put writes a box's report where the real box writes it. type offlineE2E struct { host string rel release.Store run func(c context.Context, args ...string) (int, string) put func(id, box, role, phase string, edit func(*moves.OfflineReport)) } func newOfflineE2E(t *testing.T, caps []string) *offlineE2E { const ( host = "acme.shop.example.com" addr = "paul@keymail.dev" ) tmp := t.TempDir() rel := release.NewDirStore(filepath.Join(tmp, "store")) seed, pubB64, err := release.GenerateKey() if err != nil { t.Fatal(err) } priv, err := release.ParseSigningKey(seed, "e2e") if err != nil { t.Fatal(err) } pub, err := release.ParsePublicKey(pubB64, "e2e") if err != nil { t.Fatal(err) } cst, err := console.Open(filepath.Join(tmp, "console.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { cst.Close() }) if err := cst.SetOperator(addr, true); err != nil { t.Fatal(err) } secrets, err := ecdh.P256().NewPrivateKey(bytes.Repeat([]byte{1}, 32)) if err != nil { t.Fatal(err) } fp := appconfig.KeyFingerprint(secrets.PublicKey()) var srv *console.Server ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) })) t.Cleanup(ts.Close) srv, err = console.NewServer(console.Config{ Origin: ts.URL, Keymail: "http://keymail.invalid", Registry: filepath.Join(tmp, "registry.db"), Releases: rel, ReleasePub: pub, Key: priv, AppScheme: "http", SecretsPub: base64.StdEncoding.EncodeToString(secrets.PublicKey().Bytes()), }, cst) if err != nil { t.Fatal(err) } if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, time.Now()); err != nil { t.Fatal(err) } if _, err := cst.CreateAppIn(console.OpsAccountID, "shop", addr, time.Now()); err != nil { t.Fatal(err) } token, _ := e2eDeviceLogin(t, ts.URL, cst, addr, "offline@laptop") ctx := context.Background() // A hibernating instance in ie, and the two regions' boxes published. if err := provision.PutManifest(ctx, rel, release.OpsAccount, "shop", provision.Manifest{Schema: 1, App: "shop", Domain: "shop.example.com", Channel: "stable"}); err != nil { t.Fatal(err) } req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/cli/apps/"+release.OpsAccount+"/shop/instances", strings.NewReader(`{"host":"`+host+`","region":"ie"}`)) req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Content-Type", "application/json") res, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } res.Body.Close() if res.StatusCode != http.StatusCreated { t.Fatalf("create the instance = %d", res.StatusCode) } for _, b := range []struct{ box, placement, aws, bucket string }{ {"ie-box", "ie", "eu-west-1", "data-ie"}, {"uk-box", "uk", "eu-west-2", "data-uk"}, } { body, _ := json.Marshal(map[string]any{ "v": 1, "box": b.box, "box_id": b.box, "eip": "1.2.3.4", "region": b.aws, "placement": b.placement, "data_bucket": b.bucket, "hibernation_configured": true, "serves_shared_pool": true, "accepts_inbound": true, "secrets_key_fp": fp, "capabilities": caps, "hosts": map[string]any{}, }) if _, err := rel.Put(ctx, "control/edges/"+b.box+".json", body, ""); err != nil { t.Fatal(err) } if err := status.WriteBox(ctx, rel, status.Box{Label: b.box, GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{}}); err != nil { t.Fatal(err) } } run := func(c context.Context, args ...string) (int, string) { var out bytes.Buffer deps := moveDeps{ session: func(string) (*apiSession, error) { return &apiSession{c: newConsoleClient(ts.URL, token), console: ts.URL, address: addr}, nil }, euid: func() int { return 1000 }, poll: 20 * time.Millisecond, } args = append(append([]string{"offline"}, args...), "--host", host, "--account", release.OpsAccount, "--app", "shop") code := moveMain(c, args, &out, deps) return code, out.String() } put := func(id, box, role, phase string, edit func(*moves.OfflineReport)) { t.Helper() r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: id, Host: host, Box: box, Role: role, Phase: phase, At: time.Now().UTC(), Held: true, Fenced: true, Quiesced: true} if edit != nil { edit(&r) } body, err := json.Marshal(r) if err != nil { t.Fatal(err) } key := moves.OfflineReportKey(host, id, role, box) _, etag, _ := rel.Get(ctx, key) if _, err := rel.Put(ctx, key, body, etag); err != nil { t.Fatal(err) } } return &offlineE2E{host: host, rel: rel, run: run, put: put} } // TestMoveOfflineAgainstTheRealDoors drives `carlos move offline start` and // `status` through moveMain against a real console.Server: the stub tests // type the status view by hand, and this one ties the CLI's view structs to // the JSON the console writes (next, waiting, failed, items, reports, left). // The boxes are played by reports written where the real boxes write them. func TestMoveOfflineAgainstTheRealDoors(t *testing.T) { e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1"}) host, rel, run, put := e.host, e.rel, e.run, e.put ctx := context.Background() // start waits for the source box; it runs on while the test plays it. ctxStart, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() type result struct { code int out string } started := make(chan result, 1) go func() { code, out := run(ctxStart, "start", "--to", "uk") started <- result{code, out} }() var id string for deadline := time.Now().Add(20 * time.Second); id == "" && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { if rec, err := provision.GetRecord(ctx, rel, release.OpsAccount, "shop", host); err == nil && rec.Move != nil { id = rec.Move.ID } } if id == "" { t.Fatal("start never moved the record") } // status, before the box has reported: the console's own waiting sentence. code, out := run(ctx, "status") wantWait := "Waiting for ie-box in ie to confirm it has stopped " + host + ". " + host + " is offline." wantLines(t, out, "Moving "+host+" from ie to uk.", wantWait, "Item class is ok.", "Item db is waiting.") if code != 0 { t.Fatalf("status exit %d:\n%s", code, out) } // The box reports; start's next poll sees next == seal. put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseStopped, nil) select { case r := <-started: if r.code != 0 { t.Fatalf("start exit %d:\n%s", r.code, r.out) } wantLines(t, r.out, "Moving "+host+" from ie to uk.", host+" is offline from now until the move opens or is aborted.", "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.") case <-time.After(20 * time.Second): t.Fatal("start never finished") } code, out = run(ctx, "status") wantLines(t, out, "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.", "ie-box is stopped as the source.") if code != 0 { t.Fatalf("status exit %d:\n%s", code, out) } // A target that says it failed sets the view's failed flag and its check. put(id, "uk-box", moves.OfflineRoleTarget, moves.OfflinePhaseFailed, func(r *moves.OfflineReport) { r.Items = map[string]moves.Check{"config": {OK: false, Reason: "the app settings differ"}} }) code, out = run(ctx, "status") if code != 4 { t.Fatalf("status of a failed move exit %d, want 4:\n%s", code, out) } wantLines(t, out, "uk could not take "+host+": the app settings differ. Run carlos move offline abort --host "+host+" to move it back.") // A source box that left a path in place says so, with its reason. put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseParked, func(r *moves.OfflineReport) { r.Retired = []moves.RetiredPath{{Path: "/opt/carlos/live/" + host, Action: moves.RetiredLeft, Reason: "a root-owned directory"}} }) _, out = run(ctx, "status") wantLines(t, out, "ie-box left /opt/carlos/live/"+host+" in place: a root-owned directory.") } // TestMoveOfflineStageAgainstTheRealDoors drives stage, status, start and // abort of a staged move through moveMain against a real console.Server, // tying the CLI's stage view to the JSON the console writes. The stager is // played by its report, written for the stage request the record carries. func TestMoveOfflineStageAgainstTheRealDoors(t *testing.T) { e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1", "offline-stage-1"}) host, ctx := e.host, context.Background() record := func() *provision.Record { t.Helper() rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host) if err != nil { t.Fatal(err) } return rec } // round plays the stager: one round for the record's current request. round := func(id string) { t.Helper() e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseStaged, func(r *moves.OfflineReport) { r.Held, r.Fenced, r.Quiesced = false, false, false r.StageRequest = *record().Move.StagedAt r.ReplicaObjects, r.ReplicaBytes, r.StagedObjects, r.StagedBytes = 288, 72_000_000, 288, 72_000_000 }) } type result struct { code int out string } // stage waits for its round; the test plays the stager once the record // carries the request. stage := func(wantRequestAfter time.Time) (string, result) { t.Helper() sctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() done := make(chan result, 1) go func() { code, out := e.run(sctx, "stage", "--to", "uk") done <- result{code, out} }() var id string for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { if rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host); err == nil && rec.Move != nil && rec.Move.StagedAt != nil && rec.Move.StagedAt.After(wantRequestAfter) { id = rec.Move.ID break } } if id == "" { t.Fatal("stage never wrote its request") } round(id) select { case r := <-done: return id, r case <-time.After(20 * time.Second): t.Fatal("stage never finished") } return "", result{} } staging := "Staging " + host + " in uk. " + host + " keeps serving while the data is copied." ready := host + " keeps serving. Run carlos move offline start --host " + host + " --to uk when you are ready to take it offline." id, r := stage(time.Time{}) if r.code != 0 { t.Fatalf("stage exit %d:\n%s", r.code, r.out) } wantLines(t, r.out, staging, ready) if !strings.Contains(r.out, "Staged 288 of 288 objects for "+host+" in uk (72.0 of 72.0 MB) at ") { t.Fatalf("stage output lacks the progress line:\n%s", r.out) } // Again: a new request, answered by a new round. first := *record().Move.StagedAt if id2, r := stage(first); r.code != 0 || id2 != id { t.Fatalf("second stage exit %d (move %s, want %s):\n%s", r.code, id2, id, r.out) } code, out := e.run(ctx, "status") if code != 0 { t.Fatalf("status exit %d:\n%s", code, out) } wantLines(t, out, staging, ready) // abort from staging: the staged copy goes, then finish. actx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() aborted := make(chan result, 1) go func() { code, out := e.run(actx, "abort") aborted <- result{code, out} }() for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && record().Move.State != provision.OfflineReverting; time.Sleep(10 * time.Millisecond) { } e.put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held = false }) // The pinned stager says the staged copy is gone: the finish waits for it. e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held, r.Fenced, r.Quiesced = false, false, false }) select { case r := <-aborted: if r.code != 3 { t.Fatalf("abort exit %d:\n%s", r.code, r.out) } wantLines(t, r.out, "Removing the staged copy of "+host+" from uk.", "The staged copy of "+host+" is removed. Run carlos move offline finish --host "+host+" to end the move.") case <-time.After(20 * time.Second): t.Fatal("abort never finished") } if code, out := e.run(ctx, "finish"); code != 3 { t.Fatalf("finish exit %d:\n%s", code, out) } if rec := record(); rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != provision.OfflineOutcomeAborted || rec.LastMove.OfflineS != 0 { t.Fatalf("after finish: %+v %+v", rec.Move, rec.LastMove) } }