package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "strings" "sync" "sync/atomic" "syscall" "testing" "time" ) func TestImportVerifyRetriesThroughTheWake(t *testing.T) { var calls atomic.Int32 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Carlos-Import-Verify") != "tok" { w.WriteHeader(http.StatusForbidden) return } if calls.Add(1) < 3 { w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusServiceUnavailable) return } w.WriteHeader(http.StatusNoContent) })) defer ts.Close() code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") if err != nil || code != http.StatusNoContent || calls.Load() != 3 { t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) } } func TestImportVerifyStopsOnAFailure(t *testing.T) { var calls atomic.Int32 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) w.WriteHeader(http.StatusInternalServerError) })) defer ts.Close() code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") if err != nil || code != http.StatusInternalServerError || calls.Load() != 1 { t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) } } // fakeSeedWait replaces the poll seam for one test: no real sleeps, and // every wait is recorded. func fakeSeedWait(t *testing.T, every time.Duration) *[]time.Duration { t.Helper() var mu sync.Mutex waits := &[]time.Duration{} oldWait, oldEvery := seedWait, seedPollEvery seedWait = func(ctx context.Context, d time.Duration) error { mu.Lock() *waits = append(*waits, d) mu.Unlock() return ctx.Err() } seedPollEvery = every t.Cleanup(func() { seedWait, seedPollEvery = oldWait, oldEvery }) return waits } // A gateway answer (502, 504) while the instance wakes is retried as a 503 // is, honouring Retry-After (M16). func TestImportVerifyRetriesGatewayAnswers(t *testing.T) { waits := fakeSeedWait(t, 2*time.Second) var calls atomic.Int32 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch calls.Add(1) { case 1: w.WriteHeader(http.StatusBadGateway) case 2: w.Header().Set("Retry-After", "7") w.WriteHeader(http.StatusGatewayTimeout) default: w.WriteHeader(http.StatusOK) } })) defer ts.Close() code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") if err != nil || code != http.StatusOK || calls.Load() != 3 { t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) } if len(*waits) != 2 || (*waits)[0] != 3*time.Second || (*waits)[1] != 7*time.Second { t.Fatalf("waits = %v, want 3s (no Retry-After) then 7s", *waits) } } func TestImportVerifyGivesUpAfterItsTries(t *testing.T) { fakeSeedWait(t, 2*time.Second) var calls atomic.Int32 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) w.WriteHeader(http.StatusServiceUnavailable) })) defer ts.Close() code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") if err != nil || code != http.StatusServiceUnavailable || calls.Load() != importVerifyTries+1 { t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) } } // A redirect is the verify's answer, never followed: /up answering 302 to a // login page that says 200 must not commit (M16). func TestImportVerifyDoesNotFollowARedirect(t *testing.T) { var landed atomic.Bool ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/login" { landed.Store(true) w.WriteHeader(http.StatusOK) return } http.Redirect(w, r, "/login", http.StatusFound) })) defer ts.Close() client := ts.Client() // follows redirects by default code, err := verifyImportURL(context.Background(), client, ts.URL+"/up", "tok") if err != nil || code != http.StatusFound || landed.Load() { t.Fatalf("code %d err %v followed %v", code, err, landed.Load()) } if client.CheckRedirect != nil { t.Fatal("verifyImportURL changed the caller's client") } } func TestImportVerifyBuildsTheHostURL(t *testing.T) { var path, tok string ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { path, tok = r.URL.Path, r.Header.Get("Carlos-Import-Verify") w.WriteHeader(http.StatusOK) })) defer ts.Close() host := strings.TrimPrefix(ts.URL, "https://") code, err := verifyImport(context.Background(), ts.Client(), host, "up", "tok") if err != nil || code != http.StatusOK || path != "/up" || tok != "tok" { t.Fatalf("code %d err %v path %q token %q", code, err, path, tok) } } func TestImportClientCallsTheDoor(t *testing.T) { var seen []string var begin map[string]any ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { seen = append(seen, r.Method+" "+r.URL.RequestURI()) switch { case r.Method == http.MethodPost && r.URL.Path == "/api/cli/apps/bab/shop/instances": raw, _ := io.ReadAll(r.Body) _ = json.Unmarshal(raw, &begin) w.WriteHeader(http.StatusCreated) w.Write([]byte(`{"host":"old.shop.example.com","verify_token":"tok","seed":{"id":"sd_imp","kind":"import","state":"awaiting-upload"}}`)) default: w.WriteHeader(http.StatusOK) w.Write([]byte(`{"state":"ok"}`)) } })) defer ts.Close() s := newTestSession(ts.URL) a := apiAcct{Seg: "bab"} rec, tok, err := s.importBegin(context.Background(), a, "shop", "old.shop.example.com", "staging", "aa", 3, "127.0.0.1:8401", "moving the shop") if err != nil || tok != "tok" || rec.Seed == nil || rec.Seed.ID != "sd_imp" { t.Fatalf("begin: %q %+v %v", tok, rec, err) } imp, _ := begin["import"].(map[string]any) if begin["host"] != "old.shop.example.com" || begin["environment"] != "staging" || begin["message"] != "moving the shop" || imp["sha256"] != "aa" || imp["bytes"] != float64(3) || imp["replaces"] != "127.0.0.1:8401" { t.Fatalf("begin body = %v", begin) } for _, verb := range []string{"stopped", "commit", "abort"} { if err := s.importVerb(context.Background(), a, "shop", "old.shop.example.com", "sd_imp", verb); err != nil { t.Fatalf("%s: %v", verb, err) } } // Every verb names the import it means (final review F1). want := []string{"POST /api/cli/apps/bab/shop/instances", "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/stopped?seed=sd_imp", "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/commit?seed=sd_imp", "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/abort?seed=sd_imp"} if len(seen) != len(want) { t.Fatalf("calls = %q", seen) } for i := range want { if seen[i] != want[i] { t.Fatalf("call %d = %q, want %q", i, seen[i], want[i]) } } } func TestImportBeginRefusalAndUnreadableAnswer(t *testing.T) { for _, c := range []struct { status int body string wantErr string }{ {http.StatusBadRequest, `{"error":"the sha256 is not 64 hex characters"}`, "the sha256 is not 64 hex characters"}, {http.StatusCreated, `{"host":"old.shop.example.com"}`, "instances import: unreadable response"}, {http.StatusCreated, `not json`, "instances import: unreadable response"}, } { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(c.status) io.WriteString(w, c.body) })) _, tok, err := newTestSession(ts.URL).importBegin(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "", "aa", 3, "127.0.0.1:8401") ts.Close() if err == nil || !strings.HasPrefix(err.Error(), c.wantErr) || tok != "" { t.Errorf("%d %s: token %q err %v", c.status, c.body, tok, err) } } } // Begin can provision a private store (up to about 75 s), so it must not // ride the session's 30 s JSON client (M4). The session's client is shrunk // below the server's delay to show begin does not use it, and is left as // it was. func TestImportBeginOutlastsTheJSONClient(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(150 * time.Millisecond) w.WriteHeader(http.StatusCreated) io.WriteString(w, `{"host":"old.shop.example.com","verify_token":"tok"}`) })) defer ts.Close() s := newTestSession(ts.URL) s.c.httpc.Timeout = 20 * time.Millisecond if _, tok, err := s.importBegin(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "", "aa", 3, "127.0.0.1:8401"); err != nil || tok != "tok" { t.Fatalf("token %q err %v", tok, err) } if s.c.httpc.Timeout != 20*time.Millisecond { t.Fatalf("the session's client timeout became %v", s.c.httpc.Timeout) } } func snapshotFile(t *testing.T, body string) *os.File { t.Helper() p := filepath.Join(t.TempDir(), "main.db") if err := os.WriteFile(p, []byte(body), 0o600); err != nil { t.Fatal(err) } f, err := os.Open(p) if err != nil { t.Fatal(err) } t.Cleanup(func() { f.Close() }) return f } func TestImportUploadStreamsTheFile(t *testing.T) { var got struct { method, path, seed, expect, ctype string length int64 body string } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { raw, _ := io.ReadAll(r.Body) got.method, got.path, got.seed, got.expect, got.ctype = r.Method, r.URL.Path, r.URL.Query().Get("seed"), r.Header.Get("Expect"), r.Header.Get("Content-Type") got.length, got.body = r.ContentLength, string(raw) time.Sleep(150 * time.Millisecond) w.WriteHeader(http.StatusAccepted) io.WriteString(w, `{"state":"checking"}`) })) defer ts.Close() s := newTestSession(ts.URL) s.c.httpc.Timeout = 20 * time.Millisecond // the upload never rides the JSON client if err := s.importUpload(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "sd_imp", snapshotFile(t, "abc"), 3); err != nil { t.Fatal(err) } if got.method != http.MethodPut || got.path != "/api/cli/apps/bab/shop/instances/old.shop.example.com/import" || got.seed != "sd_imp" || got.expect != "100-continue" || got.ctype != "application/octet-stream" || got.length != 3 || got.body != "abc" { t.Fatalf("request = %+v", got) } } func TestImportUploadRefusal(t *testing.T) { const sdN51 = "the upload does not match the length and sha256 this import began with, so it was not kept." ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.Copy(io.Discard, r.Body) w.WriteHeader(http.StatusUnprocessableEntity) fmt.Fprintf(w, `{"error":%q}`, sdN51) })) defer ts.Close() err := newTestSession(ts.URL).importUpload(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "sd_imp", snapshotFile(t, "abc"), 3) if err == nil || err.Error() != sdN51 { t.Fatalf("err = %v", err) } } // importConsole is a console for one import: begin, upload and the verbs // answer as configured, and the record walks through gets, one per look // (the last one repeats). type importConsole struct { uploadHold func(r *http.Request) // runs before the upload answers, outside mu mu sync.Mutex calls []string beginStatus int beginBody string upload func(w http.ResponseWriter) verbs map[string]func(w http.ResponseWriter) gets []func(w http.ResponseWriter) looks int seeds []string // the seed each upload and verb named, in order } const importBeginBody = `{"host":"main.shop.example.com","verify_token":"s3cret-token","seed":{"id":"sd_imp","kind":"import","state":"awaiting-upload"}}` func newImportConsole(gets ...func(w http.ResponseWriter)) *importConsole { return &importConsole{beginStatus: http.StatusCreated, beginBody: importBeginBody, gets: gets, verbs: map[string]func(http.ResponseWriter){}} } func (c *importConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPut && c.uploadHold != nil { io.Copy(io.Discard, r.Body) c.uploadHold(r) } c.mu.Lock() defer c.mu.Unlock() io.Copy(io.Discard, r.Body) path := r.URL.Path if i := strings.Index(path, "/instances"); i >= 0 { path = path[i:] } switch { case r.Method == http.MethodPost && path == "/instances": c.calls = append(c.calls, "begin") w.WriteHeader(c.beginStatus) io.WriteString(w, c.beginBody) case r.Method == http.MethodPut: c.calls = append(c.calls, "upload") c.seeds = append(c.seeds, r.URL.Query().Get("seed")) if c.upload != nil { c.upload(w) return } w.WriteHeader(http.StatusAccepted) io.WriteString(w, `{"state":"checking"}`) case r.Method == http.MethodPost: verb := path[strings.LastIndex(path, "/")+1:] c.calls = append(c.calls, verb) c.seeds = append(c.seeds, r.URL.Query().Get("seed")) if f := c.verbs[verb]; f != nil { f(w) return } fmt.Fprintf(w, `{"state":%q}`, verb) case r.Method == http.MethodGet: c.calls = append(c.calls, "get") i := c.looks if c.looks++; i >= len(c.gets) { i = len(c.gets) - 1 } c.gets[i](w) } } func (c *importConsole) called() []string { c.mu.Lock() defer c.mu.Unlock() return append([]string(nil), c.calls...) } // called reports whether the console saw verb. func (c *importConsole) saw(verb string) bool { for _, v := range c.called() { if v == verb { return true } } return false } func importRecord(state, reason string) func(w http.ResponseWriter) { return importRecordID("sd_imp", state, reason) } func importRecordID(id, state, reason string) func(w http.ResponseWriter) { return func(w http.ResponseWriter) { fmt.Fprintf(w, `{"host":"main.shop.example.com","seed":{"id":%q,"kind":"import","sha256":"ab","bytes":3,"state":%q,"reason":%q}}`, id, state, reason) } } // importRun runs an import of a three-byte file against c, with verify // (when not empty) answered by verifyStatus through a TLS server standing // in for the instance's host. It returns the output and the error. func importRun(t *testing.T, c *importConsole, verify string, verifyStatus ...int) (string, error) { t.Helper() return importRunCtx(t, context.Background(), c, verify, verifyStatus...) } func importRunCtx(t *testing.T, ctx context.Context, c *importConsole, verify string, verifyStatus ...int) (string, error) { t.Helper() ts := httptest.NewServer(c) defer ts.Close() j := importJob{acct: apiAcct{Seg: "bab"}, app: "shop", host: "main.shop.example.com", file: filepath.Join(t.TempDir(), "main.db"), replaces: "127.0.0.1:8401", verify: verify} if err := os.WriteFile(j.file, []byte("abc"), 0o600); err != nil { t.Fatal(err) } if verify != "" { var n atomic.Int32 vs := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Carlos-Import-Verify") != "s3cret-token" { w.WriteHeader(http.StatusForbidden) return } i := int(n.Add(1)) - 1 if i >= len(verifyStatus) { i = len(verifyStatus) - 1 } w.WriteHeader(verifyStatus[i]) })) defer vs.Close() j.host, j.verifyClient = strings.TrimPrefix(vs.URL, "https://"), vs.Client() } var out bytes.Buffer err := newTestSession(ts.URL).runImport(ctx, &out, j) // The output names the file and the host as the tests know them. return strings.NewReplacer(j.file, "main.db", j.host, "main.shop.example.com").Replace(out.String()), err } func wantExactLines(t *testing.T, out string, want ...string) { t.Helper() got := strings.Split(strings.TrimSuffix(out, "\n"), "\n") if len(got) != len(want) { t.Fatalf("output:\n%s\nwant:\n%s", out, strings.Join(want, "\n")) } for i := range want { if got[i] != want[i] { t.Fatalf("line %d = %q, want %q\noutput:\n%s", i, got[i], want[i], out) } } } const ( importCommitLine = "commit with: carlos instances import commit --app shop --host main.shop.example.com. Abort with: carlos instances import abort --app shop --host main.shop.example.com." importTokenLine = "verify token: s3cret-token. It is shown only this once." ) func TestImportWithoutVerifyPrintsTheTokenAndTheCommands(t *testing.T) { waits := fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("awaiting-upload", ""), importRecord("uploaded", ""), importRecord("ready", ""), importRecord("swapped", "")) out, err := importRun(t, c, "") if err != nil { t.Fatal(err) } wantExactLines(t, out, "uploading main.db (3B)", importTokenLine, "uploaded. The old process of main.shop.example.com is marked stopped, and the box swaps it on its next pass.", "waiting for the box to swap main.shop.example.com", "swapped. Until it is committed, main.shop.example.com answers only requests that carry the verify token.", importCommitLine) if got := strings.Join(c.called(), " "); got != "begin upload get get stopped get get" { t.Fatalf("calls = %s", got) } for _, d := range *waits { if d != 2*time.Second { t.Fatalf("waits = %v", *waits) } } } func TestImportWithVerifyCommits(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), importRecord("swapped", ""), importRecord("placed", "")) out, err := importRun(t, c, "/up", http.StatusServiceUnavailable, http.StatusNoContent) if err != nil { t.Fatal(err) } if strings.Contains(out, "s3cret-token") { t.Fatalf("a verified import never shows the token:\n%s", out) } lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") wantExactLines(t, strings.Join(lines[len(lines)-3:], "\n"), "verify passed (204). Committing.", "waiting for the box to finish placing the snapshot", "committed. main.shop.example.com is served by the platform now.") if !c.saw("commit") || c.saw("abort") { t.Fatalf("calls = %v", c.called()) } c.requireSeeds(t, "sd_imp", 3) // upload, stopped, commit } // requireSeeds: the console saw n uploads and verbs, and each named the // seed begin answered (final review F1). func (c *importConsole) requireSeeds(t *testing.T, id string, n int) { t.Helper() c.mu.Lock() defer c.mu.Unlock() if len(c.seeds) != n { t.Fatalf("seeds named = %q, want %d of %q", c.seeds, n, id) } for _, got := range c.seeds { if got != id { t.Fatalf("seeds named = %q, want every one %q", c.seeds, id) } } } func TestImportWithVerifyAbortsOnAFailure(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", "")) out, err := importRun(t, c, "/up", http.StatusInternalServerError) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") wantExactLines(t, strings.Join(lines[len(lines)-2:], "\n"), "verify failed (500). Aborting the import.", "aborted. The box puts the old route of main.shop.example.com back on its next pass.") c.requireSeeds(t, "sd_imp", 3) // upload, stopped, abort if strings.Contains(out, "s3cret-token") || !c.saw("abort") || c.saw("commit") { t.Fatalf("calls = %v, output:\n%s", c.called(), out) } } func TestImportRefusedByTheBox(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("refused", "the route belongs to app kass")) out, err := importRun(t, c, "/up", http.StatusOK) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } if !strings.HasSuffix(out, "waiting for the box to swap "+hostOf(out)+"\nthe box refused the import: the route belongs to app kass\n") { t.Fatalf("output:\n%s", out) } if strings.Contains(out, "s3cret-token") || c.saw("commit") || c.saw("abort") { t.Fatalf("calls = %v, output:\n%s", c.called(), out) } } // hostOf reads the host back out of the "waiting for the box to swap" line, // for the tests whose host is a TLS server's address. func hostOf(out string) string { const p = "waiting for the box to swap " i := strings.Index(out, p) if i < 0 { return "" } rest := out[i+len(p):] return rest[:strings.Index(rest, "\n")] } // The swap wait is ten minutes of looks; at its end the token is shown // (with --verify) and SD-N79 names the way on (M17). func TestImportSwapDeadline(t *testing.T) { waits := fakeSeedWait(t, time.Minute) c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", "")) out, err := importRun(t, c, "/up", http.StatusOK) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } host := hostOf(out) if !strings.HasSuffix(out, "waiting for the box to swap "+host+"\n"+importTokenLine+"\n"+ "the box has not swapped "+host+" after 10 minutes. Check carlos instances list, then commit or abort.\n") { t.Fatalf("output:\n%s", out) } // One wait before the upload's look, then ten minutes of one-minute waits. if len(*waits) != 11 { t.Fatalf("waits = %v", *waits) } } func TestImportPlaceDeadline(t *testing.T) { fakeSeedWait(t, time.Minute) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", "")) out, err := importRun(t, c, "/up", http.StatusOK) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } host := hostOf(out) if !strings.HasSuffix(out, "waiting for the box to finish placing the snapshot\n"+importTokenLine+"\n"+ "commit with: carlos instances import commit --app shop --host "+host+". Abort with: carlos instances import abort --app shop --host "+host+".\n") { t.Fatalf("output:\n%s", out) } if c.saw("commit") || c.saw("abort") { t.Fatalf("calls = %v", c.called()) } } func TestImportAbortedPrintsItsReason(t *testing.T) { const sdN52 = "the upload is not a database that passes an integrity check, so it was not kept." for _, c := range []struct { name, reason, want string }{ {"with a reason", sdN52, sdN52}, {"without one", "", "aborting"}, } { t.Run(c.name, func(t *testing.T) { fakeSeedWait(t, 2*time.Second) ic := newImportConsole(importRecord("awaiting-upload", ""), importRecord("aborted", c.reason)) out, err := importRun(t, ic, "/up", http.StatusOK) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } if !strings.HasSuffix(out, "\n"+c.want+"\n") || strings.Contains(out, "s3cret-token") || ic.saw("stopped") { t.Fatalf("calls = %v, output:\n%s", ic.called(), out) } }) } } func TestImportRolledBackWhileWaitingForTheSwap(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("rolled-back", "")) out, err := importRun(t, c, "") if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "waiting for the box to swap main.shop.example.com\naborting\n") { t.Fatalf("err = %v, output:\n%s", err, out) } } const importEnded = "the import of main.shop.example.com has ended. It was deleted, or another import of main.shop.example.com began. Check carlos instances list." func TestImportAnotherImportOnTheHostEndsTheWait(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecordID("sd_other", "ready", "")) out, err := importRun(t, c, "") if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "\n"+importEnded+"\n") || strings.Contains(out, "aborting") { t.Fatalf("err = %v, output:\n%s", err, out) } } // A record that goes while the CLI waits for the check, the swap or the // placing says the import ended, not the membership sentence. func TestImportRecordGoneEndsTheWait(t *testing.T) { gone := answerWith(http.StatusNotFound, "") for _, c := range []struct { name string gets []func(http.ResponseWriter) }{ {"waiting for the check", []func(http.ResponseWriter){importRecord("awaiting-upload", ""), gone}}, {"waiting for the swap", []func(http.ResponseWriter){importRecord("uploaded", ""), importRecord("ready", ""), gone}}, {"waiting for the placing", []func(http.ResponseWriter){importRecord("uploaded", ""), importRecord("swapped", ""), gone}}, } { t.Run(c.name, func(t *testing.T) { fakeSeedWait(t, 2*time.Second) out, err := importRun(t, newImportConsole(c.gets...), "/up", http.StatusOK) if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "\n"+importEnded+"\n") || strings.Contains(out, "not found") { t.Fatalf("err = %v, output:\n%s", err, out) } }) } } // The check wait is 55 minutes of looks; at its end the token is shown and // the line names the way on, not Go's "context deadline exceeded". func TestImportCheckDeadline(t *testing.T) { waits := fakeSeedWait(t, time.Minute) c := newImportConsole(importRecord("awaiting-upload", "")) out, err := importRun(t, c, "/up", http.StatusOK) if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } const want = "the console has not kept the upload of main.shop.example.com after 55 minutes. Check carlos instances list. To start over, run carlos instances import abort --app shop --host main.shop.example.com, then run the import again." if !strings.HasSuffix(out, importTokenLine+"\n"+want+"\n") || strings.Contains(out, "deadline") { t.Fatalf("output:\n%s", out) } if c.saw("stopped") || len(*waits) != 55 { t.Fatalf("calls = %v, waits = %d", c.called(), len(*waits)) } } // The usage block of carlos instances import, byte for byte. func TestImportUsageIsTheApprovedBlock(t *testing.T) { const want = "usage: carlos instances import --app --host --file --replaces \n" + " [--verify ] [--environment ]\n" + " carlos instances import commit --app --host \n" + " carlos instances import abort --app --host \n" + "\n" + "carlos instances import moves an instance the platform never ran onto the platform." if got := importUsage(); got != want { t.Fatalf("usage = %q, want %q", got, want) } } // A stopped whose first answer was lost answers 409 SD-N54 naming a state // past it; the record says the stop landed, so the import goes on. func TestImportStoppedThatAlreadyLanded(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("swapped", "")) c.verbs["stopped"] = answerWith(http.StatusConflict, `{"error":"main.shop.example.com is ready. Mark it stopped once, after its upload."}`) out, err := importRun(t, c, "") if err != nil || !strings.HasSuffix(out, importCommitLine+"\n") { t.Fatalf("err = %v, output:\n%s", err, out) } } func TestImportStoppedRefusedForReal(t *testing.T) { fakeSeedWait(t, 2*time.Second) const refusal = "carlos run is busy on main.shop.example.com. Wait for it to finish, then abort." c := newImportConsole(importRecord("uploaded", "")) c.verbs["stopped"] = answerWith(http.StatusConflict, fmt.Sprintf(`{"error":%q}`, refusal)) out, err := importRun(t, c, "/up", http.StatusOK) if err == nil || err.Error() != refusal { t.Fatalf("err = %v", err) } // With --verify, an exit that leaves the import live shows the token (M17). if !strings.HasSuffix(out, importTokenLine+"\n") { t.Fatalf("output:\n%s", out) } } func TestImportUploadRefusedShowsTheTokenWithVerify(t *testing.T) { fakeSeedWait(t, 2*time.Second) const sdN51 = "the upload does not match the length and sha256 this import began with, so it was not kept." c := newImportConsole(importRecord("awaiting-upload", "")) c.upload = answerWith(http.StatusUnprocessableEntity, fmt.Sprintf(`{"error":%q}`, sdN51)) out, err := importRun(t, c, "/up", http.StatusOK) if err == nil || err.Error() != sdN51 || strings.Contains(err.Error(), "s3cret-token") { t.Fatalf("err = %v", err) } if !strings.HasSuffix(out, importTokenLine+"\n") || c.saw("get") { t.Fatalf("calls = %v, output:\n%s", c.called(), out) } } func TestImportBeginRefusedShowsNothingMore(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole() c.beginStatus, c.beginBody = http.StatusConflict, `{"error":"instance main.shop.example.com already exists"}` out, err := importRun(t, c, "") if err == nil || err.Error() != "instance main.shop.example.com already exists" { t.Fatalf("err = %v", err) } wantExactLines(t, out, "uploading main.db (3B)") if got := c.called(); len(got) != 1 { t.Fatalf("calls = %v", got) } } // A commit that lost its answer is settled by the record (an overlapping // commit answers SD-N60; the record is committed). func TestImportCommitSettledByTheRecord(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), importRecord("placed", ""), importRecord("committed", "")) c.verbs["commit"] = answerWith(http.StatusConflict, `{"error":"main.shop.example.com changed while this request was made. Read its state with carlos instances list and try again."}`) out, err := importRun(t, c, "/up", http.StatusOK) if err != nil || !strings.HasSuffix(out, " is served by the platform now.\n") { t.Fatalf("err = %v, output:\n%s", err, out) } } // An abort that answers 404 after the import's own record has gone is done. func TestImportAbortOfAFinishedImport(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), answerWith(http.StatusNotFound, `{"error":"x is not an instance of shop"}`)) c.verbs["abort"] = answerWith(http.StatusNotFound, `{"error":"x is not an instance of shop"}`) out, err := importRun(t, c, "/up", http.StatusBadGateway, http.StatusForbidden) if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "Aborting the import.\naborted. The box puts the old route of "+hostOf(out)+" back on its next pass.\n") { t.Fatalf("err = %v, output:\n%s", err, out) } } // The verb alone acts on the import the host holds now: it reads the // record first and names that seed (final review F1). A host with no // record names none, and the console's answer is the error. func TestImportVerbAlone(t *testing.T) { for _, c := range []struct { name, verb string answer func(http.ResponseWriter) record func(http.ResponseWriter) seed string // the seed the verb named want string wantErr string }{ {"commit", "commit", nil, importRecord("placed", ""), "sd_imp", "committed. main.shop.example.com is served by the platform now.", ""}, {"abort", "abort", nil, importRecord("swapped", ""), "sd_imp", "aborted. The box puts the old route of main.shop.example.com back on its next pass.", ""}, {"abort of a re-begun import", "abort", nil, importRecordID("sd_other", "uploaded", ""), "sd_other", "aborted. The box puts the old route of main.shop.example.com back on its next pass.", ""}, {"commit refused", "commit", answerWith(http.StatusConflict, `{"error":"main.shop.example.com is swapped. Commit it once it is placed."}`), importRecord("swapped", ""), "sd_imp", "", "main.shop.example.com is swapped. Commit it once it is placed."}, {"abort that lost to its own retry", "abort", answerWith(http.StatusConflict, `{"error":"changed"}`), importRecord("aborted", ""), "sd_imp", "aborted. The box puts the old route of main.shop.example.com back on its next pass.", ""}, {"abort of a host with no record", "abort", answerWith(http.StatusNotFound, `{"error":"main.shop.example.com is not an instance of shop"}`), answerWith(http.StatusNotFound, ""), "", "", "main.shop.example.com is not an instance of shop, so there is no import to abort. An import that was aborted and cleaned up leaves no instance behind."}, {"commit of a host with no record", "commit", answerWith(http.StatusNotFound, `{"error":"main.shop.example.com is not an instance of shop"}`), answerWith(http.StatusNotFound, ""), "", "", "main.shop.example.com is not an instance of shop"}, } { t.Run(c.name, func(t *testing.T) { ic := newImportConsole(c.record) if c.answer != nil { ic.verbs[c.verb] = c.answer } ts := httptest.NewServer(ic) defer ts.Close() var out bytes.Buffer err := newTestSession(ts.URL).runImportVerb(context.Background(), &out, apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", c.verb) if calls := ic.called(); len(calls) < 2 || calls[0] != "get" || calls[1] != c.verb { t.Fatalf("calls = %q, want the record read before the %s", calls, c.verb) } ic.requireSeeds(t, c.seed, 1) if c.wantErr != "" { if err == nil || err.Error() != c.wantErr || out.Len() != 0 { t.Fatalf("err = %v, output %q", err, out.String()) } return } if err != nil || out.String() != c.want+"\n" { t.Fatalf("err = %v, output %q", err, out.String()) } }) } } // The shared poller (M18): a bounded wait ends with DeadlineExceeded after // its share of looks; failed looks count as Plan 5's do. func TestPollSeedLimitAndFailures(t *testing.T) { waits := fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("ready", "")) ts := httptest.NewServer(c) defer ts.Close() s := newTestSession(ts.URL) _, err := s.pollSeed(context.Background(), apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", 10*time.Second, importLook("main.shop.example.com", "sd_imp", "swapped")) if !errors.Is(err, context.DeadlineExceeded) || len(*waits) != 5 { t.Fatalf("err = %v, waits = %v", err, *waits) } bad := newImportConsole(answerWith(http.StatusBadGateway, `{"error":"could not read the instance"}`)) ts2 := httptest.NewServer(bad) defer ts2.Close() _, err = newTestSession(ts2.URL).pollSeed(context.Background(), apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", 0, importLook("main.shop.example.com", "sd_imp", "swapped")) bad.mu.Lock() looks := bad.looks bad.mu.Unlock() if err == nil || err.Error() != "could not read the instance" || looks != seedPollFailures { t.Fatalf("err = %v, looks = %d", err, looks) } } // cancelOnWait makes the seedWait seam cancel ctx on the first wait that // matches, as Ctrl-C would mid-wait; earlier waits pass. func cancelOnWait(t *testing.T, cancel context.CancelFunc, match func(n int, d time.Duration) bool) { t.Helper() oldWait, oldEvery := seedWait, seedPollEvery n := 0 seedWait = func(ctx context.Context, d time.Duration) error { if n++; match(n, d) { cancel() } return ctx.Err() } seedPollEvery = 2 * time.Second t.Cleanup(func() { seedWait, seedPollEvery = oldWait, oldEvery }) } // wantCancelledLive is an interrupted import that is left live: the token // and the commands to finish by hand, no commit or abort, exit non-zero. func wantCancelledLive(t *testing.T, c *importConsole, out string, err error) { t.Helper() if !errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } // Without --verify the token was shown at begin; with it, just before // the commands. Either way once, and the commands last. if !strings.HasSuffix(out, "\n"+importCommitLine+"\n") || strings.Count(out, importTokenLine) != 1 || strings.Contains(out, "verify failed") { t.Fatalf("output:\n%s", out) } if c.saw("commit") || c.saw("abort") { t.Fatalf("calls = %v", c.called()) } } func TestImportInterruptedDuringTheSwapWait(t *testing.T) { for _, verify := range []string{"", "/up"} { t.Run("verify="+verify, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cancelOnWait(t, cancel, func(n int, _ time.Duration) bool { return n == 3 }) // the upload's look, then two swap looks c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", "")) out, err := importRunCtx(t, ctx, c, verify, http.StatusOK) wantCancelledLive(t, c, out, err) if !strings.Contains(out, "waiting for the box to swap main.shop.example.com\n") { t.Fatalf("output:\n%s", out) } }) } } // A verify cut short is not a verdict: no "verify failed", no abort. func TestImportInterruptedDuringTheVerifyRetries(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cancelOnWait(t, cancel, func(_ int, d time.Duration) bool { return d == importVerifyRetry }) c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", "")) out, err := importRunCtx(t, ctx, c, "/up", http.StatusServiceUnavailable) wantCancelledLive(t, c, out, err) } func TestImportInterruptedDuringTheUpload(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", "")) c.uploadHold = func(r *http.Request) { cancel() <-r.Context().Done() // the client gives up the request } out, err := importRunCtx(t, ctx, c, "/up", http.StatusOK) wantCancelledLive(t, c, out, err) if c.saw("stopped") || c.saw("get") { t.Fatalf("calls = %v", c.called()) } } func TestImportCancelledBeforeBeginJustExits(t *testing.T) { fakeSeedWait(t, 2*time.Second) ctx, cancel := context.WithCancel(context.Background()) cancel() c := newImportConsole() out, err := importRunCtx(t, ctx, c, "/up", http.StatusOK) if !errors.Is(err, context.Canceled) || errors.Is(err, errImportFailed) { t.Fatalf("err = %v", err) } wantExactLines(t, out, "uploading main.db (3B)") if len(c.called()) != 0 { t.Fatalf("calls = %v", c.called()) } } // An import committed by hand between two looks ends the swap wait with // SD-N75 rather than running to SD-N79 (review minor 3). func TestImportCommittedByHandDuringTheSwapWait(t *testing.T) { fakeSeedWait(t, 2*time.Second) c := newImportConsole(importRecord("uploaded", ""), importRecord("committed", "")) out, err := importRun(t, c, "") if err != nil || !strings.HasSuffix(out, "waiting for the box to swap main.shop.example.com\ncommitted. main.shop.example.com is served by the platform now.\n") { t.Fatalf("err = %v, output:\n%s", err, out) } } func TestImportVerifyTarget(t *testing.T) { for _, c := range []struct{ path, want string }{ {"/up", "https://main.shop.example.com/up"}, {"up", "https://main.shop.example.com/up"}, } { if got := importVerifyTarget("main.shop.example.com", c.path); got != c.want { t.Errorf("importVerifyTarget(%q) = %q, want %q", c.path, got, c.want) } } // The CLI refuses a --verify that cannot become a request before it // begins (review minor 2), with the parse the request itself uses. if _, err := url.Parse(importVerifyTarget("main.shop.example.com", "/%zz")); err == nil { t.Fatal("a path with a bad escape parsed") } if _, err := http.NewRequest(http.MethodGet, importVerifyTarget("main.shop.example.com", "/%zz"), nil); err == nil { t.Fatal("the request would have been made") } } // The import's own signal set: Ctrl-C, SIGTERM and SIGHUP cancel it, and // SIGPIPE never does, since a reset connection raises it mid-upload. func TestImportStopSignalsLeaveOutSIGPIPE(t *testing.T) { want := map[os.Signal]bool{os.Interrupt: true, syscall.SIGTERM: true, syscall.SIGHUP: true} if len(importStopSignals) != len(want) { t.Fatalf("importStopSignals = %v", importStopSignals) } for _, sig := range importStopSignals { if sig == syscall.SIGPIPE || !want[sig] { t.Fatalf("importStopSignals = %v, want exactly Interrupt, SIGTERM and SIGHUP", importStopSignals) } } }