package main // 🤖 The live end-to-end hibernation test ("the money test", #550 Part 4 v2 // Phase 2): a REAL instance process behind the REAL router, replicating to // REAL S3. It proves the whole loop on one box — wake fresh, write data // through the public API, idle out (final checkpoint + lease release), DELETE // the local DB, wake again (restore from S3), and read the data back. // // Gated on LCHAT_TEST_S3_BUCKET like the activator package's spike tests, so // `go test ./...` stays green without AWS credentials. Run it with: // // eval "$(aws configure export-credentials --profile eleven-dev --format env)" // LCHAT_TEST_S3_BUCKET= go test -run TestActivatorLiveHibernation -v . import ( "bytes" "context" "crypto/rand" "encoding/hex" "encoding/json" "fmt" "io" "net" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "lchat/internal/activator" ) func TestActivatorLiveHibernation(t *testing.T) { bucket := os.Getenv("LCHAT_TEST_S3_BUCKET") if bucket == "" { t.Skip("LCHAT_TEST_S3_BUCKET not set; skipping live S3 hibernation test") } region := os.Getenv("LCHAT_TEST_S3_REGION") if region == "" { region = "eu-west-1" } rb := make([]byte, 4) _, _ = rand.Read(rb) prefix := fmt.Sprintf("activator-live/%d-%s", time.Now().Unix(), hex.EncodeToString(rb)) t.Cleanup(func() { cleanupS3Prefix(t, bucket, prefix, region) }) const host = "hib-live.test" // Sandbox the box-layout globals (control.go / versions.go) in temp dirs. // The socket dir must stay short — sun_path is ~104 bytes on macOS. dataDir := t.TempDir() sockDir := fmt.Sprintf("/tmp/lchat-act-%d", os.Getpid()) if err := os.MkdirAll(sockDir, 0o755); err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(sockDir) }) optTmp := t.TempDir() oldData, oldSock, oldOpt := instanceDataDir, instanceSocketDir, optDir instanceDataDir, instanceSocketDir, optDir = dataDir, sockDir, optTmp t.Cleanup(func() { instanceDataDir, instanceSocketDir, optDir = oldData, oldSock, oldOpt }) // The activator execs the same live-symlinked binary a systemd unit would, // so build the real lchat binary (the test binary must never be spawned). bin := filepath.Join(optTmp, "lchat-built") build := exec.Command("go", "build", "-o", bin, ".") build.Env = append(os.Environ(), "GOTOOLCHAIN=auto") if out, err := build.CombinedOutput(); err != nil { t.Fatalf("go build: %v\n%s", err, out) } if err := stageVersion(bin, "livetest"); err != nil { t.Fatalf("stage: %v", err) } if err := pointLive(host, "livetest"); err != nil { t.Fatalf("point live: %v", err) } // The instance's EnvironmentFile, as add-instance would have written it. adminToken := "livetest-" + hex.EncodeToString(rb) if err := writeEnvFile(filepath.Join(dataDir, host+".env"), map[string]string{"LCHAT_ADMIN_TOKEN": adminToken}); err != nil { t.Fatalf("env file: %v", err) } reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } route := Route{Host: host, Kind: "instance", Socket: filepath.Join(sockDir, host+".sock"), DBPath: filepath.Join(dataDir, host+".db"), // Per-host opt-in: only hibernate-flagged routes are activator-managed. Hibernate: true} if err := reg.upsert(route); err != nil { t.Fatalf("upsert: %v", err) } cfg := hibernateConfig{Bucket: bucket, Region: region, Prefix: prefix, Idle: 3 * time.Second} act := newActivator(cfg, &s3Backend{cfg: cfg}, reg) act.start() t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() act.shutdown(ctx) }) rt, err := newRouter(reg, act) if err != nil { t.Fatalf("new router: %v", err) } front := httptest.NewServer(rt) t.Cleanup(front.Close) get := func(path string) (int, string) { t.Helper() req, _ := http.NewRequest("GET", front.URL+path, nil) req.Host = host resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("GET %s: %v", path, err) } defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) return resp.StatusCode, string(body) } // --- Act 1: first wake is a FRESH boot (nothing in S3, no local DB). --- start := time.Now() if code, _ := get("/api/group"); code != 200 { t.Fatalf("first wake: GET /api/group = %d, want 200", code) } t.Logf("wake #1 (fresh boot, no replica) served in %s", time.Since(start).Truncate(time.Millisecond)) // Write real data through the public path: the admin names the space. groupName := "Hibernation survives " + hex.EncodeToString(rb) body, _ := json.Marshal(map[string]string{"name": groupName}) req, _ := http.NewRequest("POST", front.URL+"/api/admin/group", bytes.NewReader(body)) req.Host = host req.Header.Set("Authorization", "Bearer "+adminToken) req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("set group name: %v", err) } if resp.StatusCode != 200 { b, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Fatalf("set group name: %d %s", resp.StatusCode, b) } resp.Body.Close() t.Logf("wrote group name %q through the admin API", groupName) // The due-time probe answers on the real instance (admin-gated; nothing // scheduled → 0). The sleep below re-runs the same call over the socket. nreq, _ := http.NewRequest("GET", front.URL+"/api/next-due", nil) nreq.Host = host nreq.Header.Set("Authorization", "Bearer "+adminToken) nresp, err := http.DefaultClient.Do(nreq) if err != nil { t.Fatalf("GET /api/next-due: %v", err) } nbody, _ := io.ReadAll(nresp.Body) nresp.Body.Close() if nresp.StatusCode != 200 || !strings.Contains(string(nbody), `"due":0`) { t.Fatalf("next-due probe: %d %s, want 200 with due 0", nresp.StatusCode, nbody) } // --- Act 2: idle out. Final checkpoint to S3, lease released, process gone. --- waitForLive(t, "hibernation after idle", 90*time.Second, func() bool { return act.entryCount() == 0 }) // The sleep recorded the (empty) due-time index in the registry. if rt, err := reg.get(host); err != nil || rt == nil || rt.NextDue != 0 { t.Fatalf("registry next_due after sleep = %+v, %v; want 0", rt, err) } if _, err := net.DialTimeout("unix", route.Socket, 300*time.Millisecond); err == nil { t.Fatal("instance socket still accepting after hibernation") } if _, err := os.Stat(route.DBPath); err != nil { t.Fatalf("local DB should remain as a warm cache after sleep: %v", err) } t.Logf("instance hibernated (idle %s); local DB left as warm cache", cfg.Idle) // --- Act 3: destroy the local state; the wake path must restore from S3. --- matches, _ := filepath.Glob(route.DBPath + "*") for _, m := range matches { if err := os.RemoveAll(m); err != nil { t.Fatalf("remove %s: %v", m, err) } } t.Logf("deleted local state: %v", matches) start = time.Now() code, groupJSON := get("/api/group") if code != 200 { t.Fatalf("wake from S3: GET /api/group = %d, want 200", code) } if !strings.Contains(groupJSON, groupName) { t.Fatalf("restored instance lost the data: %s does not contain %q", groupJSON, groupName) } t.Logf("wake #2 (restored from s3://%s/%s) served in %s; data survived: %s", bucket, prefix, time.Since(start).Truncate(time.Millisecond), groupJSON) // --- Act 4: idle out again and prove the lease was released cleanly. --- waitForLive(t, "second hibernation", 90*time.Second, func() bool { return act.entryCount() == 0 }) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() held, err := activator.AcquireLease(ctx, bucket, cfg.hibernatePrefix(host), region) if err != nil { t.Fatalf("lease not cleanly released after sleep: %v", err) } if err := held.Release(ctx); err != nil { t.Fatalf("release probe lease: %v", err) } t.Log("second sleep released the lease cleanly (probe acquire+release succeeded)") } // waitForLive is waitFor with a caller-chosen deadline — S3 round trips make // the live test's transitions slower than the fakes'. func waitForLive(t *testing.T, what string, timeout time.Duration, cond func() bool) { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if cond() { return } time.Sleep(100 * time.Millisecond) } t.Fatalf("timed out waiting for %s", what) } // cleanupS3Prefix deletes every object AND version under the test prefix so // live runs leave nothing behind (mirrors the activator package's helper). func cleanupS3Prefix(t *testing.T, bucket, prefix, region string) { t.Helper() ctx := context.Background() cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) if err != nil { t.Logf("cleanup aws config: %v", err) return } client := awss3.NewFromConfig(cfg) p := awss3.NewListObjectVersionsPaginator(client, &awss3.ListObjectVersionsInput{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), }) for p.HasMorePages() { out, err := p.NextPage(ctx) if err != nil { t.Logf("cleanup list versions: %v", err) return } var objs []s3types.ObjectIdentifier for _, v := range out.Versions { objs = append(objs, s3types.ObjectIdentifier{Key: v.Key, VersionId: v.VersionId}) } for _, m := range out.DeleteMarkers { objs = append(objs, s3types.ObjectIdentifier{Key: m.Key, VersionId: m.VersionId}) } if len(objs) == 0 { continue } if _, err := client.DeleteObjects(ctx, &awss3.DeleteObjectsInput{ Bucket: aws.String(bucket), Delete: &s3types.Delete{Objects: objs}, }); err != nil { t.Logf("cleanup delete: %v", err) } } }