package console import ( "bytes" "context" "log" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/mail" "github.com/carlosframework/platform/internal/release" ) // Spec §9: expire -> trash -> purge at 3 days, on a fake clock. func TestExpiryPassExpiresThenPurgesAtThreeDays(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second).Add(time.Hour) tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) fx.srv.expiryPass(ctx, now) tomb := markerOf(t, fx, "tmp-a") if tomb == nil || !tomb.DeletedAt.Equal(now) || tomb.Actor != expiryActor { t.Fatalf("marker = %+v, want one written by the pass at %v", tomb, now) } if l := lifeOf(t, fx, "tmp-a"); l.DeletedAt != now.Unix() || l.PurgeAfter != now.Unix()+72*3600 { t.Fatalf("row = %+v, want trashed with the 3-day window", l) } fx.srv.expiryPass(ctx, now.Add(72*time.Hour-time.Second)) if markerOf(t, fx, "tmp-a") == nil { t.Fatal("purged before the 3 days were up") } fx.srv.expiryPass(ctx, now.Add(72*time.Hour)) if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) != 0 { t.Fatalf("objects left after the early purge: %v", keys) } if l := lifeOf(t, fx, "tmp-a"); l != nil { t.Fatalf("row = %+v, want it gone with the name", l) } } // Review Focus 3. func TestExpiryBoundaryIsInclusive(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) tempApp(t, fx, "tmp-now", now) tempApp(t, fx, "tmp-later", now.Add(time.Second)) fx.srv.expiryPass(context.Background(), now) if l := lifeOf(t, fx, "tmp-now"); !l.Trashed() { t.Error("expires_at == now did not expire") } if l := lifeOf(t, fx, "tmp-later"); l.Trashed() { t.Error("expires_at == now+1s expired early") } } // Spec §9: a keep between candidate listing and lock acquisition; the pass // must skip. func TestExpirySkipsAKeepThatLandedAfterTheListing(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) rows, _ := fx.st.TemporaryAppRows() fx.st.KeepApp(OpsAccountID, "tmp-a") for _, r := range rows { fx.srv.expireOne(context.Background(), r, now, always(true)) } if markerOf(t, fx, "tmp-a") != nil { t.Fatal("the pass trashed an app a keep had made permanent") } if l := lifeOf(t, fx, "tmp-a"); l.Trashed() || l.ExpiresAt != 0 { t.Fatalf("row = %+v, want live and permanent", l) } } // Spec §9: failure at step 3 retries. func TestExpiryRetriesAFailedTrash(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) fx.srv.cfg.Releases = &hookStore{Store: fx.rel, failPut: map[string]int{release.AppTombstoneKey(fx.sqid, "tmp-a"): 1}} fx.srv.expiryPass(context.Background(), now) if markerOf(t, fx, "tmp-a") != nil || lifeOf(t, fx, "tmp-a").Trashed() { t.Fatal("a failed marker write changed something") } fx.srv.expiryPass(context.Background(), now.Add(5*time.Minute)) if l := lifeOf(t, fx, "tmp-a"); !l.Trashed() || l.DeletedAt != now.Add(5*time.Minute).Unix() { t.Fatalf("row after the retry = %+v", l) } } // Spec §9: restore -> keep -> trash between decision and purge; the stale // decision refuses through PurgeAppIfTrashedAt. func TestEarlyPurgeRefusesAMarkerFromAnotherTrash(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) decided := now.Add(-72 * time.Hour) trashedTemp(t, fx, "tmp-a", decided) // The bucket moved on: restored, kept and trashed again by hand. release.RestoreApp(ctx, fx.rel, fx.sqid, "tmp-a") release.TrashApp(ctx, fx.rel, fx.sqid, "tmp-a", "paul@keymail.dev", now.Add(-time.Hour)) rows, _ := fx.st.PurgeDueApps(now) for _, r := range rows { fx.srv.purgeOne(ctx, r, now, always(true)) } if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) < 2 { t.Fatalf("a stale decision destroyed objects: %v", keys) } if lifeOf(t, fx, "tmp-a") == nil { t.Fatal("a stale decision freed the name") } } // Spec §9: an interrupted purge resumes. func TestEarlyPurgeResumesAfterAnInterruption(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) fx.rel.Put(ctx, "apps/"+fx.sqid+"/tmp-a/releases/v1/site.tar.gz", []byte("x"), "") fx.srv.cfg.Releases = &hookStore{Store: fx.rel, failDelete: map[string]int{"apps/" + fx.sqid + "/tmp-a/releases/v1/site.tar.gz": 1}} fx.srv.expiryPass(ctx, now) if lifeOf(t, fx, "tmp-a") == nil || markerOf(t, fx, "tmp-a") == nil { t.Fatal("an interrupted purge lost its marker or its row") } fx.srv.expiryPass(ctx, now.Add(5*time.Minute)) if l := lifeOf(t, fx, "tmp-a"); l != nil { t.Fatalf("row = %+v after the resumed purge, want gone", l) } } // Spec §9: ErrNotTrashed with an empty prefix finalises; with objects, restores. // // The branch is only reached when the marker disappears AFTER purgeOne's // settle read it and BEFORE PurgeAppIfTrashedAt reads it: a marker gone // before the pass is repaired by the settle sweep and never gets that far. // So the hook deletes the marker on its second read. Reads of the marker key // in one pass over a due, consistent row: purgeOne's settleApp (1), then // PurgeAppIfTrashedAt's purgeReady (2). The sweep reads nothing for a row // whose trash state already agrees with the listing. func TestEarlyPurgeNotTrashedBranches(t *testing.T) { for _, objects := range []bool{false, true} { t.Run(map[bool]string{false: "no objects finalises", true: "objects restore"}[objects], func(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) if !objects { // The marker is the only key left under the prefix. kass's // objects keep the tripwire satisfied. if err := fx.rel.Delete(ctx, "apps/"+fx.sqid+"/tmp-a/channels/edge.json"); err != nil { t.Fatal(err) } } marker := release.AppTombstoneKey(fx.sqid, "tmp-a") reads := 0 fx.srv.cfg.Releases = &hookStore{Store: fx.rel, onGet: func(key string) { if key != marker { return } reads++ if reads == 2 { fx.rel.Delete(ctx, marker) } }} var logged bytes.Buffer log.SetOutput(&logged) t.Cleanup(func() { log.SetOutput(os.Stderr) }) fx.srv.expiryPass(ctx, now) if reads < 2 { t.Fatalf("the marker was read %d time(s); the purge never reached its own precondition read", reads) } if !strings.Contains(logged.String(), "left the trash before its purge") { t.Fatalf("the ErrNotTrashed branch did not run; log:\n%s", logged.String()) } l := lifeOf(t, fx, "tmp-a") if !objects { if l != nil { t.Fatalf("row = %+v, want finalised (no marker, no objects, evidence)", l) } return } if l == nil || l.Trashed() || l.Restoring || l.PurgeAfter != 0 || l.ExpiresAt != now.Add(temporaryTTLDefault).Unix() { t.Fatalf("row = %+v, want restored: live, no deadline, expiry now+7d", l) } if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) == 0 { t.Fatal("the restore branch destroyed objects") } }) } } // Spec §9: control-plane and email-estate backstops refuse without tearing down. func TestEarlyPurgeBackstops(t *testing.T) { ctx := context.Background() now := time.Now().Truncate(time.Second) t.Run("control plane", func(t *testing.T) { fx := newConsoleFixture(t) trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) // Unreachable through any door (Task 6); set by hand to prove the backstop. fx.st.db.Exec(`UPDATE apps SET control_plane = 1 WHERE app = 'tmp-a'`) fx.srv.expiryPass(ctx, now) if markerOf(t, fx, "tmp-a") == nil || lifeOf(t, fx, "tmp-a") == nil { t.Fatal("the pass purged the control plane") } }) t.Run("email estate", func(t *testing.T) { fx := newConsoleFixture(t) trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) putMailRecord(t, fx, "tmp-a", &mail.Record{ Declared: &mail.Declared{EnvPrefix: "CARLOS_SMTP", From: "hi@example.com", At: now}, Provisioned: &mail.Provisioned{KeyID: "AKIA1", Active: true, At: now}, }) fx.srv.expiryPass(ctx, now) if markerOf(t, fx, "tmp-a") == nil || lifeOf(t, fx, "tmp-a") == nil { t.Fatal("the pass purged an app with email estate") } }) } // Spec §9 "Empty bucket": startup materialize and several passes against an // empty replacement bucket, then one holding only trash markers. No marker is // written and both rows survive unchanged. func TestExpiryPassAgainstTheWrongBucket(t *testing.T) { for _, markersOnly := range []bool{false, true} { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-due", now.Add(-72*time.Hour)) tempApp(t, fx, "tmp-expired", now.Add(-time.Hour)) before := map[string]AppLife{"tmp-due": *lifeOf(t, fx, "tmp-due"), "tmp-expired": *lifeOf(t, fx, "tmp-expired")} dir := filepath.Join(t.TempDir(), "wrong") wrong := &deletableStore{Store: release.NewDirStore(dir), root: dir} if markersOnly { release.TrashApp(ctx, wrong, fx.sqid, "someone-else", "paul@keymail.dev", now) } fx.srv.cfg.Releases = wrong materializeQuietly(ctx, wrong, fx.st) for i := 0; i < 3; i++ { fx.srv.expiryPass(ctx, now.Add(time.Duration(i)*5*time.Minute)) } keys, _ := wrong.List(ctx, "apps/") if want := map[bool]int{false: 0, true: 1}[markersOnly]; len(keys) != want { t.Fatalf("markersOnly=%v: the wrong bucket now holds %v", markersOnly, keys) } for app, b := range before { if l := lifeOf(t, fx, app); l == nil || *l != b { t.Fatalf("markersOnly=%v: %s = %+v, want unchanged %+v", markersOnly, app, l, b) } } } } // Spec §9 "Settle sweep": a manual trash of a temporary app with 30 days left // whose row update failed. No restart, no user action: the next pass settles // it, and the early purge runs at 3 days. func TestSettleSweepRepairsAFailedManualTrash(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) tempApp(t, fx, "tmp-a", now.Add(30*24*time.Hour)) release.TrashApp(ctx, fx.rel, fx.sqid, "tmp-a", "paul@keymail.dev", now) fx.srv.expiryPass(ctx, now.Add(time.Minute)) if l := lifeOf(t, fx, "tmp-a"); l.DeletedAt != now.Unix() || l.PurgeAfter != now.Unix()+72*3600 { t.Fatalf("row after the sweep = %+v, want trashed at the marker's clock with the 3-day window", l) } fx.srv.expiryPass(ctx, now.Add(72*time.Hour)) if l := lifeOf(t, fx, "tmp-a"); l != nil { t.Fatalf("row = %+v, want purged at 3 days", l) } } // Spec §9: an early purge whose final marker delete fails, with no other keys // in the bucket. The next pass trips the tripwire, still retries the purge, // finishes, and removes the row. func TestEarlyPurgeOfTheLastAppRetriesUnderTheTripwire(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) dir := filepath.Join(t.TempDir(), "only") only := &deletableStore{Store: release.NewDirStore(dir), root: dir} fx.srv.cfg.Releases = only fx.rel = only trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) marker := release.AppTombstoneKey(fx.sqid, "tmp-a") fx.srv.cfg.Releases = &hookStore{Store: only, failDelete: map[string]int{marker: 1}} fx.srv.expiryPass(ctx, now) if keys, _ := only.List(ctx, "apps/"); len(keys) != 1 || keys[0] != marker { t.Fatalf("after the first pass the bucket holds %v, want only the marker", keys) } if lifeOf(t, fx, "tmp-a") == nil { t.Fatal("the row went before the marker") } fx.srv.expiryPass(ctx, now.Add(5*time.Minute)) if keys, _ := only.List(ctx, "apps/"); len(keys) != 0 { t.Fatalf("the retry left %v", keys) } if l := lifeOf(t, fx, "tmp-a"); l != nil { t.Fatalf("row = %+v, want gone", l) } } // Task 7 review: a restore whose marker delete landed and whose row update // failed is finished by the settle sweep, not by the door. A door retry after // that answers "not in the trash" and audits nothing, so the sweep must say // what it did, and bump the serial so hosts bring the routes back promptly. func TestSettleSweepReportsACatchUpRestore(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) if ok, err := fx.st.SetRestoring(OpsAccountID, "tmp-a", true); err != nil || !ok { t.Fatal(ok, err) } if err := release.RestoreApp(ctx, fx.rel, fx.sqid, "tmp-a"); err != nil { t.Fatal(err) } serialBefore, _, _ := fx.rel.Get(ctx, release.SerialKey) var logged bytes.Buffer log.SetOutput(&logged) t.Cleanup(func() { log.SetOutput(os.Stderr) }) fx.srv.expiryPass(ctx, now) if l := lifeOf(t, fx, "tmp-a"); l == nil || l.Trashed() || l.Restoring { t.Fatalf("row = %+v, want the restore finished", l) } if !strings.Contains(logged.String(), "expiry pass: finished a restore of "+fx.sqid+"/tmp-a") { t.Fatalf("the catch-up restore was silent; log:\n%s", logged.String()) } if serialAfter, _, _ := fx.rel.Get(ctx, release.SerialKey); string(serialAfter) == string(serialBefore) { t.Fatal("the catch-up restore did not bump the serial") } }