package boxvolumes import ( "context" "errors" "strings" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" smithymiddleware "github.com/aws/smithy-go/middleware" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/release" ) var ( cloud = boxid.Cloud{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"} entry = Entry{Box: "annie-welding", Enrollment: "enr-annie-1", Cloud: cloud} writer = WrittenBy{Box: "bert-sawing", Enrollment: "enr-bert-1"} t0 = time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) ) type vol struct { dev, id string dot bool } func reply(owner string, state types.InstanceStateName, vols ...vol) *ec2.DescribeInstancesOutput { var maps []types.InstanceBlockDeviceMapping for _, v := range vols { maps = append(maps, types.InstanceBlockDeviceMapping{DeviceName: aws.String(v.dev), Ebs: &types.EbsInstanceBlockDevice{VolumeId: aws.String(v.id), DeleteOnTermination: aws.Bool(v.dot), Status: types.AttachmentStatusAttached}}) } var md smithymiddleware.Metadata awsmiddleware.SetRequestIDMetadata(&md, "req-1") return &ec2.DescribeInstancesOutput{ResultMetadata: md, Reservations: []types.Reservation{{OwnerId: aws.String(owner), Instances: []types.Instance{{InstanceId: aws.String("i-0abc"), State: &types.InstanceState{Name: state}, BlockDeviceMappings: maps}}}}} } var root, data, scratch = vol{"/dev/xvda", "vol-root", true}, vol{"/dev/xvdb", "vol-data", true}, vol{"/dev/xvdc", "vol-keep", false} func observe(t *testing.T, out *ec2.DescribeInstancesOutput) Observation { t.Helper() o, err := Observe(out, cloud) if err != nil { t.Fatal(err) } return o } func TestObserveReadsEBSMappingsInVolumeOrder(t *testing.T) { out := reply("111122223333", types.InstanceStateNameRunning, root, data) out.Reservations[0].Instances[0].BlockDeviceMappings = append(out.Reservations[0].Instances[0].BlockDeviceMappings, types.InstanceBlockDeviceMapping{DeviceName: aws.String("/dev/sdz")}) // instance store: no Ebs o := observe(t, out) want := []Device{{"/dev/xvdb", "vol-data", true, "attached"}, {"/dev/xvda", "vol-root", true, "attached"}} if o.State != "running" || o.RequestID != "req-1" || len(o.Devices) != 2 || o.Devices[0] != want[0] || o.Devices[1] != want[1] { t.Fatalf("observation = %+v", o) } } // Review Focus 3: a terminated instance's reply has no mappings. Recorded, // it would claim every volume was removed; it must not be recordable. func TestObserveRefusesWhatIsNotTheInstancesMapping(t *testing.T) { for name, out := range map[string]*ec2.DescribeInstancesOutput{ "terminated": reply("111122223333", types.InstanceStateNameTerminated), "another owner": reply("999999999999", types.InstanceStateNameRunning, root), "another instance": {Reservations: []types.Reservation{{OwnerId: aws.String("111122223333"), Instances: []types.Instance{{InstanceId: aws.String("i-0zzz")}}}}}, "empty reply": {}, "nil reply": nil, } { t.Run(name, func(t *testing.T) { if o, err := Observe(out, cloud); !errors.Is(err, ErrNotRecordable) { t.Fatalf("observation = %+v, err = %v; want ErrNotRecordable", o, err) } }) } // A shutting-down read still lists the mappings and may be recorded. if o := observe(t, reply("111122223333", types.InstanceStateNameShuttingDown, root)); len(o.Devices) != 1 { t.Fatalf("shutting-down: %+v", o) } } func TestNextWritesOnChangeAndAtLeastHourly(t *testing.T) { first, ok := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0) if !ok || first.ObservedAt != "2026-10-06T09:00:00Z" || len(first.RemovedVolumes) != 0 || first.RemovedVolumes == nil { t.Fatalf("first = %+v ok=%v", first, ok) } same := observe(t, reply("111122223333", types.InstanceStateNameRunning, root)) if _, ok := Next(&first, entry, writer, same, t0.Add(59*time.Minute)); ok { t.Fatal("wrote an unchanged mapping inside the hour") } if _, ok := Next(&first, entry, writer, same, t0.Add(time.Hour)); !ok { t.Fatal("no record after an hour") } grown, ok := Next(&first, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root, scratch)), t0.Add(time.Minute)) if !ok || len(grown.BlockDevices) != 2 { t.Fatalf("an attached volume was not recorded: %+v", grown) } shrunk, ok := Next(&grown, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0.Add(2*time.Minute)) if !ok || len(shrunk.RemovedVolumes) != 1 || shrunk.RemovedVolumes[0] != "vol-keep" { t.Fatalf("a detached volume is not in removed_volumes: %+v", shrunk) } if _, ok := Next(&first, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameShuttingDown, root)), t0.Add(time.Minute)); !ok { t.Fatal("a state change was not recorded") } } func TestWriteIsCreateOnlyAndListReturnsObservationOrder(t *testing.T) { ctx, st := context.Background(), release.NewDirStore(t.TempDir()) a, _ := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0) b, _ := Next(&a, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root, data)), t0.Add(10*time.Minute)) for _, r := range []Record{b, a} { if _, err := Write(ctx, st, r); err != nil { t.Fatal(err) } } if _, err := Write(ctx, st, a); err != nil { t.Fatalf("the same record twice: %v", err) } other := a other.RequestID = "req-other" if _, err := Write(ctx, st, other); !errors.Is(err, release.ErrCASConflict) { t.Fatalf("a different record under the same key: err = %v", err) } got, err := List(ctx, st, entry.Enrollment) if err != nil { t.Fatal(err) } if len(got) != 2 || got[0].ObservedAt != a.ObservedAt || got[1].ObservedAt != b.ObservedAt { t.Fatalf("list = %+v", got) } if _, err := Write(ctx, st, Record{Enrollment: "../edges", ObservedAt: a.ObservedAt}); err == nil { t.Fatal("wrote outside the prefix") } } func TestListFailsOnARecordItCannotRead(t *testing.T) { ctx, st := context.Background(), release.NewDirStore(t.TempDir()) if _, err := st.Put(ctx, Key(entry.Enrollment, t0), []byte(`{"v":1,"surprise":true}`), ""); err != nil { t.Fatal(err) } if _, err := List(ctx, st, entry.Enrollment); err == nil { t.Fatal("an unreadable record read as a gap") } } func records(t *testing.T, steps ...[]vol) []Record { t.Helper() var out []Record var prev *Record for i, vs := range steps { rec, _ := Next(prev, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, vs...)), t0.Add(time.Duration(i)*10*time.Minute)) out = append(out, rec) prev = &out[len(out)-1] } return out } func TestDischarge(t *testing.T) { enrolled := t0.Add(-5 * time.Minute) terminated := t0.Add(30 * time.Minute) if v := Discharge(records(t, []vol{root}, []vol{root, data}, []vol{root, data}), entry, enrolled, terminated); !v.Discharge { t.Fatalf("all true, nothing removed: %+v", v) } cases := map[string]struct { recs []Record at time.Time }{ "no record at all": {nil, enrolled}, "a false flag": {records(t, []vol{root}, []vol{root, scratch}), enrolled}, "a removed volume": {records(t, []vol{root, data}, []vol{root}), enrolled}, "a replaced root volume": {records(t, []vol{root}, []vol{{"/dev/xvda", "vol-root2", true}}), enrolled}, "first record long after": {records(t, []vol{root}), t0.Add(-2 * time.Hour)}, "only another enrollment's": {func() []Record { r := records(t, []vol{root}); r[0].Enrollment = "enr-other"; return r }(), enrolled}, "only another instance's": {func() []Record { r := records(t, []vol{root}); r[0].Cloud.InstanceID = "i-0zzz"; return r }(), enrolled}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { if v := Discharge(tc.recs, entry, tc.at, terminated); v.Discharge || v.Reason == "" { t.Fatalf("verdict = %+v, want the chores left open with a reason", v) } }) } } // A removal is caught from the sequence even when the record that should // have listed it was never written (two workers, one missed write). func TestDischargeSeesARemovalNoRecordListed(t *testing.T) { recs := records(t, []vol{root, data}, []vol{root}) recs[1].RemovedVolumes = []string{} if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); v.Discharge { t.Fatal("discharged across a silent removal") } } // Controller ruling, 2026-10-06: the newest record must be no more than // LastRecordWindow older than the termination observation. A recorder that // stopped long before the box died cannot speak for its last hours. func TestDischargeNeedsARecentLastRecord(t *testing.T) { recs := records(t, []vol{root}, []vol{root}) // newest at t0+10m newest := t0.Add(10 * time.Minute) if v := Discharge(recs, entry, t0, newest.Add(LastRecordWindow)); !v.Discharge { t.Fatalf("exactly at the window: %+v", v) } if v := Discharge(recs, entry, t0, newest.Add(LastRecordWindow+time.Second)); v.Discharge || v.Reason == "" { t.Fatalf("a newest record older than the window discharged: %+v", v) } } // Astra plan R1-6: what List returns is whatever is stored. A stored record // that is not evidence of a running instance's mappings keeps the chores // open, however it got there. Each bad record sits beside a good one so // that no removal is seen between them: only the record's own check can // refuse it. func TestDischargeRefusesStoredRecordsThatAreNotEvidence(t *testing.T) { ctx := context.Background() good := records(t, []vol{root}) const before = `"enrollment":"enr-annie-1","cloud":{"provider":"aws","account":"111122223333","region":"eu-west-1","instance_id":"i-0abc"},"observed_at":"2026-10-06T08:55:00Z"` const after = `"enrollment":"enr-annie-1","cloud":{"provider":"aws","account":"111122223333","region":"eu-west-1","instance_id":"i-0abc"},"observed_at":"2026-10-06T09:05:00Z"` type bad struct { body string at time.Duration // from t0; the good record is at t0 } cases := map[string]bad{ // Before the good record: an empty mapping followed by one volume is // an attach, not a removal. "only the binding fields": {`{"v":1,` + before + `}`, -5 * time.Minute}, "a terminated read": {`{"v":1,` + before + `,"instance_state":"terminated","block_devices":[],"removed_volumes":[]}`, -5 * time.Minute}, // After it, keeping vol-root, so nothing is removed. "a volume with no id": {`{"v":1,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"},{"device":"/dev/xvdb","volume_id":"","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute}, "a version 2 record": {`{"v":2,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute}, // Astra plan R2-2: one case per check, each failing only that check. "a running read with no volume": {`{"v":1,` + before + `,"instance_state":"running","block_devices":[],"removed_volumes":[]}`, -5 * time.Minute}, "a terminated read with its root mapping": {`{"v":1,` + after + `,"instance_state":"terminated","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute}, "a volume with no device": {`{"v":1,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"},{"device":"","volume_id":"vol-data","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute}, } const goodAfter = `{"v":1,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}` swap := func(old, new string) string { if !strings.Contains(goodAfter, old) { t.Fatalf("no %q in the template", old) } return strings.Replace(goodAfter, old, new, 1) } cases["a detaching volume"] = bad{swap(`"status":"attached"`, `"status":"detaching"`), 5 * time.Minute} cases["a detached volume"] = bad{swap(`"status":"attached"`, `"status":"detached"`), 5 * time.Minute} cases["no status"] = bad{swap(`"status":"attached"`, `"status":""`), 5 * time.Minute} cases["no enrollment"] = bad{swap(`"enrollment":"enr-annie-1"`, `"enrollment":""`), 5 * time.Minute} cases["another enrollment"] = bad{swap(`"enrollment":"enr-annie-1"`, `"enrollment":"enr-other"`), 5 * time.Minute} cases["another instance"] = bad{swap(`"instance_id":"i-0abc"`, `"instance_id":"i-0zzz"`), 5 * time.Minute} cases["a region in another case"] = bad{swap(`"region":"eu-west-1"`, `"region":"EU-WEST-1"`), 5 * time.Minute} cases["an unlabelled version 2 record"] = bad{`{"v":2}`, 5 * time.Minute} for name, tc := range cases { t.Run(name, func(t *testing.T) { st := release.NewDirStore(t.TempDir()) if _, err := Write(ctx, st, good[0]); err != nil { t.Fatal(err) } if _, err := st.Put(ctx, Key(entry.Enrollment, t0.Add(tc.at)), []byte(tc.body), ""); err != nil { t.Fatal(err) } recs, err := List(ctx, st, entry.Enrollment) if err != nil { t.Fatal(err) } if len(recs) != 2 { t.Fatalf("listed %d records, want 2", len(recs)) } if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); v.Discharge || v.Reason == "" { t.Fatalf("verdict = %+v, want the chores left open", v) } }) } // The good record alone, stored and listed, still discharges. st := release.NewDirStore(t.TempDir()) if _, err := Write(ctx, st, good[0]); err != nil { t.Fatal(err) } recs, _ := List(ctx, st, entry.Enrollment) if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); !v.Discharge { t.Fatalf("a stored good record: %+v", v) } } // A gap longer than LastRecordWindow between two records is a stretch the // recorder did not cover. func TestDischargeRefusesAnInteriorGap(t *testing.T) { a, _ := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0) late := t0.Add(30 * 24 * time.Hour) b, _ := Next(&a, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), late) if v := Discharge([]Record{a, b}, entry, t0, late.Add(time.Hour)); v.Discharge || v.Reason == "" { t.Fatalf("a 30 day gap discharged: %+v", v) } c, _ := Next(&a, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0.Add(LastRecordWindow)) if v := Discharge([]Record{a, c}, entry, t0, t0.Add(LastRecordWindow)); !v.Discharge { t.Fatalf("a gap of exactly the window: %+v", v) } } // Records must lie between enrollment and termination, and both times must // be real and in order. func TestDischargeRefusesTimesThatDoNotFit(t *testing.T) { recs := records(t, []vol{root}, []vol{root}) // t0, t0+10m var zero time.Time cases := map[string]struct{ enrolled, terminated time.Time }{ "zero termination": {t0, zero}, "zero enrollment": {zero, t0.Add(time.Hour)}, "terminated before enrolled": {t0.Add(time.Hour), t0}, "a record before enrollment": {t0.Add(time.Minute), t0.Add(time.Hour)}, "a record after termination": {t0, t0.Add(5 * time.Minute)}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { if v := Discharge(recs, entry, tc.enrolled, tc.terminated); v.Discharge || v.Reason == "" { t.Fatalf("verdict = %+v", v) } }) } } // Final review 1: as EC2 shuts an instance down, its DeleteOnTermination // volumes show "detaching" on the way to deletion. A shutting-down record // listing them is the normal last record of a box and must discharge; a // detaching volume with a false flag, or one in any other state, must not. func TestDischargeAcceptsDetachingDeleteOnTerminationVolumesWhileShuttingDown(t *testing.T) { running, _ := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root, data)), t0) shutting := func(dot bool) Record { out := reply("111122223333", types.InstanceStateNameShuttingDown, root, vol{data.dev, data.id, dot}) for _, m := range out.Reservations[0].Instances[0].BlockDeviceMappings { m.Ebs.Status = types.AttachmentStatusDetaching } rec, ok := Next(&running, entry, writer, observe(t, out), t0.Add(10*time.Minute)) if !ok { t.Fatal("the shutting-down read was not a new record") } return rec } if v := Discharge([]Record{running, shutting(true)}, entry, t0, t0.Add(15*time.Minute)); !v.Discharge { t.Fatalf("shutting down, every volume detaching and DeleteOnTermination: %+v", v) } if v := Discharge([]Record{running, shutting(false)}, entry, t0, t0.Add(15*time.Minute)); v.Discharge || v.Reason == "" { t.Fatalf("shutting down with a detaching volume that is not DeleteOnTermination: %+v", v) } if reason := incomplete(shutting(false)); reason == "" { t.Fatal("incomplete accepted a detaching volume with a false flag") } stopping := shutting(true) stopping.InstanceState = "stopping" if v := Discharge([]Record{running, stopping}, entry, t0, t0.Add(15*time.Minute)); v.Discharge || v.Reason == "" { t.Fatalf("a stopping record with detaching volumes: %+v", v) } detached := shutting(true) detached.BlockDevices[0].Status = "detached" if v := Discharge([]Record{running, detached}, entry, t0, t0.Add(15*time.Minute)); v.Discharge || v.Reason == "" { t.Fatalf("a shutting-down record with a detached volume: %+v", v) } }