package hostfence import ( "context" "crypto/ed25519" "crypto/rand" "encoding/json" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/carlosframework/platform/internal/awsinv" "github.com/carlosframework/platform/internal/boxvolumes" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "strings" "testing" "time" ) type terminatedEC2 struct { state types.InstanceStateName missing bool owner string after func() } func (f terminatedEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { out := &ec2.DescribeInstancesOutput{} if !f.missing { out.Reservations = []types.Reservation{{OwnerId: aws.String(f.owner), Instances: []types.Instance{{InstanceId: aws.String(in.InstanceIds[0]), State: &types.InstanceState{Name: f.state}}}}} } if f.after != nil { f.after() } awsmiddleware.SetRequestIDMetadata(&out.ResultMetadata, "request") return out, nil } func terminationFixture(t *testing.T) (TerminationWorker, release.Store) { t.Helper() ctx := context.Background() pub, key, _ := ed25519.GenerateKey(rand.Reader) s := release.NewDirStore(t.TempDir()) now := time.Now().UTC() r := hostactors.Roster{V: 1, Revision: 1, DeploymentBucket: "bucket", Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} for i, ref := range []string{"worker", "dead"} { en := strings.Repeat(string(rune('a'+i)), 32) r.Boxes[ref] = hostactors.Box{Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("e", 64), State: "active"} } raw, _ := release.SignRootEnvelope(release.RootSigner(key), r) if _, err := s.Put(ctx, hostactors.Key, raw, ""); err != nil { t.Fatal(err) } return TerminationWorker{Fences: s, Enabled: true, Directory: hostactors.Directory{Store: s, PublicKey: pub, Bucket: "bucket"}, Self: Worker{Box: "worker", Enrollment: r.Boxes["worker"].Enrollment}, Now: func() time.Time { return now }}, s } func TestTerminationAloneFencesAndAbsenceNeverDoes(t *testing.T) { for _, test := range []struct { name string state types.InstanceStateName missing bool owner string ok bool }{ {"terminated-no-volumes", types.InstanceStateNameTerminated, false, "123", true}, {"stopped", types.InstanceStateNameStopped, false, "123", false}, {"shutting-down", types.InstanceStateNameShuttingDown, false, "123", false}, {"absent", types.InstanceStateNameTerminated, true, "123", false}, {"wrong-account", types.InstanceStateNameTerminated, false, "456", false}, } { t.Run(test.name, func(t *testing.T) { w, s := terminationFixture(t) w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) { return terminatedEC2{state: test.state, missing: test.missing, owner: test.owner}, nil } ref, err := w.Describe(context.Background(), "dead") if (err == nil) != test.ok { t.Fatal(ref, err) } if test.ok { raw, _, err := s.Get(context.Background(), ref.Key) if err != nil { t.Fatal(err) } var f Termination _ = json.Unmarshal(raw, &f) old, err := time.Parse(time.RFC3339Nano, f.ObservedAt) if err != nil { t.Fatal(err) } w.Now = func() time.Time { return old.Add(time.Hour) } again, err := w.Describe(context.Background(), "dead") if err != nil || again != ref { t.Fatal("retry moved evidence", again, err) } } else { keys, _ := s.List(context.Background(), "control/box-fences/") if len(keys) != 0 { t.Fatal("refusal wrote a fence") } } }) } } func TestDestroyedLocalIsIndependentAndNeedsHistory(t *testing.T) { now := time.Now().UTC() enrolledAt := now.Add(-time.Minute) e := boxvolumes.Entry{Box: "dead", Enrollment: "enrollment", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-dead"}} if boxvolumes.Discharge(nil, e, enrolledAt, now).Discharge { t.Fatal("empty terminated mappings proved destroyed storage") } r := boxvolumes.Record{V: 1, Box: e.Box, Enrollment: e.Enrollment, Cloud: e.Cloud, InstanceState: "running", ObservedAt: enrolledAt.Format(time.RFC3339Nano), RequestID: "request", BlockDevices: []boxvolumes.Device{{Device: "root", VolumeID: "volume", DeleteOnTermination: true, Status: "attached"}}} if !boxvolumes.Discharge([]boxvolumes.Record{r}, e, enrolledAt, now).Discharge { t.Fatal("complete true history refused") } r.BlockDevices[0].DeleteOnTermination = false if boxvolumes.Discharge([]boxvolumes.Record{r}, e, enrolledAt, now).Discharge { t.Fatal("surviving volume discharged") } } func TestFencedWorkerCannotProduceNewEvidence(t *testing.T) { w, s := terminationFixture(t) o, err := w.Directory.Read(context.Background()) if err != nil { t.Fatal(err) } self := o.Roster.Boxes["worker"] dead := o.Roster.Boxes["dead"] fence := Termination{V: 1, Box: "worker", Enrollment: self.Enrollment, Cloud: self.Cloud, State: "terminated", ObservedAt: w.Now().Format(time.RFC3339Nano), Via: "describe", RequestID: "request", WrittenBy: Worker{Box: "dead", Enrollment: dead.Enrollment}} raw, _ := json.Marshal(fence) if _, err := s.Put(context.Background(), FenceKey(self.Enrollment), raw, ""); err != nil { t.Fatal(err) } w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) { return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil } if _, err := w.Describe(context.Background(), "dead"); err == nil { t.Fatal("fenced worker produced new execution authority") } keys, err := s.List(context.Background(), "control/box-fences/") if err != nil || len(keys) != 1 { t.Fatal("refused worker wrote another fence", keys, err) } } func TestTerminationReferenceIncludesExactEnrollmentAndBytes(t *testing.T) { w, s := terminationFixture(t) w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) { return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil } ref, err := w.Describe(context.Background(), "dead") if err != nil { t.Fatal(err) } raw, _, err := s.Get(context.Background(), ref.Key) if err != nil { t.Fatal(err) } var fence Termination if json.Unmarshal(raw, &fence) != nil || ref.Enrollment != fence.Enrollment || ref.Key != FenceKey(fence.Enrollment) || ref.SHA256 != hostfloor.Digest(raw) { t.Fatal("reference is not bound to the verified enrollment and bytes", ref) } wire, err := json.Marshal(ref) if err != nil { t.Fatal(err) } var fields map[string]string if json.Unmarshal(wire, &fields) != nil || len(fields) != 3 || fields["enrollment"] != fence.Enrollment || fields["key"] != ref.Key || fields["sha256"] != ref.SHA256 { t.Fatal("journal reference lacks the adopted D2 wire contract", string(wire)) } } func TestTerminationFenceTimeIsAfterTheVerifiedRead(t *testing.T) { w, s := terminationFixture(t) start := w.Now() now := start w.Now = func() time.Time { return now } w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) { return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123", after: func() { now = start.Add(time.Minute) }}, nil } ref, err := w.Describe(context.Background(), "dead") if err != nil { t.Fatal(err) } raw, _, err := s.Get(context.Background(), ref.Key) if err != nil { t.Fatal(err) } var fence Termination if err = json.Unmarshal(raw, &fence); err != nil { t.Fatal(err) } if fence.ObservedAt != now.Format(time.RFC3339Nano) { t.Fatal("fence time predates completion of the terminal-state read", fence.ObservedAt, now) } }