// Package termfence decides whether an observation proves that a box's // machine has been terminated (region HA spec 1 §5.9, invariants HA7 and // H19). // // Only the state "terminated" counts, only for the instance, account and // region the box's roster entry records under cloud, and only when the // worker that read it is in that same region. Silence is never evidence, // and no disk condition applies: volume records decide only whether the // dead box's local chores are discharged (internal/boxvolumes). // // This package reads and decides. Writing the fence record and citing it // in a claim CAS wait for the claim's acceptance of L2 to L5. package termfence import ( "context" "encoding/json" "errors" "fmt" "time" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/smithy-go" "github.com/carlosframework/platform/internal/boxid" ) const ( ViaDescribe = "describe" ViaEvent = "event" stateTerminated = "terminated" eventSource = "aws.ec2" eventDetailType = "EC2 Instance State-change Notification" ) // ErrNotEvidence wraps every reason an observation does not fence a box. var ErrNotEvidence = errors.New("termfence: not evidence of termination") // Observation is what one source said about one instance. Found is false // when the source said nothing about it: an empty reply, another instance. type Observation struct { Via string Found bool InstanceID string Account string Region string State string EventID string RequestID string } // Box is a roster entry as the fence needs it: the reference, the // enrollment, and the cloud identity enrollment proved (boxid.Enroll). type Box struct { Ref string Enrollment string Cloud boxid.Cloud } func (b Box) complete() bool { return b.Ref != "" && b.Enrollment != "" && b.Cloud.Complete() } // WrittenBy names the regional worker that made the read. type WrittenBy struct { Box string `json:"box"` Enrollment string `json:"enrollment"` } // Record is §5.9's fence record. Its key (control/box-fences/.json) // and its writer belong to the claim's L4 and are not built here. type Record struct { V int `json:"v"` Box string `json:"box"` Enrollment string `json:"enrollment"` Cloud boxid.Cloud `json:"cloud"` State string `json:"state"` ObservedAt string `json:"observed_at"` Via string `json:"via"` RequestID string `json:"request_id,omitempty"` EventID string `json:"event_id,omitempty"` WrittenBy WrittenBy `json:"written_by"` } // FromDescribe reads one instance out of a DescribeInstances reply. region // is the region the call was made in: the reply does not carry it. A reply // that does not mention instanceID yields Found false, which Verify // refuses: AWS forgets a terminated instance within about an hour, so // absence proves nothing. func FromDescribe(out *ec2.DescribeInstancesOutput, instanceID, region string) Observation { o := Observation{Via: ViaDescribe, Region: region} if out == nil || instanceID == "" { return o } if id, ok := awsmiddleware.GetRequestIDMetadata(out.ResultMetadata); ok { o.RequestID = id } for _, r := range out.Reservations { for _, inst := range r.Instances { if aws.ToString(inst.InstanceId) != instanceID { continue } o.Found, o.InstanceID, o.Account = true, instanceID, aws.ToString(r.OwnerId) if inst.State != nil { o.State = string(inst.State.Name) } return o } } return o } // FromEvent reads an EventBridge "EC2 Instance State-change Notification". // It refuses anything else: the queue's policy admits only the rule that // forwards these, and this is the second check. func FromEvent(body []byte) (Observation, error) { var ev struct { ID string `json:"id"` DetailType string `json:"detail-type"` Source string `json:"source"` Account string `json:"account"` Region string `json:"region"` Detail struct { InstanceID string `json:"instance-id"` State string `json:"state"` } `json:"detail"` } if err := json.Unmarshal(body, &ev); err != nil { return Observation{}, fmt.Errorf("termfence: event is not JSON: %w", err) } switch { case ev.Source != eventSource: return Observation{}, fmt.Errorf("termfence: event source %q is not %s", ev.Source, eventSource) case ev.DetailType != eventDetailType: return Observation{}, fmt.Errorf("termfence: event detail-type %q is not a state change", ev.DetailType) case ev.ID == "": return Observation{}, errors.New("termfence: event has no id") } return Observation{Via: ViaEvent, Found: ev.Detail.InstanceID != "", InstanceID: ev.Detail.InstanceID, Account: ev.Account, Region: ev.Region, State: ev.Detail.State, EventID: ev.ID}, nil } // EC2 is the one call this package makes. *ec2.Client satisfies it; tests // use a fake. It is a read: the box role holds no EC2 write right. type EC2 interface { DescribeInstances(ctx context.Context, in *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) } // DescribeInstance asks EC2 about c.InstanceID alone, in c.Region whatever // region the client was built for. InvalidInstanceID.NotFound comes back // as an empty reply, which FromDescribe turns into Found false: absence, // never termination. Every other error is returned. func DescribeInstance(ctx context.Context, api EC2, c boxid.Cloud) (*ec2.DescribeInstancesOutput, error) { if !c.Complete() { return nil, errors.New("termfence: describe of an incomplete cloud identity") } region := c.Region out, err := api.DescribeInstances(ctx, &ec2.DescribeInstancesInput{InstanceIds: []string{c.InstanceID}}, func(o *ec2.Options) { o.Region = region }) var apiErr smithy.APIError if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidInstanceID.NotFound" { return &ec2.DescribeInstancesOutput{}, nil } return out, err } func refuse(format string, a ...any) error { return fmt.Errorf("%w: %s", ErrNotEvidence, fmt.Sprintf(format, a...)) } // Verify returns box's fence record when o proves that the instance its // roster entry records is terminated, read by worker, a regional worker of // the same AWS region (spec §6: the fence refuses any observation not made // in the box's own region). Whether worker is active in the roster is the // caller's check, made before it reads. func Verify(box, worker Box, o Observation, now time.Time) (Record, error) { c := box.Cloud switch { case !box.complete(): return Record{}, refuse("the box has no proven cloud identity") case !worker.complete(): return Record{}, refuse("the worker has no proven cloud identity") case worker.Cloud.Region != c.Region: return Record{}, refuse("worker %s is in %q, the box in %q", worker.Ref, worker.Cloud.Region, c.Region) case o.Via != ViaDescribe && o.Via != ViaEvent: return Record{}, refuse("source %q is not one this binary accepts", o.Via) case !o.Found: return Record{}, refuse("the source said nothing about %s; absence is not termination", c.InstanceID) case o.InstanceID != c.InstanceID: return Record{}, refuse("observed %s, the box is %s", o.InstanceID, c.InstanceID) case o.Account != c.Account: return Record{}, refuse("observed in account %q, the box is in %q", o.Account, c.Account) case o.Region != c.Region: return Record{}, refuse("observed in region %q, the box is in %q", o.Region, c.Region) case o.State != stateTerminated: return Record{}, refuse("state is %q; only %q counts", o.State, stateTerminated) } return Record{V: 1, Box: box.Ref, Enrollment: box.Enrollment, Cloud: c, State: o.State, ObservedAt: now.UTC().Format(time.RFC3339), Via: o.Via, RequestID: o.RequestID, EventID: o.EventID, WrittenBy: WrittenBy{Box: worker.Ref, Enrollment: worker.Enrollment}}, nil }