package host import ( "context" "errors" "log" "net/http" "os" "strconv" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/carlosframework/platform/internal/rootaws" ) // memwatch (memory-pressure spec §2.4) publishes two CloudWatch metrics a // minute so a box that is running out of memory pages someone BEFORE the // kernel starts thrashing — the state the flagship sat in for fifteen // minutes on 2026-08-28 with every AWS-side check green. The status // snapshot already carries memory to the bucket every five minutes, but a // bucket write alarms nobody; CloudWatch does. // // Carlos/Host MemAvailablePercent MemAvailable / MemTotal, /proc/meminfo // Carlos/Host MemoryPressureSome60 some avg60 from /proc/pressure/memory // // Both keyed by InstanceId (IMDS). The region-host module's alarm reads the // first; the second is the early warning — it climbs minutes before // MemAvailable hits the floor. // // Per-box, not a housekeeper duty: every box reports itself. Armed only by // CARLOS_HOST_METRICS=cloudwatch in host.env, so a Pi or a customer fleet // box never tries to reach an API it has no credentials for. const ( memwatchNamespace = "Carlos/Host" memwatchPeriod = time.Minute ) // metricPutter is the one CloudWatch call this file makes. type metricPutter interface { PutMetricData(context.Context, *cloudwatch.PutMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) } // putMetricFunc is the metricPutter over one bound method value. Storing the // *cloudwatch.Client itself in the interface would make the linker keep every // exported CloudWatch method once html/template is linked in (discussion // #275); a method value keeps PutMetricData alone. type putMetricFunc func(context.Context, *cloudwatch.PutMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) func (f putMetricFunc) PutMetricData(ctx context.Context, in *cloudwatch.PutMetricDataInput, opts ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) { return f(ctx, in, opts...) } type memwatch struct { put metricPutter instanceID string mem func() (*int64, *int64) // MemTotal, MemAvailable in bytes pressure func() (string, error) // /proc/pressure/memory content logf func(string, ...any) // lastPutErr is the text of the put failure last logged, "" while // puts succeed. Failures log on CHANGE (and recovery logs once), not // once a minute: a box rolled before its IAM grant lands would // otherwise write 1,440 identical lines a day. lastPutErr string } // memwatchArmed is the host.env gate, split out so the three answers it // can give are table-testable: armed, off (unset), or off with a value // this duty doesn't understand — the last is the one an operator wants to // see in the journal, because they typed it. func memwatchArmed(getenv func(string) string) (armed bool, why string) { switch v := strings.TrimSpace(getenv("CARLOS_HOST_METRICS")); v { case "cloudwatch": return true, "" case "": return false, "CARLOS_HOST_METRICS unset" default: return false, "CARLOS_HOST_METRICS=" + strconv.Quote(v) + ` (the only value is "cloudwatch")` } } // memwatchLoop arms the duty when host.env asks for it. One log line either // way; a box that can't resolve its instance id or build a client logs why // and stays quiet rather than retrying — the operator set the env, so a // silent duty is a bug they'd want to see at arm time, in the journal. // Process-lifetime, like statusLoop: nothing stops the ticker. func memwatchLoop() { if armed, why := memwatchArmed(os.Getenv); !armed { log.Printf("agent: memwatch disabled (%s)", why) return } ctx := context.Background() // Region comes from the default chain — AWS_REGION in host.env, else // IMDS. A compatible-store deployment that points AWS_REGION at its // own object store (internal/lease/endpoint.go's note) would aim the // CloudWatch client at nonsense, which is one more reason this duty // is opt-in per box rather than on wherever credentials resolve. cfg, err := rootaws.LoadDefaultConfig(ctx) if err != nil { log.Printf("agent: memwatch disabled: load aws config: %v", err) return } // The metric goes to the box's OWN region, not host.env's AWS_REGION: // the flagship sets AWS_REGION=eu-west-1 fleet-wide (it names the // deployment bucket's region), and on the first roll all four boxes // dutifully published into Ireland — while the region-host module // creates each box's alarm in the box's region, so three alarms would // have watched an empty namespace forever. IMDS placement wins; "" (not // on EC2) keeps the chain's answer. if r := readPlacementRegion(); r != "" { cfg.Region = r } id, err := imdsInstanceID() if err != nil { log.Printf("agent: memwatch disabled: instance id: %v", err) return } w := memwatch{ put: putMetricFunc(cloudwatch.NewFromConfig(cfg).PutMetricData), instanceID: id, mem: readMemBytes, pressure: readPressureFile, logf: log.Printf, } log.Printf("agent: memwatch armed (%s in %s, every %s, instance %s)", memwatchNamespace, cfg.Region, memwatchPeriod, id) for now := range time.Tick(memwatchPeriod) { w.tick(ctx, now) } } // tick reads, builds, puts. A failed read skips the tick (a zero would look // like a full box); a failed put is logged when it first happens or changes, // and the next tick tries again. func (w *memwatch) tick(ctx context.Context, now time.Time) { total, avail := w.mem() if total == nil || avail == nil || *total <= 0 { w.logf("memwatch: /proc/meminfo unreadable this tick — skipped") return } availPct := float64(*avail) / float64(*total) * 100 psi, hasPSI := 0.0, false if s, err := w.pressure(); err == nil { psi, hasPSI = parsePressureSome60(s) } ctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() _, err := w.put.PutMetricData(ctx, memwatchData(w.instanceID, availPct, psi, hasPSI, now)) switch { case err != nil && err.Error() != w.lastPutErr: w.lastPutErr = err.Error() w.logf("memwatch: put: %v (logged once until it changes)", err) case err == nil && w.lastPutErr != "": w.lastPutErr = "" w.logf("memwatch: put recovered") } } // memwatchData is the pure half: the exact PutMetricData input for one // tick. Standard resolution — a one-minute period at standard resolution // is free-tier territory; high resolution is not and buys nothing for a // three-minute alarm. func memwatchData(instanceID string, availPct, psi float64, hasPSI bool, now time.Time) *cloudwatch.PutMetricDataInput { dims := []types.Dimension{{Name: aws.String("InstanceId"), Value: aws.String(instanceID)}} data := []types.MetricDatum{{ MetricName: aws.String("MemAvailablePercent"), Dimensions: dims, Timestamp: aws.Time(now), Unit: types.StandardUnitPercent, Value: aws.Float64(availPct), }} if hasPSI { data = append(data, types.MetricDatum{ MetricName: aws.String("MemoryPressureSome60"), Dimensions: dims, Timestamp: aws.Time(now), Unit: types.StandardUnitPercent, Value: aws.Float64(psi), }) } return &cloudwatch.PutMetricDataInput{Namespace: aws.String(memwatchNamespace), MetricData: data} } // imdsInstanceID asks the metadata service which instance this is — the // dimension the alarm keys on — through the same IMDSv2 token dance // readPlacementRegion uses (imdsToken/imdsLeaf). The IMDS uid-gate // (harden-host) lets the carlos user through by construction. func imdsInstanceID() (string, error) { client := &http.Client{Timeout: 2 * time.Second} token := imdsToken(client) if token == "" { return "", errors.New("no IMDSv2 token (not on EC2, or IMDS unreachable)") } id, code := imdsLeaf(client, token, "/latest/meta-data/instance-id") if code != http.StatusOK || id == "" { return "", errors.New("instance-id leaf did not answer 200") } return id, nil } // readPressureFile is the /proc/pressure/memory read; absent on a kernel // without PSI (CONFIG_PSI) — the tick then publishes MemAvailable alone. func readPressureFile() (string, error) { b, err := os.ReadFile("/proc/pressure/memory") return string(b), err } // parsePressureSome60 pulls "some ... avg60=N" out of /proc/pressure/memory. // Not-ok for any shape it doesn't recognise: a parse bug must not report a // calm box. func parsePressureSome60(s string) (float64, bool) { for _, line := range strings.Split(s, "\n") { fields := strings.Fields(line) if len(fields) == 0 || fields[0] != "some" { continue } for _, f := range fields[1:] { if v, ok := strings.CutPrefix(f, "avg60="); ok { n, err := strconv.ParseFloat(v, 64) return n, err == nil } } return 0, false } return 0, false }