//go:build linux && rootrunner package host import ( "bytes" "crypto/ed25519" "crypto/rand" "debug/buildinfo" "encoding/base64" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "io" "net" "net/http" "net/http/httptest" "os" "os/exec" "os/user" "path/filepath" "reflect" "strings" "sync" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/claimactivation" "github.com/carlosframework/platform/internal/console" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) var activationConsoleImage, activationCarlosImage, activationCarlosOldImage []byte // activationImagesBuild is how the three embedded images are built, from the // worktree root, before `go test -c -tags rootrunner,activationconsole` // (controller ruling C5). All are stripped (-s -w), as release builds are: // an unstripped Go binary carries compressed DWARF sections, which the // bounded ELF pre-check refuses (F1). The release under test is v1.0.0: // guard-arm compares the carlos build with the identity, and the console's // own floor report (Disk.Running) compares the console's build with it, so // publish-identity refuses a console stamped otherwise. v0.9.0 is the older // release the box is first installed with, so the update leaves it in // carlos.prev (prev-rehearsal R2). checkActivationImages asserts all three // images match this command. const activationImagesBuild = `GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0" -o internal/host/activation-carlos-runner.bin . GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v0.9.0" -o internal/host/activation-carlos-old-runner.bin . GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0" -o internal/host/activation-console-runner.bin ./cmd/console` func checkActivationImages(t *testing.T) { t.Helper() for _, img := range []struct { raw []byte path, ldflag string }{ {activationCarlosImage, "github.com/carlosframework/platform", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0"}, {activationCarlosOldImage, "github.com/carlosframework/platform", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v0.9.0"}, {activationConsoleImage, "github.com/carlosframework/platform/cmd/console", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0"}, } { info, err := buildinfo.Read(bytes.NewReader(img.raw)) if err != nil { t.Fatal("embedded image has no build info; build it with:\n"+activationImagesBuild, err) } settings := map[string]string{} for _, s := range info.Settings { settings[s.Key] = s.Value } if info.Path != img.path || settings["GOOS"] != "linux" || settings["GOARCH"] != "arm64" || settings["CGO_ENABLED"] != "0" || settings["-ldflags"] != img.ldflag { t.Fatalf("embedded image %s (%v) is not the one this command builds:\n%s", info.Path, settings, activationImagesBuild) } } } // activationInspected is `carlos system claims inspect`'s output. type activationInspected struct { Roster hostactors.Roster `json:"roster"` ETag string `json:"etag"` SignedRoster json.RawMessage `json:"signed_roster"` } // activationPhaseResult is `carlos system claims run`'s output. type activationPhaseResult struct { claimactivation.Result Out string `json:"out,omitempty"` } // The claim trial's steps 6 and 7, rehearsed on a box the platform made // (prev-rehearsal ruling R2) with no hand-made input on the trial's path // (untagged-pointer ruling R2). Every pointer, identity, floor, policy, // roster and slot comes from the platform's own commands and code paths, as // on dev: // // - the box is installed at v0.9.0 (README steps 1 to 4: the binary, // host.env, `carlos bootstrap`, the edge); // - the operator's system promote of v1.0.0 is an in-process store.Put of // the artifact (scaffolding) and release.PromoteSystem with the untagged // release {Version, Source "release:", Artifacts}, so the pointer // carries no release tag and no moves protocol, as dev's v0.28.3 // pointers do. It is tied to `carlos system update`'s door only through // the console's TestSystemUpdateWritesAnUntaggedPointerBeforeTheFloor, // which drives the real door with no floor and pins that shape. The real agent and `carlos system apply` then update the box // to v1.0.0, leaving v0.9.0 in carlos.prev; // - its AWS identity record is published by `carlos system box // publish-record` as the agent user (step 5.0); // - the console release is step 5.0a's: `carlos channels digests on`, // `carlos ship` and `carlos promote` run by the operator's CLI, and the // box stages the promoted head through the adopt pass's staging // (release.StageSignedHead) into /opt/carlos/versions; // - the roster is made by `claims run` initialize to enroll-console, with // the runner's real AWS identity; sign-policy is the real binary; // publish-identity is claims run's own producing path // (reserveActivationOutput, runProducingPhase, activationRunner) with // only the sums key swapped, because the vendor key cannot sign a // fixture (the real binary's refusal of the fixture sums is checked); // - the trial's 6.2 stop is reproduced, then `install-channel --policy // --identity` (R1) makes carlos.prev the release over the untagged // pointer, and step 6 and step 7 run as the runbook gives them, with // the real agent's box reports as evidence. // // One AWS instance can be enrolled as one box only (boxid refuses a second // reference for an instance), so this box is both the trial's hosting box // and its console box: guard-arm runs in leo's form, then in sophie's. // What the witness still writes itself, all outside the trial's path, is // listed in the untagged-pointer report: the loopback S3 bucket, the // operator's inputs (host.env, the first binary, plan files, floor.json, the // fixture sums), the file transfers the runbook does with box_put and jq, // the console unit and its database, and the claim state behind the // authenticated reset after step 7. Network: every carlos unit gets a // runtime drop-in allowing only loopback and the instance metadata service. func TestRootActivationConsoleRuntimePayload(t *testing.T) { if os.Geteuid() != 0 || len(activationConsoleImage) == 0 || len(activationCarlosImage) == 0 || len(activationCarlosOldImage) == 0 { t.Fatal("requires root and embedded activation binaries") } checkActivationImages(t) if pid1, err := os.ReadFile("/proc/1/comm"); err != nil || strings.TrimSpace(string(pid1)) != "systemd" { t.Fatal("root runner requires systemd", err) } instance := status.InstanceID() if instance == "" { t.Fatal("runner instance identity unavailable") } // The box's name is the one its agent publishes under. box := status.Label(readMachineID()) if !boxid.ValidBox(box) { t.Fatal("runner box name", box) } const bucket = "activation-fixture" const consoleID = "dev-console" opt := release.OptDir() owned := []string{"/opt/carlos", "/etc/carlos", "/data", "/run/carlos", "/etc/systemd/system/carlos-edge.service", "/etc/systemd/system/console.service", "/etc/systemd/system/console.service.d"} for _, p := range owned { if _, err := os.Lstat(p); !os.IsNotExist(err) { t.Fatal("pre-existing fixture path", p, err) } } if _, err := user.Lookup(serviceUser); err == nil { t.Fatal("the service user exists before the box is made") } // Everything bootstrap and the steps add is found by name afterwards. listed := []string{"/etc/systemd/system", "/run/systemd/system", "/etc/tmpfiles.d", "/etc/sysusers.d", "/etc/systemd/journald.conf.d"} before := map[string]map[string]bool{} for _, dir := range listed { before[dir] = activationDirNames(dir) } d := activationRootFixture(t) check := func(err error) { t.Helper() if err != nil { t.Fatal(err) } } write := func(path string, b []byte, mode os.FileMode) { t.Helper() check(os.MkdirAll(filepath.Dir(path), 0755)) check(os.WriteFile(path, b, mode)) check(os.Chmod(path, mode)) } absent := func(path string) { t.Helper() if _, e := os.Lstat(path); !os.IsNotExist(e) { t.Fatal("present", path, e) } } journal := func(unit string, n int) string { out, _ := exec.Command("journalctl", "-u", unit, "--no-pager", "-n", fmt.Sprint(n)).CombinedOutput() return string(out) } t.Cleanup(func() { for _, u := range []string{"carlos-edge.service", "carlos-system-update.service", "console.service"} { t.Log(u, "journal:\n"+journal(u, 80)) } var added []string for _, dir := range listed { for name := range activationDirNames(dir) { if !before[dir][name] { added = append(added, filepath.Join(dir, name)) } } } var unitNames []string for _, p := range added { if filepath.Dir(p) == "/etc/systemd/system" && !strings.HasSuffix(p, ".d") && !strings.HasSuffix(p, ".wants") { unitNames = append(unitNames, filepath.Base(p)) } } exec.Command("systemctl", "stop", "console.service", "carlos-edge.service").Run() if len(unitNames) > 0 { exec.Command("systemctl", append([]string{"stop"}, unitNames...)...).Run() exec.Command("systemctl", append([]string{"disable"}, unitNames...)...).Run() } for _, m := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} { exec.Command("systemctl", "stop", m).Run() } for _, p := range added { os.RemoveAll(p) } for _, p := range owned { os.RemoveAll(p) } exec.Command("systemctl", "daemon-reload").Run() exec.Command("systemctl", "reset-failed").Run() exec.Command("userdel", serviceUser).Run() exec.Command("groupdel", socketsGroup).Run() }) store := release.NewDirStore(filepath.Join(d, "bucket")) endpoint := activationS3Fixture(t, store) pub, key, _ := ed25519.GenerateKey(nil) signer := release.RootSigner(key) pubArg := base64.StdEncoding.EncodeToString(pub) ctx := t.Context() // host.env (README step 2), with the hibernation block a hosting box // carries, as leo's does. No release key: a box only verifies. (An agent // with no CARLOS_HIBERNATE_BUCKET panics at start on this tree: runAgent // dereferences the nil activator config at registerProductionMovesParks. // Reported, not fixed here.) hostEnv := []string{"CARLOS_DEPLOYMENT_BUCKET=" + bucket, "CARLOS_HIBERNATE_BUCKET=" + bucket + "-ie", "CARLOS_HIBERNATE_REGION=eu-west-1", "CARLOS_HIBERNATE_PREFIX=instances", "CARLOS_RELEASE_PUBKEY=" + pubArg, "AWS_REGION=eu-west-1", "AWS_DEFAULT_REGION=eu-west-1", "AWS_ACCESS_KEY_ID=fixture", "AWS_SECRET_ACCESS_KEY=fixture", "AWS_EC2_METADATA_DISABLED=true", "AWS_CONFIG_FILE=/nonexistent", "AWS_SHARED_CREDENTIALS_FILE=/nonexistent", "CARLOS_S3_ENDPOINT=" + endpoint} for _, kv := range hostEnv { if k, v, _ := strings.Cut(kv, "="); strings.HasPrefix(k, "AWS_") || k == "CARLOS_S3_ENDPOINT" { t.Setenv(k, v) } } // A root shell on the box after `set -a; . /etc/carlos/host.env; set +a`, // as the runbook's box blocks run. The signer's commands get the same // bucket and credentials. shellEnv := append([]string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin"}, hostEnv...) try := func(bin string, args ...string) (string, string, error) { t.Helper() cmd := exec.CommandContext(ctx, bin, args...) cmd.Env = shellEnv cmd.Dir = d var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr err := cmd.Run() return stdout.String(), stderr.String(), err } run := func(want bool, bin string, args ...string) (string, string) { t.Helper() stdout, stderr, err := try(bin, args...) if (err == nil) != want { t.Fatal(bin, args, "\nstdout:", stdout, "\nstderr:", stderr, err) } return stdout, stderr } carlos := func(want bool, args ...string) (string, string) { t.Helper() return run(want, "/opt/carlos/carlos", args...) } ctl := func(args ...string) string { t.Helper() out, e := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() if e != nil { t.Fatal(args, string(out), e) } return strings.TrimSpace(string(out)) } readers := func() (string, string) { t.Helper() cur, _ := run(true, systemBinaryPath(opt), "version") prev, _ := run(true, systemPrevPath(opt), "version") return strings.TrimSpace(cur), strings.TrimSpace(prev) } // The box, made by the platform's own install and update path. // README step 1: the binary. The box's first build is the older release. write(systemBinaryPath(opt), activationCarlosOldImage, 0755) // Step 2: host.env. write("/etc/carlos/host.env", []byte(strings.Join(hostEnv, "\n")+"\n"), 0600) // Step 3: carlos bootstrap. out, _ := carlos(true, "bootstrap") t.Log("bootstrap:", out) for _, u := range []string{"carlos-edge.service", "carlos-system-update.service", "carlos-system-recover.service", "carlos-adopt.service", "carlos-stage.service", "carlos-restart.service", "carlos-config-sync.service", "carlos-instance-control.service"} { write("/run/systemd/system/"+u+".d/10-rootrunner-network.conf", []byte("[Service]\nIPAddressDeny=any\nIPAddressAllow=127.0.0.1 169.254.169.254\n"), 0644) } ctl("daemon-reload") // The operator's promote of v1.0.0, before any floor is published. This // is not the console door or the carlos system update CLI: it is an // in-process store.Put of the artifact (scaffolding, standing in for the // door's mirror) and release.PromoteSystem with the untagged release // {Version, Source "release:", Artifacts}. That shape is tied to the // real door only through the console's // TestSystemUpdateWritesAnUntaggedPointerBeforeTheFloor, which drives the // door with no floor and pins exactly this pointer. artifact := release.SystemArtifact{Key: release.SystemArtifactKey("v1.0.0", "linux-arm64"), SHA256: hostfloor.Digest(activationCarlosImage)} _, err := store.Put(ctx, artifact.Key, activationCarlosImage, "") check(err) _, err = release.PromoteSystem(ctx, store, signer, release.SystemComponentRouter, "stable", release.SystemRelease{Version: "v1.0.0", Source: "release:v1.0.0", Artifacts: map[string]release.SystemArtifact{"linux-arm64": artifact}}, release.SystemPromoteOptions{Actor: "operator@example.test"}) check(err) // The pointer is dev's: no release tag, no moves protocol. if p, _, e := release.GetSystemPointer(ctx, store, pub, release.SystemComponentRouter, "stable"); e != nil || p.Current.ReleaseTag != "" || p.Current.MovesProtocol != 0 || p.Current.Source != "release:v1.0.0" { t.Fatal("the operator's promote is not the untagged pointer the trial found", p, e) } // Step 4: the edge. Its agent finds the promote, fetches the build and // asks root, and carlos-system-update.service applies it. ctl("start", "carlos-edge.service") deadline := time.Now().Add(5 * time.Minute) for { r, _ := readSystemResult(opt) if r != nil && r.Version == "v1.0.0" && r.Outcome == systemOutcomeApplied { break } if r != nil && r.Version == "v1.0.0" && r.Outcome != systemOutcomeDeferred { t.Fatalf("the update to v1.0.0 ended %s: %s\n%s", r.Outcome, r.Reason, journal("carlos-system-update.service", 120)) } if time.Now().After(deadline) { t.Fatalf("the update to v1.0.0 did not apply: %+v\n%s\n%s", r, journal("carlos-edge.service", 120), journal("carlos-system-update.service", 120)) } time.Sleep(2 * time.Second) } if j, _ := readSystemJournal(opt); j != nil { t.Fatal("the update left its journal", j) } cur, _ := os.ReadFile(systemBinaryPath(opt)) prev, _ := os.ReadFile(systemPrevPath(opt)) if !bytes.Equal(cur, activationCarlosImage) || !bytes.Equal(prev, activationCarlosOldImage) { t.Fatal("the update did not leave v1.0.0 in carlos and v0.9.0 in carlos.prev") } if applied, e := readSystemApplied(opt); e != nil || applied == nil || applied.Version != "v1.0.0" { t.Fatal("applied", applied, e) } absent(channelIdentityPath(opt)) if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v0.9.0" { t.Fatal("readers before install-channel", c, p) } if state := ctl("is-active", "carlos-edge.service"); state != "active" { t.Fatal("edge", state) } t.Log("box made: carlos v1.0.0, carlos.prev v0.9.0, by bootstrap and the real update") // Step 5.0: the box's AWS identity record, as the agent user. publishRecord := func() string { t.Helper() out, _ := run(true, "systemd-run", "--wait", "--pipe", "--quiet", "-p", "EnvironmentFile=/etc/carlos/host.env", "-p", "User="+serviceUser, "-p", "IPAddressDeny=any", "-p", "IPAddressAllow=127.0.0.1 169.254.169.254", systemBinaryPath(opt), "system", "box", "publish-record") return strings.TrimSpace(out) } if line := publishRecord(); line != fmt.Sprintf("published the identity record for %s (instance %s) to control/boxes/%s.json", box, instance, box) { t.Fatal("publish-record", line) } if line := publishRecord(); !strings.HasPrefix(line, fmt.Sprintf("the record at control/boxes/%s.json is valid AWS-signed evidence for %s (instance %s), so nothing was written", box, box, instance)) { t.Fatal("publish-record verify", line) } // The trusted signing host: its CLI, the key in a root 0600 file. sd := filepath.Join(d, "signer") check(os.MkdirAll(sd, 0700)) write(sd+"/carlos", activationCarlosImage, 0755) write(sd+"/release.key", []byte(base64.StdEncoding.EncodeToString(key.Seed())+"\n"), 0600) signerCLI := func(want bool, args ...string) (string, string) { t.Helper() return run(want, sd+"/carlos", args...) } inspectWith := func(bin string) activationInspected { t.Helper() out, _ := run(true, bin, "system", "claims", "inspect", "--bucket", bucket, "--aws-region", "eu-west-1", "--public-key", pubArg) var o activationInspected check(json.Unmarshal([]byte(out), &o)) return o } refresh := func() activationInspected { t.Helper(); return inspectWith(sd + "/carlos") } // The control objects a phase may write; the agent writes status beside // them throughout. controlSnapshot := func() map[string]string { t.Helper() all := activationStoreSnapshot(t, store) for k := range all { keep := k == hostactors.Key || k == hostfloor.Key || k == hostfloor.AuthKey for _, prefix := range []string{"control/host-actor-invitations/", "control/host-console-invitations/", "control/system/router/releases/"} { keep = keep || strings.HasPrefix(k, prefix) } if !keep { delete(all, k) } } return all } runArgs := func(out string) []string { args := []string{"system", "claims", "run", "--plan", sd + "/plan.json", "--signing-key", sd + "/release.key", "--public-key", pubArg, "--aws-region", "eu-west-1"} if out != "" { args = append(args, "--out", out) } return args } // apply_plan (and phase_out, with out): the dry run, then the apply. tryPlan := func(p claimactivation.Plan, out string) (activationPhaseResult, error) { t.Helper() raw, _ := json.Marshal(p) write(sd+"/plan.json", raw, 0600) before := controlSnapshot() if stdout, stderr, e := try(sd+"/carlos", append(runArgs(out), "--dry-run")...); e != nil { return activationPhaseResult{}, fmt.Errorf("%s dry run: %v: %s %s", p.Phase, e, stdout, stderr) } if !reflect.DeepEqual(before, controlSnapshot()) { t.Fatal("dry run wrote", p.Phase) } if out != "" { absent(out) } stdout, stderr, e := try(sd+"/carlos", runArgs(out)...) if e != nil { return activationPhaseResult{}, fmt.Errorf("%s: %v: %s %s", p.Phase, e, stdout, stderr) } var r activationPhaseResult check(json.Unmarshal([]byte(stdout), &r)) t.Log("phase", p.Phase, "revision", r.Revision, "writes", len(r.Writes)) return r, nil } applyPlan := func(p claimactivation.Plan, out string) activationPhaseResult { t.Helper() r, e := tryPlan(p, out) check(e) return r } phase := func(name string, p claimactivation.Plan) activationPhaseResult { t.Helper() p.Phase, p.Bucket, p.ExpectedRevision = name, bucket, refresh().Roster.Revision return applyPlan(p, "") } // Step 5.1: roster and enrollment. applyPlan(claimactivation.Plan{Phase: "initialize", Bucket: bucket}, "") phase("configure", claimactivation.Plan{Regions: map[string]hostactors.Region{"ie": {Account: "640012953448", AWSRegion: "eu-west-1", Namespaces: []string{"platform"}, Pools: []string{"cohort1", "console"}}}}) boxInput := claimactivation.BoxInput{Reference: box, Namespace: "platform", Label: box, Region: "ie", Pool: "cohort1", AWSRegion: "eu-west-1", InstanceID: instance, Operator: "root-runner"} phase("invite-box", claimactivation.Plan{Box: &boxInput}) phase("enroll-box", claimactivation.Plan{Box: &boxInput}) enrolled, ok := refresh().Roster.Boxes[box] if !ok || enrolled.Pool != "cohort1" || enrolled.Cloud.InstanceID != instance || enrolled.State != "active" { t.Fatal("enrolled box", enrolled) } token := make([]byte, 16) _, err = rand.Read(token) check(err) consoleIn := hostactors.Console{InstanceID: consoleID, Enrollment: hex.EncodeToString(token), DeploymentBucket: bucket, Namespaces: []string{"platform"}, Compute: &hostactors.ComputeBinding{Box: box, Enrollment: enrolled.Enrollment}} phase("invite-console", claimactivation.Plan{Console: &consoleIn}) phase("enroll-console", claimactivation.Plan{Console: &consoleIn}) if o := refresh(); o.Roster.Consoles[consoleID].Compute == nil || len(o.Roster.Boxes) != 1 { t.Fatal("inspect", o.Roster) } // The box's staged inputs (/root/claim-trial on the trial's boxes). The // enrollment file is 5.1's jq over the signer's inspect output, // '{box:$box,enrollment:.roster.boxes[$box].enrollment,instance_id:.roster.boxes[$box].cloud.instance_id}', // as jq prints it, then box_put. enrollmentRaw is what guard-arm then // writes to host-actor.json. bd := filepath.Join(d, "box") check(os.MkdirAll(bd, 0700)) inspected := refresh().Roster.Boxes[box] enrollmentFile, err := json.MarshalIndent(struct { Box string `json:"box"` Enrollment string `json:"enrollment"` InstanceID string `json:"instance_id"` }{box, inspected.Enrollment, inspected.Cloud.InstanceID}, "", " ") check(err) enrollmentFile = append(enrollmentFile, '\n') write(sd+"/"+box+"-enrollment.json", enrollmentFile, 0600) write(bd+"/"+box+"-enrollment.json", enrollmentFile, 0600) enrollmentRaw, _ := json.Marshal(hostfloor.Enrollment{Box: box, Enrollment: enrolled.Enrollment, InstanceID: enrolled.Cloud.InstanceID}) // Step 6.1: the armed policy, through the real binary's sign-policy. floor := hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"} floorRaw, _ := json.Marshal(floor) signed := applyPlan(claimactivation.Plan{Phase: "sign-policy", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Floor: &floor}, sd+"/policy.v1.0.0.json") policyRaw, err := os.ReadFile(sd + "/policy.v1.0.0.json") check(err) if len(signed.Writes) != 0 || signed.FloorSHA256 != hostfloor.Digest(floorRaw) || signed.OutputSHA256 != hostfloor.Digest(policyRaw) { t.Fatal("sign-policy", signed.Result) } signerCLI(false, runArgs(sd+"/policy.v1.0.0.json")...) // --out is never replaced var policy hostfloor.Policy check(json.Unmarshal(policyRaw, &policy)) if st, e := os.Stat(sd + "/policy.v1.0.0.json"); e != nil || st.Mode().Perm() != 0600 || !bytes.Equal(policy.Floor, floorRaw) || policy.Mode != "armed" { t.Fatal("policy file", st, e) } // Step 5.0a, the console release, by the operator's CLI in direct-bucket // mode as ship-app.sh runs it: pointer digests on for the account, then // carlos ship and carlos promote of the prebuilt console. od := filepath.Join(d, "operator") check(os.MkdirAll(od+"/home", 0700)) write(od+"/console-linux-arm64", activationConsoleImage, 0755) operator := func(args ...string) string { t.Helper() cmd := exec.CommandContext(ctx, sd+"/carlos", args...) cmd.Env = append(append([]string{}, shellEnv...), "HOME="+od+"/home", "CARLOS_RELEASE_KEY="+base64.StdEncoding.EncodeToString(key.Seed())) cmd.Dir = od out, e := cmd.CombinedOutput() if e != nil { t.Fatal(args, string(out), e) } return string(out) } t.Log(operator("channels", "digests", "on", "--account-id", "bab")) t.Log(operator("ship", "--app", "console", "--account-id", "bab", "--version", "abc1234", "--kind", "binary", od+"/console-linux-arm64")) t.Log(operator("promote", "-app", "console", "--account-id", "bab", "abc1234", "canary/console")) // publish-identity through claims run's own producing path (phase_out: // --out reserved before the phase, the dry run, then the apply, the file // read back) over the production S3 store against the loopback bucket. // The runner is the CLI's (activationRunner) with only its sums key // swapped: the vendor key cannot sign a fixture checksum list. The real // binary's own refusal of that list is checked below. s3, err := release.NewS3StoreIn(ctx, bucket, "eu-west-1") check(err) sumsPub, sumsKey, _ := ed25519.GenerateKey(nil) identityRunner := activationRunner(s3, pub, bucket, key) identityRunner.SumsKey = base64.StdEncoding.EncodeToString(sumsPub) sums := hostfloor.Digest(activationCarlosImage) + " carlos-linux-arm64\n" + hostfloor.Digest([]byte("amd64 deb")) + " carlos_1.0.0_amd64.deb\n" + hostfloor.Digest([]byte("arm64 deb")) + " carlos_1.0.0_arm64.deb\n" identityPlan := claimactivation.Plan{Phase: "publish-identity", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Identity: &claimactivation.IdentityInput{ReleaseTag: "v1.0.0", MovesProtocol: 1, Sums: sums, SumsSignature: base64.StdEncoding.EncodeToString(ed25519.Sign(sumsKey, []byte(sums))), Console: claimactivation.ConsoleRelease{Account: "bab", App: "console", Channel: "canary/console", Version: "abc1234"}}} producing := func(out string, dry bool) (claimactivation.Result, error) { t.Helper() reserved, e := reserveActivationOutput(out, dry) if e != nil { return claimactivation.Result{}, e } return runProducingPhase(ctx, identityRunner, &putAttempts{Store: s3}, identityPlan, reserved, dry) } beforeIdentity := controlSnapshot() preview, err := producing(sd+"/identity.v1.0.0.json", true) check(err) if _, _, e := store.Get(ctx, "control/system/router/releases/v1.0.0.json"); !errors.Is(e, os.ErrNotExist) || preview.Output != nil || !reflect.DeepEqual(beforeIdentity, controlSnapshot()) { t.Fatal("identity dry run published", e) } absent(sd + "/identity.v1.0.0.json") published, err := producing(sd+"/identity.v1.0.0.json", false) check(err) identityRaw, err := os.ReadFile(sd + "/identity.v1.0.0.json") check(err) if published.OutputSHA256 != hostfloor.Digest(identityRaw) || published.IdentityKey != "control/system/router/releases/v1.0.0.json" { t.Fatal("publish-identity's --out file", published) } planRaw, _ := json.Marshal(identityPlan) write(sd+"/plan.json", planRaw, 0600) if _, stderr := signerCLI(false, runArgs(sd+"/identity-vendor.v1.0.0.json")...); !strings.Contains(stderr, sd+"/identity-vendor.v1.0.0.json was kept. run it again with a new --out path") { t.Fatal("vendor-key refusal", stderr) } if st, e := os.Stat(sd + "/identity-vendor.v1.0.0.json"); e != nil || st.Size() != 0 { t.Fatal("the refused run's reservation", st, e) } // Staged on the box, byte for byte (box_put). write(bd+"/policy.v1.0.0.json", policyRaw, 0600) write(bd+"/identity.v1.0.0.json", identityRaw, 0600) // Step 6.2 as the trial ran it: guard-arm refuses the updated box, whose // carlos.prev is the release before, and writes nothing. armLeo := []string{"system", "guard-arm", "--policy", bd + "/policy.v1.0.0.json", "--identity", bd + "/identity.v1.0.0.json", "--enrollment", bd + "/" + box + "-enrollment.json"} _, guardBefore := os.Lstat(systemGuardDir(opt)) if _, stderr := carlos(false, armLeo...); !strings.Contains(stderr, "hostfloor: authenticated floor required") { t.Fatal("the trial's stop did not reproduce", stderr) } absent("/etc/carlos/host-actor.json") absent("/etc/systemd/system/carlos-edge.service.d/90-host-floor.conf") if _, e := os.Lstat(systemGuardDir(opt)); os.IsNotExist(e) != os.IsNotExist(guardBefore) { t.Fatal("the refused guard-arm changed the guard directory", e) } t.Log("step 6.2 stop reproduced: guard-arm refuses carlos.prev v0.9.0") // install-channel without --policy is the trial's loop: no floor yet. if _, stderr := carlos(false, "system", "install-channel"); !strings.HasSuffix("\n"+strings.TrimSpace(stderr), "\nno host floor is published, so this box could not arm its guard and nothing was installed") { t.Fatal("install-channel without --policy", stderr) } // --policy alone is refused up front (untagged-pointer R1), and changes // nothing. slotsBefore := activationSlotFiles(t, opt) if _, stderr, e := try("/opt/carlos/carlos", "system", "install-channel", "--policy", bd+"/policy.v1.0.0.json"); e == nil || !strings.HasSuffix("\n"+strings.TrimSpace(stderr), "\n--policy and --identity go together, so nothing was installed") { t.Fatal("install-channel --policy without --identity", stderr, e) } if !reflect.DeepEqual(slotsBefore, activationSlotFiles(t, opt)) { t.Fatal("the refused install-channel changed the box") } // 6.1a: install-channel --policy --identity over the untagged pointer. installChannel := []string{"system", "install-channel", "--policy", bd + "/policy.v1.0.0.json", "--identity", bd + "/identity.v1.0.0.json"} if out, _ := carlos(true, installChannel...); strings.TrimSpace(out) != "replaced the rollback copy carlos.prev with v1.0.0 (v1.0.0) from the stable channel, the release carlos already runs. the release carlos.prev held before is no longer on this box" { t.Fatal("install-channel --policy --identity", out) } cur, _ = os.ReadFile(systemBinaryPath(opt)) prev, _ = os.ReadFile(systemPrevPath(opt)) if !bytes.Equal(cur, activationCarlosImage) || !bytes.Equal(prev, activationCarlosImage) { t.Fatal("install-channel --policy did not leave v1.0.0 in both slots") } if held, e := os.ReadFile(channelIdentityPath(opt)); e != nil || !bytes.Equal(held, identityRaw) { t.Fatal("the channel identity is not the published one", e) } if out, _ := carlos(true, installChannel...); strings.TrimSpace(out) != "v1.0.0 (v1.0.0) from the stable channel is already installed" { t.Fatal("install-channel --policy --identity again", out) } // Readers (P3's box line): both slots are the trial release. if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v1.0.0" { t.Fatal("readers after install-channel", c, p) } // Step 6.2: arm the hosting box (leo's form). armed := time.Now() carlos(true, armLeo...) if raw, e := os.ReadFile("/etc/carlos/host-actor.json"); e != nil || !bytes.Equal(raw, enrollmentRaw) { t.Fatal("host-actor.json", e) } if raw, e := os.ReadFile("/etc/systemd/system/carlos-edge.service.d/90-host-floor.conf"); e != nil || !strings.Contains(string(raw), "guard-exec edge") { t.Fatal("edge drop-in", e) } t.Log("step 6.2 passed: guard-arm armed the updated box") // Step 6.3: the console box. The console release's build, staged in a // carlos-owned versions tree as on the dev console box. uid, gid := activationServiceIDs(t) check(os.Chmod(d, 0755)) // 5.0a on the console box: the adopt pass stages the promoted head into // the box's versions tree (StageSignedHead: the channel's signed // digest, the manifest that hashes to it, then the artefact by its // sha256), and install-console takes it as --source, as sophie's // /opt/carlos/versions/50f5b268/console. _, stagedManifest, err := release.StageSignedHead(ctx, s3, pub, nil, opt, registry.Route{Account: "bab", App: "console", Channel: "canary/console", Host: "platform.example.test"}) check(err) source := release.VersionPath(opt, "bab", "abc1234", "console") if body, e := os.ReadFile(source); e != nil || !bytes.Equal(body, activationConsoleImage) || stagedManifest.Version != "abc1234" { t.Fatal("the staged console is not the promoted build", source, e) } // A refusal input only: a build that is not the release. write(d+"/versions/other/console", append([]byte("not the release "), activationConsoleImage[:64]...), 0755) // The box-only console.service and its database. dbPath := "/data/console.db" st, err := console.Open(dbPath) check(err) now := time.Now().UTC() check(st.CreateAccount(123, "bab", "fixture", "operator@example.test", now)) _, err = st.CreateAppIn(123, "trial", "operator@example.test", now) check(err) check(st.Close()) st, err = console.OpenGated(dbPath) check(err) check(st.Close()) st, err = console.Open(dbPath) check(err) check(st.BindClaimAppIdentity(123, "trial", "trial-app-id")) tokenFor := func(address string) string { user, device, e := st.NewCLIAuth("fixture", "127.0.0.1", now) check(e) check(st.ApproveCLIAuth(user, address, now)) _, token, _, e := st.PollCLIAuth(device, now) check(e) return token } operatorToken := tokenFor("operator@example.test") memberToken := tokenFor("member@example.test") check(st.SetOperator("operator@example.test", true)) check(st.Close()) for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { if _, e := os.Stat(p); e == nil { check(os.Chown(p, uid, gid)) } } ln, err := net.Listen("tcp", "127.0.0.1:0") check(err) addr := ln.Addr().String() check(ln.Close()) write("/etc/carlos/console-fixture.env", []byte("CARLOS_RELEASE_KEY="+base64.StdEncoding.EncodeToString(key.Seed())+"\n"), 0600) unit := "[Service]\nType=notify\nUser=carlos\nGroup=carlos\nEnvironmentFile=/etc/carlos/host.env\nEnvironmentFile=/etc/carlos/console-fixture.env\nEnvironment=CONSOLE_ADDR=" + addr + " CONSOLE_ORIGIN=http://" + addr + " CONSOLE_KEYMAIL=http://127.0.0.1:1 CONSOLE_DB=" + dbPath + "\nExecStart=/opt/carlos/system/console/console serve\nRestart=no\nTimeoutStartSec=30s\nNoNewPrivileges=yes\nProtectSystem=strict\nReadWritePaths=/data\nPrivateTmp=yes\nIPAddressDeny=any\nIPAddressAllow=127.0.0.1 169.254.169.254\n" write("/etc/systemd/system/console.service", []byte(unit), 0644) ctl("daemon-reload") // P9: no console mail name anywhere the console's environment comes from. for _, p := range []string{"/etc/carlos/host.env", "/etc/carlos/console-fixture.env", "/etc/systemd/system/console.service"} { raw, e := os.ReadFile(p) check(e) for _, name := range []string{"CARLOS_CONSOLE_MAIL_FROM", "CARLOS_MAIL_REGION", "CARLOS_MAIL_PLATFORM_SUFFIX", "CARLOS_MAIL_DMARC_RUA", "CARLOS_MAIL_SNS_TOPIC_ARN", "CARLOS_MAIL_OPERATORS", "CARLOS_MAIL_LEGACY_SWEEP"} { if bytes.Contains(raw, []byte(name)) { t.Fatal("P9: SET", name, p) } } } // box_inspect: the box reads the roster with its own credentials. boxInspect := func() uint64 { t.Helper() o := inspectWith(systemBinaryPath(opt)) write(bd+"/roster.json", o.SignedRoster, 0600) return o.Roster.Revision } rev := boxInspect() install := func(src string) []string { return []string{"system", "claims", "install-console", "--identity", bd + "/identity.v1.0.0.json", "--source", src, "--bucket", bucket, "--aws-region", "eu-west-1", "--public-key", pubArg} } carlos(false, install(d+"/versions/other/console")...) carlos(false, install(systemBinaryPath(opt))...) absent("/opt/carlos/system/console") carlos(true, append(install(source), "--dry-run")...) absent("/opt/carlos/system/console") out, _ = carlos(true, install(source)...) t.Log("install-console", out) for _, p := range []string{"/opt/carlos/system/console/console", "/opt/carlos/system/console/console.prev"} { body, e := os.ReadFile(p) st, se := os.Stat(p) if e != nil || se != nil || !bytes.Equal(body, activationConsoleImage) || st.Mode().Perm() != 0755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 { t.Fatal("protected console", p, e, se) } } if out, _ := carlos(true, install(source)...); !strings.Contains(out, `"already_installed": true`) { t.Fatal("rerun was not a no-op", out) } // guard-arm on the console box (sophie's form). carlos(true, append(append([]string{}, armLeo...), "--console", "/opt/carlos/system/console/console")...) dropIn, err := os.ReadFile("/etc/systemd/system/console.service.d/90-host-floor.conf") check(err) if string(dropIn) != hostfloor.ConsoleGuardDropIn("/opt/carlos/system/console/console") { t.Fatal("guard drop-in", string(dropIn)) } consoleConfig := func(enable bool) { t.Helper() args := []string{"system", "claims", "console-config", "--roster", bd + "/roster.json", "--expect-revision", fmt.Sprint(rev), "--bucket", bucket, "--console-id", consoleID, "--public-key", pubArg} if enable { args = append(args, "--enable-console-runtime") } carlos(true, append(append([]string{}, args...), "--dry-run")...) carlos(true, args...) } consoleConfig(false) ctl("restart", "console.service") if state := ctl("is-active", "console.service"); state != "active" { t.Fatal("console", state) } consoleReported := func(enabled bool, previous string) hostactors.Report { t.Helper() deadline := time.Now().Add(30 * time.Second) for time.Now().Before(deadline) { raw, _, e := store.Get(ctx, "status/consoles/"+consoleID+".json") var r hostactors.Report if e == nil && json.Unmarshal(raw, &r) == nil && r.HostClaimEnabled == enabled && r.ProcessID != "" && r.ProcessID != previous { return r } time.Sleep(200 * time.Millisecond) } t.Fatal("console did not report", enabled, journal("console.service", 60)) return hostactors.Report{} } first := consoleReported(false, "") // The console runs under the guard drop-in as the service user, from the // protected copy's bytes. pidOut := ctl("show", "-p", "MainPID", "--value", "console.service") statusRaw, err := os.ReadFile("/proc/" + pidOut + "/status") check(err) if !strings.Contains(string(statusRaw), fmt.Sprintf("\nUid:\t%d\t%d\t", uid, uid)) { t.Fatal("console is not the service user", string(statusRaw)) } exe, err := os.ReadFile("/proc/" + pidOut + "/exe") check(err) if hostfloor.Digest(exe) != hostfloor.Digest(activationConsoleImage) { t.Fatal("console process image differs from the protected copy") } // The agent's own box report, once it has seen the armed guard: the // evidence the console report and the floor publication read. Nothing // here writes it. boxReported := func() { t.Helper() deadline := time.Now().Add(9 * time.Minute) for { raw, _, e := store.Get(ctx, "status/boxes/"+box+".json") var b status.Box if e == nil && json.Unmarshal(raw, &b) == nil && b.Metrics != nil && b.Metrics.FloorGuardReady && b.Metrics.FloorSHA256 == hostfloor.Digest(floorRaw) && b.Metrics.Enrollment == enrolled.Enrollment && b.GeneratedAt.After(armed) { t.Log("agent box report", b.GeneratedAt, b.Metrics.Release, b.Metrics.BinarySHA256) return } if time.Now().After(deadline) { t.Fatalf("the agent did not report the armed guard: %s\n%s", raw, journal("carlos-edge.service", 80)) } time.Sleep(5 * time.Second) } } boxReported() // console-report: dry run, then the report. The agent's next box report // replaces the console process entry, so a report that loses that race // is run again (the runbook's rule). consoleReport := func() { t.Helper() var last error for attempt := 1; attempt <= 3; attempt++ { rev = boxInspect() args := []string{"system", "claims", "console-report", "--bucket", bucket, "--aws-region", "eu-west-1", "--expect-revision", fmt.Sprint(rev), "--console-id", consoleID, "--public-key", pubArg} if stdout, stderr, e := try(systemBinaryPath(opt), append(append([]string{}, args...), "--dry-run")...); e != nil { last = fmt.Errorf("dry run: %v: %s %s", e, stdout, stderr) } else if stdout, stderr, e := try(systemBinaryPath(opt), args...); e != nil { last = fmt.Errorf("%v: %s %s", e, stdout, stderr) } else { t.Log("console-report", stdout) return } t.Log("console-report attempt", attempt, last) time.Sleep(3 * time.Second) } t.Fatal("console-report", last) } consoleReport() t.Log("step 6 passed") // Step 7.1: readers. Every slot is the trial release. if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v1.0.0" { t.Fatal("readers", c, p) } // Step 7.2: floor and home, immediately after the console's report. floorPlan := func() claimactivation.Plan { return claimactivation.Plan{Phase: "publish-floor", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Floor: &floor} } for attempt := 1; ; attempt++ { if _, e := tryPlan(floorPlan(), ""); e == nil { break } else if attempt == 3 { t.Fatal("publish-floor", e) } else { t.Log("publish-floor attempt", attempt, e) } consoleReport() } // The floor publish-floor wrote is the armed policy's, byte for byte. for k, want := range map[string][]byte{hostfloor.Key: policy.Floor, hostfloor.AuthKey: policy.Authorization} { got, _, e := store.Get(ctx, k) if e != nil || !bytes.Equal(got, want) { t.Fatal("published floor differs from the armed policy", k, e) } } phase("home", claimactivation.Plan{HomeRegion: "ie"}) // Step 7.3: the box's evidence key, then bind-key on the signer. rev = boxInspect() evidence := []string{"system", "claims", "evidence-key", "--roster", bd + "/roster.json", "--expect-revision", fmt.Sprint(rev), "--bucket", bucket, "--box", box, "--enrollment", enrolled.Enrollment, "--public-key", pubArg} carlos(true, append(append([]string{}, evidence...), "--dry-run")...) absent("/etc/carlos/host-evidence.key") proofOut, _ := carlos(true, evidence...) write(bd+"/key-proof.json", []byte(proofOut), 0600) var keyProof struct { Proof activationKeyProof `json:"key_proof"` } check(json.Unmarshal([]byte(proofOut), &keyProof)) if fi, e := os.Lstat("/etc/carlos/host-evidence.key"); e != nil || fi.Mode().Perm() != 0600 || fi.Sys().(*syscall.Stat_t).Uid != 0 || fi.Sys().(*syscall.Stat_t).Nlink != 1 { t.Fatal("host-evidence.key", fi, e) } phase("bind-key", claimactivation.Plan{Binding: &keyProof.Proof.Binding, Proof: keyProof.Proof.Proof}) if o := refresh(); o.Roster.HomeRegion != "ie" || o.Roster.Publication != "published" || o.Roster.RequestLimitNS != 120000000000 { t.Fatal("roster after step 7.3", o.Roster.HomeRegion, o.Roster.Publication, o.Roster.RequestLimitNS) } // Step 7.4: the console runtime. rev = boxInspect() consoleConfig(true) ctl("restart", "console.service") if state := ctl("is-active", "console.service"); state != "active" { t.Fatal("console", state) } second := consoleReported(true, first.ProcessID) consoleReport() t.Log("step 7 passed") // The console runtime's own refusals, as before. raw, err := os.ReadFile("/etc/carlos/host.env") check(err) if bytes.Contains(raw, []byte("CARLOS_HOST_CLAIM=")) { t.Fatal("enabled host execution") } o, err := (hostactors.Directory{Store: store, PublicKey: pub, Bucket: bucket}).Read(ctx) check(err) forged := second forged.ProcessID = first.ProcessID if _, err = activationConsoleProcess(ctx, pub, bucket, o, consoleID, forged); err == nil { t.Fatal("accepted old process") } forged = second forged.BinarySHA256 = strings.Repeat("a", 64) if _, err = activationConsoleProcess(ctx, pub, bucket, o, consoleID, forged); err == nil { t.Fatal("accepted false process digest") } activationResetFixture(t, store, "http://"+addr, operatorToken, memberToken, box, enrolled.Enrollment) // Observation for the report: the agent's next box report is written // whole (status.WriteBox), so whether the console process entry // console-report added survives it is recorded here, not asserted. reportedAt := time.Now() for deadline := time.Now().Add(7 * time.Minute); time.Now().Before(deadline); time.Sleep(5 * time.Second) { raw, _, e := store.Get(ctx, "status/boxes/"+box+".json") var b status.Box if e == nil && json.Unmarshal(raw, &b) == nil && b.Metrics != nil && b.GeneratedAt.After(reportedAt) { t.Log("OBSERVATION: the agent's next box report at", b.GeneratedAt, "carries", len(b.Metrics.ConsoleProcesses), "console process entries") break } } ctl("stop", "console.service") if _, err = publishActivationConsoleReport(ctx, store, pub, bucket, consoleID, o.Roster.Revision, false); err == nil { t.Fatal("reported stopped console") } t.Log("PASS steps 6 and 7 on a box made by bootstrap and the real update: the 6.2 stop reproduced, install-channel --policy --identity over the untagged pointer, guard-arm in both forms, the guarded console, floor, home, key and console runtime, with the agent's own box reports") } // activationServiceIDs is the service user bootstrap created. func activationServiceIDs(t *testing.T) (int, int) { t.Helper() u, err := user.Lookup(serviceUser) if err != nil { t.Fatal(err) } var uid, gid int if _, err := fmt.Sscan(u.Uid, &uid); err != nil { t.Fatal(err) } if _, err := fmt.Sscan(u.Gid, &gid); err != nil { t.Fatal(err) } return uid, gid } // activationDirNames is the set of names in dir; a missing dir is empty. func activationDirNames(dir string) map[string]bool { names := map[string]bool{} ents, _ := os.ReadDir(dir) for _, e := range ents { names[e.Name()] = true } return names } func activationStoreSnapshot(t *testing.T, s release.Store) map[string]string { t.Helper() keys, err := s.List(t.Context(), "") if err != nil { t.Fatal(err) } result := map[string]string{} for _, k := range keys { raw, tag, e := s.Get(t.Context(), k) if errors.Is(e, os.ErrNotExist) { continue // removed between the list and the read } if e != nil { t.Fatal(e) } result[k] = tag + ":" + string(raw) } return result } func activationS3Fixture(t *testing.T, s release.Store) string { t.Helper() var mu sync.Mutex server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() defer mu.Unlock() _, key, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") fail := func(code int, name string) { w.Header().Set("Content-Type", "application/xml") w.WriteHeader(code) fmt.Fprintf(w, "%s", name) } switch r.Method { case "GET", "HEAD": if key == "" { prefix := r.URL.Query().Get("prefix") keys, e := s.List(r.Context(), prefix) if e != nil { fail(500, "InternalError") return } type item struct{ Key string } type common struct{ Prefix string } result := struct { XMLName xml.Name `xml:"ListBucketResult"` IsTruncated bool Contents []item CommonPrefixes []common }{} seen := map[string]bool{} for _, k := range keys { if delim := r.URL.Query().Get("delimiter"); delim != "" { rest := strings.TrimPrefix(k, prefix) if at := strings.Index(rest, delim); at >= 0 { p := prefix + rest[:at+len(delim)] if !seen[p] { result.CommonPrefixes = append(result.CommonPrefixes, common{p}) seen[p] = true } continue } } result.Contents = append(result.Contents, item{k}) } w.Header().Set("Content-Type", "application/xml") xml.NewEncoder(w).Encode(result) return } raw, tag, e := s.Get(r.Context(), key) if e != nil { fail(404, "NoSuchKey") return } w.Header().Set("ETag", `"`+tag+`"`) w.Header().Set("Content-Length", fmt.Sprint(len(raw))) if r.Method == "GET" { w.Write(raw) } case "PUT": raw, e := io.ReadAll(r.Body) if e != nil { fail(400, "IncompleteBody") return } _, old, e := s.Get(r.Context(), key) if e != nil && !errors.Is(e, os.ErrNotExist) { fail(500, "InternalError") return } before := strings.Trim(r.Header.Get("If-Match"), `"`) if r.Header.Get("If-None-Match") == "*" && old != "" || before != "" && before != old { fail(412, "PreconditionFailed") return } tag, e := s.Put(r.Context(), key, raw, old) if e != nil { fail(412, "PreconditionFailed") return } w.Header().Set("ETag", `"`+tag+`"`) case "DELETE": if e := s.Delete(r.Context(), key); e != nil && !errors.Is(e, os.ErrNotExist) { fail(500, "InternalError") return } w.WriteHeader(204) default: fail(400, "InvalidRequest") } })) t.Cleanup(server.Close) return server.URL } // activationResetFixture is the authenticated reset of an exhausted wake // retry through the guarded console. Its claim state is fixture state; box // and enrollment are the enrolled box's. func activationResetFixture(t *testing.T, s release.Store, origin, token, other, box, enrollment string) { t.Helper() ctx := t.Context() check := func(e error) { t.Helper() if e != nil { t.Fatal(e) } } now := time.Now().UTC().Add(-time.Minute) const host = "trial.bab.example.test" record := []byte(`{"schema":1,"host":"trial.bab.example.test","account":"bab","app":"trial","region":"ie","created_at":"` + now.Format(time.RFC3339Nano) + `"}`) o, err := hostclaim.Read(ctx, s, host) check(err) o, err = hostclaim.Update(ctx, s, o, hostclaim.BeginCreate(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/trial/instances/" + host + ".json", Account: "bab", App: "trial", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: now}, record, "create", "dev-console", now)) check(err) o, err = hostclaim.Update(ctx, s, o, hostclaim.CompleteCreate("create", box)) check(err) for i := 0; i < hostclaim.WakeRetryLimit; i++ { a := hostclaim.WakeRetryAttempt{ID: fmt.Sprint(i), Box: box, Enrollment: enrollment, BootID: "fixture", ControllerID: "fixture", InvocationID: "fixture", LaunchInputs: "fixture"} o, err = hostclaim.Update(ctx, s, o, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: o.Claim.Holder.Incarnation, EpisodeID: "exhausted", Attempt: a, Now: now})) check(err) o, err = hostclaim.Update(ctx, s, o, hostclaim.FailWakeRetryAttempt(hostclaim.WakeRetryResult{HolderIncarnation: o.Claim.Holder.Incarnation, EpisodeID: "exhausted", Attempt: a, Now: now})) check(err) now = now.Add(20 * time.Second) } admission := hostops.AppAdmission{V: 1, AppID: "trial-app-id", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]hostops.Registration{}, Effects: map[string]hostops.Effect{}} raw, _ := json.Marshal(admission) key, _ := hostops.AdmissionKey("bab", "trial") _, err = s.Put(ctx, key, raw, "") check(err) payload := `{"action":"reset-wake-retry","incarnation":"` + o.Claim.Holder.Incarnation + `","episode":"exhausted","request_id":"fixture-reset"}` call := func(auth, body string) int { t.Helper() req, _ := http.NewRequestWithContext(ctx, "POST", origin+"/api/cli/ops/hosts/"+host+"/resolve", strings.NewReader(body)) req.Header.Set("Content-Type", "application/json") if auth != "" { req.Header.Set("Authorization", "Bearer "+auth) } client := http.Client{Timeout: 10 * time.Second} resp, e := client.Do(req) check(e) defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) t.Log("reset response", resp.StatusCode, string(b)) return resp.StatusCode } for _, test := range []struct { token string status int }{{"", 401}, {other, 404}} { if code := call(test.token, payload); code != test.status { t.Fatal(code, test.status) } } before, _ := hostclaim.Read(ctx, s, host) if before.ETag != o.ETag { t.Fatal("unauthorized reset changed claim") } if code := call(token, payload); code != 200 { t.Fatal("reset failed", code) } after, err := hostclaim.Read(ctx, s, host) check(err) retry := after.Claim.Holder.WakeRetry if retry.EpisodeID == "exhausted" || retry.AttemptsReserved != 0 || retry.State != "open" || retry.ResetRequest != `{"Actor":"operator@example.test","RequestID":"fixture-reset"}` { t.Fatal("reset result", retry) } previous := *o.Claim.Holder previous.WakeRetry = retry if !reflect.DeepEqual(&previous, after.Claim.Holder) || !reflect.DeepEqual(o.Claim.Op, after.Claim.Op) { t.Fatal("reset changed execution authority") } if code := call(token, payload); code != 409 { t.Fatal("stale reset accepted", code) } final, err := hostclaim.Read(ctx, s, host) check(err) if final.ETag != after.ETag { t.Fatal("stale reset mutated claim") } t.Log("exhausted retry reset through production runtime; holder/session/stamp preserved") } // activationSlotFiles is every file in the box's /opt/carlos and in root's // system/router directory, by name and content, so a refused command can be // shown to have changed nothing. func activationSlotFiles(t *testing.T, opt string) map[string]string { t.Helper() got := map[string]string{} for _, dir := range []string{opt, systemRootDir(opt)} { ents, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } for _, e := range ents { path := filepath.Join(dir, e.Name()) if fi, err := os.Lstat(path); err == nil && fi.Mode().IsRegular() { b, _ := os.ReadFile(path) got[path] = hostfloor.Digest(b) } else if err == nil { got[path] = fi.Mode().String() } } } return got }