package host import ( "bytes" "context" "crypto" "crypto/rand" "crypto/rsa" "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/json" "errors" "fmt" "net" "net/http" "os" "path/filepath" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) type handoverRecoveryRoster struct{ actor hostbox.Actor } func (r handoverRecoveryRoster) Actor(context.Context, string) (hostbox.Actor, error) { return r.actor, nil } type handoverRecoveryEvidence []byte func (e handoverRecoveryEvidence) Load(context.Context, string) ([]byte, error) { return e, nil } // handoverRecoveryKey is the RSA key every test identity signs its document // with. Generating a 2048-bit key costs tens of milliseconds and this helper // has dozens of callers, many in loops, so the process makes one. Each // provider still carries its own document, signature and certificate map, and // no test verifies one provider's evidence against another's certificates. var handoverRecoveryKey = sync.OnceValues(func() (*rsa.PrivateKey, error) { return rsa.GenerateKey(rand.Reader, 2048) }) func handoverRecoveryIdentity(t *testing.T, actor hostbox.Actor) hostbox.IdentityProvider { t.Helper() key, err := handoverRecoveryKey() if err != nil { t.Fatal(err) } actor.Bucket, actor.Namespace, actor.Eligible = "data", "platform", true if actor.Binding == "" { // Fixture shorthand: these fixtures stamp installed state with the // actor's numeral, so the opaque binding defaults to that string. The // production roster computes hostactors.BoxBinding; the binding // witnesses (claim_binding_*_test.go) run on it. actor.Binding = fmt.Sprint(actor.Revision) } actor.Cloud = hostbox.CloudIdentity{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"} doc, err := json.Marshal(map[string]string{"accountId": actor.Cloud.Account, "region": actor.Cloud.Region, "instanceId": actor.Cloud.InstanceID}) if err != nil { t.Fatal(err) } sum := sha256.Sum256(doc) signature, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:]) if err != nil { t.Fatal(err) } raw, err := json.Marshal(boxid.Record{V: 1, Box: actor.Box, IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: base64.StdEncoding.EncodeToString(signature)}) if err != nil { t.Fatal(err) } return hostbox.IdentityProvider{Account: actor.Cloud.Account, Box: actor.Box, Region: actor.Region, AWSRegion: actor.Cloud.Region, Bucket: actor.Bucket, Namespace: actor.Namespace, Roster: handoverRecoveryRoster{actor}, Evidence: handoverRecoveryEvidence(raw), Certs: map[string]*x509.Certificate{actor.Cloud.Region: {PublicKey: &key.PublicKey, PublicKeyAlgorithm: x509.RSA}}} } func postRecoveryHandover(t *testing.T, socket string, request coldHandoverRequest) int { t.Helper() body, err := json.Marshal(request) if err != nil { t.Fatal(err) } transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", socket) }} defer transport.CloseIdleConnections() req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://instance-control/claim/handover", bytes.NewReader(body)) if err != nil { t.Fatal(err) } response, err := (&http.Client{Transport: transport}).Do(req) if err != nil { t.Fatal(err) } defer response.Body.Close() return response.StatusCode } // The kernel and lease are fixtures. Recovery, persisted receipts, the Unix // root endpoint, cold handover, writer admission and next restore are real. func TestClaimColdHandoverAfterControllerRecoveryReachesNextWake(t *testing.T) { t.Parallel() for _, boot := range []string{"boot-1", "boot-2"} { t.Run(boot, func(t *testing.T) { ctx := t.Context() f := newRecoveryFixture(t) // Production reads claims and publishes evidence in the same bucket. f.evidence = f.claims recordedBoot, parkRuntime := servingRecoveryFilesystem(t, f.guard.DataDir) if boot == "boot-1" { boot = recordedBoot } former := f.controller(recordedBoot, "controller-1", 4101, "start-1") admitted := former.admit(t, f) f.revision = 8 restarted := f.controller(boot, "controller-2", 4202, "start-2") route := registry.Route{Host: recoveryHost, Kind: "instance", Account: admitted.Claim.Holder.Account, App: admitted.Claim.Holder.App, DBPath: provision.NestedDBPath(f.guard.DataDir, recoveryHost), Socket: provision.NestedSocket(recoveryHost), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: admitted.Claim.Holder.Incarnation, ClaimStamp: admitted.Claim.Holder.Stamp, ClaimOwner: "old-installation", ClaimRowID: "old-route"}} f.kernel.alive = true if err := restarted.runtime.Prerequisites(ctx, route, f.read(), recoveryPolicy); !errors.Is(err, ErrClaimStopControllerAlive) { t.Fatalf("live predecessor accepted: %v", err) } f.kernel.alive = false if err := restarted.runtime.Prerequisites(ctx, route, f.read(), recoveryPolicy); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("restart recovery: %v", err) } if retry := f.retry(); retry.State != "open" || retry.AttemptsReserved != 1 || retry.Attempt != nil { t.Fatalf("recovery refunded or stranded attempt: %+v", retry) } prior := f.read() receipt, err := f.guard.CompletedColdPredecessor(ctx, prior.Host, hostguard.Captured{Incarnation: prior.Claim.Holder.Incarnation, Stamp: prior.Claim.Holder.Stamp}, prior.Claim.Holder.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase) if err != nil { t.Fatal(err) } if receipt.Target.ControllerID != "controller-1" || receipt.Target.BootID != recordedBoot || receipt.Target.DirectoryRevision != "7" { t.Fatalf("recovered tuple was rewritten: %+v", receipt.Target) } base := t.TempDir() inventory, err := openLegacyInventory(t, filepath.Join(base, "inventory.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { inventory.Close() }) reg, err := registry.Open(filepath.Join(base, "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) if err = reg.Upsert(route); err != nil { t.Fatal(err) } owner := route.ClaimOwnership in := hostbox.StopInventory{Complete: true, Host: route.Host, Incarnation: owner.ClaimIncarnation, InstallationStamp: owner.ClaimStamp, Box: "box-1", Enrollment: "enrollment-1", DirectoryRevision: "7", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: route.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}}} if err = os.MkdirAll(filepath.Dir(route.DBPath), 0700); err != nil { t.Fatal(err) } rel, err := filepath.Rel(f.guard.DataDir, filepath.Dir(route.DBPath)) if err != nil { t.Fatal(err) } roots := map[string]string{"data": f.guard.DataDir} a, err := (hostbox.TypedParker{Roots: roots, ParkedRoots: roots, Incarnation: owner.ClaimIncarnation, Host: route.Host, Retirement: "fixture"}).CaptureArtifact("data", rel, owner.ClaimOwner, hostbox.ArtifactPath) if err != nil { t.Fatal(err) } in.Entries = append(in.Entries, hostbox.StopInventoryEntry{Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: rel, Ownership: owner.ClaimOwner, Materialization: "materialized", Device: a.Dev, Inode: a.Ino, Type: a.Type}) holder := prior.Claim.Holder body, err := json.Marshal(provision.Record{Schema: 1, Host: prior.Host, Account: holder.Account, App: holder.App, CreatedAt: holder.CreatedAt, Channel: "edge"}) if err != nil { t.Fatal(err) } if _, err = f.claims.Put(ctx, holder.Key, body, ""); err != nil { t.Fatal(err) } if err = inventory.WriteStopInventory(ctx, in); err != nil { t.Fatal(err) } rootGuard := f.guard rootGuard.Root = hostguard.NewRootAuthorityWithInventory(&rollbackDisk{dir: t.TempDir()}, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { _, revision, err := inventory.Snapshot(ctx, host, c) return revision, err }) held := prior.Claim.Holder if err := rootGuard.EstablishRoot(ctx, hostguard.RootInstallation{Host: prior.Host, Incarnation: held.Incarnation, Stamp: held.Stamp, Account: held.Account, App: held.App, Record: held.Key, Box: "box-1", Enrollment: "enrollment-1", Migration: 1, InventoryRevision: 1}, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } observedFence := prior observedFence.ETag = receipt.Target.ClaimETag rootGuard.Claims = rootViewClaimReader{obs: observedFence} if _, err := rootGuard.BeginFence(ctx, receipt.Target); err != nil { t.Fatal("root fence setup", err) } if err := rootGuard.CompleteFence(ctx, receipt.Target, receipt.Bytes, receipt.EvidenceAt, func(_ context.Context, target hostguard.FenceTarget, body []byte, at time.Time) error { return verifyRetirementReceipt(ctx, hostguard.FenceReceipt{Target: target, Bytes: body, EvidenceAt: at}) }); err != nil { t.Fatal("root receipt setup", err) } rootGuard.Claims = f.guard.Claims coordinator := f.root(recoveryStopDrain{}) coordinator.Guard = rootGuard root := ClaimRootHTTP{Launches: f.journal, Stop: ClaimStopHTTP{Coordinator: coordinator, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil }}, Roots: map[string]string{"data": f.guard.DataDir}, Reconcile: hostbox.ReconcileAdapter{Box: "box-1", Enrollment: "enrollment-1", Registry: reg, Inventory: inventory, Retirement: hostbox.TypedRetirement{JournalDir: filepath.Join(base, "retire")}}} root.servingParks = parkRuntime root.RequireNewWork = func(context.Context) error { return nil } root.InventoryActor = func(context.Context) (hostbox.Actor, error) { actor := restarted.local.Actor actor.Eligible = true return actor, nil } socket := startRetirementRecoveryTransport(t, root) unauthenticated := root unauthenticated.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, errors.New("unknown requester") } refusedSocket := startRetirementRecoveryTransport(t, unauthenticated) hook := coldAdmissionHook(f.guard, handoverRecoveryIdentity(t, restarted.local.Actor), socket) restarted.wake.AfterAdmission = func(ctx context.Context, before, after hostclaim.Observed) error { h := after.Claim.Holder request := coldHandoverRequest{Prior: receipt.Target, Next: hostbox.ColdSuccessor{Host: after.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Generation: h.Generation, Box: "box-1", Enrollment: "enrollment-1", DirectoryRevision: recoveryFixtureBinding, ETag: after.ETag, Op: after.Claim.Op.ID, Phase: after.Claim.Op.Phase}} if status := postRecoveryHandover(t, refusedSocket, request); status != http.StatusForbidden { t.Fatalf("receipt authenticated an unknown requester: %d", status) } for _, field := range []string{"controller", "boot", "revision", "box", "enrollment"} { forged := request switch field { case "controller": forged.Prior.ControllerID = "unknown-controller" case "boot": forged.Prior.BootID = "unknown-boot" case "revision": forged.Prior.DirectoryRevision = "8" case "box": forged.Prior.Box = "other-box" case "enrollment": forged.Prior.Enrollment = "other-enrollment" } if status := postRecoveryHandover(t, socket, forged); status != http.StatusBadRequest { t.Fatalf("forged %s accepted: %d", field, status) } } if row, err := reg.Get(route.Host); err != nil || row == nil || row.ClaimRowID != route.ClaimRowID { t.Fatalf("refusal changed predecessor row: %+v %v", row, err) } if status := postRecoveryHandover(t, socket, request); status != http.StatusNoContent { t.Fatalf("exact completed predecessor receipt refused: HTTP %d", status) } return hook(ctx, before, after) } // The fixture supplies the lease; the real Admit invokes root's cold // handover before activation, prelaunch persistence and retry binding. f.now = f.retry().NotBefore next := restarted.admit(t, f) if next.Claim.Holder.Session <= admitted.Claim.Holder.Session { t.Fatal("next wake reused predecessor session") } if retry := f.retry(); retry.AttemptsReserved != 2 || retry.Attempt == nil || retry.Attempt.ControllerID != "controller-2" || retry.Attempt.BootID != boot { t.Fatalf("next attempt not charged to current requester: %+v", retry) } currentRow, err := reg.Get(route.Host) if err != nil || currentRow == nil || currentRow.ClaimStamp != next.Claim.Holder.Stamp || currentRow.ClaimRowID == route.ClaimRowID { t.Fatalf("next wake ownership: %+v %v", currentRow, err) } prepared, err := restarted.runtime.Prepare(ctx, *currentRow, next, recoveryPolicy) if err != nil { t.Fatal(err) } if err = prepared.Restore(ctx); err != nil { t.Fatalf("next wake restore: %v", err) } marker := filepath.Join(f.guard.DataDir, "state", "data.db.gen") if body, err := os.ReadFile(marker); err != nil || strings.TrimSpace(string(body)) != next.Claim.Holder.Generation { t.Fatalf("next wake generation: %q %v", body, err) } replayed, err := rootGuard.FenceReceiptFor(ctx, receipt.Target) if err != nil || !bytes.Equal(replayed.Bytes, receipt.Bytes) || !replayed.EvidenceAt.Equal(receipt.EvidenceAt) || replayed.Target != receipt.Target { t.Fatalf("handover rewrote predecessor evidence: %+v %v", replayed, err) } launch, err := f.journal.Read(ctx, recoveryTuple(next, restarted.local)) if err != nil || launch.ControllerPID != 4202 || launch.ProcessStart != "start-2" || launch.Target.DirectoryRevision != recoveryFixtureBinding { t.Fatalf("next wake launch tuple: %+v %v", launch, err) } }) } }