package main // πŸ€– `lchat relocate --to ` β€” move a space to another region // while keeping its STABLE ADDRESS (multi-edge Phase 1, docs/multi-edge.md). // // A space's address is its identity (invite links, per-origin keys bind to it), // so it must never change; its region is a mutable property (`home_region`), so // it must be free to move. Relocation is the operator/admin action that changes // where the one copy lives without changing its name. Three moving parts, in the // order they run: // // 1. Store copy. The destination region's box wakes the space by RESTORING its // DB from that region's object store, so the replica must exist there first. // On the dev mesh both regions share ONE bucket, so the bytes are already // visible β€” the copy is a NO-OP (relocationStorePlan detects same // bucket+prefix+endpoint β†’ skip). In production each region has its own // store, so it is a real prefix-to-prefix copy of s3:////… // (copyS3Prefix), done while the space sleeps. // 2. Flip home_region + re-fan-out. setHomeRegion writes the replicated routes // column; fanOutMeshRoute pushes the updated route to peers so every box's // candidatesFor (mesh.go) re-homes the space onto a box in the destination // region. The local router needs no reload β€” its mesh reads home_region live // from the shared registry each placement decision. // 3. (Optional) A-record move. Repointing 's A record at the destination // box avoids the proxy hop, but is NOT required for correctness: the // *. wildcard + the mesh's proxy-to-owner already route any request // to the new owner through the DNS-TTL window. Off by default; --move-dns // opts in, and only Route53 implements it today (route53dns.go). // // WHY THIS IS DATA-SAFE β€” the single-writer lease. Relocation never // split-brains because the S3 lease (internal/activator: lock.json under the // space's prefix, acquired via a conditional write) is the ONE truth of who may // write, and home_region/DNS are only hints that steer placement: // // - Shared bucket (dev): source and destination contend on the SAME lock.json, // so the lease is globally singular. After the flip the destination becomes // the HRW owner; if the source is momentarily still awake it holds the lease, // the destination's AcquireLease fails loudly (LeaseExistsError β†’ 503 + // Retry-After), and the client's retry lands once the source sleeps and // releases. Never two writers, ever. // - Per-region buckets (prod): the copy must run while the source is ASLEEP // (lease released, replica fully flushed by the sleep-time SyncAndWait), so // the destination restores a complete copy and acquires a FRESH lease in its // own bucket. Once home_region is flipped the source's owns() returns false, // so it never wakes there again β€” a stray request proxies to the new owner // instead. copyS3Prefix deliberately skips lock.json so a copied, live lease // can't stall the destination's first wake. // // IDEMPOTENCY + the copy-clobber guard. Re-running a completed relocation is a // no-op: once home_region already equals the target the space's authoritative // copy is the DESTINATION's, so relocate must NOT re-copy sourceβ†’dest (that // would overwrite newer destination writes with stale source bytes). So the // copy + flip run ONLY on the forward move (current != target); an // already-home re-run just heals peers by re-fanning-out the route. A crash // between copy and flip converges safely: the source is still authoritative // (not yet flipped), so the next run re-copies (idempotent overwrite) and flips. import ( "context" "flag" "fmt" "log" "os" "path" "strings" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "path/filepath" ) // storeLocation names one region's object store for a space's replica prefix. // Endpoint distinguishes same-named buckets across accounts/providers (AWS S3 // vs an R2 endpoint) so the skip test in relocationStorePlan is never fooled by // a bucket-name coincidence. type storeLocation struct { Bucket string Prefix string // the per-host key prefix, e.g. "/" Endpoint string // "" = AWS default endpoint; set for R2 / custom endpoints } // storePlan is relocationStorePlan's decision: whether the cross-region store // copy is a no-op (the dev shared bucket β€” the bytes are already there) or a // real prefix-to-prefix copy (prod's per-region stores). type storePlan struct { Skip bool Reason string Src storeLocation Dst storeLocation } // relocationStorePlan is the PURE decision at the heart of the store-copy step, // split out so it is unit-testable with no S3. It returns skip=true exactly when // the source and destination name the same bytes β€” same bucket, same per-host // prefix, and same endpoint β€” which is the dev mesh's one-shared-bucket case: // both regions already read the same replica, so there is nothing to copy. // Anything else is a real copy plan the caller executes with copyS3Prefix. func relocationStorePlan(src, dst storeLocation) storePlan { if src.Bucket == dst.Bucket && src.Prefix == dst.Prefix && src.Endpoint == dst.Endpoint { return storePlan{ Skip: true, Reason: "source and destination share the same bucket, prefix and endpoint β€” the replica bytes are already visible to the destination region (the dev shared-bucket case)", Src: src, Dst: dst, } } return storePlan{ Skip: false, Reason: fmt.Sprintf("cross-region prefix copy: s3://%s/%s/ β†’ s3://%s/%s/", src.Bucket, src.Prefix, dst.Bucket, dst.Prefix), Src: src, Dst: dst, } } // s3CopyAPI is the slice of the S3 client copyS3Prefix needs β€” an interface so // the copy is unit-tested against a fake without reaching AWS (the same shape // s3blobstore.go and route53dns.go use for their clients). type s3CopyAPI interface { ListObjectsV2(ctx context.Context, in *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) CopyObject(ctx context.Context, in *awss3.CopyObjectInput, optFns ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) } // copyS3Prefix copies every object under src's per-host prefix to dst's, // preserving the relative key layout, via server-side CopyObject (no bytes // transit this process). It EXCLUDES the lease object (lock.json): a copied, // still-valid lease would make the destination's first AcquireLease see a held // lock and 503 until the TTL expires β€” the destination must start with a clean // lease and take its own. Returns the count copied. // // Same-account, same-credential copy is the realistic prod shape (regional // buckets in one AWS account; CopyObject works cross-region). Cross-ACCOUNT or // cross-ENDPOINT (e.g. an R2 destination) needs a download+reupload variant β€” // a documented follow-on (docs/multi-edge.md), not this MVP. func copyS3Prefix(ctx context.Context, client s3CopyAPI, srcBucket, srcPrefix, dstBucket, dstPrefix string) (int, error) { srcPrefix = strings.Trim(srcPrefix, "/") dstPrefix = strings.Trim(dstPrefix, "/") listPrefix := srcPrefix + "/" p := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{ Bucket: aws.String(srcBucket), Prefix: aws.String(listPrefix), }) copied := 0 for p.HasMorePages() { page, err := p.NextPage(ctx) if err != nil { return copied, fmt.Errorf("relocate: list s3://%s/%s: %w", srcBucket, listPrefix, err) } for _, obj := range page.Contents { srcKey := aws.ToString(obj.Key) rel := strings.TrimPrefix(srcKey, listPrefix) if rel == "" || path.Base(rel) == "lock.json" { continue // never copy the lease β€” the destination must take a fresh one } dstKey := dstPrefix + "/" + rel // CopySource is "/"; our keys are ASCII-safe (host FQDN // prefixes + litestream LTX / base64url leaves), so no URL-escaping // is needed. A cross-account/cross-endpoint copier would encode here. copySource := srcBucket + "/" + srcKey if _, err := client.CopyObject(ctx, &awss3.CopyObjectInput{ Bucket: aws.String(dstBucket), Key: aws.String(dstKey), CopySource: aws.String(copySource), }); err != nil { return copied, fmt.Errorf("relocate: copy %s β†’ s3://%s/%s: %w", copySource, dstBucket, dstKey, err) } copied++ } } return copied, nil } // aRecordUpserter is the OPTIONAL A-record half of relocation β€” a sibling to the // dnsProvider seam (wildcard.go), kept separate so the ACME-TXT path stays // untouched and a provider can support DNS-01 without supporting A moves. // route53DNS implements it (route53dns.go); DNSimple does not yet, so --move-dns // refuses on DNSimple with a clear message. Relocation is correct WITHOUT it // (the wildcard + proxy route to the new owner), so this is never on the MVP's // critical path. type aRecordUpserter interface { upsertA(ctx context.Context, zone, host, ipv4 string) error } // relocateOpts parameterizes one relocation. Reusable so a future controld RPC // or member-facing "change region" button can call relocate() with the same // core (the CLI runRelocate is one caller). type relocateOpts struct { Host string ToRegion string // Dst names the destination region's object store for the cross-region copy. // The zero value means "the same store as the source" β€” the dev shared-bucket // case, where the copy is a no-op. Prefix defaults to the source's per-host // prefix when empty. Dst storeLocation // MoveDNS opts into repointing 's A record at DNSTarget (off by // default). DNSZone overrides the hosted zone; empty derives it from the // wildcard/mesh domains the host falls under. MoveDNS bool DNSTarget string DNSZone string } // relocateResult reports what happened, for the CLI to print. type relocateResult struct { Host string FromRegion string ToRegion string Plan storePlan Copied int // objects copied (0 on the skip path) AlreadyHome bool // home_region already == ToRegion β€” a converging no-op DNSMoved bool } // relocate is the shared relocation core behind `lchat relocate` (and any future // RPC/button). It validates, does the store copy (or skips it), flips // home_region, re-fans-out the route, and optionally moves the A record. mesh // answers the "is the destination region live?" precondition and is beaten once // here for a fresh view; the actual route replication reuses fanOutMeshRoute. func relocate(reg *Registry, mesh *meshSvc, hc *hibernateConfig, opts relocateOpts) (*relocateResult, error) { host := strings.ToLower(strings.TrimSpace(opts.Host)) if !validHost(host) { return nil, fmt.Errorf("%w: %q", errInvalidHost, host) } region := strings.TrimSpace(opts.ToRegion) if region == "" { return nil, fmt.Errorf("relocate: --to is required (clearing a space's region back to agnostic is a separate operation, not relocate)") } if hc == nil { return nil, fmt.Errorf("relocate: hibernation is not configured on this box (LCHAT_HIBERNATE_BUCKET unset) β€” relocation is hibernation-backed: the space's state must live in S3 for the destination region to restore it") } if mesh == nil { return nil, fmt.Errorf("relocate: the activator mesh is not configured (LCHAT_MESH_SELF/_DOMAINS/_TOKEN/_REGION) β€” placement-by-region needs it; run on a mesh box with /data/router.env sourced") } rt, err := reg.get(host) if err != nil { return nil, err } if rt == nil { return nil, fmt.Errorf("%w: %q (run relocate on a box whose registry has the route β€” typically the source region's box)", errUnknownHost, host) } if rt.Kind != "instance" { return nil, fmt.Errorf("relocate: %q is a %s route, not an instance", host, rt.Kind) } if rt.Status != "ready" { return nil, fmt.Errorf("relocate: %s is %s, not ready β€” fix the instance first", host, rt.Status) } if !rt.Hibernate { return nil, fmt.Errorf("relocate: %s is systemd-managed, not hibernating β€” a live systemd instance is pinned to this box; run `lchat hibernate %s --on` first so its state lives in S3 and can be restored in the destination region", host, host) } if !mesh.inScope(host) { return nil, fmt.Errorf("relocate: %s is not under a configured mesh domain (%s) β€” only in-scope hosts are placed by region", host, strings.Join(mesh.domains, ", ")) } res := &relocateResult{Host: host, FromRegion: rt.HomeRegion, ToRegion: region} // Idempotent + copy-clobber guard: once the space is already homed to the // target, the DESTINATION holds the authoritative copy. Re-copying sourceβ†’dest // would clobber newer destination writes, so DON'T. Just heal any peer that // missed the earlier flip by re-fanning-out the route, and return. // The instance env rides every fan-out (meshRoutePayload doc) so the new // owner wakes this space with its real admin token, not a token-less boot. instEnv, _ := readEnvFile(filepath.Join(instanceDataDir, host+".env")) if rt.HomeRegion == region { res.AlreadyHome = true here := storeLocation{Bucket: hc.Bucket, Prefix: hc.hibernatePrefix(host)} res.Plan = relocationStorePlan(here, here) // same store β†’ Skip fanOutMeshRoute(hc, *rt, instEnv) // best-effort heal; a no-op when the mesh is off return res, nil } // The destination region must have a live box, or the flip re-homes the space // onto nothing (candidatesFor would just fall back to the whole fleet and the // space would not actually move). beat() first, exactly like fanOutMeshRoute, // so the membership + peer-region view is fresh. mesh.beat() if !mesh.regionLive(region) { return nil, fmt.Errorf("relocate: no live mesh box advertises region %q β€” bring up a box with LCHAT_MESH_REGION=%s (and confirm this box has its own LCHAT_MESH_REGION set so it can read peers' regions)", region, region) } // Step 1 β€” the store copy (or the dev no-op). src := storeLocation{Bucket: hc.Bucket, Prefix: hc.hibernatePrefix(host)} dst := opts.Dst if dst.Bucket == "" { dst = src // no destination store named β†’ same store (dev shared bucket) } if dst.Prefix == "" { dst.Prefix = hc.hibernatePrefix(host) } res.Plan = relocationStorePlan(src, dst) if !res.Plan.Skip { copied, err := runStoreCopy(context.Background(), hc.Region, res.Plan) if err != nil { return nil, err } res.Copied = copied } // Step 2 β€” flip home_region and re-fan-out the updated route to peers. if err := reg.setHomeRegion(host, region); err != nil { return nil, fmt.Errorf("relocate: set home_region: %w", err) } updated := *rt updated.HomeRegion = region fanOutMeshRoute(hc, updated, instEnv) // Step 3 β€” optional A-record move. if opts.MoveDNS { if err := moveARecord(context.Background(), mesh, host, opts); err != nil { // The flip already succeeded, so relocation is correct via the proxy; // surface the DNS failure but don't unwind the (correct) move. return res, fmt.Errorf("relocate: home_region flipped to %s, but the optional A-record move failed: %w (relocation is still correct β€” the wildcard + proxy route to the new owner)", region, err) } res.DNSMoved = true } return res, nil } // runStoreCopy builds one S3 client and runs copyS3Prefix. Only reached on the // cross-bucket (prod) path β€” the dev shared-bucket case skips before here. Uses // the source region as the signing region (a same-account client copies // cross-region via CopyObject regardless) and, when the destination names a // custom endpoint (e.g. R2), addresses it path-style. Optional static // credentials for the destination store ride LCHAT_RELOCATE_DST_ACCESS_KEY / // _SECRET_KEY, mirroring s3blobstore.go's mirror-store pattern; unset = the // default AWS chain. func runStoreCopy(ctx context.Context, srcRegion string, plan storePlan) (int, error) { var opts []func(*awsconfig.LoadOptions) error if srcRegion != "" { opts = append(opts, awsconfig.WithRegion(srcRegion)) } if ak := strings.TrimSpace(os.Getenv("LCHAT_RELOCATE_DST_ACCESS_KEY")); ak != "" { opts = append(opts, awsconfig.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(ak, strings.TrimSpace(os.Getenv("LCHAT_RELOCATE_DST_SECRET_KEY")), ""))) } cfg, err := awsconfig.LoadDefaultConfig(ctx, opts...) if err != nil { return 0, fmt.Errorf("relocate: load aws config for copy: %w", err) } client := awss3.NewFromConfig(cfg, func(o *awss3.Options) { if plan.Dst.Endpoint != "" { o.BaseEndpoint = aws.String(plan.Dst.Endpoint) o.UsePathStyle = true } }) log.Printf("relocate: %s", plan.Reason) return copyS3Prefix(ctx, client, plan.Src.Bucket, plan.Src.Prefix, plan.Dst.Bucket, plan.Dst.Prefix) } // moveARecord repoints host's A record via whichever DNS provider is configured, // if it supports A upserts. Off unless the caller passed --move-dns. func moveARecord(ctx context.Context, mesh *meshSvc, host string, opts relocateOpts) error { if strings.TrimSpace(opts.DNSTarget) == "" { return fmt.Errorf("--move-dns needs --dns-target (the destination box's address)") } prov, reason := dnsProviderFromEnv() if prov == nil { return fmt.Errorf("no DNS provider configured: %s", reason) } up, ok := prov.(aRecordUpserter) if !ok { return fmt.Errorf("the configured DNS provider can't move A records yet (only LCHAT_DNS_PROVIDER=route53 does) β€” omit --move-dns: relocation is correct without it (the wildcard + proxy route to the new owner)") } zone := strings.TrimSpace(opts.DNSZone) if zone == "" { zone = zoneForHost(host, mesh.domains) } if zone == "" { return fmt.Errorf("could not derive a hosted zone for %s from the mesh domains β€” pass --dns-zone", host) } return up.upsertA(ctx, zone, host, strings.TrimSpace(opts.DNSTarget)) } // zoneForHost picks the hosted zone for host from the configured domains: the // longest domain that host is a subdomain of (so a delegated sub-zone wins over // its parent). "" when none matches. func zoneForHost(host string, domains []string) string { host = strings.ToLower(strings.TrimSuffix(host, ".")) best := "" for _, d := range domains { d = strings.ToLower(strings.TrimSpace(d)) if d == "" { continue } if strings.HasSuffix(host, "."+d) && len(d) > len(best) { best = d } } return best } // runRelocate: `lchat relocate --to [flags]`. func runRelocate(args []string) { fs := flag.NewFlagSet("relocate", flag.ExitOnError) registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") to := fs.String("to", "", "destination region (matches a live box's LCHAT_MESH_REGION) β€” required") dstBucket := fs.String("dst-bucket", "", "destination region's object-store bucket (prod per-region stores); empty = same store as source (the dev shared-bucket no-op)") dstPrefix := fs.String("dst-prefix", "", "destination per-host key prefix; empty = the same prefix the source uses") dstEndpoint := fs.String("dst-endpoint", "", "destination S3 endpoint (e.g. an R2 URL); empty = AWS default") moveDNS := fs.Bool("move-dns", false, "also repoint the host's A record at the destination box (OPTIONAL β€” off by default; relocation is correct without it, the wildcard + proxy route to the new owner)") dnsTarget := fs.String("dns-target", "", "destination box IPv4 for the A record (required with --move-dns)") dnsZone := fs.String("dns-zone", "", "hosted zone for the A record; empty = derived from the mesh domains") _ = fs.Parse(reorderFlagsFirst(fs, args)) host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) if !validHost(host) { die("relocate: invalid host %q (usage: lchat relocate --to )", host) } region := strings.TrimSpace(*to) if region == "" { die("relocate: --to is required") } hc, err := hibernateConfigFromEnv() if err != nil { die("relocate: %v", err) } mesh, err := meshFromEnv(hc) if err != nil { die("relocate: mesh config: %v", err) } reg, err := openRegistry(*registryPath) if err != nil { die("relocate: open registry: %v", err) } if mesh != nil { mesh.reg = reg // so any region read resolves against the local registry } res, err := relocate(reg, mesh, hc, relocateOpts{ Host: host, ToRegion: region, Dst: storeLocation{ Bucket: strings.TrimSpace(*dstBucket), Prefix: strings.TrimSpace(*dstPrefix), Endpoint: strings.TrimSpace(*dstEndpoint), }, MoveDNS: *moveDNS, DNSTarget: strings.TrimSpace(*dnsTarget), DNSZone: strings.TrimSpace(*dnsZone), }) if err != nil { die("relocate: %v", err) } if res.AlreadyHome { fmt.Printf("%s is already homed to %s β€” re-fanned the route out to heal any peer that missed it (no copy, no re-flip)\n", res.Host, res.ToRegion) return } from := res.FromRegion if from == "" { from = "(agnostic)" } fmt.Printf("relocated %s: %s β†’ %s\n", res.Host, from, res.ToRegion) if res.Plan.Skip { fmt.Printf(" store: no copy β€” %s\n", res.Plan.Reason) } else { fmt.Printf(" store: copied %d object(s) β€” %s\n", res.Copied, res.Plan.Reason) } fmt.Printf(" home_region flipped + route re-fanned-out to peers β€” the mesh now places %s on a %s box (the single-writer lease keeps one writer through the move)\n", res.Host, res.ToRegion) if res.DNSMoved { fmt.Printf(" A record moved to the destination box (%s)\n", strings.TrimSpace(*dnsTarget)) } else { fmt.Printf(" A record unchanged β€” the wildcard + mesh proxy route to the new owner through the DNS-TTL window (use --move-dns --dns-target to avoid the proxy hop)\n") } }