//go:build linux package layout import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/registry" ) // seedAppEntries writes what an app makes for itself beside its flat // database (discussion #256): a directory named for the host holding its // key, a `-blobs` directory, and a dot-suffixed file. func seedAppEntries(t *testing.T, dd, h string) { t.Helper() for path, body := range map[string]string{ filepath.Join(dd, h, "instance.key"): "key " + h, filepath.Join(dd, h, "report.token"): "token " + h, filepath.Join(dd, h+"-blobs", "ab", "cd"): "blob " + h, filepath.Join(dd, h+".cache"): "cache " + h, } { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(body), 0o600); err != nil { t.Fatal(err) } } } // Discussion #256: the first migration moved the nine platform names and // left behind /data//instance.key and /data/-blobs; the app // started nested without them and minted a new key. Every entry the flat // naming rules make the host's now moves into the instance directory at // the same relative name, and comes back on revert. An entry whose name a // longer host owns (`-x` is the host `-x` itself) stays, and so // does a shared directory no host owns, which is reported. func TestConvergeMovesAppCreatedEntriesWithTheHost(t *testing.T) { p, f := box(t) var logged []string p.Logf = func(format string, a ...any) { logged = append(logged, fmt.Sprintf(format, a...)) } seedAppEntries(t, p.DataDir, host) // A route whose name is host plus "-x": an externally supervised one // (not activator-managed), so its own entries must be left alone. decoy := host + "-x" if err := p.Registry.Upsert(registry.Route{Host: decoy, Kind: "instance", Socket: filepath.Join(p.SocketDir, decoy+".sock")}); err != nil { t.Fatal(err) } seedAppEntries(t, p.DataDir, decoy) // What a flat app with cwd / wrote as ./blobs. if err := os.MkdirAll(filepath.Join(p.DataDir, "blobs", "aa"), 0o700); err != nil { t.Fatal(err) } res := byHost(t, p.Converge(context.Background(), ModeNested)) if res[host].Outcome != "migrated" { t.Fatalf("%s: %+v", host, res[host]) } raw := rawInstanceDir(p.DataDir, host) for rel, want := range map[string]string{ filepath.Join(host, "instance.key"): "key " + host, filepath.Join(host, "report.token"): "token " + host, filepath.Join(host+"-blobs", "ab", "cd"): "blob " + host, host + ".cache": "cache " + host, } { if b, err := os.ReadFile(filepath.Join(raw, rel)); err != nil || string(b) != want { t.Errorf("%s not in the instance directory: %q %v", rel, b, err) } if _, err := os.Lstat(filepath.Join(p.DataDir, rel)); !errors.Is(err, os.ErrNotExist) { t.Errorf("%s left behind in the data directory: %v", rel, err) } } for _, rel := range []string{filepath.Join(decoy, "instance.key"), filepath.Join(decoy+"-blobs", "ab", "cd"), decoy + ".cache", filepath.Join("blobs", "aa")} { if _, err := os.Lstat(filepath.Join(p.DataDir, rel)); err != nil { t.Errorf("%s, not this host's, was moved: %v", rel, err) } if _, err := os.Lstat(filepath.Join(raw, rel)); !errors.Is(err, os.ErrNotExist) { t.Errorf("%s, not this host's, is in the instance directory: %v", rel, err) } } if JournalPresent(p.DataDir, host) || f.fences[host] != 0 { t.Fatal("journal or fence left after the move") } if n := p.Note(host); !strings.Contains(n, "moved with it: "+host+", "+host+"-blobs, "+host+".cache") { t.Errorf("note = %q", n) } // The aside directory holds nothing: these were moved, not quarantined. if q, err := os.ReadDir(asideDir(p.DataDir, host)); err == nil && len(q) > 0 { t.Errorf("quarantined instead of moved: %v", q) } joined := strings.Join(logged, "\n") if !strings.Contains(joined, "owned by no host on this box") || !strings.Contains(joined, filepath.Join(p.DataDir, "blobs")) { t.Errorf("the shared directory was not reported:\n%s", joined) } if strings.Contains(joined, filepath.Join(p.DataDir, decoy)) { t.Errorf("the decoy host's own directory was reported as unowned:\n%s", joined) } if !strings.Contains(joined, "named for another host: "+decoy+" ("+decoy+"'s), "+decoy+"-blobs ("+decoy+"'s)") { t.Errorf("the decoy host's entries were not reported as left for it:\n%s", joined) } // Revert brings the same names back, and only those. res = byHost(t, p.Converge(context.Background(), ModeFlat)) if res[host].Outcome != "reverted" { t.Fatalf("revert %s: %+v", host, res[host]) } assertFlatUntouched(t, p, host) for rel, want := range map[string]string{ filepath.Join(host, "instance.key"): "key " + host, filepath.Join(host+"-blobs", "ab", "cd"): "blob " + host, host + ".cache": "cache " + host, } { if b, err := os.ReadFile(filepath.Join(p.DataDir, rel)); err != nil || string(b) != want { t.Errorf("%s not back in the data directory: %q %v", rel, b, err) } } if entries, _ := os.ReadDir(raw); len(entries) != 0 { t.Errorf("instance directory not emptied by the revert: %v", entries) } } // The same protections the platform names get, on the way in: a file // inside an app's directory that has another name OUTSIDE it would be // bound read-write into the namespace while something else still reaches // it, so the move is refused with the inode and the find, and the host is // held (a tenant can plant exactly this). A hard link whose every name is // inside the directory is the app's own deduplication and moves. func TestConvergeRefusesAnAppEntryWithANameOutsideIt(t *testing.T) { p, f := box(t) seedAppEntries(t, p.DataDir, host) secret := filepath.Join(p.DataDir, "litestream.env") if err := os.WriteFile(secret, []byte("AWS_SECRET_ACCESS_KEY=x\n"), 0o600); err != nil { t.Fatal(err) } if err := os.Link(secret, filepath.Join(p.DataDir, host, "creds")); err != nil { t.Skipf("hard links unavailable here: %v", err) } // And a legitimate pair, both names inside the blob store. if err := os.Link(filepath.Join(p.DataDir, host+"-blobs", "ab", "cd"), filepath.Join(p.DataDir, host+"-blobs", "dup")); err != nil { t.Fatal(err) } res := byHost(t, p.Converge(context.Background(), ModeNested)) r := res[host] if r.Outcome != "held" || !strings.Contains(r.Reason, "creds") || !strings.Contains(r.Reason, "2 names") || !strings.Contains(r.Reason, "only 1 of them are inside") || !strings.Contains(r.Reason, "-samefile") { t.Fatalf("linked outside: %+v", r) } assertFlatUntouched(t, p, host) if f.fences[host] == 0 { t.Fatal("the fence was lifted; the host would wake flat with the link in place") } if _, err := os.Stat(filepath.Join(p.DataDir, host, "instance.key")); err != nil { t.Fatalf("a refused move touched the app's directory: %v", err) } // The operator removes the outside name; the inside pair is fine. os.Remove(filepath.Join(p.DataDir, host, "creds")) res = byHost(t, p.Converge(context.Background(), ModeNested)) if res[host].Outcome != "migrated" || f.fences[host] != 0 { t.Fatalf("after the link is gone: %+v fence=%d", res[host], f.fences[host]) } raw := rawInstanceDir(p.DataDir, host) a, err1 := os.Stat(filepath.Join(raw, host+"-blobs", "ab", "cd")) b, err2 := os.Stat(filepath.Join(raw, host+"-blobs", "dup")) if err1 != nil || err2 != nil || !os.SameFile(a, b) { t.Fatalf("the app's own hard-link pair did not move together: %v %v", err1, err2) } } // A symlink inside an app's directory is refused like one anywhere else // the pass moves, and the host is held. func TestConvergeRefusesASymlinkInsideAnAppEntry(t *testing.T) { p, f := box(t) seedAppEntries(t, p.DataDir, host) if err := os.Symlink("/etc/passwd", filepath.Join(p.DataDir, host, "link")); err != nil { t.Fatal(err) } res := byHost(t, p.Converge(context.Background(), ModeNested)) if r := res[host]; r.Outcome != "held" || !strings.Contains(r.Reason, "link") || !strings.Contains(r.Reason, "not a regular file") { t.Fatalf("symlink: %+v", r) } if f.fences[host] == 0 { t.Fatal("the fence was lifted") } assertFlatUntouched(t, p, host) } // An extra whose destination is already taken is refused before the // journal, naming both, never a choice. On the way to nested the // destination check has just cleared the instance directory, so a taken // destination was planted since (here: inside the serialized section, // after the check and before the scan), and the host is held. func TestConvergeRefusesAnAppEntryWhoseDestinationIsTaken(t *testing.T) { p, f := box(t) seedAppEntries(t, p.DataDir, host) p.Serialize = func(fn func()) { if !JournalPresent(p.DataDir, host) { os.MkdirAll(filepath.Join(rawInstanceDir(p.DataDir, host), host), 0o700) } fn() } res := byHost(t, p.Converge(context.Background(), ModeNested)) r := res[host] if r.Outcome != "held" || !strings.Contains(r.Reason, "exists at both") || !strings.Contains(r.Reason, "refusing to choose") { t.Fatalf("taken destination: %+v", r) } if f.fences[host] == 0 { t.Fatal("the fence was lifted") } assertFlatUntouched(t, p, host) if _, err := os.Stat(filepath.Join(p.DataDir, host, "instance.key")); err != nil { t.Fatalf("a refused move touched the app's directory: %v", err) } } // Recovery completes a move with extras from its journal: the names are // recorded, re-derived on recovery, and moved with the rest. func TestRecoverCompletesAMoveWithAppEntries(t *testing.T) { for _, step := range []string{"after-journal", "mid-files", "after-files"} { t.Run(step, func(t *testing.T) { p, _ := box(t) seedAppEntries(t, p.DataDir, host) p.abortAt = step p.Converge(context.Background(), ModeNested) p.abortAt = "" j, found, err := ReadJournal(p.DataDir, host) if err != nil || !found || len(j.Extra) != 3 { t.Fatalf("journal = %+v %v %v, want three extras recorded", j, found, err) } res := byHost(t, p.Recover(context.Background(), ModeNested)) if res[host].Outcome != "recovered" { t.Fatalf("recover = %+v", res[host]) } raw := rawInstanceDir(p.DataDir, host) for _, rel := range []string{filepath.Join(host, "instance.key"), filepath.Join(host+"-blobs", "ab", "cd"), host + ".cache"} { if _, err := os.Stat(filepath.Join(raw, rel)); err != nil { t.Errorf("%s not at destination after recovery from %s: %v", rel, step, err) } if _, err := os.Lstat(filepath.Join(p.DataDir, rel)); !errors.Is(err, os.ErrNotExist) { t.Errorf("%s left behind after recovery from %s", rel, step) } } if JournalPresent(p.DataDir, host) { t.Error("journal remains") } }) } } // A forged journal for host H that names H-x, the directory of the host // H-x itself, is refused on recovery: the name passes the shape rule, but // ownership is judged against the box's rows as they stand, and it is not // H's. The host stays fenced, nothing moves. func TestRecoverRefusesAJournalNamingAnotherHostsEntry(t *testing.T) { p, _ := box(t) decoy := host + "-x" if err := p.Registry.Upsert(registry.Route{Host: decoy, Kind: "instance", Socket: filepath.Join(p.SocketDir, decoy+".sock")}); err != nil { t.Fatal(err) } seedAppEntries(t, p.DataDir, decoy) plan, err := PlanFor(p.DataDir, p.SocketDir, host, ModeNested) if err != nil { t.Fatal(err) } present := make([]bool, len(plan.Artifacts)) present[0] = true j := plan.Journal(present) j.Extra = []JournalArtifact{{Name: decoy, Present: true}} if err := WriteJournal(p.DataDir, j); err != nil { t.Fatal(err) } res := byHost(t, p.Recover(context.Background(), ModeNested)) if r := res[host]; r.Outcome != "refused" || !strings.Contains(r.Reason, "belongs to "+decoy) || !strings.HasSuffix(r.Reason, "not acting") { t.Fatalf("forged extra: %+v", r) } if _, err := os.Stat(filepath.Join(p.DataDir, decoy, "instance.key")); err != nil { t.Fatalf("the other host's directory was moved: %v", err) } if !JournalPresent(p.DataDir, host) { t.Fatal("the refused journal was removed") } } // Review (#256 B): the hosts migrated before the pass carried the app's // entries along (the 104 on IE) still have them in the data directory, // where the namespace cannot see them. The pass never moves a nested // host's files, but it says so on every pass, in the log and in the // host's status note, until an operator has moved them. func TestConvergeReportsFlatLeftoversOfANestedHost(t *testing.T) { p, _ := box(t) if res := byHost(t, p.Converge(context.Background(), ModeNested)); res[host].Outcome != "migrated" { t.Fatalf("setup: %+v", res[host]) } // What an earlier release left behind: the app's directory, and a // platform-named file minted at the flat path since. seedAppEntries(t, p.DataDir, host) if err := os.WriteFile(filepath.Join(p.DataDir, host+".env"), []byte("CARLOS_ADMIN_TOKEN=old\n"), 0o600); err != nil { t.Fatal(err) } var logged []string p.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } res := byHost(t, p.Converge(context.Background(), ModeNested)) if res[host].Outcome != "unchanged" { t.Fatalf("%+v", res[host]) } want := "still in " + p.DataDir + " from before its move, where the instance cannot see them: " + filepath.Join(p.DataDir, host) + ", " + filepath.Join(p.DataDir, host+"-blobs") + ", " + filepath.Join(p.DataDir, host+".cache") + ", " + filepath.Join(p.DataDir, host+".env") + ". move what the app needs into " + rawInstanceDir(p.DataDir, host) + " by hand" if j := strings.Join(logged, "\n"); !strings.Contains(j, "layout: "+host+": "+want) { t.Errorf("log:\n%s\nwant a line with:\n%s", j, want) } if n := p.Note(host); n != want { t.Errorf("note = %q", n) } for _, rel := range []string{filepath.Join(host, "instance.key"), host + ".env"} { if _, err := os.Stat(filepath.Join(p.DataDir, rel)); err != nil { t.Errorf("a report moved %s: %v", rel, err) } } if n := p.Note(replica); strings.Contains(n, "still in") { t.Errorf("the replica, with nothing left behind, got the note: %q", n) } }