package instancectl import ( "context" "errors" "fmt" "net/http" "net/netip" "os" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/provision" ) // ensureExe lays out exactly the two links the templates bind (#214 item // 2): app at the host's live link, live/ at ../app, nothing else in // live/, and a second call repoints rather than failing. func TestEnsureExeLaysOutOneBuild(t *testing.T) { root := filepath.Join(t.TempDir(), "exe") if err := ensureExe(root, "h.example.com", "/opt/carlos/live/h.example.com", "h.example.com"); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "h.example.com", "live", "stray"), nil, 0o644); err != nil { t.Fatal(err) } if err := ensureExe(root, "h.example.com", "/opt/carlos/live/h.example.com", "h.example.com"); err != nil { t.Fatal(err) } if got, _ := os.Readlink(filepath.Join(root, "h.example.com", "app")); got != "/opt/carlos/live/h.example.com" { t.Errorf("app -> %q", got) } if got, _ := os.Readlink(filepath.Join(root, "h.example.com", "live", "h.example.com")); got != "../app" { t.Errorf("live/h.example.com -> %q", got) } entries, _ := os.ReadDir(filepath.Join(root, "h.example.com", "live")) if len(entries) != 1 { t.Errorf("live/ holds %d entries, want 1", len(entries)) } for _, bad := range [][2]string{{"..", "h"}, {"a/b", "h"}, {"x", "../h"}, {"", "h"}} { if err := ensureExe(root, bad[0], "/t", bad[1]); err == nil { t.Errorf("ensureExe(%q, %q) accepted", bad[0], bad[1]) } } } // A long host's hashed identifier keys the directory; the link inside is // still named for the host, the path the trampoline execs. func TestStartWritesTheHostsExeBeforeStarting(t *testing.T) { exe := filepath.Join(t.TempDir(), "exe") ctl := &fakeCtl{} _, c := newServer(t, ctl, func(s *Server) { s.ExeRoot = exe }) host := "a-very-long-hostname-that-gets-hashed.bdm.oncarlos.com" resp := post(t, c, "/start?host="+host+"&role=run&memory_max=536870912") if resp.StatusCode != 200 { t.Fatalf("status %d", resp.StatusCode) } iid := provision.InstanceID(host) if iid == host { t.Fatal("fixture host is not long enough to be hashed") } if got, _ := os.Readlink(filepath.Join(exe, iid, "live", host)); got != "../app" { t.Errorf("live/%s -> %q (dir %s)", host, got, iid) } } func TestResolverAllowPolicy(t *testing.T) { conf := []byte("nameserver 172.31.0.2\nnameserver 127.0.0.53\nnameserver 127.0.0.54\nnameserver 127.0.0.1\nnameserver ::1\n" + "nameserver 169.254.169.254\nnameserver 169.254.169.253\nnameserver 169.254.1.1\nnameserver fd00:ec2::253\n" + "nameserver fe80::1%eth0\nnameserver 0.0.0.0\nnameserver 172.31.0.2\noptions edns0\n") allow, refused := resolverAllow(conf, boxPrefixes) if got, want := strings.Join(allow, " "), "169.254.169.253/32 172.31.0.2/32 fd00:ec2::253/128"; got != want { t.Errorf("allow = %q, want %q", got, want) } if len(refused) != 0 { t.Errorf("refused = %q", refused) } // review F1: resolved's stubs are loopback, and an allow on them is // an allow on every wildcard listener of the box. for _, stub := range []string{"127.0.0.53", "127.0.0.54"} { if ok, why := ResolverAllowed(netip.MustParseAddr(stub)); ok || !strings.Contains(why, "upstreams") { t.Errorf("%s allowed=%v %q", stub, ok, why) } } } // Where resolved runs, its upstream list is what instances see (the // template binds it over /etc/resolv.conf), so it is what gets opened; // without it, the box's own file. func TestAllowFollowsTheFileInstancesSee(t *testing.T) { dir := t.TempDir() etc := filepath.Join(dir, "resolv.conf") up := filepath.Join(dir, "upstream.conf") if err := os.WriteFile(etc, []byte("nameserver 127.0.0.53\n"), 0o644); err != nil { t.Fatal(err) } ctl := &fakeCtl{} s := &Server{Systemctl: ctl.run, Logf: t.Logf, ResolvConf: etc, ResolvedConf: up, LocalAddrs: boxAddrs} if _, err := s.allowResolvers(context.Background()); err != nil { t.Fatal(err) } if got := ctl.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow=" { t.Errorf("stub only: %q", got) } if err := os.WriteFile(up, []byte("nameserver 10.0.0.2\n"), 0o644); err != nil { t.Fatal(err) } ctl2 := &fakeCtl{} s.Systemctl = ctl2.run if _, err := s.allowResolvers(context.Background()); err != nil { t.Fatal(err) } if got := ctl2.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow= IPAddressAllow=10.0.0.2/32" { t.Errorf("upstreams: %q", got) } } // The allow is reset and re-written on carlos-instances.slice before the // probe unit runs (#214 item 3). func TestProbeRunReopensTheResolversFirst(t *testing.T) { dir := t.TempDir() resolv := filepath.Join(dir, "resolv.conf") if err := os.WriteFile(resolv, []byte("nameserver 10.0.0.2\n"), 0o644); err != nil { t.Fatal(err) } ctl := &fakeCtl{} s, _ := newServer(t, ctl, func(s *Server) { s.ResolvConf = resolv s.ExeRoot = filepath.Join(dir, "exe") s.LocalAddrs = boxAddrs }) s.RunProbe(context.Background()) calls := ctl.joined() want := "set-property --runtime carlos-instances.slice IPAddressAllow= IPAddressAllow=10.0.0.2/32\nstart carlos-instance-probe.service" if !strings.Contains(calls, want) { t.Errorf("calls:\n%s\nwant in order:\n%s", calls, want) } if got, _ := os.Readlink(filepath.Join(dir, "exe", "_probe", "app")); got != SystemBinary { t.Errorf("probe app -> %q", got) } } // A box on EC2: its own VPC address, loopback as a local range, and the // v6 loopback and link-local. var boxPrefixes = []netip.Prefix{ netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("172.31.7.19/32"), netip.MustParsePrefix("::1/128"), netip.MustParsePrefix("fe80::4a1:2ff:fe3b:5c7d/128"), } func boxAddrs() ([]netip.Prefix, error) { return boxPrefixes, nil } // The box's own address is never opened as a resolver, however it got // into resolv.conf (#214 item 3, review F1). The control service makes // this check where it writes the allow, from procfs: the probe used to // make it from inside its unit, where listing interfaces takes netlink // and is refused (#126). func TestResolverAllowRefusesTheBoxsOwnAddress(t *testing.T) { conf := []byte("nameserver 172.31.7.19\nnameserver 172.31.0.2\nnameserver 10.9.0.53\n") local := append([]netip.Prefix{netip.MustParsePrefix("10.9.0.0/24")}, boxPrefixes...) allow, refused := resolverAllow(conf, local) if got := strings.Join(allow, " "); got != "172.31.0.2/32" { t.Errorf("allow = %q", got) } if got := strings.Join(refused, "; "); got != "172.31.7.19 is an address of this box (172.31.7.19/32); 10.9.0.53 is an address of this box (10.9.0.0/24)" { t.Errorf("refused = %q", got) } } // A refused resolver is a failing line in the report, so the box is not // ready, whatever the probe says; and the allow written leaves it out. func TestProbeRunFailsOnALocalResolver(t *testing.T) { dir := t.TempDir() resolv := filepath.Join(dir, "resolv.conf") if err := os.WriteFile(resolv, []byte("nameserver 172.31.7.19\n"), 0o644); err != nil { t.Fatal(err) } ctl := &fakeCtl{} s, _ := newServer(t, ctl, func(s *Server) { s.ResolvConf = resolv; s.LocalAddrs = boxAddrs }) probeOK(t, s) if got := ctl.joined(); !strings.HasPrefix(got, "set-property --runtime carlos-instances.slice IPAddressAllow=\n") { t.Errorf("calls:\n%s", got) } rep, ready := s.Ready() if ready || rep.Ready { t.Errorf("ready with a local resolver: %+v", rep) } if l := rep.Lines[len(rep.Lines)-1]; l.Check != "resolver-local" || l.Verdict != "fail" || !strings.Contains(l.Detail, "172.31.7.19 is an address of this box") { t.Errorf("last line = %+v", l) } // And a good resolver is an ok line. if err := os.WriteFile(resolv, []byte("nameserver 172.31.0.2\n"), 0o644); err != nil { t.Fatal(err) } probeOK(t, s) rep, ready = s.Ready() if l := rep.Lines[len(rep.Lines)-1]; !ready || l.Check != "resolver-local" || l.Verdict != "ok" { t.Errorf("ready=%v report=%+v", ready, rep) } } // When the box's addresses cannot be read, nothing is opened: the allow // is reset to empty and the probe does not run. func TestAllowClosesWhenTheBoxsAddressesCannotBeRead(t *testing.T) { dir := t.TempDir() resolv := filepath.Join(dir, "resolv.conf") if err := os.WriteFile(resolv, []byte("nameserver 172.31.0.2\n"), 0o644); err != nil { t.Fatal(err) } for name, local := range map[string]func() ([]netip.Prefix, error){ "error": func() ([]netip.Prefix, error) { return nil, errors.New("no procfs") }, "nil": nil, } { ctl := &fakeCtl{} s, _ := newServer(t, ctl, func(s *Server) { s.ResolvConf = resolv; s.LocalAddrs = local }) probeOK(t, s) if got := ctl.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow=" { t.Errorf("%s: calls:\n%s", name, got) } rep, ready := s.Ready() if ready || len(rep.Lines) != 1 || rep.Lines[0].Check != "resolver-allow" || !strings.Contains(rep.Lines[0].Detail, "every resolver is closed") { t.Errorf("%s: ready=%v report=%+v", name, ready, rep) } } } func TestParseTheBoxsAddresses(t *testing.T) { dir := t.TempDir() fib := filepath.Join(dir, "fib_trie") v6 := filepath.Join(dir, "if_inet6") // An EC2 box: /proc/net/fib_trie (Main and Local tables) and // /proc/net/if_inet6, trimmed. if err := os.WriteFile(fib, []byte(`Main: +-- 0.0.0.0/0 3 0 5 |-- 0.0.0.0 /0 universe UNICAST +-- 127.0.0.0/8 2 0 2 +-- 127.0.0.0/31 1 0 0 |-- 127.0.0.0 /8 host LOCAL |-- 127.0.0.1 /32 host LOCAL |-- 127.255.255.255 /32 link BROADCAST +-- 172.31.0.0/20 2 0 2 |-- 172.31.0.0 /20 link UNICAST |-- 172.31.7.19 /32 host LOCAL |-- 172.31.15.255 /32 link BROADCAST Local: +-- 0.0.0.0/0 3 0 5 |-- 172.31.7.19 /32 host LOCAL `), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(v6, []byte("00000000000000000000000000000001 01 80 10 80 lo\nfe8000000000000004a102fffe3b5c7d 02 40 20 80 ens5\n"), 0o644); err != nil { t.Fatal(err) } got, err := localPrefixes(fib, v6) if err != nil { t.Fatal(err) } if s := fmt.Sprint(got); s != "[127.0.0.0/8 127.0.0.1/32 172.31.7.19/32 172.31.7.19/32 ::1/128 fe80::4a1:2ff:fe3b:5c7d/128]" { t.Errorf("local = %s", s) } // The resolver, the subnet's broadcast and network addresses are not // local; anything on loopback is. for a, want := range map[string]bool{"172.31.0.2": false, "172.31.15.255": false, "172.31.0.0": false, "127.0.0.53": true, "172.31.7.19": true} { if _, ok := isLocal(netip.MustParseAddr(a), got); ok != want { t.Errorf("isLocal(%s) = %v", a, ok) } } // No IPv6 in the kernel is no v6 address; a fib_trie with no LOCAL // route is not one this parser knows, and is refused. if _, err := localPrefixes(fib, filepath.Join(dir, "absent")); err != nil { t.Errorf("no if_inet6: %v", err) } if err := os.WriteFile(fib, []byte("Main:\n +-- 0.0.0.0/0 3 0 5\n"), 0o644); err != nil { t.Fatal(err) } if _, err := localPrefixes(fib, v6); err == nil { t.Error("a fib_trie with no LOCAL route was read as no addresses") } if _, err := localPrefixes(filepath.Join(dir, "absent"), v6); err == nil { t.Error("no fib_trie was read as no addresses") } } func TestDiskFloor(t *testing.T) { if f := DiskFloor(8 << 30); f != 1<<30 { t.Errorf("8 GiB: %d", f) } if f := DiskFloor(100 << 30); f != 10<<30 { t.Errorf("100 GiB: %d", f) } } // Below the floor a start and a seed are refused with 507 and no unit is // touched (#214 item 1). func TestStartAndSeedRefuseBelowTheDiskFloor(t *testing.T) { ctl := &fakeCtl{} _, c := newServer(t, ctl, func(s *Server) { s.Statfs = func(string) (uint64, uint64, error) { return 900 << 20, 8 << 30, nil } }) for _, path := range []string{"/start?host=h.example.com&role=run&memory_max=536870912", "/seed?host=h.example.com&memory_max=536870912"} { resp := post(t, c, path) if resp.StatusCode != http.StatusInsufficientStorage { t.Errorf("%s: status %d", path, resp.StatusCode) } } if calls := ctl.joined(); calls != "" { t.Errorf("systemctl called: %s", calls) } } // The guard warns and never stops (review F2): usage cannot be attributed // reliably from here, and a stop from here would bypass the activator's // fence, so a full disk refuses new starts and leaves awake instances be. func TestFullDiskWarnsAndStopsNothing(t *testing.T) { ctl := &fakeCtl{} var logged []string s := &Server{Systemctl: ctl.run, DataRoot: t.TempDir(), Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, Statfs: func(string) (uint64, uint64, error) { return 100 << 20, 8 << 30, nil }} if !s.warnIfFull() { t.Fatal("no warning below the floor") } if len(logged) != 1 || !strings.Contains(logged[0], "refusing every instance start") { t.Errorf("logged %q", logged) } if calls := ctl.joined(); calls != "" { t.Errorf("systemctl called: %s", calls) } s.Statfs = func(string) (uint64, uint64, error) { return 4 << 30, 8 << 30, nil } if s.warnIfFull() { t.Error("warned with room") } }