diff --git a/coderd/rbac/roles_test.go b/coderd/rbac/roles_test.go index d90f045284c5b..e6680d4d628cc 100644 --- a/coderd/rbac/roles_test.go +++ b/coderd/rbac/roles_test.go @@ -77,7 +77,7 @@ func TestOwnerExec(t *testing.T) { }) } -// nolint:tparallel,paralleltest -- subtests share a map, just run sequentially. +// nolint:tparallel,paralleltest // subtests share a map, just run sequentially. func TestRolePermissions(t *testing.T) { t.Parallel() @@ -557,7 +557,7 @@ func TestRolePermissions(t *testing.T) { // nolint:tparallel,paralleltest for _, c := range testCases { c := c - // nolint:tparallel,paralleltest -- These share the same remainingPermissions map + // nolint:tparallel,paralleltest // These share the same remainingPermissions map t.Run(c.Name, func(t *testing.T) { remainingSubjs := make(map[string]struct{}) for _, subj := range requiredSubjects { @@ -600,7 +600,7 @@ func TestRolePermissions(t *testing.T) { // Only run these if the tests on top passed. Otherwise, the error output is too noisy. if passed { for rtype, v := range remainingPermissions { - // nolint:tparallel,paralleltest -- Making a subtest for easier diagnosing failures. + // nolint:tparallel,paralleltest // Making a subtest for easier diagnosing failures. t.Run(fmt.Sprintf("%s-AllActions", rtype), func(t *testing.T) { if len(v) > 0 { assert.Equal(t, map[policy.Action]bool{}, v, "remaining permissions should be empty for type %q", rtype) diff --git a/tailnet/test/integration/integration.go b/tailnet/test/integration/integration.go index b26365ea3ee8b..3877542c8eafc 100644 --- a/tailnet/test/integration/integration.go +++ b/tailnet/test/integration/integration.go @@ -41,12 +41,34 @@ import ( "github.com/coder/coder/v2/testutil" ) -// IDs used in tests. -var ( - Client1ID = uuid.MustParse("00000000-0000-0000-0000-000000000001") - Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002") +type ClientNumber int + +const ( + ClientNumber1 ClientNumber = 1 + ClientNumber2 ClientNumber = 2 ) +type Client struct { + Number ClientNumber + ID uuid.UUID + ListenPort uint16 + ShouldRunTests bool +} + +var Client1 = Client{ + Number: ClientNumber1, + ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + ListenPort: client1Port, + ShouldRunTests: true, +} + +var Client2 = Client{ + Number: ClientNumber2, + ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"), + ListenPort: client2Port, + ShouldRunTests: false, +} + type TestTopology struct { Name string // SetupNetworking creates interfaces and network namespaces for the test. @@ -59,12 +81,12 @@ type TestTopology struct { Server ServerStarter // StartClient gets called in each client subprocess. It's expected to // create the tailnet.Conn and ensure connectivity to it's peer. - StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn + StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn // RunTests is the main test function. It's called in each of the client // subprocesses. If tests can only run once, they should check the client ID // and return early if it's not the expected one. - RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn) + RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, conn *tailnet.Conn, me Client, peer Client) } type ServerStarter interface { @@ -264,13 +286,14 @@ http { // StartClientDERP creates a client connection to the server for coordination // and creates a tailnet.Conn which will only use DERP to connect to the peer. -func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: true, Logger: logger, DERPForceWebSockets: false, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, @@ -279,13 +302,14 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, // StartClientDERPWebSockets does the same thing as StartClientDERP but will // only use DERP WebSocket fallback. -func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: true, Logger: logger, DERPForceWebSockets: true, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, @@ -295,20 +319,21 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url. // StartClientDirect does the same thing as StartClientDERP but disables // BlockEndpoints (which enables Direct connections), and waits for a direct // connection to be established between the two peers. -func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: false, Logger: logger, DERPForceWebSockets: true, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, }) // Wait for direct connection to be established. - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) require.Eventually(t, func() bool { t.Log("attempting ping to peer to judge direct connection") ctx := testutil.Context(t, testutil.WaitShort) @@ -332,8 +357,8 @@ type ClientStarter struct { Options *tailnet.Options } -func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, options *tailnet.Options) *tailnet.Conn { - u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String())) +func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn { + u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", me.ID.String())) require.NoError(t, err) //nolint:bodyclose ws, _, err := websocket.Dial(context.Background(), u.String(), nil) @@ -357,7 +382,7 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my _ = conn.Close() }) - coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peerID) + coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peer.ID) t.Cleanup(func() { _ = coordination.Close() }) @@ -365,10 +390,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my return conn } -func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap { +func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) { + serverURL, err := url.Parse(serverURLStr) + if err != nil { + return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err) + } + portStr := serverURL.Port() port, err := strconv.Atoi(portStr) - require.NoError(t, err, "parse server port") + if err != nil { + return nil, xerrors.Errorf("parse port %q: %w", portStr, err) + } hostname := serverURL.Hostname() ipv4 := "" @@ -399,7 +431,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap { }, }, }, - } + }, nil } // ExecBackground starts a subprocess with the given flags and returns a diff --git a/tailnet/test/integration/integration_test.go b/tailnet/test/integration/integration_test.go index 45d88145216c1..e23b716096048 100644 --- a/tailnet/test/integration/integration_test.go +++ b/tailnet/test/integration/integration_test.go @@ -4,19 +4,27 @@ package integration_test import ( + "context" + "encoding/json" "flag" "fmt" + "net" "net/http" "net/url" "os" "os/signal" + "path/filepath" "runtime" + "strconv" "syscall" "testing" "time" - "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "tailscale.com/net/stun/stuntest" + "tailscale.com/tailcfg" + "tailscale.com/types/nettype" "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" @@ -30,17 +38,19 @@ const runTestEnv = "CODER_TAILNET_TESTS" var ( isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess") testID = flag.String("test-name", "", "Which test is being run") - role = flag.String("role", "", "The role of the test subprocess: server, client") + role = flag.String("role", "", "The role of the test subprocess: server, stun, client") // Role: server serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server") + // Role: stun + stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server") + // Role: client - clientName = flag.String("client-name", "", "The name of the client for logs") - clientServerURL = flag.String("client-server-url", "", "The url to connect to the server") - clientMyID = flag.String("client-id", "", "The id of the client") - clientPeerID = flag.String("client-peer-id", "", "The id of the other client") - clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess") + clientName = flag.String("client-name", "", "The name of the client for logs") + clientNumber = flag.Int("client-number", 0, "The number of the client") + clientServerURL = flag.String("client-server-url", "", "The url to connect to the server") + clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client") ) func TestMain(m *testing.M) { @@ -87,7 +97,7 @@ var topologies = []integration.TestTopology{ // endpoints to connect as routing is enabled between client 1 and // client 2. Name: "EasyNATDirect", - SetupNetworking: integration.SetupNetworkingEasyNAT, + SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN, Server: integration.SimpleServerOptions{}, StartClient: integration.StartClientDirect, RunTests: integration.TestSuite, @@ -143,17 +153,41 @@ func TestIntegration(t *testing.T) { log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) networking := topo.SetupNetworking(t, log) - // Fork the three child processes. + // Useful for debugging network namespaces by avoiding cleanup. + // t.Cleanup(func() { + // time.Sleep(time.Minute * 15) + // }) + closeServer := startServerSubprocess(t, topo.Name, networking) + + closeSTUN := func() error { return nil } + if networking.STUN.ListenAddr != "" { + closeSTUN = startSTUNSubprocess(t, topo.Name, networking) + } + + // Write the DERP maps to a file. + tempDir := t.TempDir() + client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json") + client1DERPMap, err := networking.Client1.ResolveDERPMap() + require.NoError(t, err, "resolve client 1 DERP map") + err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap) + require.NoError(t, err, "write client 1 DERP map") + client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json") + client2DERPMap, err := networking.Client2.ResolveDERPMap() + require.NoError(t, err, "resolve client 2 DERP map") + err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap) + require.NoError(t, err, "write client 2 DERP map") + // client1 runs the tests. - client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1) - _, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2) + client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, integration.Client1, client1DERPMapPath) + _, closeClient2 := startClientSubprocess(t, topo.Name, networking, integration.Client2, client2DERPMapPath) // Wait for client1 to exit. require.NoError(t, <-client1ErrCh, "client 1 exited") // Close client2 and the server. require.NoError(t, closeClient2(), "client 2 exited") + require.NoError(t, closeSTUN(), "stun exited") require.NoError(t, closeServer(), "server exited") }) } @@ -169,10 +203,11 @@ func handleTestSubprocess(t *testing.T) { } } require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID) + require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role) testName := topo.Name + "/" - if *role == "server" { - testName += "server" + if *role == "server" || *role == "stun" { + testName += *role } else { testName += *clientName } @@ -185,27 +220,44 @@ func handleTestSubprocess(t *testing.T) { topo.Server.StartServer(t, logger, *serverListenAddr) // no exit + case "stun": + launchSTUNServer(t, *stunListenAddr) + // no exit + case "client": logger = logger.Named(*clientName) + if *clientNumber != int(integration.ClientNumber1) && *clientNumber != int(integration.ClientNumber2) { + t.Fatalf("invalid client number %d", clientNumber) + } + me, peer := integration.Client1, integration.Client2 + if *clientNumber == int(integration.ClientNumber2) { + me, peer = peer, me + } + serverURL, err := url.Parse(*clientServerURL) require.NoErrorf(t, err, "parse server url %q", *clientServerURL) - myID, err := uuid.Parse(*clientMyID) - require.NoErrorf(t, err, "parse client id %q", *clientMyID) - peerID, err := uuid.Parse(*clientPeerID) - require.NoErrorf(t, err, "parse peer id %q", *clientPeerID) + + // Load the DERP map. + var derpMap tailcfg.DERPMap + derpMapPath := *clientDERPMapPath + f, err := os.Open(derpMapPath) + require.NoErrorf(t, err, "open DERP map %q", derpMapPath) + err = json.NewDecoder(f).Decode(&derpMap) + _ = f.Close() + require.NoErrorf(t, err, "decode DERP map %q", derpMapPath) waitForServerAvailable(t, serverURL) - conn := topo.StartClient(t, logger, serverURL, myID, peerID) + conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer) - if *clientRunTests { + if me.ShouldRunTests { // Wait for connectivity. - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) { t.Fatalf("peer %v did not become reachable", peerIP) } - topo.RunTests(t, logger, serverURL, myID, peerID, conn) + topo.RunTests(t, logger, serverURL, conn, me, peer) // then exit return } @@ -218,6 +270,23 @@ func handleTestSubprocess(t *testing.T) { }) } +type forcedAddrPacketListener struct { + addr string +} + +var _ nettype.PacketListener = forcedAddrPacketListener{} + +func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) { + return nettype.Std{}.ListenPacket(ctx, network, ln.addr) +} + +func launchSTUNServer(t *testing.T, listenAddr string) { + ln := forcedAddrPacketListener{addr: listenAddr} + addr, cleanup := stuntest.ServeWithPacketListener(t, ln) + t.Cleanup(cleanup) + assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr") +} + func waitForServerAvailable(t *testing.T, serverURL *url.URL) { const delay = 100 * time.Millisecond const reqTimeout = 2 * time.Second @@ -247,29 +316,32 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) { } func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error { - _, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{ + _, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{ "--subprocess", "--test-name=" + topologyName, "--role=server", - "--server-listen-addr=" + networking.ServerListenAddr, + "--server-listen-addr=" + networking.Server.ListenAddr, }) return closeFn } -func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) { - require.True(t, clientNumber == 1 || clientNumber == 2) +func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error { + _, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{ + "--subprocess", + "--test-name=" + topologyName, + "--role=stun", + "--stun-listen-addr=" + networking.STUN.ListenAddr, + }) + return closeFn +} +func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, me integration.Client, derpMapPath string) (<-chan error, func() error) { var ( - clientName = fmt.Sprintf("client%d", clientNumber) - myID = integration.Client1ID - peerID = integration.Client2ID - accessURL = networking.ServerAccessURLClient1 - netNS = networking.ProcessClient1.NetNS + clientName = fmt.Sprintf("client%d", me.Number) + clientProcessConfig = networking.Client1 ) - if clientNumber == 2 { - myID, peerID = peerID, myID - accessURL = networking.ServerAccessURLClient2 - netNS = networking.ProcessClient2.NetNS + if me.Number == integration.ClientNumber2 { + clientProcessConfig = networking.Client2 } flags := []string{ @@ -277,15 +349,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra "--test-name=" + topologyName, "--role=client", "--client-name=" + clientName, - "--client-server-url=" + accessURL, - "--client-id=" + myID.String(), - "--client-peer-id=" + peerID.String(), - } - if clientNumber == 1 { - flags = append(flags, "--client-run-tests") + "--client-number=" + strconv.Itoa(int(me.Number)), + "--client-server-url=" + clientProcessConfig.ServerAccessURL, + "--client-derp-map-path=" + derpMapPath, } - return startSubprocess(t, clientName, netNS, flags) + return startSubprocess(t, clientName, clientProcessConfig.Process.NetNS, flags) } // startSubprocess launches the test binary with the same flags as the test, but @@ -295,6 +364,22 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func() error) { name := os.Args[0] // Always use verbose mode since it gets piped to the parent test anyways. - args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) + args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) //nolint:gocritic return integration.ExecBackground(t, processName, netNS, name, args) } + +func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + err = enc.Encode(derpMap) + if err != nil { + return err + } + return nil +} diff --git a/tailnet/test/integration/network.go b/tailnet/test/integration/network.go index 80eeb6048bd66..e0d8f7109c167 100644 --- a/tailnet/test/integration/network.go +++ b/tailnet/test/integration/network.go @@ -13,27 +13,55 @@ import ( "github.com/stretchr/testify/require" "github.com/tailscale/netlink" "golang.org/x/xerrors" + "tailscale.com/tailcfg" "cdr.dev/slog" "github.com/coder/coder/v2/cryptorand" ) +const ( + client1Port = 48001 + client1RouterPort = 48011 + client2Port = 48002 + client2RouterPort = 48012 +) + type TestNetworking struct { - // ServerListenAddr is the IP address and port that the server listens on, - // passed to StartServer. - ServerListenAddr string - // ServerAccessURLClient1 is the hostname and port that the first client - // uses to access the server. - ServerAccessURLClient1 string - // ServerAccessURLClient2 is the hostname and port that the second client - // uses to access the server. - ServerAccessURLClient2 string - - // Networking settings for each subprocess. - ProcessServer TestNetworkingProcess - ProcessClient1 TestNetworkingProcess - ProcessClient2 TestNetworkingProcess + Server TestNetworkingServer + STUN TestNetworkingSTUN + Client1 TestNetworkingClient + Client2 TestNetworkingClient +} + +type TestNetworkingServer struct { + Process TestNetworkingProcess + ListenAddr string +} + +type TestNetworkingSTUN struct { + Process TestNetworkingProcess + // If empty, no STUN subprocess is launched. + ListenAddr string +} + +type TestNetworkingClient struct { + Process TestNetworkingProcess + // ServerAccessURL is the hostname and port that the client uses to access + // the server over HTTP for coordination. + ServerAccessURL string + // DERPMap is the DERP map that the client uses. If nil, a basic DERP map + // containing only a single DERP with `ServerAccessURL` is used with no + // STUN servers. + DERPMap *tailcfg.DERPMap +} + +func (c TestNetworkingClient) ResolveDERPMap() (*tailcfg.DERPMap, error) { + if c.DERPMap != nil { + return c.DERPMap, nil + } + + return basicDERPMap(c.ServerAccessURL) } type TestNetworkingProcess struct { @@ -46,14 +74,9 @@ type TestNetworkingProcess struct { // namespace only exists for isolation on the host and doesn't serve any routing // purpose. func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { - netNSName := "codertest_netns_" - randStr, err := cryptorand.String(4) - require.NoError(t, err, "generate random string for netns name") - netNSName += randStr - // Create a single network namespace for all tests so we can have an // isolated loopback interface. - netNSFile := createNetNS(t, netNSName) + netNSFile := createNetNS(t, uniqNetName(t)) var ( listenAddr = "127.0.0.1:8080" @@ -62,176 +85,323 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { } ) return TestNetworking{ - ServerListenAddr: listenAddr, - ServerAccessURLClient1: "http://" + listenAddr, - ServerAccessURLClient2: "http://" + listenAddr, - ProcessServer: process, - ProcessClient1: process, - ProcessClient2: process, + Server: TestNetworkingServer{ + Process: process, + ListenAddr: listenAddr, + }, + Client1: TestNetworkingClient{ + Process: process, + ServerAccessURL: "http://" + listenAddr, + }, + Client2: TestNetworkingClient{ + Process: process, + ServerAccessURL: "http://" + listenAddr, + }, } } -// SetupNetworkingEasyNAT creates a network namespace with a router that NATs -// packets between two clients and a server. -// See createFakeRouter for the full topology. +func easyNAT(t *testing.T) fakeInternet { + internet := createFakeInternet(t) + + _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") + + // Set up iptables masquerade rules to allow each router to NAT packets. + leaves := []struct { + fakeRouterLeaf + clientPort int + natPort int + }{ + {internet.Client1, client1Port, client1RouterPort}, + {internet.Client2, client2Port, client2RouterPort}, + } + for _, leaf := range leaves { + _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") + + // All non-UDP traffic should use regular masquerade e.g. for HTTP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "POSTROUTING", + // Every interface except loopback. + "!", "-o", "lo", + // Every protocol except UDP. + "!", "-p", "udp", + "-j", "MASQUERADE", + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") + + // Outgoing traffic should get NATed to the router's IP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "POSTROUTING", + "-p", "udp", + "--sport", fmt.Sprint(leaf.clientPort), + "-j", "SNAT", + "--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort), + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") + + // Incoming traffic should be forwarded to the client's IP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "PREROUTING", + "-p", "udp", + "--dport", fmt.Sprint(leaf.natPort), + "-j", "DNAT", + "--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort), + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") + } + + return internet +} + +// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT" +// forwarding rules. +// See createFakeInternet. // NAT is achieved through a single iptables masquerade rule. func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking { - router := createFakeRouter(t) - - // Set up iptables masquerade rules to allow the router to NAT packets - // between the Three Kingdoms. - _, err := commandInNetNS(router.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() - require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") - _, err = commandInNetNS(router.RouterNetNS, "iptables", []string{ - "-t", "nat", - "-A", "POSTROUTING", - // Every interface except loopback. - "!", "-o", "lo", - "-j", "MASQUERADE", - }).Output() - require.NoError(t, wrapExitErr(err), "add iptables masquerade rule") - - return router.Net + return easyNAT(t).Net } -type fakeRouter struct { - Net TestNetworking +// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but +// also creates a namespace and bridge address for a STUN server. +func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking { + internet := easyNAT(t) - RouterNetNS *os.File - RouterVeths struct { - Server string - Client1 string - Client2 string + // Create another network namespace for the STUN server. + stunNetNS := createNetNS(t, internet.NamePrefix+"stun") + internet.Net.STUN.Process = TestNetworkingProcess{ + NetNS: stunNetNS, + } + + const ip = "10.0.0.64" + err := joinBridge(joinBridgeOpts{ + bridgeNetNS: internet.BridgeNetNS, + netNS: stunNetNS, + bridgeName: internet.BridgeName, + vethPair: vethPair{ + Outer: internet.NamePrefix + "b-stun", + Inner: internet.NamePrefix + "stun-b", + }, + ip: ip, + }) + require.NoError(t, err, "join bridge with STUN server") + internet.Net.STUN.ListenAddr = ip + ":3478" + + // Define custom DERP map. + stunRegion := &tailcfg.DERPRegion{ + RegionID: 10000, + RegionCode: "stun0", + RegionName: "STUN0", + Nodes: []*tailcfg.DERPNode{ + { + Name: "stun0a", + RegionID: 1, + IPv4: ip, + IPv6: "none", + STUNPort: 3478, + STUNOnly: true, + }, + }, } - ServerNetNS *os.File - ServerVeth string - Client1NetNS *os.File - Client1Veth string - Client2NetNS *os.File - Client2Veth string + client1DERP, err := internet.Net.Client1.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 1") + client1DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client1.DERPMap = client1DERP + client2DERP, err := internet.Net.Client2.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 2") + client2DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client2.DERPMap = client2DERP + + return internet.Net } -// fakeRouter creates multiple namespaces with veth pairs between them with -// the following topology: -// -// namespaces: -// - router -// - server -// - client1 -// - client2 +type vethPair struct { + Outer string + Inner string +} + +type fakeRouterLeaf struct { + // RouterIP is the IP address of the router on the bridge. + RouterIP string + // ClientIP is the IP address of the client on the router. + ClientIP string + // RouterNetNS is the router for this specific leaf. + RouterNetNS *os.File + // ClientNetNS is where the "user" is. + ClientNetNS *os.File + // Veth pair between the router and the bridge. + OuterVethPair vethPair + // Veth pair between the user and the router. + InnerVethPair vethPair +} + +type fakeInternet struct { + Net TestNetworking + + NamePrefix string + BridgeNetNS *os.File + BridgeName string + ServerNetNS *os.File + ServerVethPair vethPair // between bridge and server NS + Client1 fakeRouterLeaf + Client2 fakeRouterLeaf +} + +// createFakeInternet creates multiple namespaces with veth pairs between them +// with the following topology: // -// veth pairs: -// - router-server (10.0.1.1) <-> server-router (10.0.1.2) -// - router-client1 (10.0.2.1) <-> client1-router (10.0.2.2) -// - router-client2 (10.0.3.1) <-> client2-router (10.0.3.2) +// . veth ┌────────┐ veth +// . ┌─────────────────┤ Bridge ├───────────────────┐ +// . │ └───┬────┘ │ +// . │ │ │ +// . │10.0.0.1 veth│10.0.0.2 │10.0.0.3 +// . ┌───────┴───────┐ ┌───────┴─────────┐ ┌────────┴────────┐ +// . │ Server │ │ Client 1 router │ │ Client 2 router │ +// . └───────────────┘ └───────┬─────────┘ └────────┬────────┘ +// . │10.0.2.1 │10.0.3.1 +// . veth│ veth│ +// . │10.0.2.2 │10.0.3.2 +// . ┌───────┴─────────┐ ┌────────┴────────┐ +// . │ Client 1 │ │ Client 2 │ +// . └─────────────────┘ └─────────────────┘ // // No iptables rules are created, so packets will not be forwarded out of the -// box. Routes are created between all namespaces based on the veth pairs, -// however. -func createFakeRouter(t *testing.T) fakeRouter { +// box. Default routes are created from the edge namespaces (client1, client2) +// to their respective routers, but no NAT rules are created. +func createFakeInternet(t *testing.T) fakeInternet { t.Helper() const ( - routerServerPrefix = "10.0.1." - routerServerIP = routerServerPrefix + "1" - serverIP = routerServerPrefix + "2" - routerClient1Prefix = "10.0.2." - routerClient1IP = routerClient1Prefix + "1" - client1IP = routerClient1Prefix + "2" - routerClient2Prefix = "10.0.3." - routerClient2IP = routerClient2Prefix + "1" - client2IP = routerClient2Prefix + "2" + bridgePrefix = "10.0.0." + serverIP = bridgePrefix + "1" + client1Prefix = "10.0.2." + client2Prefix = "10.0.3." ) + var ( + namePrefix = uniqNetName(t) + "_" + router = fakeInternet{ + NamePrefix: namePrefix, + BridgeName: namePrefix + "b", + } + ) + + // Create bridge namespace and bridge interface. + router.BridgeNetNS = createNetNS(t, router.BridgeName) + err := createBridge(router.BridgeNetNS, router.BridgeName) + require.NoError(t, err, "create bridge in netns") - prefix := uniqNetName(t) + "_" - router := fakeRouter{} - router.RouterVeths.Server = prefix + "r-s" - router.RouterVeths.Client1 = prefix + "r-c1" - router.RouterVeths.Client2 = prefix + "r-c2" - router.ServerVeth = prefix + "s-r" - router.Client1Veth = prefix + "c1-r" - router.Client2Veth = prefix + "c2-r" - - // Create namespaces. - router.RouterNetNS = createNetNS(t, prefix+"r") - serverNS := createNetNS(t, prefix+"s") - client1NS := createNetNS(t, prefix+"c1") - client2NS := createNetNS(t, prefix+"c2") - - vethPairs := []struct { - parentName string - peerName string - parentNS *os.File - peerNS *os.File - parentIP string - peerIP string + // Create server namespace and veth pair between bridge and server. + router.ServerNetNS = createNetNS(t, namePrefix+"s") + router.ServerVethPair = vethPair{ + Outer: namePrefix + "b-s", + Inner: namePrefix + "s-b", + } + err = joinBridge(joinBridgeOpts{ + bridgeNetNS: router.BridgeNetNS, + netNS: router.ServerNetNS, + bridgeName: router.BridgeName, + vethPair: router.ServerVethPair, + ip: serverIP, + }) + require.NoError(t, err, "join bridge with server") + + leaves := []struct { + leaf *fakeRouterLeaf + routerName string + clientName string + routerBridgeIP string + routerClientIP string + clientIP string }{ { - parentName: router.RouterVeths.Server, - peerName: router.ServerVeth, - parentNS: router.RouterNetNS, - peerNS: serverNS, - parentIP: routerServerIP, - peerIP: serverIP, - }, - { - parentName: router.RouterVeths.Client1, - peerName: router.Client1Veth, - parentNS: router.RouterNetNS, - peerNS: client1NS, - parentIP: routerClient1IP, - peerIP: client1IP, + leaf: &router.Client1, + routerName: "c1r", + clientName: "c1", + routerBridgeIP: bridgePrefix + "2", + routerClientIP: client1Prefix + "1", + clientIP: client1Prefix + "2", }, { - parentName: router.RouterVeths.Client2, - peerName: router.Client2Veth, - parentNS: router.RouterNetNS, - peerNS: client2NS, - parentIP: routerClient2IP, - peerIP: client2IP, + leaf: &router.Client2, + routerName: "c2r", + clientName: "c2", + routerBridgeIP: bridgePrefix + "3", + routerClientIP: client2Prefix + "1", + clientIP: client2Prefix + "2", }, } - for _, vethPair := range vethPairs { - err := createVethPair(vethPair.parentName, vethPair.peerName) - require.NoErrorf(t, err, "create veth pair %q <-> %q", vethPair.parentName, vethPair.peerName) - - // Move the veth interfaces to the respective network namespaces. - err = setVethNetNS(vethPair.parentName, int(vethPair.parentNS.Fd())) - require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.parentName) - err = setVethNetNS(vethPair.peerName, int(vethPair.peerNS.Fd())) - require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.peerName) + for _, leaf := range leaves { + leaf.leaf.RouterIP = leaf.routerBridgeIP + leaf.leaf.ClientIP = leaf.clientIP - // Set IP addresses on the interfaces. - err = setInterfaceIP(vethPair.parentNS, vethPair.parentName, vethPair.parentIP) - require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.parentIP, vethPair.parentName) - err = setInterfaceIP(vethPair.peerNS, vethPair.peerName, vethPair.peerIP) - require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.peerIP, vethPair.peerName) + // Create two network namespaces for each leaf: one for the router and + // one for the "client". + leaf.leaf.RouterNetNS = createNetNS(t, namePrefix+leaf.routerName) + leaf.leaf.ClientNetNS = createNetNS(t, namePrefix+leaf.clientName) - // Bring up both interfaces. - err = setInterfaceUp(vethPair.parentNS, vethPair.parentName) - require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName) - err = setInterfaceUp(vethPair.peerNS, vethPair.peerName) - require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName) + // Join the bridge. + leaf.leaf.OuterVethPair = vethPair{ + Outer: namePrefix + "b-" + leaf.routerName, + Inner: namePrefix + leaf.routerName + "-b", + } + err = joinBridge(joinBridgeOpts{ + bridgeNetNS: router.BridgeNetNS, + netNS: leaf.leaf.RouterNetNS, + bridgeName: router.BridgeName, + vethPair: leaf.leaf.OuterVethPair, + ip: leaf.routerBridgeIP, + }) + require.NoError(t, err, "join bridge with router") + + // Create inner veth pair between the router and the client. + leaf.leaf.InnerVethPair = vethPair{ + Outer: namePrefix + leaf.routerName + "-" + leaf.clientName, + Inner: namePrefix + leaf.clientName + "-" + leaf.routerName, + } + err = createVethPair(leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) + require.NoErrorf(t, err, "create veth pair %q <-> %q", leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) + + // Move the network interfaces to the respective network namespaces. + err = setVethNetNS(leaf.leaf.InnerVethPair.Outer, int(leaf.leaf.RouterNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Outer) + err = setVethNetNS(leaf.leaf.InnerVethPair.Inner, int(leaf.leaf.ClientNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Inner) + + // Set router's "local" IP on the veth. + err = setInterfaceIP(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer, leaf.routerClientIP) + require.NoErrorf(t, err, "set IP %q on interface %q", leaf.routerClientIP, leaf.leaf.InnerVethPair.Outer) + // Set client's IP on the veth. + err = setInterfaceIP(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner, leaf.clientIP) + require.NoErrorf(t, err, "set IP %q on interface %q", leaf.clientIP, leaf.leaf.InnerVethPair.Inner) + + // Bring up the interfaces. + err = setInterfaceUp(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer) + require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.OuterVethPair.Outer) + err = setInterfaceUp(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner) + require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.InnerVethPair.Inner) // We don't need to add a route from parent to peer since the kernel // already adds a default route for the /24. We DO need to add a default // route from peer to parent, however. - err = addRouteInNetNS(vethPair.peerNS, []string{"default", "via", vethPair.parentIP, "dev", vethPair.peerName}) - require.NoErrorf(t, err, "add peer default route to %q", vethPair.peerName) + err = addRouteInNetNS(leaf.leaf.ClientNetNS, []string{"default", "via", leaf.routerClientIP, "dev", leaf.leaf.InnerVethPair.Inner}) + require.NoErrorf(t, err, "add peer default route to %q", leaf.leaf.InnerVethPair.Inner) } router.Net = TestNetworking{ - ServerListenAddr: serverIP + ":8080", - ServerAccessURLClient1: "http://" + serverIP + ":8080", - ServerAccessURLClient2: "http://" + serverIP + ":8080", - ProcessServer: TestNetworkingProcess{ - NetNS: serverNS, + Server: TestNetworkingServer{ + Process: TestNetworkingProcess{NetNS: router.ServerNetNS}, + ListenAddr: serverIP + ":8080", }, - ProcessClient1: TestNetworkingProcess{ - NetNS: client1NS, + Client1: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: router.Client1.ClientNetNS}, + ServerAccessURL: "http://" + serverIP + ":8080", }, - ProcessClient2: TestNetworkingProcess{ - NetNS: client2NS, + Client2: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: router.Client2.ClientNetNS}, + ServerAccessURL: "http://" + serverIP + ":8080", }, } return router @@ -246,6 +416,60 @@ func uniqNetName(t *testing.T) string { return netNSName } +type joinBridgeOpts struct { + bridgeNetNS *os.File + netNS *os.File + bridgeName string + // This vethPair will be created and should not already exist. + vethPair vethPair + ip string +} + +// joinBridge joins the given network namespace to the bridge. It creates a veth +// pair between the specified NetNS and the bridge NetNS, sets the IP address on +// the "child" veth, and brings up the interfaces. +func joinBridge(opts joinBridgeOpts) error { + // Create outer veth pair between the router and the bridge. + err := createVethPair(opts.vethPair.Outer, opts.vethPair.Inner) + if err != nil { + return xerrors.Errorf("create veth pair %q <-> %q: %w", opts.vethPair.Outer, opts.vethPair.Inner, err) + } + + // Move the network interfaces to the respective network namespaces. + err = setVethNetNS(opts.vethPair.Outer, int(opts.bridgeNetNS.Fd())) + if err != nil { + return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Outer, err) + } + err = setVethNetNS(opts.vethPair.Inner, int(opts.netNS.Fd())) + if err != nil { + return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Inner, err) + } + + // Connect the outer veth to the bridge. + err = setInterfaceBridge(opts.bridgeNetNS, opts.vethPair.Outer, opts.bridgeName) + if err != nil { + return xerrors.Errorf("set interface %q master to %q: %w", opts.vethPair.Outer, opts.bridgeName, err) + } + + // Set the bridge IP on the inner veth. + err = setInterfaceIP(opts.netNS, opts.vethPair.Inner, opts.ip) + if err != nil { + return xerrors.Errorf("set IP %q on interface %q: %w", opts.ip, opts.vethPair.Inner, err) + } + + // Bring up the interfaces. + err = setInterfaceUp(opts.bridgeNetNS, opts.vethPair.Outer) + if err != nil { + return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Outer, err) + } + err = setInterfaceUp(opts.netNS, opts.vethPair.Inner) + if err != nil { + return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Inner, err) + } + + return nil +} + // createNetNS creates a new network namespace with the given name. The returned // file is a file descriptor to the network namespace. // Note: all cleanup is handled for you, you do not need to call Close on the @@ -283,18 +507,48 @@ func createNetNS(t *testing.T, name string) *os.File { return file } +// createBridge creates a bridge in the given network namespace. The bridge is +// automatically brought up. +func createBridge(netNS *os.File, name string) error { + // While it might be possible to create a bridge directly in a NetNS or move + // an existing bridge to a NetNS, I couldn't figure out a way to do it. + // Creating it directly within the NetNS is the simplest way. + _, err := commandInNetNS(netNS, "ip", []string{"link", "add", name, "type", "bridge"}).Output() + if err != nil { + return xerrors.Errorf("create bridge %q in netns: %w", name, wrapExitErr(err)) + } + + _, err = commandInNetNS(netNS, "ip", []string{"link", "set", name, "up"}).Output() + if err != nil { + return xerrors.Errorf("set bridge %q up in netns: %w", name, wrapExitErr(err)) + } + + return nil +} + +// setInterfaceBridge sets the master of the given interface to the specified +// bridge. +func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error { + _, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "master", bridgeName}).Output() + if err != nil { + return xerrors.Errorf("set interface %q master to %q in netns: %w", ifaceName, bridgeName, wrapExitErr(err)) + } + + return nil +} + // createVethPair creates a veth pair with the given names. func createVethPair(parentVethName, peerVethName string) error { - vethLinkAttrs := netlink.NewLinkAttrs() - vethLinkAttrs.Name = parentVethName + linkAttrs := netlink.NewLinkAttrs() + linkAttrs.Name = parentVethName veth := &netlink.Veth{ - LinkAttrs: vethLinkAttrs, + LinkAttrs: linkAttrs, PeerName: peerVethName, } err := netlink.LinkAdd(veth) if err != nil { - return xerrors.Errorf("LinkAdd(name: %q, peerName: %q): %w", parentVethName, peerVethName, err) + return xerrors.Errorf("LinkAdd(type: veth, name: %q, peerName: %q): %w", parentVethName, peerVethName, err) } return nil diff --git a/tailnet/test/integration/remove_test_ns.sh b/tailnet/test/integration/remove_test_ns.sh new file mode 100755 index 0000000000000..464aac6c8eff0 --- /dev/null +++ b/tailnet/test/integration/remove_test_ns.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $(id -u) -ne 0 ]]; then + echo "Please run with sudo" + exit 1 +fi + +to_delete=$(ip netns list | grep -o 'cdr_.*_.*' | cut -d' ' -f1) +echo "Will delete:" +for ns in $to_delete; do + echo "- $ns" +done + +read -p "Continue? [y/N] " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 +fi + +for ns in $to_delete; do + ip netns delete "$ns" +done diff --git a/tailnet/test/integration/suite.go b/tailnet/test/integration/suite.go index 54fb0856a21af..32d9adb2e4a14 100644 --- a/tailnet/test/integration/suite.go +++ b/tailnet/test/integration/suite.go @@ -7,7 +7,6 @@ import ( "net/url" "testing" - "github.com/google/uuid" "github.com/stretchr/testify/require" "cdr.dev/slog" @@ -17,12 +16,12 @@ import ( // TODO: instead of reusing one conn for each suite, maybe we should make a new // one for each subtest? -func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, _, peerID uuid.UUID, conn *tailnet.Conn) { +func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) { t.Parallel() t.Run("Connectivity", func(t *testing.T) { t.Parallel() - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) _, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP) require.NoError(t, err, "ping peer") })