diff --git a/tailnet/test/integration/integration.go b/tailnet/test/integration/integration.go index 1190a3aa98b0d..70320567841a9 100644 --- a/tailnet/test/integration/integration.go +++ b/tailnet/test/integration/integration.go @@ -28,8 +28,10 @@ import ( "golang.org/x/xerrors" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/net/packet" "tailscale.com/tailcfg" "tailscale.com/types/key" + "tailscale.com/wgengine/capture" "cdr.dev/slog" "github.com/coder/coder/v2/coderd/httpapi" @@ -54,6 +56,7 @@ type Client struct { ID uuid.UUID ListenPort uint16 ShouldRunTests bool + TunnelSrc bool } var Client1 = Client{ @@ -61,6 +64,7 @@ var Client1 = Client{ ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), ListenPort: client1Port, ShouldRunTests: true, + TunnelSrc: true, } var Client2 = Client{ @@ -68,21 +72,20 @@ var Client2 = Client{ ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"), ListenPort: client2Port, ShouldRunTests: false, + TunnelSrc: false, } type TestTopology struct { Name string - // SetupNetworking creates interfaces and network namespaces for the test. - // The most simple implementation is NetworkSetupDefault, which only creates - // a network namespace shared for all tests. - SetupNetworking func(t *testing.T, logger slog.Logger) TestNetworking + + NetworkingProvider NetworkingProvider // Server is the server starter for the test. It is executed in the server // subprocess. Server ServerStarter - // StartClient gets called in each client subprocess. It's expected to + // ClientStarter.StartClient gets called in each client subprocess. It's expected to // create the tailnet.Conn and ensure connectivity to it's peer. - StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn + ClientStarter ClientStarter // RunTests is the main test function. It's called in each of the client // subprocesses. If tests can only run once, they should check the client ID @@ -97,6 +100,17 @@ type ServerStarter interface { StartServer(t *testing.T, logger slog.Logger, listenAddr string) } +type NetworkingProvider interface { + // SetupNetworking creates interfaces and network namespaces for the test. + // The most simple implementation is NetworkSetupDefault, which only creates + // a network namespace shared for all tests. + SetupNetworking(t *testing.T, logger slog.Logger) TestNetworking +} + +type ClientStarter interface { + StartClient(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn +} + type SimpleServerOptions struct { // FailUpgradeDERP will make the DERP server fail to handle the initial DERP // upgrade in a way that causes the client to fallback to @@ -369,77 +383,107 @@ http { _, _ = ExecBackground(t, "server.nginx", nil, "nginx", []string{"-c", cfgPath}) } -// StartClientDERP creates a client connection to the server for coordination -// and creates a tailnet.Conn which will only use DERP to connect to the peer. -func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ - Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)}, - DERPMap: derpMap, - BlockEndpoints: true, - Logger: logger, - DERPForceWebSockets: false, - ListenPort: me.ListenPort, - // These tests don't have internet connection, so we need to force - // magicsock to do anything. - ForceNetworkUp: true, - }) +type BasicClientStarter struct { + BlockEndpoints bool + DERPForceWebsockets bool + // WaitForConnection means wait for (any) peer connection before returning from StartClient + WaitForConnection bool + // WaitForConnection means wait for a direct peer connection before returning from StartClient + WaitForDirect bool + // Service is a network service (e.g. an echo server) to start on the client. If Wait* is set, the service is + // started prior to waiting. + Service NetworkService + LogPackets bool } -// StartClientDERPWebSockets does the same thing as StartClientDERP but will -// only use DERP WebSocket fallback. -func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ - Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)}, - DERPMap: derpMap, - BlockEndpoints: true, - Logger: logger, - DERPForceWebSockets: true, - ListenPort: me.ListenPort, - // These tests don't have internet connection, so we need to force - // magicsock to do anything. - ForceNetworkUp: true, - }) +type NetworkService interface { + StartService(t *testing.T, logger slog.Logger, conn *tailnet.Conn) } -// StartClientDirect does the same thing as StartClientDERP but disables -// BlockEndpoints (which enables Direct connections), and waits for a direct -// connection to be established between the two peers. -func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { +func (b BasicClientStarter) StartClient(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + var hook capture.Callback + if b.LogPackets { + pktLogger := packetLogger{logger} + hook = pktLogger.LogPacket + } conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ Addresses: []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)}, DERPMap: derpMap, - BlockEndpoints: false, + BlockEndpoints: b.BlockEndpoints, Logger: logger, - DERPForceWebSockets: true, + DERPForceWebSockets: b.DERPForceWebsockets, ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, + CaptureHook: hook, }) - // Wait for direct connection to be established. - peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID) - require.Eventually(t, func() bool { - t.Log("attempting ping to peer to judge direct connection") - ctx := testutil.Context(t, testutil.WaitShort) - _, p2p, pong, err := conn.Ping(ctx, peerIP) - if err != nil { - t.Logf("ping failed: %v", err) - return false - } - if !p2p { - t.Log("ping succeeded, but not direct yet") - return false - } - t.Logf("ping succeeded, direct connection established via %s", pong.Endpoint) - return true - }, testutil.WaitLong, testutil.IntervalMedium) + if b.Service != nil { + b.Service.StartService(t, logger, conn) + } + + if b.WaitForConnection || b.WaitForDirect { + // Wait for connection to be established. + peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID) + require.Eventually(t, func() bool { + t.Log("attempting ping to peer to judge direct connection") + ctx := testutil.Context(t, testutil.WaitShort) + _, p2p, pong, err := conn.Ping(ctx, peerIP) + if err != nil { + t.Logf("ping failed: %v", err) + return false + } + if !p2p && b.WaitForDirect { + t.Log("ping succeeded, but not direct yet") + return false + } + t.Logf("ping succeeded, p2p=%t, endpoint=%s", p2p, pong.Endpoint) + return true + }, testutil.WaitLong, testutil.IntervalMedium) + } return conn } -type ClientStarter struct { - Options *tailnet.Options +const EchoPort = 2381 + +type UDPEchoService struct{} + +func (UDPEchoService) StartService(t *testing.T, logger slog.Logger, _ *tailnet.Conn) { + // tailnet doesn't handle UDP connections "in-process" the way we do for TCP, so we need to listen in the OS, + // and tailnet will forward packets. + l, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.IPv6zero, // all interfaces + Port: EchoPort, + }) + require.NoError(t, err) + logger.Info(context.Background(), "started UDPEcho server") + t.Cleanup(func() { + lCloseErr := l.Close() + if lCloseErr != nil { + t.Logf("error closing UDPEcho listener: %v", lCloseErr) + } + }) + go func() { + buf := make([]byte, 1500) + for { + n, remote, readErr := l.ReadFromUDP(buf) + if readErr != nil { + logger.Info(context.Background(), "error reading UDPEcho listener", slog.Error(readErr)) + return + } + logger.Info(context.Background(), "received UDPEcho packet", + slog.F("len", n), slog.F("remote", remote)) + n, writeErr := l.WriteToUDP(buf[:n], remote) + if writeErr != nil { + logger.Info(context.Background(), "error writing UDPEcho listener", slog.Error(writeErr)) + return + } + logger.Info(context.Background(), "wrote UDPEcho packet", + slog.F("len", n), slog.F("remote", remote)) + } + }() } func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn { @@ -467,9 +511,16 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me _ = conn.Close() }) - ctrl := tailnet.NewTunnelSrcCoordController(logger, conn) - ctrl.AddDestination(peer.ID) - coordination := ctrl.New(coord) + var coordination tailnet.CloserWaiter + if me.TunnelSrc { + ctrl := tailnet.NewTunnelSrcCoordController(logger, conn) + ctrl.AddDestination(peer.ID) + coordination = ctrl.New(coord) + } else { + // use the "Agent" controller so that we act as a tunnel destination and send "ReadyForHandshake" acks. + ctrl := tailnet.NewAgentCoordinationController(logger, conn) + coordination = ctrl.New(coord) + } t.Cleanup(func() { cctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) defer cancel() @@ -492,11 +543,17 @@ func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) { } hostname := serverURL.Hostname() - ipv4 := "" + ipv4 := "none" + ipv6 := "none" ip, err := netip.ParseAddr(hostname) if err == nil { hostname = "" - ipv4 = ip.String() + if ip.Is4() { + ipv4 = ip.String() + } + if ip.Is6() { + ipv6 = ip.String() + } } return &tailcfg.DERPMap{ @@ -511,7 +568,7 @@ func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) { RegionID: 1, HostName: hostname, IPv4: ipv4, - IPv6: "none", + IPv6: ipv6, DERPPort: port, STUNPort: -1, ForceHTTP: true, @@ -648,3 +705,35 @@ func (w *testWriter) Flush() { } w.capturedLines = nil } + +type packetLogger struct { + l slog.Logger +} + +func (p packetLogger) LogPacket(path capture.Path, when time.Time, pkt []byte, _ packet.CaptureMeta) { + q := new(packet.Parsed) + q.Decode(pkt) + p.l.Info(context.Background(), "Packet", + slog.F("path", pathString(path)), + slog.F("when", when), + slog.F("decode", q.String()), + slog.F("len", len(pkt)), + ) +} + +func pathString(path capture.Path) string { + switch path { + case capture.FromLocal: + return "Local" + case capture.FromPeer: + return "Peer" + case capture.SynthesizedToLocal: + return "SynthesizedToLocal" + case capture.SynthesizedToPeer: + return "SynthesizedToPeer" + case capture.PathDisco: + return "Disco" + default: + return "<>" + } +} diff --git a/tailnet/test/integration/integration_test.go b/tailnet/test/integration/integration_test.go index b2cfa900674f0..260c21a6458f5 100644 --- a/tailnet/test/integration/integration_test.go +++ b/tailnet/test/integration/integration_test.go @@ -76,70 +76,90 @@ func TestMain(m *testing.M) { var topologies = []integration.TestTopology{ { // Test that DERP over loopback works. - Name: "BasicLoopbackDERP", - SetupNetworking: integration.SetupNetworkingLoopback, - Server: integration.SimpleServerOptions{}, - StartClient: integration.StartClientDERP, - RunTests: integration.TestSuite, + Name: "BasicLoopbackDERP", + NetworkingProvider: integration.NetworkingLoopback{}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{BlockEndpoints: true}, + RunTests: integration.TestSuite, }, { // Test that DERP over "easy" NAT works. The server, client 1 and client // 2 are on different networks with their own routers, which are joined // by a bridge. - Name: "EasyNATDERP", - SetupNetworking: integration.SetupNetworkingEasyNAT, - Server: integration.SimpleServerOptions{}, - StartClient: integration.StartClientDERP, - RunTests: integration.TestSuite, + Name: "EasyNATDERP", + NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{BlockEndpoints: true}, + RunTests: integration.TestSuite, }, { // Test that direct over "easy" NAT works with IP/ports grabbed from // STUN. - Name: "EasyNATDirect", - SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN, - Server: integration.SimpleServerOptions{}, - StartClient: integration.StartClientDirect, - RunTests: integration.TestSuite, + Name: "EasyNATDirect", + NetworkingProvider: integration.NetworkingNAT{StunCount: 1, Client1Hard: false, Client2Hard: false}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{WaitForDirect: true}, + RunTests: integration.TestSuite, }, { // Test that direct over hard NAT <=> easy NAT works. - Name: "HardNATEasyNATDirect", - SetupNetworking: integration.SetupNetworkingHardNATEasyNATDirect, - Server: integration.SimpleServerOptions{}, - StartClient: integration.StartClientDirect, - RunTests: integration.TestSuite, + Name: "HardNATEasyNATDirect", + NetworkingProvider: integration.NetworkingNAT{StunCount: 2, Client1Hard: true, Client2Hard: false}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{WaitForDirect: true}, + RunTests: integration.TestSuite, + }, + { + // Test that direct over normal MTU works. + Name: "DirectMTU1500", + NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1500}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{ + WaitForDirect: true, + Service: integration.UDPEchoService{}, + LogPackets: true, + }, + RunTests: integration.TestBigUDP, + }, + { + // Test that small MTU works. + Name: "MTU1280", + NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1280}, + Server: integration.SimpleServerOptions{}, + ClientStarter: integration.BasicClientStarter{Service: integration.UDPEchoService{}, LogPackets: true}, + RunTests: integration.TestBigUDP, }, { // Test that DERP over WebSocket (as well as DERPForceWebSockets works). // This does not test the actual DERP failure detection code and // automatic fallback. - Name: "DERPForceWebSockets", - SetupNetworking: integration.SetupNetworkingEasyNAT, + Name: "DERPForceWebSockets", + NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false}, Server: integration.SimpleServerOptions{ FailUpgradeDERP: false, DERPWebsocketOnly: true, }, - StartClient: integration.StartClientDERPWebSockets, - RunTests: integration.TestSuite, + ClientStarter: integration.BasicClientStarter{BlockEndpoints: true, DERPForceWebsockets: true}, + RunTests: integration.TestSuite, }, { // Test that falling back to DERP over WebSocket works. - Name: "DERPFallbackWebSockets", - SetupNetworking: integration.SetupNetworkingEasyNAT, + Name: "DERPFallbackWebSockets", + NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false}, Server: integration.SimpleServerOptions{ FailUpgradeDERP: true, DERPWebsocketOnly: false, }, // Use a basic client that will try `Upgrade: derp` first. - StartClient: integration.StartClientDERP, - RunTests: integration.TestSuite, + ClientStarter: integration.BasicClientStarter{BlockEndpoints: true}, + RunTests: integration.TestSuite, }, { - Name: "BasicLoopbackDERPNGINX", - SetupNetworking: integration.SetupNetworkingLoopback, - Server: integration.NGINXServerOptions{}, - StartClient: integration.StartClientDERP, - RunTests: integration.TestSuite, + Name: "BasicLoopbackDERPNGINX", + NetworkingProvider: integration.NetworkingLoopback{}, + Server: integration.NGINXServerOptions{}, + ClientStarter: integration.BasicClientStarter{BlockEndpoints: true}, + RunTests: integration.TestSuite, }, } @@ -151,7 +171,6 @@ func TestIntegration(t *testing.T) { } for _, topo := range topologies { - topo := topo t.Run(topo.Name, func(t *testing.T) { // These can run in parallel because every test should be in an // isolated NetNS. @@ -166,7 +185,11 @@ func TestIntegration(t *testing.T) { } log := testutil.Logger(t) - networking := topo.SetupNetworking(t, log) + networking := topo.NetworkingProvider.SetupNetworking(t, log) + + tempDir := t.TempDir() + // useful for debugging: + // networking.Client1.Process.CapturePackets(t, "client1", tempDir) // Useful for debugging network namespaces by avoiding cleanup. // t.Cleanup(func() { @@ -181,7 +204,6 @@ func TestIntegration(t *testing.T) { } // Write the DERP maps to a file. - tempDir := t.TempDir() client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json") client1DERPMap, err := networking.Client1.ResolveDERPMap() require.NoError(t, err, "resolve client 1 DERP map") @@ -270,7 +292,7 @@ func handleTestSubprocess(t *testing.T) { waitForServerAvailable(t, serverURL) - conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer) + conn := topo.ClientStarter.StartClient(t, logger, serverURL, &derpMap, me, peer) if me.ShouldRunTests { // Wait for connectivity. diff --git a/tailnet/test/integration/network.go b/tailnet/test/integration/network.go index b496879fd1219..871423974f3eb 100644 --- a/tailnet/test/integration/network.go +++ b/tailnet/test/integration/network.go @@ -5,9 +5,11 @@ package integration import ( "bytes" + "context" "fmt" "os" "os/exec" + "path" "testing" "github.com/stretchr/testify/require" @@ -71,11 +73,21 @@ type TestNetworkingProcess struct { NetNS *os.File } -// SetupNetworkingLoopback creates a network namespace with a loopback interface +func (p TestNetworkingProcess) CapturePackets(t *testing.T, name, dir string) { + dumpfile := path.Join(dir, name+".pcap") + _, _ = ExecBackground(t, name+".pcap", p.NetNS, "tcpdump", []string{ + "-i", "any", + "-w", dumpfile, + }) +} + +// NetworkingLoopback creates a network namespace with a loopback interface // for all tests to share. This is the simplest networking setup. The network // namespace only exists for isolation on the host and doesn't serve any routing // purpose. -func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { +type NetworkingLoopback struct{} + +func (NetworkingLoopback) SetupNetworking(t *testing.T, _ slog.Logger) TestNetworking { // Create a single network namespace for all tests so we can have an // isolated loopback interface. netNSFile := createNetNS(t, uniqNetName(t)) @@ -102,91 +114,25 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { } } -func easyNAT(t *testing.T) fakeInternet { - internet := createFakeInternet(t) - - _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() - require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") - - // Set up iptables masquerade rules to allow each router to NAT packets. - leaves := []struct { - fakeRouterLeaf - clientPort int - natPort int - }{ - {internet.Client1, client1Port, client1RouterPort}, - {internet.Client2, client2Port, client2RouterPort}, - } - for _, leaf := range leaves { - _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() - require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") - - // All non-UDP traffic should use regular masquerade e.g. for HTTP. - _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ - "-t", "nat", - "-A", "POSTROUTING", - // Every interface except loopback. - "!", "-o", "lo", - // Every protocol except UDP. - "!", "-p", "udp", - "-j", "MASQUERADE", - }).Output() - require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") - - // Outgoing traffic should get NATed to the router's IP. - _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ - "-t", "nat", - "-A", "POSTROUTING", - "-p", "udp", - "--sport", fmt.Sprint(leaf.clientPort), - "-j", "SNAT", - "--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort), - }).Output() - require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") - - // Incoming traffic should be forwarded to the client's IP. - _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ - "-t", "nat", - "-A", "PREROUTING", - "-p", "udp", - "--dport", fmt.Sprint(leaf.natPort), - "-j", "DNAT", - "--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort), - }).Output() - require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") - } - - return internet -} - -// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT" -// forwarding rules. +// NetworkingNAT creates a fake internet and sets up "NAT" +// forwarding rules, either easy or hard. // See createFakeInternet. // NAT is achieved through a single iptables masquerade rule. -func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking { - return easyNAT(t).Net +type NetworkingNAT struct { + StunCount int + Client1Hard bool + Client2Hard bool } -// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but -// also creates a namespace and bridge address for a STUN server. -func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking { - internet := easyNAT(t) - internet.Net.STUNs = []TestNetworkingSTUN{ - prepareSTUNServer(t, &internet, 0), - } - - return internet.Net -} - -// hardNAT creates a fake internet with multiple STUN servers and sets up "hard -// NAT" forwarding rules. If bothHard is false, only the first client will have -// hard NAT rules, and the second client will have easy NAT rules. -// -//nolint:revive -func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { +// SetupNetworking creates a fake internet with multiple STUN servers and sets up +// NAT forwarding rules. Client NATs are controlled by the switches ClientXHard, which if true, sets up hard +// nat. +func (n NetworkingNAT) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking { + logger := l.Named("setup-networking").Leveled(slog.LevelDebug) internet := createFakeInternet(t) - internet.Net.STUNs = make([]TestNetworkingSTUN, stunCount) - for i := 0; i < stunCount; i++ { + logger.Debug(context.Background(), "preparing STUN", slog.F("stun_count", n.StunCount)) + internet.Net.STUNs = make([]TestNetworkingSTUN, n.StunCount) + for i := 0; i < n.StunCount; i++ { internet.Net.STUNs[i] = prepareSTUNServer(t, &internet, i) } @@ -202,8 +148,14 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { natStartPortSTUN int }{ { - fakeRouterLeaf: internet.Client1, - peerIP: internet.Client2.RouterIP, + fakeRouterLeaf: internet.Client1, + // If peerIP is empty, we do easy NAT (even for STUN) + peerIP: func() string { + if n.Client1Hard { + return internet.Client2.RouterIP + } + return "" + }(), clientPort: client1Port, natPortPeer: client1RouterPort, natStartPortSTUN: client1RouterPortSTUN, @@ -212,7 +164,7 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { fakeRouterLeaf: internet.Client2, // If peerIP is empty, we do easy NAT (even for STUN) peerIP: func() string { - if bothHard { + if n.Client2Hard { return internet.Client1.RouterIP } return "" @@ -235,6 +187,9 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { // NAT from this client to each STUN server. Only do this if we're doing // hard NAT, as the rule above will also touch STUN traffic in easy NAT. if leaf.peerIP != "" { + logger.Debug(context.Background(), "creating NAT to STUN", + slog.F("client_ip", leaf.ClientIP), slog.F("peer_ip", leaf.peerIP), + ) for i, stun := range internet.Net.STUNs { natPort := leaf.natStartPortSTUN + i iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, natPort, stun.IP) @@ -242,11 +197,7 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { } } - return internet -} - -func SetupNetworkingHardNATEasyNATDirect(t *testing.T, _ slog.Logger) TestNetworking { - return hardNAT(t, 2, false).Net + return internet.Net } type vethPair struct { @@ -438,6 +389,149 @@ func createFakeInternet(t *testing.T) fakeInternet { return router } +type TriangleNetwork struct { + InterClientMTU int +} + +type fakeTriangleNetwork struct { + NamePrefix string + ServerNetNS *os.File + Client1NetNS *os.File + Client2NetNS *os.File + ServerClient1VethPair vethPair + ServerClient2VethPair vethPair + Client1Client2VethPair vethPair +} + +// SetupNetworking creates multiple namespaces with veth pairs between them +// with the following topology: +// . +// . ┌────────────────────────────────────────────┐ +// . │ Server │ +// . └─────┬───────────────────────────────────┬──┘ +// . │fdac:38fa:ffff:2::3 │fdac:38fa:ffff:3::3 +// . veth│ veth│ +// . │fdac:38fa:ffff:2::1 │fdac:38fa:ffff:3::2 +// . ┌───────┴──────┐ ┌─────┴───────┐ +// . │ │ fdac:38fa:ffff:1::2│ │ +// . │ Client 1 ├──────────────────────┤ Client 2 │ +// . │ │fdac:38fa:ffff:1::1 │ │ +// . └──────────────┘ └─────────────┘ +func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking { + logger := l.Named("setup-networking").Leveled(slog.LevelDebug) + t.Helper() + var ( + namePrefix = uniqNetName(t) + "_" + network = fakeTriangleNetwork{ + NamePrefix: namePrefix, + } + // Unique Local Address prefix + ula = "fdac:38fa:ffff:" + ) + + // Create three network namespaces for server, client1, and client2 + network.ServerNetNS = createNetNS(t, namePrefix+"server") + network.Client1NetNS = createNetNS(t, namePrefix+"client1") + network.Client2NetNS = createNetNS(t, namePrefix+"client2") + + // Create veth pair between server and client1 + network.ServerClient1VethPair = vethPair{ + Outer: namePrefix + "s-1", + Inner: namePrefix + "1-s", + } + err := createVethPair(network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner) + require.NoErrorf(t, err, "create veth pair %q <-> %q", + network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner) + + // Move server-client1 veth ends to their respective namespaces + err = setVethNetNS(network.ServerClient1VethPair.Outer, int(network.ServerNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient1VethPair.Outer) + err = setVethNetNS(network.ServerClient1VethPair.Inner, int(network.Client1NetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.ServerClient1VethPair.Inner) + + // Create veth pair between server and client2 + network.ServerClient2VethPair = vethPair{ + Outer: namePrefix + "s-2", + Inner: namePrefix + "2-s", + } + err = createVethPair(network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner) + require.NoErrorf(t, err, "create veth pair %q <-> %q", + network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner) + + // Move server-client2 veth ends to their respective namespaces + err = setVethNetNS(network.ServerClient2VethPair.Outer, int(network.ServerNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient2VethPair.Outer) + err = setVethNetNS(network.ServerClient2VethPair.Inner, int(network.Client2NetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.ServerClient2VethPair.Inner) + + // Create veth pair between client1 and client2 + network.Client1Client2VethPair = vethPair{ + Outer: namePrefix + "1-2", + Inner: namePrefix + "2-1", + } + logger.Debug(context.Background(), "creating inter-client link", slog.F("mtu", n.InterClientMTU)) + err = createVethPair(network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner, + withMTU(n.InterClientMTU)) + require.NoErrorf(t, err, "create veth pair %q <-> %q", + network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner) + + // Move client1-client2 veth ends to their respective namespaces + err = setVethNetNS(network.Client1Client2VethPair.Outer, int(network.Client1NetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client1Client2VethPair.Outer) + err = setVethNetNS(network.Client1Client2VethPair.Inner, int(network.Client2NetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1Client2VethPair.Inner) + + // Set IP addresses according to the diagram: + err = setInterfaceIP6(network.ServerNetNS, network.ServerClient1VethPair.Outer, ula+"2::3") + require.NoErrorf(t, err, "set IP on server-client1 interface") + err = setInterfaceIP6(network.ServerNetNS, network.ServerClient2VethPair.Outer, ula+"3::3") + require.NoErrorf(t, err, "set IP on server-client2 interface") + + err = setInterfaceIP6(network.Client1NetNS, network.ServerClient1VethPair.Inner, ula+"2::1") + require.NoErrorf(t, err, "set IP on client1-server interface") + err = setInterfaceIP6(network.Client1NetNS, network.Client1Client2VethPair.Outer, ula+"1::1") + require.NoErrorf(t, err, "set IP on client1-client2 interface") + + err = setInterfaceIP6(network.Client2NetNS, network.ServerClient2VethPair.Inner, ula+"3::2") + require.NoErrorf(t, err, "set IP on client2-server interface") + err = setInterfaceIP6(network.Client2NetNS, network.Client1Client2VethPair.Inner, ula+"1::2") + require.NoErrorf(t, err, "set IP on client2-client1 interface") + + // Bring up all interfaces + interfaces := []struct { + netNS *os.File + ifaceName string + }{ + {network.ServerNetNS, network.ServerClient1VethPair.Outer}, + {network.ServerNetNS, network.ServerClient2VethPair.Outer}, + {network.Client1NetNS, network.ServerClient1VethPair.Inner}, + {network.Client1NetNS, network.Client1Client2VethPair.Outer}, + {network.Client2NetNS, network.ServerClient2VethPair.Inner}, + {network.Client2NetNS, network.Client1Client2VethPair.Inner}, + } + for _, iface := range interfaces { + err = setInterfaceUp(iface.netNS, iface.ifaceName) + require.NoErrorf(t, err, "bring up interface %q", iface.ifaceName) + // Note: routes are not needed as we are fully connected, so nothing needs to forward IP to a further + // destination. + } + + return TestNetworking{ + Server: TestNetworkingServer{ + Process: TestNetworkingProcess{NetNS: network.ServerNetNS}, + ListenAddr: "[::]:8080", // Server listens on all IPs + }, + Client1: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: network.Client1NetNS}, + ServerAccessURL: "http://[" + ula + "2::3]:8080", // Client1 accesses server directly + }, + Client2: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: network.Client2NetNS}, + ServerAccessURL: "http://[" + ula + "3::3]:8080", // Client2 accesses server directly + }, + } +} + func uniqNetName(t *testing.T) string { t.Helper() netNSName := "cdr_" @@ -522,8 +616,8 @@ func createNetNS(t *testing.T, name string) *os.File { }) // Open /run/netns/$name to get a file descriptor to the network namespace. - path := fmt.Sprintf("/run/netns/%s", name) - file, err := os.OpenFile(path, os.O_RDONLY, 0) + netnsPath := fmt.Sprintf("/run/netns/%s", name) + file, err := os.OpenFile(netnsPath, os.O_RDONLY, 0) require.NoError(t, err, "open network namespace file") t.Cleanup(func() { _ = file.Close() @@ -568,10 +662,22 @@ func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error { return nil } +type linkOption func(attrs netlink.LinkAttrs) netlink.LinkAttrs + +func withMTU(mtu int) linkOption { + return func(attrs netlink.LinkAttrs) netlink.LinkAttrs { + attrs.MTU = mtu + return attrs + } +} + // createVethPair creates a veth pair with the given names. -func createVethPair(parentVethName, peerVethName string) error { +func createVethPair(parentVethName, peerVethName string, options ...linkOption) error { linkAttrs := netlink.NewLinkAttrs() linkAttrs.Name = parentVethName + for _, option := range options { + linkAttrs = option(linkAttrs) + } veth := &netlink.Veth{ LinkAttrs: linkAttrs, PeerName: peerVethName, @@ -611,6 +717,17 @@ func setInterfaceIP(netNS *os.File, ifaceName, ip string) error { return nil } +// setInterfaceIP6 sets the IPv6 address on the given interface. It automatically +// adds a /64 subnet mask. +func setInterfaceIP6(netNS *os.File, ifaceName, ip string) error { + _, err := commandInNetNS(netNS, "ip", []string{"addr", "add", ip + "/64", "dev", ifaceName}).Output() + if err != nil { + return xerrors.Errorf("set IP %q on interface %q in netns: %w", ip, ifaceName, wrapExitErr(err)) + } + + return nil +} + // setInterfaceUp brings the given interface up. func setInterfaceUp(netNS *os.File, ifaceName string) error { _, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "up"}).Output() @@ -703,7 +820,9 @@ func iptablesMasqueradeNonUDP(t *testing.T, netNS *os.File) { // iptablesNAT sets up iptables rules for NAT forwarding. If destIP is // specified, the forwarding rule will only apply to traffic to/from that IP // (mapvarydest). -func iptablesNAT(t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string) { +func iptablesNAT( + t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string, +) { t.Helper() snatArgs := []string{ diff --git a/tailnet/test/integration/suite.go b/tailnet/test/integration/suite.go index eefba0eaf2ce0..9e04de03de53a 100644 --- a/tailnet/test/integration/suite.go +++ b/tailnet/test/integration/suite.go @@ -5,6 +5,7 @@ package integration import ( "net/http" + "net/netip" "net/url" "testing" "time" @@ -80,3 +81,40 @@ func TestSuite(t *testing.T, _ slog.Logger, serverURL *url.URL, conn *tailnet.Co require.NoError(t, err, "ping peer after restart") }) } + +func TestBigUDP(t *testing.T, logger slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) { + t.Run("UDPEcho", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitShort) + + peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID) + udpConn, err := conn.DialContextUDP(ctx, netip.AddrPortFrom(peerIP, uint16(EchoPort))) + require.NoError(t, err) + defer udpConn.Close() + + // 1280 max tunnel packet size + // -40 + // -8 UDP header + // ---------------------------- + // 1232 data size + logger.Info(ctx, "sending UDP test packet") + packet := make([]byte, 1232) + for i := range packet { + packet[i] = byte(i % 256) + } + err = udpConn.SetWriteDeadline(time.Now().Add(5 * time.Second)) + require.NoError(t, err) + n, err := udpConn.Write(packet) + require.NoError(t, err) + require.Equal(t, len(packet), n) + + // read the echo + logger.Info(ctx, "attempting to read UDP reply") + buf := make([]byte, 1280) + err = udpConn.SetReadDeadline(time.Now().Add(5 * time.Second)) + require.NoError(t, err) + n, err = udpConn.Read(buf) + require.NoError(t, err) + require.Equal(t, len(packet), n) + require.Equal(t, packet, buf[:n]) + }) +}