From 27e00130a9677585cafd21a5c74694e699e48b7a Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 20 May 2024 08:39:37 +0000 Subject: [PATCH] chore: hard NAT <-> easy NAT integration test --- tailnet/test/integration/integration_test.go | 46 ++-- tailnet/test/integration/network.go | 238 +++++++++++++++---- 2 files changed, 222 insertions(+), 62 deletions(-) diff --git a/tailnet/test/integration/integration_test.go b/tailnet/test/integration/integration_test.go index e23b716096048..142df60db0d5b 100644 --- a/tailnet/test/integration/integration_test.go +++ b/tailnet/test/integration/integration_test.go @@ -44,6 +44,7 @@ var ( serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server") // Role: stun + stunNumber = flag.Int("stun-number", 0, "The number of the STUN server") stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server") // Role: client @@ -84,8 +85,8 @@ var topologies = []integration.TestTopology{ }, { // Test that DERP over "easy" NAT works. The server, client 1 and client - // 2 are on different networks with a shared router, and the router - // masquerades the traffic. + // 2 are on different networks with their own routers, which are joined + // by a bridge. Name: "EasyNATDERP", SetupNetworking: integration.SetupNetworkingEasyNAT, Server: integration.SimpleServerOptions{}, @@ -93,15 +94,22 @@ var topologies = []integration.TestTopology{ RunTests: integration.TestSuite, }, { - // Test that direct over "easy" NAT works. This should use local - // endpoints to connect as routing is enabled between client 1 and - // client 2. + // Test that direct over "easy" NAT works with IP/ports grabbed from + // STUN. Name: "EasyNATDirect", SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN, Server: integration.SimpleServerOptions{}, StartClient: integration.StartClientDirect, RunTests: integration.TestSuite, }, + { + // Test that direct over hard NAT <=> easy NAT works. + Name: "HardNATEasyNATDirect", + SetupNetworking: integration.SetupNetworkingHardNATEasyNATDirect, + Server: integration.SimpleServerOptions{}, + StartClient: integration.StartClientDirect, + RunTests: integration.TestSuite, + }, { // Test that DERP over WebSocket (as well as DERPForceWebSockets works). // This does not test the actual DERP failure detection code and @@ -160,9 +168,9 @@ func TestIntegration(t *testing.T) { closeServer := startServerSubprocess(t, topo.Name, networking) - closeSTUN := func() error { return nil } - if networking.STUN.ListenAddr != "" { - closeSTUN = startSTUNSubprocess(t, topo.Name, networking) + stunClosers := make([]func() error, len(networking.STUNs)) + for i, stun := range networking.STUNs { + stunClosers[i] = startSTUNSubprocess(t, topo.Name, i, stun) } // Write the DERP maps to a file. @@ -187,7 +195,9 @@ func TestIntegration(t *testing.T) { // Close client2 and the server. require.NoError(t, closeClient2(), "client 2 exited") - require.NoError(t, closeSTUN(), "stun exited") + for i, closeSTUN := range stunClosers { + require.NoErrorf(t, closeSTUN(), "stun %v exited", i) + } require.NoError(t, closeServer(), "server exited") }) } @@ -206,10 +216,15 @@ func handleTestSubprocess(t *testing.T) { require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role) testName := topo.Name + "/" - if *role == "server" || *role == "stun" { - testName += *role - } else { + switch *role { + case "server": + testName += "server" + case "stun": + testName += fmt.Sprintf("stun%d", *stunNumber) + case "client": testName += *clientName + default: + t.Fatalf("unknown role %q", *role) } t.Run(testName, func(t *testing.T) { @@ -325,12 +340,13 @@ func startServerSubprocess(t *testing.T, topologyName string, networking integra return closeFn } -func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error { - _, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{ +func startSTUNSubprocess(t *testing.T, topologyName string, number int, stun integration.TestNetworkingSTUN) func() error { + _, closeFn := startSubprocess(t, "stun", stun.Process.NetNS, []string{ "--subprocess", "--test-name=" + topologyName, "--role=stun", - "--stun-listen-addr=" + networking.STUN.ListenAddr, + "--stun-number=" + strconv.Itoa(number), + "--stun-listen-addr=" + stun.ListenAddr, }) return closeFn } diff --git a/tailnet/test/integration/network.go b/tailnet/test/integration/network.go index e0d8f7109c167..b496879fd1219 100644 --- a/tailnet/test/integration/network.go +++ b/tailnet/test/integration/network.go @@ -21,15 +21,17 @@ import ( ) const ( - client1Port = 48001 - client1RouterPort = 48011 - client2Port = 48002 - client2RouterPort = 48012 + client1Port = 48001 + client1RouterPort = 48011 // used in easy and hard NAT + client1RouterPortSTUN = 48201 // used in hard NAT + client2Port = 48002 + client2RouterPort = 48012 // used in easy and hard NAT + client2RouterPortSTUN = 48101 // used in hard NAT ) type TestNetworking struct { Server TestNetworkingServer - STUN TestNetworkingSTUN + STUNs []TestNetworkingSTUN Client1 TestNetworkingClient Client2 TestNetworkingClient } @@ -40,8 +42,8 @@ type TestNetworkingServer struct { } type TestNetworkingSTUN struct { - Process TestNetworkingProcess - // If empty, no STUN subprocess is launched. + Process TestNetworkingProcess + IP string ListenAddr string } @@ -169,53 +171,82 @@ func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking { // also creates a namespace and bridge address for a STUN server. func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking { internet := easyNAT(t) + internet.Net.STUNs = []TestNetworkingSTUN{ + prepareSTUNServer(t, &internet, 0), + } - // Create another network namespace for the STUN server. - stunNetNS := createNetNS(t, internet.NamePrefix+"stun") - internet.Net.STUN.Process = TestNetworkingProcess{ - NetNS: stunNetNS, + return internet.Net +} + +// hardNAT creates a fake internet with multiple STUN servers and sets up "hard +// NAT" forwarding rules. If bothHard is false, only the first client will have +// hard NAT rules, and the second client will have easy NAT rules. +// +//nolint:revive +func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { + internet := createFakeInternet(t) + internet.Net.STUNs = make([]TestNetworkingSTUN, stunCount) + for i := 0; i < stunCount; i++ { + internet.Net.STUNs[i] = prepareSTUNServer(t, &internet, i) } - const ip = "10.0.0.64" - err := joinBridge(joinBridgeOpts{ - bridgeNetNS: internet.BridgeNetNS, - netNS: stunNetNS, - bridgeName: internet.BridgeName, - vethPair: vethPair{ - Outer: internet.NamePrefix + "b-stun", - Inner: internet.NamePrefix + "stun-b", - }, - ip: ip, - }) - require.NoError(t, err, "join bridge with STUN server") - internet.Net.STUN.ListenAddr = ip + ":3478" + _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") - // Define custom DERP map. - stunRegion := &tailcfg.DERPRegion{ - RegionID: 10000, - RegionCode: "stun0", - RegionName: "STUN0", - Nodes: []*tailcfg.DERPNode{ - { - Name: "stun0a", - RegionID: 1, - IPv4: ip, - IPv6: "none", - STUNPort: 3478, - STUNOnly: true, - }, + // Set up iptables masquerade rules to allow each router to NAT packets. + leaves := []struct { + fakeRouterLeaf + peerIP string + clientPort int + natPortPeer int + natStartPortSTUN int + }{ + { + fakeRouterLeaf: internet.Client1, + peerIP: internet.Client2.RouterIP, + clientPort: client1Port, + natPortPeer: client1RouterPort, + natStartPortSTUN: client1RouterPortSTUN, + }, + { + fakeRouterLeaf: internet.Client2, + // If peerIP is empty, we do easy NAT (even for STUN) + peerIP: func() string { + if bothHard { + return internet.Client1.RouterIP + } + return "" + }(), + clientPort: client2Port, + natPortPeer: client2RouterPort, + natStartPortSTUN: client2RouterPortSTUN, }, } - client1DERP, err := internet.Net.Client1.ResolveDERPMap() - require.NoError(t, err, "resolve DERP map for client 1") - client1DERP.Regions[stunRegion.RegionID] = stunRegion - internet.Net.Client1.DERPMap = client1DERP - client2DERP, err := internet.Net.Client2.ResolveDERPMap() - require.NoError(t, err, "resolve DERP map for client 2") - client2DERP.Regions[stunRegion.RegionID] = stunRegion - internet.Net.Client2.DERPMap = client2DERP + for _, leaf := range leaves { + _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") - return internet.Net + // All non-UDP traffic should use regular masquerade e.g. for HTTP. + iptablesMasqueradeNonUDP(t, leaf.RouterNetNS) + + // NAT from this client to its peer. + iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, leaf.natPortPeer, leaf.peerIP) + + // NAT from this client to each STUN server. Only do this if we're doing + // hard NAT, as the rule above will also touch STUN traffic in easy NAT. + if leaf.peerIP != "" { + for i, stun := range internet.Net.STUNs { + natPort := leaf.natStartPortSTUN + i + iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, natPort, stun.IP) + } + } + } + + return internet +} + +func SetupNetworkingHardNATEasyNATDirect(t *testing.T, _ slog.Logger) TestNetworking { + return hardNAT(t, 2, false).Net } type vethPair struct { @@ -600,6 +631,119 @@ func addRouteInNetNS(netNS *os.File, route []string) error { return nil } +// prepareSTUNServer creates a STUN server networking spec in a network +// namespace and joins it to the bridge. It also sets up the DERP map for the +// clients to use the STUN. +func prepareSTUNServer(t *testing.T, internet *fakeInternet, number int) TestNetworkingSTUN { + name := fmt.Sprintf("stn%d", number) + + stunNetNS := createNetNS(t, internet.NamePrefix+name) + stun := TestNetworkingSTUN{ + Process: TestNetworkingProcess{ + NetNS: stunNetNS, + }, + } + + stun.IP = "10.0.0." + fmt.Sprint(64+number) + err := joinBridge(joinBridgeOpts{ + bridgeNetNS: internet.BridgeNetNS, + netNS: stunNetNS, + bridgeName: internet.BridgeName, + vethPair: vethPair{ + Outer: internet.NamePrefix + "b-" + name, + Inner: internet.NamePrefix + name + "-b", + }, + ip: stun.IP, + }) + require.NoError(t, err, "join bridge with STUN server") + stun.ListenAddr = stun.IP + ":3478" + + // Define custom DERP map. + stunRegion := &tailcfg.DERPRegion{ + RegionID: 10000 + number, + RegionCode: name, + RegionName: name, + Nodes: []*tailcfg.DERPNode{ + { + Name: name + "a", + RegionID: 1, + IPv4: stun.IP, + IPv6: "none", + STUNPort: 3478, + STUNOnly: true, + }, + }, + } + client1DERP, err := internet.Net.Client1.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 1") + client1DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client1.DERPMap = client1DERP + client2DERP, err := internet.Net.Client2.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 2") + client2DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client2.DERPMap = client2DERP + + return stun +} + +func iptablesMasqueradeNonUDP(t *testing.T, netNS *os.File) { + t.Helper() + _, err := commandInNetNS(netNS, "iptables", []string{ + "-t", "nat", + "-A", "POSTROUTING", + // Every interface except loopback. + "!", "-o", "lo", + // Every protocol except UDP. + "!", "-p", "udp", + "-j", "MASQUERADE", + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") +} + +// iptablesNAT sets up iptables rules for NAT forwarding. If destIP is +// specified, the forwarding rule will only apply to traffic to/from that IP +// (mapvarydest). +func iptablesNAT(t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string) { + t.Helper() + + snatArgs := []string{ + "-t", "nat", + "-A", "POSTROUTING", + "-p", "udp", + "--sport", fmt.Sprint(clientPort), + "-j", "SNAT", + "--to-source", fmt.Sprintf("%s:%d", routerIP, routerPort), + } + if destIP != "" { + // Insert `-d $destIP` after the --sport flag+value. + newSnatArgs := append([]string{}, snatArgs[:8]...) + newSnatArgs = append(newSnatArgs, "-d", destIP) + newSnatArgs = append(newSnatArgs, snatArgs[8:]...) + snatArgs = newSnatArgs + } + _, err := commandInNetNS(netNS, "iptables", snatArgs).Output() + require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") + + // Incoming traffic should be forwarded to the client's IP. + dnatArgs := []string{ + "-t", "nat", + "-A", "PREROUTING", + "-p", "udp", + "--dport", fmt.Sprint(routerPort), + "-j", "DNAT", + "--to-destination", fmt.Sprintf("%s:%d", clientIP, clientPort), + } + if destIP != "" { + // Insert `-s $destIP` before the --dport flag+value. + newDnatArgs := append([]string{}, dnatArgs[:6]...) + newDnatArgs = append(newDnatArgs, "-s", destIP) + newDnatArgs = append(newDnatArgs, dnatArgs[6:]...) + dnatArgs = newDnatArgs + } + _, err = commandInNetNS(netNS, "iptables", dnatArgs).Output() + require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") +} + func commandInNetNS(netNS *os.File, bin string, args []string) *exec.Cmd { //nolint:gosec cmd := exec.Command("nsenter", append([]string{"--net=/proc/self/fd/3", bin}, args...)...)