Skip to content

Commit 9299e9f

Browse files
deansheatherdeansheather
authored
chore: hard NAT <-> easy NAT integration test (coder#13314)
1 parent e5d848f commit 9299e9f

File tree

2 files changed

+222
-62
lines changed

2 files changed

+222
-62
lines changed

‎tailnet/test/integration/integration_test.go

+31-15
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ var (
4444
serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server")
4545

4646
// Role: stun
47+
stunNumber = flag.Int("stun-number", 0, "The number of the STUN server")
4748
stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server")
4849

4950
// Role: client
@@ -84,24 +85,31 @@ var topologies = []integration.TestTopology{
8485
},
8586
{
8687
// Test that DERP over "easy" NAT works. The server, client 1 and client
87-
// 2 are on different networks with a shared router, and the router
88-
// masquerades the traffic.
88+
// 2 are on different networks with their own routers, which are joined
89+
// by a bridge.
8990
Name: "EasyNATDERP",
9091
SetupNetworking: integration.SetupNetworkingEasyNAT,
9192
Server: integration.SimpleServerOptions{},
9293
StartClient: integration.StartClientDERP,
9394
RunTests: integration.TestSuite,
9495
},
9596
{
96-
// Test that direct over "easy" NAT works. This should use local
97-
// endpoints to connect as routing is enabled between client 1 and
98-
// client 2.
97+
// Test that direct over "easy" NAT works with IP/ports grabbed from
98+
// STUN.
9999
Name: "EasyNATDirect",
100100
SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
101101
Server: integration.SimpleServerOptions{},
102102
StartClient: integration.StartClientDirect,
103103
RunTests: integration.TestSuite,
104104
},
105+
{
106+
// Test that direct over hard NAT <=> easy NAT works.
107+
Name: "HardNATEasyNATDirect",
108+
SetupNetworking: integration.SetupNetworkingHardNATEasyNATDirect,
109+
Server: integration.SimpleServerOptions{},
110+
StartClient: integration.StartClientDirect,
111+
RunTests: integration.TestSuite,
112+
},
105113
{
106114
// Test that DERP over WebSocket (as well as DERPForceWebSockets works).
107115
// This does not test the actual DERP failure detection code and
@@ -160,9 +168,9 @@ func TestIntegration(t *testing.T) {
160168

161169
closeServer := startServerSubprocess(t, topo.Name, networking)
162170

163-
closeSTUN := func() error { return nil }
164-
if networking.STUN.ListenAddr != "" {
165-
closeSTUN = startSTUNSubprocess(t, topo.Name, networking)
171+
stunClosers := make([]func() error, len(networking.STUNs))
172+
for i, stun := range networking.STUNs {
173+
stunClosers[i] = startSTUNSubprocess(t, topo.Name, i, stun)
166174
}
167175

168176
// Write the DERP maps to a file.
@@ -187,7 +195,9 @@ func TestIntegration(t *testing.T) {
187195

188196
// Close client2 and the server.
189197
require.NoError(t, closeClient2(), "client 2 exited")
190-
require.NoError(t, closeSTUN(), "stun exited")
198+
for i, closeSTUN := range stunClosers {
199+
require.NoErrorf(t, closeSTUN(), "stun %v exited", i)
200+
}
191201
require.NoError(t, closeServer(), "server exited")
192202
})
193203
}
@@ -206,10 +216,15 @@ func handleTestSubprocess(t *testing.T) {
206216
require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role)
207217

208218
testName := topo.Name + "/"
209-
if *role == "server" || *role == "stun" {
210-
testName += *role
211-
} else {
219+
switch *role {
220+
case "server":
221+
testName += "server"
222+
case "stun":
223+
testName += fmt.Sprintf("stun%d", *stunNumber)
224+
case "client":
212225
testName += *clientName
226+
default:
227+
t.Fatalf("unknown role %q", *role)
213228
}
214229

215230
t.Run(testName, func(t *testing.T) {
@@ -325,12 +340,13 @@ func startServerSubprocess(t *testing.T, topologyName string, networking integra
325340
return closeFn
326341
}
327342

328-
func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
329-
_, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{
343+
func startSTUNSubprocess(t *testing.T, topologyName string, number int, stun integration.TestNetworkingSTUN) func() error {
344+
_, closeFn := startSubprocess(t, "stun", stun.Process.NetNS, []string{
330345
"--subprocess",
331346
"--test-name=" + topologyName,
332347
"--role=stun",
333-
"--stun-listen-addr=" + networking.STUN.ListenAddr,
348+
"--stun-number=" + strconv.Itoa(number),
349+
"--stun-listen-addr=" + stun.ListenAddr,
334350
})
335351
return closeFn
336352
}

‎tailnet/test/integration/network.go

+191-47
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,17 @@ import (
2121
)
2222

2323
const (
24-
client1Port = 48001
25-
client1RouterPort = 48011
26-
client2Port = 48002
27-
client2RouterPort = 48012
24+
client1Port = 48001
25+
client1RouterPort = 48011 // used in easy and hard NAT
26+
client1RouterPortSTUN = 48201 // used in hard NAT
27+
client2Port = 48002
28+
client2RouterPort = 48012 // used in easy and hard NAT
29+
client2RouterPortSTUN = 48101 // used in hard NAT
2830
)
2931

3032
type TestNetworking struct {
3133
Server TestNetworkingServer
32-
STUN TestNetworkingSTUN
34+
STUNs []TestNetworkingSTUN
3335
Client1 TestNetworkingClient
3436
Client2 TestNetworkingClient
3537
}
@@ -40,8 +42,8 @@ type TestNetworkingServer struct {
4042
}
4143

4244
type TestNetworkingSTUN struct {
43-
Process TestNetworkingProcess
44-
// If empty, no STUN subprocess is launched.
45+
Process TestNetworkingProcess
46+
IP string
4547
ListenAddr string
4648
}
4749

@@ -169,53 +171,82 @@ func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking {
169171
// also creates a namespace and bridge address for a STUN server.
170172
func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking {
171173
internet := easyNAT(t)
174+
internet.Net.STUNs = []TestNetworkingSTUN{
175+
prepareSTUNServer(t, &internet, 0),
176+
}
172177

173-
// Create another network namespace for the STUN server.
174-
stunNetNS := createNetNS(t, internet.NamePrefix+"stun")
175-
internet.Net.STUN.Process = TestNetworkingProcess{
176-
NetNS: stunNetNS,
178+
return internet.Net
179+
}
180+
181+
// hardNAT creates a fake internet with multiple STUN servers and sets up "hard
182+
// NAT" forwarding rules. If bothHard is false, only the first client will have
183+
// hard NAT rules, and the second client will have easy NAT rules.
184+
//
185+
//nolint:revive
186+
func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
187+
internet := createFakeInternet(t)
188+
internet.Net.STUNs = make([]TestNetworkingSTUN, stunCount)
189+
for i := 0; i < stunCount; i++ {
190+
internet.Net.STUNs[i] = prepareSTUNServer(t, &internet, i)
177191
}
178192

179-
const ip = "10.0.0.64"
180-
err := joinBridge(joinBridgeOpts{
181-
bridgeNetNS: internet.BridgeNetNS,
182-
netNS: stunNetNS,
183-
bridgeName: internet.BridgeName,
184-
vethPair: vethPair{
185-
Outer: internet.NamePrefix + "b-stun",
186-
Inner: internet.NamePrefix + "stun-b",
187-
},
188-
ip: ip,
189-
})
190-
require.NoError(t, err, "join bridge with STUN server")
191-
internet.Net.STUN.ListenAddr = ip + ":3478"
193+
_, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
194+
require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS")
192195

193-
// Define custom DERP map.
194-
stunRegion := &tailcfg.DERPRegion{
195-
RegionID: 10000,
196-
RegionCode: "stun0",
197-
RegionName: "STUN0",
198-
Nodes: []*tailcfg.DERPNode{
199-
{
200-
Name: "stun0a",
201-
RegionID: 1,
202-
IPv4: ip,
203-
IPv6: "none",
204-
STUNPort: 3478,
205-
STUNOnly: true,
206-
},
196+
// Set up iptables masquerade rules to allow each router to NAT packets.
197+
leaves := []struct {
198+
fakeRouterLeaf
199+
peerIP string
200+
clientPort int
201+
natPortPeer int
202+
natStartPortSTUN int
203+
}{
204+
{
205+
fakeRouterLeaf: internet.Client1,
206+
peerIP: internet.Client2.RouterIP,
207+
clientPort: client1Port,
208+
natPortPeer: client1RouterPort,
209+
natStartPortSTUN: client1RouterPortSTUN,
210+
},
211+
{
212+
fakeRouterLeaf: internet.Client2,
213+
// If peerIP is empty, we do easy NAT (even for STUN)
214+
peerIP: func() string {
215+
if bothHard {
216+
return internet.Client1.RouterIP
217+
}
218+
return ""
219+
}(),
220+
clientPort: client2Port,
221+
natPortPeer: client2RouterPort,
222+
natStartPortSTUN: client2RouterPortSTUN,
207223
},
208224
}
209-
client1DERP, err := internet.Net.Client1.ResolveDERPMap()
210-
require.NoError(t, err, "resolve DERP map for client 1")
211-
client1DERP.Regions[stunRegion.RegionID] = stunRegion
212-
internet.Net.Client1.DERPMap = client1DERP
213-
client2DERP, err := internet.Net.Client2.ResolveDERPMap()
214-
require.NoError(t, err, "resolve DERP map for client 2")
215-
client2DERP.Regions[stunRegion.RegionID] = stunRegion
216-
internet.Net.Client2.DERPMap = client2DERP
225+
for _, leaf := range leaves {
226+
_, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
227+
require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
217228

218-
return internet.Net
229+
// All non-UDP traffic should use regular masquerade e.g. for HTTP.
230+
iptablesMasqueradeNonUDP(t, leaf.RouterNetNS)
231+
232+
// NAT from this client to its peer.
233+
iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, leaf.natPortPeer, leaf.peerIP)
234+
235+
// NAT from this client to each STUN server. Only do this if we're doing
236+
// hard NAT, as the rule above will also touch STUN traffic in easy NAT.
237+
if leaf.peerIP != "" {
238+
for i, stun := range internet.Net.STUNs {
239+
natPort := leaf.natStartPortSTUN + i
240+
iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, natPort, stun.IP)
241+
}
242+
}
243+
}
244+
245+
return internet
246+
}
247+
248+
func SetupNetworkingHardNATEasyNATDirect(t *testing.T, _ slog.Logger) TestNetworking {
249+
return hardNAT(t, 2, false).Net
219250
}
220251

221252
type vethPair struct {
@@ -600,6 +631,119 @@ func addRouteInNetNS(netNS *os.File, route []string) error {
600631
return nil
601632
}
602633

634+
// prepareSTUNServer creates a STUN server networking spec in a network
635+
// namespace and joins it to the bridge. It also sets up the DERP map for the
636+
// clients to use the STUN.
637+
func prepareSTUNServer(t *testing.T, internet *fakeInternet, number int) TestNetworkingSTUN {
638+
name := fmt.Sprintf("stn%d", number)
639+
640+
stunNetNS := createNetNS(t, internet.NamePrefix+name)
641+
stun := TestNetworkingSTUN{
642+
Process: TestNetworkingProcess{
643+
NetNS: stunNetNS,
644+
},
645+
}
646+
647+
stun.IP = "10.0.0." + fmt.Sprint(64+number)
648+
err := joinBridge(joinBridgeOpts{
649+
bridgeNetNS: internet.BridgeNetNS,
650+
netNS: stunNetNS,
651+
bridgeName: internet.BridgeName,
652+
vethPair: vethPair{
653+
Outer: internet.NamePrefix + "b-" + name,
654+
Inner: internet.NamePrefix + name + "-b",
655+
},
656+
ip: stun.IP,
657+
})
658+
require.NoError(t, err, "join bridge with STUN server")
659+
stun.ListenAddr = stun.IP + ":3478"
660+
661+
// Define custom DERP map.
662+
stunRegion := &tailcfg.DERPRegion{
663+
RegionID: 10000 + number,
664+
RegionCode: name,
665+
RegionName: name,
666+
Nodes: []*tailcfg.DERPNode{
667+
{
668+
Name: name + "a",
669+
RegionID: 1,
670+
IPv4: stun.IP,
671+
IPv6: "none",
672+
STUNPort: 3478,
673+
STUNOnly: true,
674+
},
675+
},
676+
}
677+
client1DERP, err := internet.Net.Client1.ResolveDERPMap()
678+
require.NoError(t, err, "resolve DERP map for client 1")
679+
client1DERP.Regions[stunRegion.RegionID] = stunRegion
680+
internet.Net.Client1.DERPMap = client1DERP
681+
client2DERP, err := internet.Net.Client2.ResolveDERPMap()
682+
require.NoError(t, err, "resolve DERP map for client 2")
683+
client2DERP.Regions[stunRegion.RegionID] = stunRegion
684+
internet.Net.Client2.DERPMap = client2DERP
685+
686+
return stun
687+
}
688+
689+
func iptablesMasqueradeNonUDP(t *testing.T, netNS *os.File) {
690+
t.Helper()
691+
_, err := commandInNetNS(netNS, "iptables", []string{
692+
"-t", "nat",
693+
"-A", "POSTROUTING",
694+
// Every interface except loopback.
695+
"!", "-o", "lo",
696+
// Every protocol except UDP.
697+
"!", "-p", "udp",
698+
"-j", "MASQUERADE",
699+
}).Output()
700+
require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule")
701+
}
702+
703+
// iptablesNAT sets up iptables rules for NAT forwarding. If destIP is
704+
// specified, the forwarding rule will only apply to traffic to/from that IP
705+
// (mapvarydest).
706+
func iptablesNAT(t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string) {
707+
t.Helper()
708+
709+
snatArgs := []string{
710+
"-t", "nat",
711+
"-A", "POSTROUTING",
712+
"-p", "udp",
713+
"--sport", fmt.Sprint(clientPort),
714+
"-j", "SNAT",
715+
"--to-source", fmt.Sprintf("%s:%d", routerIP, routerPort),
716+
}
717+
if destIP != "" {
718+
// Insert `-d $destIP` after the --sport flag+value.
719+
newSnatArgs := append([]string{}, snatArgs[:8]...)
720+
newSnatArgs = append(newSnatArgs, "-d", destIP)
721+
newSnatArgs = append(newSnatArgs, snatArgs[8:]...)
722+
snatArgs = newSnatArgs
723+
}
724+
_, err := commandInNetNS(netNS, "iptables", snatArgs).Output()
725+
require.NoError(t, wrapExitErr(err), "add iptables SNAT rule")
726+
727+
// Incoming traffic should be forwarded to the client's IP.
728+
dnatArgs := []string{
729+
"-t", "nat",
730+
"-A", "PREROUTING",
731+
"-p", "udp",
732+
"--dport", fmt.Sprint(routerPort),
733+
"-j", "DNAT",
734+
"--to-destination", fmt.Sprintf("%s:%d", clientIP, clientPort),
735+
}
736+
if destIP != "" {
737+
// Insert `-s $destIP` before the --dport flag+value.
738+
newDnatArgs := append([]string{}, dnatArgs[:6]...)
739+
newDnatArgs = append(newDnatArgs, "-s", destIP)
740+
newDnatArgs = append(newDnatArgs, dnatArgs[6:]...)
741+
dnatArgs = newDnatArgs
742+
}
743+
_, err = commandInNetNS(netNS, "iptables", dnatArgs).Output()
744+
require.NoError(t, wrapExitErr(err), "add iptables DNAT rule")
745+
}
746+
603747
func commandInNetNS(netNS *os.File, bin string, args []string) *exec.Cmd {
604748
//nolint:gosec
605749
cmd := exec.Command("nsenter", append([]string{"--net=/proc/self/fd/3", bin}, args...)...)

0 commit comments

Comments
 (0)