Skip to content

Commit 264fc56

Browse files
deansheatherdeansheather
committed
chore: add easy NAT integration tests part 2
1 parent a63d427 commit 264fc56

File tree

5 files changed

+592
-194
lines changed

5 files changed

+592
-194
lines changed

‎enterprise/coderd/roles_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func TestCustomRole(t *testing.T) {
6464

6565
// Verify the role exists in the list
6666
// TODO: Turn this assertion back on when the cli api experience is created.
67-
//allRoles, err := tmplAdmin.ListSiteRoles(ctx)
67+
// allRoles, err := tmplAdmin.ListSiteRoles(ctx)
6868
//require.NoError(t, err)
6969
//
7070
//require.True(t, slices.ContainsFunc(allRoles, func(selected codersdk.AssignableRoles) bool {

‎tailnet/test/integration/integration.go

+32-10
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ type TestTopology struct {
5959
Server ServerStarter
6060
// StartClient gets called in each client subprocess. It's expected to
6161
// create the tailnet.Conn and ensure connectivity to it's peer.
62-
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
62+
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
6363

6464
// RunTests is the main test function. It's called in each of the client
6565
// subprocesses. If tests can only run once, they should check the client ID
@@ -264,13 +264,18 @@ http {
264264

265265
// StartClientDERP creates a client connection to the server for coordination
266266
// and creates a tailnet.Conn which will only use DERP to connect to the peer.
267-
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
267+
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
268+
listenPort := uint16(client1Port)
269+
if clientNumber == 2 {
270+
listenPort = client2Port
271+
}
268272
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
269273
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
270-
DERPMap: basicDERPMap(t, serverURL),
274+
DERPMap: derpMap,
271275
BlockEndpoints: true,
272276
Logger: logger,
273277
DERPForceWebSockets: false,
278+
ListenPort: listenPort,
274279
// These tests don't have internet connection, so we need to force
275280
// magicsock to do anything.
276281
ForceNetworkUp: true,
@@ -279,13 +284,18 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID,
279284

280285
// StartClientDERPWebSockets does the same thing as StartClientDERP but will
281286
// only use DERP WebSocket fallback.
282-
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
287+
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
288+
listenPort := uint16(client1Port)
289+
if clientNumber == 2 {
290+
listenPort = client2Port
291+
}
283292
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
284293
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
285-
DERPMap: basicDERPMap(t, serverURL),
294+
DERPMap: derpMap,
286295
BlockEndpoints: true,
287296
Logger: logger,
288297
DERPForceWebSockets: true,
298+
ListenPort: listenPort,
289299
// These tests don't have internet connection, so we need to force
290300
// magicsock to do anything.
291301
ForceNetworkUp: true,
@@ -295,13 +305,18 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.
295305
// StartClientDirect does the same thing as StartClientDERP but disables
296306
// BlockEndpoints (which enables Direct connections), and waits for a direct
297307
// connection to be established between the two peers.
298-
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
308+
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
309+
listenPort := uint16(client1Port)
310+
if clientNumber == 2 {
311+
listenPort = client2Port
312+
}
299313
conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
300314
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
301-
DERPMap: basicDERPMap(t, serverURL),
315+
DERPMap: derpMap,
302316
BlockEndpoints: false,
303317
Logger: logger,
304318
DERPForceWebSockets: true,
319+
ListenPort: listenPort,
305320
// These tests don't have internet connection, so we need to force
306321
// magicsock to do anything.
307322
ForceNetworkUp: true,
@@ -365,10 +380,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
365380
return conn
366381
}
367382

368-
func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
383+
func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
384+
serverURL, err := url.Parse(serverURLStr)
385+
if err != nil {
386+
return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err)
387+
}
388+
369389
portStr := serverURL.Port()
370390
port, err := strconv.Atoi(portStr)
371-
require.NoError(t, err, "parse server port")
391+
if err != nil {
392+
return nil, xerrors.Errorf("parse port %q: %w", portStr, err)
393+
}
372394

373395
hostname := serverURL.Hostname()
374396
ipv4 := ""
@@ -399,7 +421,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
399421
},
400422
},
401423
},
402-
}
424+
}, nil
403425
}
404426

405427
// ExecBackground starts a subprocess with the given flags and returns a

‎tailnet/test/integration/integration_test.go

+123-25
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,28 @@
44
package integration_test
55

66
import (
7+
"context"
8+
"encoding/json"
79
"flag"
810
"fmt"
11+
"net"
912
"net/http"
1013
"net/url"
1114
"os"
1215
"os/signal"
16+
"path/filepath"
1317
"runtime"
18+
"strconv"
1419
"syscall"
1520
"testing"
1621
"time"
1722

1823
"github.com/google/uuid"
24+
"github.com/stretchr/testify/assert"
1925
"github.com/stretchr/testify/require"
26+
"tailscale.com/net/stun/stuntest"
27+
"tailscale.com/tailcfg"
28+
"tailscale.com/types/nettype"
2029

2130
"cdr.dev/slog"
2231
"cdr.dev/slog/sloggers/slogtest"
@@ -30,17 +39,22 @@ const runTestEnv = "CODER_TAILNET_TESTS"
3039
var (
3140
isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess")
3241
testID = flag.String("test-name", "", "Which test is being run")
33-
role = flag.String("role", "", "The role of the test subprocess: server, client")
42+
role = flag.String("role", "", "The role of the test subprocess: server, stun, client")
3443

3544
// Role: server
3645
serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server")
3746

47+
// Role: stun
48+
stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server")
49+
3850
// Role: client
39-
clientName = flag.String("client-name", "", "The name of the client for logs")
40-
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
41-
clientMyID = flag.String("client-id", "", "The id of the client")
42-
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
43-
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
51+
clientName = flag.String("client-name", "", "The name of the client for logs")
52+
clientNumber = flag.Int("client-number", 0, "The number of the client")
53+
clientMyID = flag.String("client-id", "", "The id of the client")
54+
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
55+
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
56+
clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client")
57+
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
4458
)
4559

4660
func TestMain(m *testing.M) {
@@ -87,7 +101,7 @@ var topologies = []integration.TestTopology{
87101
// endpoints to connect as routing is enabled between client 1 and
88102
// client 2.
89103
Name: "EasyNATDirect",
90-
SetupNetworking: integration.SetupNetworkingEasyNAT,
104+
SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
91105
Server: integration.SimpleServerOptions{},
92106
StartClient: integration.StartClientDirect,
93107
RunTests: integration.TestSuite,
@@ -143,17 +157,41 @@ func TestIntegration(t *testing.T) {
143157
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
144158
networking := topo.SetupNetworking(t, log)
145159

146-
// Fork the three child processes.
160+
// Useful for debugging network namespaces by avoiding cleanup.
161+
// t.Cleanup(func() {
162+
// time.Sleep(time.Minute * 15)
163+
// })
164+
147165
closeServer := startServerSubprocess(t, topo.Name, networking)
166+
167+
closeSTUN := func() error { return nil }
168+
if networking.STUN.ListenAddr != "" {
169+
closeSTUN = startSTUNSubprocess(t, topo.Name, networking)
170+
}
171+
172+
// Write the DERP maps to a file.
173+
tempDir := t.TempDir()
174+
client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json")
175+
client1DERPMap, err := networking.Client1.ResolveDERPMap()
176+
require.NoError(t, err, "resolve client 1 DERP map")
177+
err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap)
178+
require.NoError(t, err, "write client 1 DERP map")
179+
client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json")
180+
client2DERPMap, err := networking.Client2.ResolveDERPMap()
181+
require.NoError(t, err, "resolve client 2 DERP map")
182+
err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap)
183+
require.NoError(t, err, "write client 2 DERP map")
184+
148185
// client1 runs the tests.
149-
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1)
150-
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)
186+
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1, client1DERPMapPath)
187+
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2, client2DERPMapPath)
151188

152189
// Wait for client1 to exit.
153190
require.NoError(t, <-client1ErrCh, "client 1 exited")
154191

155192
// Close client2 and the server.
156193
require.NoError(t, closeClient2(), "client 2 exited")
194+
require.NoError(t, closeSTUN(), "stun exited")
157195
require.NoError(t, closeServer(), "server exited")
158196
})
159197
}
@@ -169,10 +207,11 @@ func handleTestSubprocess(t *testing.T) {
169207
}
170208
}
171209
require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID)
210+
require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role)
172211

173212
testName := topo.Name + "/"
174-
if *role == "server" {
175-
testName += "server"
213+
if *role == "server" || *role == "stun" {
214+
testName += *role
176215
} else {
177216
testName += *clientName
178217
}
@@ -185,18 +224,34 @@ func handleTestSubprocess(t *testing.T) {
185224
topo.Server.StartServer(t, logger, *serverListenAddr)
186225
// no exit
187226

227+
case "stun":
228+
launchSTUNServer(t, *stunListenAddr)
229+
// no exit
230+
188231
case "client":
189232
logger = logger.Named(*clientName)
233+
if *clientNumber != 1 && *clientNumber != 2 {
234+
t.Fatalf("invalid client number %d", clientNumber)
235+
}
190236
serverURL, err := url.Parse(*clientServerURL)
191237
require.NoErrorf(t, err, "parse server url %q", *clientServerURL)
192238
myID, err := uuid.Parse(*clientMyID)
193239
require.NoErrorf(t, err, "parse client id %q", *clientMyID)
194240
peerID, err := uuid.Parse(*clientPeerID)
195241
require.NoErrorf(t, err, "parse peer id %q", *clientPeerID)
196242

243+
// Load the DERP map.
244+
var derpMap tailcfg.DERPMap
245+
derpMapPath := *clientDERPMapPath
246+
f, err := os.Open(derpMapPath)
247+
require.NoErrorf(t, err, "open DERP map %q", derpMapPath)
248+
err = json.NewDecoder(f).Decode(&derpMap)
249+
_ = f.Close()
250+
require.NoErrorf(t, err, "decode DERP map %q", derpMapPath)
251+
197252
waitForServerAvailable(t, serverURL)
198253

199-
conn := topo.StartClient(t, logger, serverURL, myID, peerID)
254+
conn := topo.StartClient(t, logger, serverURL, &derpMap, *clientNumber, myID, peerID)
200255

201256
if *clientRunTests {
202257
// Wait for connectivity.
@@ -218,6 +273,23 @@ func handleTestSubprocess(t *testing.T) {
218273
})
219274
}
220275

276+
type forcedAddrPacketListener struct {
277+
addr string
278+
}
279+
280+
var _ nettype.PacketListener = forcedAddrPacketListener{}
281+
282+
func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) {
283+
return nettype.Std{}.ListenPacket(ctx, network, ln.addr)
284+
}
285+
286+
func launchSTUNServer(t *testing.T, listenAddr string) {
287+
ln := forcedAddrPacketListener{addr: listenAddr}
288+
addr, cleanup := stuntest.ServeWithPacketListener(t, ln)
289+
t.Cleanup(cleanup)
290+
assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr")
291+
}
292+
221293
func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
222294
const delay = 100 * time.Millisecond
223295
const reqTimeout = 2 * time.Second
@@ -247,45 +319,55 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
247319
}
248320

249321
func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
250-
_, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
322+
_, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{
251323
"--subprocess",
252324
"--test-name=" + topologyName,
253325
"--role=server",
254-
"--server-listen-addr=" + networking.ServerListenAddr,
326+
"--server-listen-addr=" + networking.Server.ListenAddr,
327+
})
328+
return closeFn
329+
}
330+
331+
func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
332+
_, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{
333+
"--subprocess",
334+
"--test-name=" + topologyName,
335+
"--role=stun",
336+
"--stun-listen-addr=" + networking.STUN.ListenAddr,
255337
})
256338
return closeFn
257339
}
258340

259-
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) {
341+
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int, derpMapPath string) (<-chan error, func() error) {
260342
require.True(t, clientNumber == 1 || clientNumber == 2)
261343

262344
var (
263-
clientName = fmt.Sprintf("client%d", clientNumber)
264-
myID = integration.Client1ID
265-
peerID = integration.Client2ID
266-
accessURL = networking.ServerAccessURLClient1
267-
netNS = networking.ProcessClient1.NetNS
345+
clientName = fmt.Sprintf("client%d", clientNumber)
346+
myID = integration.Client1ID
347+
peerID = integration.Client2ID
348+
clientConfig = networking.Client1
268349
)
269350
if clientNumber == 2 {
270351
myID, peerID = peerID, myID
271-
accessURL = networking.ServerAccessURLClient2
272-
netNS = networking.ProcessClient2.NetNS
352+
clientConfig = networking.Client2
273353
}
274354

275355
flags := []string{
276356
"--subprocess",
277357
"--test-name=" + topologyName,
278358
"--role=client",
279359
"--client-name=" + clientName,
280-
"--client-server-url=" + accessURL,
360+
"--client-number=" + strconv.Itoa(clientNumber),
361+
"--client-server-url=" + clientConfig.ServerAccessURL,
281362
"--client-id=" + myID.String(),
282363
"--client-peer-id=" + peerID.String(),
364+
"--client-derp-map-path=" + derpMapPath,
283365
}
284366
if clientNumber == 1 {
285367
flags = append(flags, "--client-run-tests")
286368
}
287369

288-
return startSubprocess(t, clientName, netNS, flags)
370+
return startSubprocess(t, clientName, clientConfig.Process.NetNS, flags)
289371
}
290372

291373
// startSubprocess launches the test binary with the same flags as the test, but
@@ -298,3 +380,19 @@ func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []s
298380
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)
299381
return integration.ExecBackground(t, processName, netNS, name, args)
300382
}
383+
384+
func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error {
385+
f, err := os.Create(path)
386+
if err != nil {
387+
return err
388+
}
389+
defer f.Close()
390+
391+
enc := json.NewEncoder(f)
392+
enc.SetIndent("", " ")
393+
err = enc.Encode(derpMap)
394+
if err != nil {
395+
return err
396+
}
397+
return nil
398+
}

0 commit comments

Comments
 (0)