Skip to content

Commit d956af0

Browse files
deansheatherdeansheather
authored
chore: add EasyNATDERP tailnet integration test (coder#13138)
1 parent 886a97b commit d956af0

File tree

4 files changed

+506
-149
lines changed

4 files changed

+506
-149
lines changed

‎tailnet/test/integration/integration.go

Lines changed: 39 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
"github.com/coder/coder/v2/coderd/httpmw"
3131
"github.com/coder/coder/v2/coderd/tracing"
3232
"github.com/coder/coder/v2/codersdk"
33-
"github.com/coder/coder/v2/cryptorand"
3433
"github.com/coder/coder/v2/tailnet"
3534
)
3635

@@ -40,78 +39,7 @@ var (
4039
Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002")
4140
)
4241

43-
type TestTopology struct {
44-
Name string
45-
// SetupNetworking creates interfaces and network namespaces for the test.
46-
// The most simple implementation is NetworkSetupDefault, which only creates
47-
// a network namespace shared for all tests.
48-
SetupNetworking func(t *testing.T, logger slog.Logger) TestNetworking
49-
50-
// StartServer gets called in the server subprocess. It's expected to start
51-
// the coordinator server in the background and return.
52-
StartServer func(t *testing.T, logger slog.Logger, listenAddr string)
53-
// StartClient gets called in each client subprocess. It's expected to
54-
// create the tailnet.Conn and ensure connectivity to it's peer.
55-
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
56-
57-
// RunTests is the main test function. It's called in each of the client
58-
// subprocesses. If tests can only run once, they should check the client ID
59-
// and return early if it's not the expected one.
60-
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn)
61-
}
62-
63-
type TestNetworking struct {
64-
// ServerListenAddr is the IP address and port that the server listens on,
65-
// passed to StartServer.
66-
ServerListenAddr string
67-
// ServerAccessURLClient1 is the hostname and port that the first client
68-
// uses to access the server.
69-
ServerAccessURLClient1 string
70-
// ServerAccessURLClient2 is the hostname and port that the second client
71-
// uses to access the server.
72-
ServerAccessURLClient2 string
73-
74-
// Networking settings for each subprocess.
75-
ProcessServer TestNetworkingProcess
76-
ProcessClient1 TestNetworkingProcess
77-
ProcessClient2 TestNetworkingProcess
78-
}
79-
80-
type TestNetworkingProcess struct {
81-
// NetNS to enter. If zero, the current network namespace is used.
82-
NetNSFd int
83-
}
84-
85-
func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
86-
netNSName := "codertest_netns_"
87-
randStr, err := cryptorand.String(4)
88-
require.NoError(t, err, "generate random string for netns name")
89-
netNSName += randStr
90-
91-
// Create a single network namespace for all tests so we can have an
92-
// isolated loopback interface.
93-
netNSFile, err := createNetNS(netNSName)
94-
require.NoError(t, err, "create network namespace")
95-
t.Cleanup(func() {
96-
_ = netNSFile.Close()
97-
})
98-
99-
var (
100-
listenAddr = "127.0.0.1:8080"
101-
process = TestNetworkingProcess{
102-
NetNSFd: int(netNSFile.Fd()),
103-
}
104-
)
105-
return TestNetworking{
106-
ServerListenAddr: listenAddr,
107-
ServerAccessURLClient1: "http://" + listenAddr,
108-
ServerAccessURLClient2: "http://" + listenAddr,
109-
ProcessServer: process,
110-
ProcessClient1: process,
111-
ProcessClient2: process,
112-
}
113-
}
114-
42+
// StartServerBasic creates a coordinator and DERP server.
11543
func StartServerBasic(t *testing.T, logger slog.Logger, listenAddr string) {
11644
coord := tailnet.NewCoordinator(logger)
11745
var coordPtr atomic.Pointer[tailnet.Coordinator]
@@ -208,42 +136,7 @@ func StartServerBasic(t *testing.T, logger slog.Logger, listenAddr string) {
208136
})
209137
}
210138

211-
func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
212-
portStr := serverURL.Port()
213-
port, err := strconv.Atoi(portStr)
214-
require.NoError(t, err, "parse server port")
215-
216-
hostname := serverURL.Hostname()
217-
ipv4 := ""
218-
ip, err := netip.ParseAddr(hostname)
219-
if err == nil {
220-
hostname = ""
221-
ipv4 = ip.String()
222-
}
223-
224-
return &tailcfg.DERPMap{
225-
Regions: map[int]*tailcfg.DERPRegion{
226-
1: {
227-
RegionID: 1,
228-
RegionCode: "test",
229-
RegionName: "test server",
230-
Nodes: []*tailcfg.DERPNode{
231-
{
232-
Name: "test0",
233-
RegionID: 1,
234-
HostName: hostname,
235-
IPv4: ipv4,
236-
IPv6: "none",
237-
DERPPort: port,
238-
ForceHTTP: true,
239-
InsecureForTests: true,
240-
},
241-
},
242-
},
243-
},
244-
}
245-
}
246-
139+
// StartClientBasic creates a client connection to the server.
247140
func StartClientBasic(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn {
248141
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String()))
249142
require.NoError(t, err)
@@ -284,3 +177,40 @@ func StartClientBasic(t *testing.T, logger slog.Logger, serverURL *url.URL, myID
284177

285178
return conn
286179
}
180+
181+
func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
182+
portStr := serverURL.Port()
183+
port, err := strconv.Atoi(portStr)
184+
require.NoError(t, err, "parse server port")
185+
186+
hostname := serverURL.Hostname()
187+
ipv4 := ""
188+
ip, err := netip.ParseAddr(hostname)
189+
if err == nil {
190+
hostname = ""
191+
ipv4 = ip.String()
192+
}
193+
194+
return &tailcfg.DERPMap{
195+
Regions: map[int]*tailcfg.DERPRegion{
196+
1: {
197+
RegionID: 1,
198+
RegionCode: "test",
199+
RegionName: "test server",
200+
Nodes: []*tailcfg.DERPNode{
201+
{
202+
Name: "test0",
203+
RegionID: 1,
204+
HostName: hostname,
205+
IPv4: ipv4,
206+
IPv6: "none",
207+
DERPPort: port,
208+
STUNPort: -1,
209+
ForceHTTP: true,
210+
InsecureForTests: true,
211+
},
212+
},
213+
},
214+
},
215+
}
216+
}

‎tailnet/test/integration/integration_test.go

Lines changed: 85 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
"os/exec"
1313
"os/signal"
1414
"runtime"
15+
"strings"
16+
"sync"
1517
"syscall"
1618
"testing"
1719
"time"
@@ -66,31 +68,36 @@ func TestMain(m *testing.M) {
6668

6769
var topologies = []integration.TestTopology{
6870
{
69-
Name: "BasicLoopback",
71+
Name: "BasicLoopbackDERP",
7072
SetupNetworking: integration.SetupNetworkingLoopback,
7173
StartServer: integration.StartServerBasic,
7274
StartClient: integration.StartClientBasic,
73-
RunTests: func(t *testing.T, log slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, conn *tailnet.Conn) {
74-
// Test basic connectivity
75-
peerIP := tailnet.IPFromUUID(peerID)
76-
_, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP)
77-
require.NoError(t, err, "ping peer")
78-
},
75+
RunTests: integration.TestSuite,
76+
},
77+
{
78+
Name: "EasyNATDERP",
79+
SetupNetworking: integration.SetupNetworkingEasyNAT,
80+
StartServer: integration.StartServerBasic,
81+
StartClient: integration.StartClientBasic,
82+
RunTests: integration.TestSuite,
7983
},
8084
}
8185

82-
//nolint:paralleltest
86+
//nolint:paralleltest,tparallel
8387
func TestIntegration(t *testing.T) {
8488
if *isSubprocess {
8589
handleTestSubprocess(t)
8690
return
8791
}
8892

8993
for _, topo := range topologies {
90-
//nolint:paralleltest
94+
topo := topo
9195
t.Run(topo.Name, func(t *testing.T) {
92-
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
96+
// These can run in parallel because every test should be in an
97+
// isolated NetNS.
98+
t.Parallel()
9399

100+
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
94101
networking := topo.SetupNetworking(t, log)
95102

96103
// Fork the three child processes.
@@ -100,13 +107,13 @@ func TestIntegration(t *testing.T) {
100107
client2ErrCh, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)
101108

102109
// Wait for client1 to exit.
103-
require.NoError(t, <-client1ErrCh)
110+
require.NoError(t, <-client1ErrCh, "client 1 exited")
104111

105112
// Close client2 and the server.
106113
closeClient2()
107-
require.NoError(t, <-client2ErrCh)
114+
require.NoError(t, <-client2ErrCh, "client 2 exited")
108115
closeServer()
109-
require.NoError(t, <-serverErrCh)
116+
require.NoError(t, <-serverErrCh, "server exited")
110117
})
111118
}
112119
}
@@ -152,8 +159,14 @@ func handleTestSubprocess(t *testing.T) {
152159
conn := topo.StartClient(t, log, serverURL, myID, peerID)
153160

154161
if *clientRunTests {
162+
// Wait for connectivity.
163+
peerIP := tailnet.IPFromUUID(peerID)
164+
if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) {
165+
t.Fatalf("peer %v did not become reachable", peerIP)
166+
}
167+
155168
topo.RunTests(t, log, serverURL, myID, peerID, conn)
156-
// and exit
169+
// then exit
157170
return
158171
}
159172
}
@@ -194,7 +207,7 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
194207
}
195208

196209
func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) (<-chan error, func()) {
197-
return startSubprocess(t, networking.ProcessServer.NetNSFd, []string{
210+
return startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
198211
"--subprocess",
199212
"--test-name=" + topologyName,
200213
"--role=server",
@@ -210,10 +223,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
210223
myID = integration.Client1ID
211224
peerID = integration.Client2ID
212225
accessURL = networking.ServerAccessURLClient1
226+
netNS = networking.ProcessClient1.NetNS
213227
)
214228
if clientNumber == 2 {
215229
myID, peerID = peerID, myID
216230
accessURL = networking.ServerAccessURLClient2
231+
netNS = networking.ProcessClient2.NetNS
217232
}
218233

219234
flags := []string{
@@ -229,14 +244,15 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
229244
flags = append(flags, "--client-run-tests")
230245
}
231246

232-
return startSubprocess(t, networking.ProcessClient1.NetNSFd, flags)
247+
return startSubprocess(t, clientName, netNS, flags)
233248
}
234249

235-
func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, func()) {
250+
func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func()) {
236251
name := os.Args[0]
237-
args := append(os.Args[1:], flags...)
252+
// Always use verbose mode since it gets piped to the parent test anyways.
253+
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)
238254

239-
if netNSFd > 0 {
255+
if netNS != nil {
240256
// We use nsenter to enter the namespace.
241257
// We can't use `setns` easily from Golang in the parent process because
242258
// you can't execute the syscall in the forked child thread before it
@@ -249,11 +265,17 @@ func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, f
249265
}
250266

251267
cmd := exec.Command(name, args...)
252-
if netNSFd > 0 {
253-
cmd.ExtraFiles = []*os.File{os.NewFile(uintptr(netNSFd), "")}
268+
if netNS != nil {
269+
cmd.ExtraFiles = []*os.File{netNS}
270+
}
271+
272+
out := &testWriter{
273+
name: processName,
274+
t: t,
254275
}
255-
cmd.Stdout = os.Stdout
256-
cmd.Stderr = os.Stderr
276+
t.Cleanup(out.Flush)
277+
cmd.Stdout = out
278+
cmd.Stderr = out
257279
cmd.SysProcAttr = &syscall.SysProcAttr{
258280
Pdeathsig: syscall.SIGTERM,
259281
}
@@ -293,3 +315,43 @@ func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, f
293315

294316
return waitErr, closeFn
295317
}
318+
319+
type testWriter struct {
320+
mut sync.Mutex
321+
name string
322+
t *testing.T
323+
324+
capturedLines []string
325+
}
326+
327+
func (w *testWriter) Write(p []byte) (n int, err error) {
328+
w.mut.Lock()
329+
defer w.mut.Unlock()
330+
str := string(p)
331+
split := strings.Split(str, "\n")
332+
for _, s := range split {
333+
if s == "" {
334+
continue
335+
}
336+
337+
// If a line begins with "\s*--- (PASS|FAIL)" or is just PASS or FAIL,
338+
// then it's a test result line. We want to capture it and log it later.
339+
trimmed := strings.TrimSpace(s)
340+
if strings.HasPrefix(trimmed, "--- PASS") || strings.HasPrefix(trimmed, "--- FAIL") || trimmed == "PASS" || trimmed == "FAIL" {
341+
w.capturedLines = append(w.capturedLines, s)
342+
continue
343+
}
344+
345+
w.t.Logf("%s output: \t%s", w.name, s)
346+
}
347+
return len(p), nil
348+
}
349+
350+
func (w *testWriter) Flush() {
351+
w.mut.Lock()
352+
defer w.mut.Unlock()
353+
for _, s := range w.capturedLines {
354+
w.t.Logf("%s output: \t%s", w.name, s)
355+
}
356+
w.capturedLines = nil
357+
}

0 commit comments

Comments
 (0)