Skip to content

chore: add EasyNATDERP tailnet integration test #13138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 6, 2024
Merged

chore: add EasyNATDERP tailnet integration test #13138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 39 additions & 109 deletions tailnet/test/integration/integration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/coderd/tracing"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/cryptorand"
"github.com/coder/coder/v2/tailnet"
)

Expand All @@ -40,78 +39,7 @@ var (
Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002")
)

type TestTopology struct {
Name string
// SetupNetworking creates interfaces and network namespaces for the test.
// The most simple implementation is NetworkSetupDefault, which only creates
// a network namespace shared for all tests.
SetupNetworking func(t *testing.T, logger slog.Logger) TestNetworking

// StartServer gets called in the server subprocess. It's expected to start
// the coordinator server in the background and return.
StartServer func(t *testing.T, logger slog.Logger, listenAddr string)
// StartClient gets called in each client subprocess. It's expected to
// create the tailnet.Conn and ensure connectivity to it's peer.
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn

// RunTests is the main test function. It's called in each of the client
// subprocesses. If tests can only run once, they should check the client ID
// and return early if it's not the expected one.
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn)
}

type TestNetworking struct {
// ServerListenAddr is the IP address and port that the server listens on,
// passed to StartServer.
ServerListenAddr string
// ServerAccessURLClient1 is the hostname and port that the first client
// uses to access the server.
ServerAccessURLClient1 string
// ServerAccessURLClient2 is the hostname and port that the second client
// uses to access the server.
ServerAccessURLClient2 string

// Networking settings for each subprocess.
ProcessServer TestNetworkingProcess
ProcessClient1 TestNetworkingProcess
ProcessClient2 TestNetworkingProcess
}

type TestNetworkingProcess struct {
// NetNS to enter. If zero, the current network namespace is used.
NetNSFd int
}

func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
netNSName := "codertest_netns_"
randStr, err := cryptorand.String(4)
require.NoError(t, err, "generate random string for netns name")
netNSName += randStr

// Create a single network namespace for all tests so we can have an
// isolated loopback interface.
netNSFile, err := createNetNS(netNSName)
require.NoError(t, err, "create network namespace")
t.Cleanup(func() {
_ = netNSFile.Close()
})

var (
listenAddr = "127.0.0.1:8080"
process = TestNetworkingProcess{
NetNSFd: int(netNSFile.Fd()),
}
)
return TestNetworking{
ServerListenAddr: listenAddr,
ServerAccessURLClient1: "http://" + listenAddr,
ServerAccessURLClient2: "http://" + listenAddr,
ProcessServer: process,
ProcessClient1: process,
ProcessClient2: process,
}
}

// StartServerBasic creates a coordinator and DERP server.
func StartServerBasic(t *testing.T, logger slog.Logger, listenAddr string) {
coord := tailnet.NewCoordinator(logger)
var coordPtr atomic.Pointer[tailnet.Coordinator]
Expand Down Expand Up @@ -208,42 +136,7 @@ func StartServerBasic(t *testing.T, logger slog.Logger, listenAddr string) {
})
}

func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
portStr := serverURL.Port()
port, err := strconv.Atoi(portStr)
require.NoError(t, err, "parse server port")

hostname := serverURL.Hostname()
ipv4 := ""
ip, err := netip.ParseAddr(hostname)
if err == nil {
hostname = ""
ipv4 = ip.String()
}

return &tailcfg.DERPMap{
Regions: map[int]*tailcfg.DERPRegion{
1: {
RegionID: 1,
RegionCode: "test",
RegionName: "test server",
Nodes: []*tailcfg.DERPNode{
{
Name: "test0",
RegionID: 1,
HostName: hostname,
IPv4: ipv4,
IPv6: "none",
DERPPort: port,
ForceHTTP: true,
InsecureForTests: true,
},
},
},
},
}
}

// StartClientBasic creates a client connection to the server.
func StartClientBasic(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn {
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String()))
require.NoError(t, err)
Expand Down Expand Up @@ -284,3 +177,40 @@ func StartClientBasic(t *testing.T, logger slog.Logger, serverURL *url.URL, myID

return conn
}

func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
portStr := serverURL.Port()
port, err := strconv.Atoi(portStr)
require.NoError(t, err, "parse server port")

hostname := serverURL.Hostname()
ipv4 := ""
ip, err := netip.ParseAddr(hostname)
if err == nil {
hostname = ""
ipv4 = ip.String()
}

return &tailcfg.DERPMap{
Regions: map[int]*tailcfg.DERPRegion{
1: {
RegionID: 1,
RegionCode: "test",
RegionName: "test server",
Nodes: []*tailcfg.DERPNode{
{
Name: "test0",
RegionID: 1,
HostName: hostname,
IPv4: ipv4,
IPv6: "none",
DERPPort: port,
STUNPort: -1,
ForceHTTP: true,
InsecureForTests: true,
},
},
},
},
}
}
108 changes: 85 additions & 23 deletions tailnet/test/integration/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"os/exec"
"os/signal"
"runtime"
"strings"
"sync"
"syscall"
"testing"
"time"
Expand Down Expand Up @@ -66,31 +68,36 @@ func TestMain(m *testing.M) {

var topologies = []integration.TestTopology{
{
Name: "BasicLoopback",
Name: "BasicLoopbackDERP",
SetupNetworking: integration.SetupNetworkingLoopback,
StartServer: integration.StartServerBasic,
StartClient: integration.StartClientBasic,
RunTests: func(t *testing.T, log slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, conn *tailnet.Conn) {
// Test basic connectivity
peerIP := tailnet.IPFromUUID(peerID)
_, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP)
require.NoError(t, err, "ping peer")
},
RunTests: integration.TestSuite,
},
{
Name: "EasyNATDERP",
SetupNetworking: integration.SetupNetworkingEasyNAT,
StartServer: integration.StartServerBasic,
StartClient: integration.StartClientBasic,
RunTests: integration.TestSuite,
},
}

//nolint:paralleltest
//nolint:paralleltest,tparallel
func TestIntegration(t *testing.T) {
if *isSubprocess {
handleTestSubprocess(t)
return
}

for _, topo := range topologies {
//nolint:paralleltest
topo := topo
t.Run(topo.Name, func(t *testing.T) {
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
// These can run in parallel because every test should be in an
// isolated NetNS.
t.Parallel()

log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
networking := topo.SetupNetworking(t, log)

// Fork the three child processes.
Expand All @@ -100,13 +107,13 @@ func TestIntegration(t *testing.T) {
client2ErrCh, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)

// Wait for client1 to exit.
require.NoError(t, <-client1ErrCh)
require.NoError(t, <-client1ErrCh, "client 1 exited")

// Close client2 and the server.
closeClient2()
require.NoError(t, <-client2ErrCh)
require.NoError(t, <-client2ErrCh, "client 2 exited")
closeServer()
require.NoError(t, <-serverErrCh)
require.NoError(t, <-serverErrCh, "server exited")
})
}
}
Expand Down Expand Up @@ -152,8 +159,14 @@ func handleTestSubprocess(t *testing.T) {
conn := topo.StartClient(t, log, serverURL, myID, peerID)

if *clientRunTests {
// Wait for connectivity.
peerIP := tailnet.IPFromUUID(peerID)
if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) {
t.Fatalf("peer %v did not become reachable", peerIP)
}

topo.RunTests(t, log, serverURL, myID, peerID, conn)
// and exit
// then exit
return
}
}
Expand Down Expand Up @@ -194,7 +207,7 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
}

func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) (<-chan error, func()) {
return startSubprocess(t, networking.ProcessServer.NetNSFd, []string{
return startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=server",
Expand All @@ -210,10 +223,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
myID = integration.Client1ID
peerID = integration.Client2ID
accessURL = networking.ServerAccessURLClient1
netNS = networking.ProcessClient1.NetNS
)
if clientNumber == 2 {
myID, peerID = peerID, myID
accessURL = networking.ServerAccessURLClient2
netNS = networking.ProcessClient2.NetNS
}

flags := []string{
Expand All @@ -229,14 +244,15 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
flags = append(flags, "--client-run-tests")
}

return startSubprocess(t, networking.ProcessClient1.NetNSFd, flags)
return startSubprocess(t, clientName, netNS, flags)
}

func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, func()) {
func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func()) {
name := os.Args[0]
args := append(os.Args[1:], flags...)
// Always use verbose mode since it gets piped to the parent test anyways.
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)

if netNSFd > 0 {
if netNS != nil {
// We use nsenter to enter the namespace.
// We can't use `setns` easily from Golang in the parent process because
// you can't execute the syscall in the forked child thread before it
Expand All @@ -249,11 +265,17 @@ func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, f
}

cmd := exec.Command(name, args...)
if netNSFd > 0 {
cmd.ExtraFiles = []*os.File{os.NewFile(uintptr(netNSFd), "")}
if netNS != nil {
cmd.ExtraFiles = []*os.File{netNS}
}

out := &testWriter{
name: processName,
t: t,
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
t.Cleanup(out.Flush)
cmd.Stdout = out
cmd.Stderr = out
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGTERM,
}
Expand Down Expand Up @@ -293,3 +315,43 @@ func startSubprocess(t *testing.T, netNSFd int, flags []string) (<-chan error, f

return waitErr, closeFn
}

type testWriter struct {
mut sync.Mutex
name string
t *testing.T

capturedLines []string
}

func (w *testWriter) Write(p []byte) (n int, err error) {
w.mut.Lock()
defer w.mut.Unlock()
str := string(p)
split := strings.Split(str, "\n")
for _, s := range split {
if s == "" {
continue
}

// If a line begins with "\s*--- (PASS|FAIL)" or is just PASS or FAIL,
// then it's a test result line. We want to capture it and log it later.
trimmed := strings.TrimSpace(s)
if strings.HasPrefix(trimmed, "--- PASS") || strings.HasPrefix(trimmed, "--- FAIL") || trimmed == "PASS" || trimmed == "FAIL" {
w.capturedLines = append(w.capturedLines, s)
continue
}

w.t.Logf("%s output: \t%s", w.name, s)
}
return len(p), nil
}

func (w *testWriter) Flush() {
w.mut.Lock()
defer w.mut.Unlock()
for _, s := range w.capturedLines {
w.t.Logf("%s output: \t%s", w.name, s)
}
w.capturedLines = nil
}
Loading