Skip to content

Commit af4a668

Browse files
spikecurtisspikecurtis
authored
fix: use tailscale that avoids small MTU paths (#18323)
Fixes #15523 Uses latest https://github.com/coder/tailscale which includes coder/tailscale#85 to stop selecting paths with small MTU for direct connections. Also updates the tailnet integration test to reproduce the issue. The previous version had the 2 peers connected by a single veth, but this allows the OS to fragment the packet. In the new version, the 2 peers (and server) are all connected by a central router. The link between peer 1 and the router has an adjustable MTU. IPv6 does not allow packets to be fragmented by intermediate routers, so sending a too-large packet in this scenario forces the router to drop packets and reproduce the issue (without the tailscale changes).
1 parent 2377d76 commit af4a668

File tree

5 files changed

+109
-85
lines changed

5 files changed

+109
-85
lines changed

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ replace github.com/tcnksm/go-httpstat => github.com/coder/go-httpstat v0.0.0-202
3636

3737
// There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here:
3838
// https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main
39-
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e
39+
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c
4040

4141
// This is replaced to include
4242
// 1. a fix for a data race: c.f. https://github.com/tailscale/wireguard-go/pull/25

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -920,8 +920,8 @@ github.com/coder/serpent v0.10.0 h1:ofVk9FJXSek+SmL3yVE3GoArP83M+1tX+H7S4t8BSuM=
920920
github.com/coder/serpent v0.10.0/go.mod h1:cZFW6/fP+kE9nd/oRkEHJpG6sXCtQ+AX7WMMEHv0Y3Q=
921921
github.com/coder/ssh v0.0.0-20231128192721-70855dedb788 h1:YoUSJ19E8AtuUFVYBpXuOD6a/zVP3rcxezNsoDseTUw=
922922
github.com/coder/ssh v0.0.0-20231128192721-70855dedb788/go.mod h1:aGQbuCLyhRLMzZF067xc84Lh7JDs1FKwCmF1Crl9dxQ=
923-
github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e h1:nope/SZfoLB9MCOB9wdCE6gW5+8l3PhFrDC5IWPL8bk=
924-
github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e/go.mod h1:1ggFFdHTRjPRu9Yc1yA7nVHBYB50w9Ce7VIXNqcW6Ko=
923+
github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c h1:d/qBIi3Ez7KkopRgNtfdvTMqvqBg47d36qVfkd3C5EQ=
924+
github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c/go.mod h1:l7ml5uu7lFh5hY28lGYM4b/oFSmuPHYX6uk4RAu23Lc=
925925
github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e h1:JNLPDi2P73laR1oAclY6jWzAbucf70ASAvf5mh2cME0=
926926
github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
927927
github.com/coder/terraform-provider-coder/v2 v2.5.3 h1:EwqIIQKe/j8bsR4WyDJ3bD0dVdkfVqJ43TwClyGneUU=

‎tailnet/test/integration/integration.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/go-chi/chi/v5"
2626
"github.com/google/uuid"
2727
"github.com/stretchr/testify/require"
28+
"golang.org/x/sys/unix"
2829
"golang.org/x/xerrors"
2930
"tailscale.com/derp"
3031
"tailscale.com/derp/derphttp"
@@ -458,6 +459,16 @@ func (UDPEchoService) StartService(t *testing.T, logger slog.Logger, _ *tailnet.
458459
Port: EchoPort,
459460
})
460461
require.NoError(t, err)
462+
463+
// set path MTU discovery so that we don't fragment the responses.
464+
c, err := l.SyscallConn()
465+
require.NoError(t, err)
466+
var sockErr error
467+
err = c.Control(func(fd uintptr) {
468+
sockErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_MTU_DISCOVER, unix.IP_PMTUDISC_DO)
469+
})
470+
require.NoError(t, err)
471+
require.NoError(t, sockErr)
461472
logger.Info(context.Background(), "started UDPEcho server")
462473
t.Cleanup(func() {
463474
lCloseErr := l.Close()

‎tailnet/test/integration/integration_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ var topologies = []integration.TestTopology{
112112
{
113113
// Test that direct over normal MTU works.
114114
Name: "DirectMTU1500",
115-
NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1500},
115+
NetworkingProvider: integration.TriangleNetwork{Client1MTU: 1500},
116116
Server: integration.SimpleServerOptions{},
117117
ClientStarter: integration.BasicClientStarter{
118118
WaitForDirect: true,
@@ -124,7 +124,7 @@ var topologies = []integration.TestTopology{
124124
{
125125
// Test that small MTU works.
126126
Name: "MTU1280",
127-
NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1280},
127+
NetworkingProvider: integration.TriangleNetwork{Client1MTU: 1280},
128128
Server: integration.SimpleServerOptions{},
129129
ClientStarter: integration.BasicClientStarter{Service: integration.UDPEchoService{}, LogPackets: true},
130130
RunTests: integration.TestBigUDP,

‎tailnet/test/integration/network.go

Lines changed: 93 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -390,33 +390,38 @@ func createFakeInternet(t *testing.T) fakeInternet {
390390
}
391391

392392
type TriangleNetwork struct {
393-
InterClientMTU int
393+
Client1MTU int
394394
}
395395

396396
type fakeTriangleNetwork struct {
397-
NamePrefix string
398-
ServerNetNS *os.File
399-
Client1NetNS *os.File
400-
Client2NetNS *os.File
401-
ServerClient1VethPair vethPair
402-
ServerClient2VethPair vethPair
403-
Client1Client2VethPair vethPair
397+
NamePrefix string
398+
ServerNetNS *os.File
399+
Client1NetNS *os.File
400+
Client2NetNS *os.File
401+
RouterNetNS *os.File
402+
ServerVethPair vethPair
403+
Client1VethPair vethPair
404+
Client2VethPair vethPair
404405
}
405406

406-
// SetupNetworking creates multiple namespaces with veth pairs between them
407-
// with the following topology:
407+
// SetupNetworking creates multiple namespaces with a central router in the following topology
408408
// .
409-
// . ┌────────────────────────────────────────────┐
410-
// . │ Server │
411-
// . └─────┬───────────────────────────────────┬──┘
412-
// . │fdac:38fa:ffff:2::3 │fdac:38fa:ffff:3::3
413-
// . veth│ veth│
414-
// . │fdac:38fa:ffff:2::1 │fdac:38fa:ffff:3::2
415-
// . ┌───────┴──────┐ ┌─────┴───────┐
416-
// . │ │ fdac:38fa:ffff:1::2│ │
417-
// . │ Client 1 ├──────────────────────┤ Client 2 │
418-
// . │ │fdac:38fa:ffff:1::1 │ │
419-
// . └──────────────┘ └─────────────┘
409+
// . ┌──────────────┐
410+
// . │ │
411+
// . │ Server ├─────────────────────────────────────┐
412+
// . │ │fdac:38fa:ffff:3::2 │
413+
// . └──────────────┘ │ fdac:38fa:ffff:3::1
414+
// . ┌──────────────┐ ┌─────┴───────┐
415+
// . │ │ fdac:38fa:ffff:1::1│ │
416+
// . │ Client 1 ├───────────────────────────────┤ Router │
417+
// . │ │fdac:38fa:ffff:1::2 │ │
418+
// . └──────────────┘ └─────┬───────┘
419+
// . ┌──────────────┐ │ fdac:38fa:ffff:2::1
420+
// . │ │ │
421+
// . │ Client 2 ├─────────────────────────────────────┘
422+
// . │ │fdac:38fa:ffff:2::2
423+
// . └──────────────┘
424+
// The veth link between Client 1 and the router has a configurable MTU via Client1MTU.
420425
func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking {
421426
logger := l.Named("setup-networking").Leveled(slog.LevelDebug)
422427
t.Helper()
@@ -433,101 +438,109 @@ func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetwor
433438
network.ServerNetNS = createNetNS(t, namePrefix+"server")
434439
network.Client1NetNS = createNetNS(t, namePrefix+"client1")
435440
network.Client2NetNS = createNetNS(t, namePrefix+"client2")
441+
network.RouterNetNS = createNetNS(t, namePrefix+"router")
436442

437-
// Create veth pair between server and client1
438-
network.ServerClient1VethPair = vethPair{
439-
Outer: namePrefix + "s-1",
440-
Inner: namePrefix + "1-s",
443+
// Create veth pair between server and router
444+
network.ServerVethPair = vethPair{
445+
Outer: namePrefix + "s-r",
446+
Inner: namePrefix + "r-s",
441447
}
442-
err := createVethPair(network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
448+
err := createVethPair(network.ServerVethPair.Outer, network.ServerVethPair.Inner)
443449
require.NoErrorf(t, err, "create veth pair %q <-> %q",
444-
network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
450+
network.ServerVethPair.Outer, network.ServerVethPair.Inner)
445451

446-
// Move server-client1 veth ends to their respective namespaces
447-
err = setVethNetNS(network.ServerClient1VethPair.Outer, int(network.ServerNetNS.Fd()))
448-
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient1VethPair.Outer)
449-
err = setVethNetNS(network.ServerClient1VethPair.Inner, int(network.Client1NetNS.Fd()))
450-
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.ServerClient1VethPair.Inner)
452+
// Move server-router veth ends to their respective namespaces
453+
err = setVethNetNS(network.ServerVethPair.Outer, int(network.ServerNetNS.Fd()))
454+
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerVethPair.Outer)
455+
err = setVethNetNS(network.ServerVethPair.Inner, int(network.RouterNetNS.Fd()))
456+
require.NoErrorf(t, err, "set veth %q to router NetNS", network.ServerVethPair.Inner)
451457

452-
// Create veth pair between server and client2
453-
network.ServerClient2VethPair = vethPair{
454-
Outer: namePrefix + "s-2",
455-
Inner: namePrefix + "2-s",
458+
// Create veth pair between client1 and router
459+
network.Client1VethPair = vethPair{
460+
Outer: namePrefix + "1-r",
461+
Inner: namePrefix + "r-1",
456462
}
457-
err = createVethPair(network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
463+
logger.Debug(context.Background(), "creating client1 link", slog.F("mtu", n.Client1MTU))
464+
err = createVethPair(network.Client1VethPair.Outer, network.Client1VethPair.Inner, withMTU(n.Client1MTU))
458465
require.NoErrorf(t, err, "create veth pair %q <-> %q",
459-
network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
466+
network.Client1VethPair.Outer, network.Client1VethPair.Inner)
460467

461-
// Move server-client2 veth ends to their respective namespaces
462-
err = setVethNetNS(network.ServerClient2VethPair.Outer, int(network.ServerNetNS.Fd()))
463-
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient2VethPair.Outer)
464-
err = setVethNetNS(network.ServerClient2VethPair.Inner, int(network.Client2NetNS.Fd()))
465-
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.ServerClient2VethPair.Inner)
468+
// Move client1-router veth ends to their respective namespaces
469+
err = setVethNetNS(network.Client1VethPair.Outer, int(network.Client1NetNS.Fd()))
470+
require.NoErrorf(t, err, "set veth %q to server NetNS", network.Client1VethPair.Outer)
471+
err = setVethNetNS(network.Client1VethPair.Inner, int(network.RouterNetNS.Fd()))
472+
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1VethPair.Inner)
466473

467474
// Create veth pair between client1 and client2
468-
network.Client1Client2VethPair = vethPair{
469-
Outer: namePrefix + "1-2",
470-
Inner: namePrefix + "2-1",
475+
network.Client2VethPair = vethPair{
476+
Outer: namePrefix + "2-r",
477+
Inner: namePrefix + "r-2",
471478
}
472-
logger.Debug(context.Background(), "creating inter-client link", slog.F("mtu", n.InterClientMTU))
473-
err = createVethPair(network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner,
474-
withMTU(n.InterClientMTU))
479+
480+
err = createVethPair(network.Client2VethPair.Outer, network.Client2VethPair.Inner)
475481
require.NoErrorf(t, err, "create veth pair %q <-> %q",
476-
network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner)
482+
network.Client2VethPair.Outer, network.Client2VethPair.Inner)
477483

478484
// Move client1-client2 veth ends to their respective namespaces
479-
err = setVethNetNS(network.Client1Client2VethPair.Outer, int(network.Client1NetNS.Fd()))
480-
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client1Client2VethPair.Outer)
481-
err = setVethNetNS(network.Client1Client2VethPair.Inner, int(network.Client2NetNS.Fd()))
482-
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1Client2VethPair.Inner)
485+
err = setVethNetNS(network.Client2VethPair.Outer, int(network.Client2NetNS.Fd()))
486+
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client2VethPair.Outer)
487+
err = setVethNetNS(network.Client2VethPair.Inner, int(network.RouterNetNS.Fd()))
488+
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client2VethPair.Inner)
483489

484490
// Set IP addresses according to the diagram:
485-
err = setInterfaceIP6(network.ServerNetNS, network.ServerClient1VethPair.Outer, ula+"2::3")
486-
require.NoErrorf(t, err, "set IP on server-client1 interface")
487-
err = setInterfaceIP6(network.ServerNetNS, network.ServerClient2VethPair.Outer, ula+"3::3")
488-
require.NoErrorf(t, err, "set IP on server-client2 interface")
489-
490-
err = setInterfaceIP6(network.Client1NetNS, network.ServerClient1VethPair.Inner, ula+"2::1")
491-
require.NoErrorf(t, err, "set IP on client1-server interface")
492-
err = setInterfaceIP6(network.Client1NetNS, network.Client1Client2VethPair.Outer, ula+"1::1")
493-
require.NoErrorf(t, err, "set IP on client1-client2 interface")
494-
495-
err = setInterfaceIP6(network.Client2NetNS, network.ServerClient2VethPair.Inner, ula+"3::2")
496-
require.NoErrorf(t, err, "set IP on client2-server interface")
497-
err = setInterfaceIP6(network.Client2NetNS, network.Client1Client2VethPair.Inner, ula+"1::2")
498-
require.NoErrorf(t, err, "set IP on client2-client1 interface")
491+
err = setInterfaceIP6(network.ServerNetNS, network.ServerVethPair.Outer, ula+"3::2")
492+
require.NoErrorf(t, err, "set IP on server interface")
493+
err = setInterfaceIP6(network.Client1NetNS, network.Client1VethPair.Outer, ula+"1::2")
494+
require.NoErrorf(t, err, "set IP on client1 interface")
495+
err = setInterfaceIP6(network.Client2NetNS, network.Client2VethPair.Outer, ula+"2::2")
496+
require.NoErrorf(t, err, "set IP on client2 interface")
497+
498+
err = setInterfaceIP6(network.RouterNetNS, network.ServerVethPair.Inner, ula+"3::1")
499+
require.NoErrorf(t, err, "set IP on router-server interface")
500+
err = setInterfaceIP6(network.RouterNetNS, network.Client1VethPair.Inner, ula+"1::1")
501+
require.NoErrorf(t, err, "set IP on router-client1 interface")
502+
err = setInterfaceIP6(network.RouterNetNS, network.Client2VethPair.Inner, ula+"2::1")
503+
require.NoErrorf(t, err, "set IP on router-client2 interface")
499504

500505
// Bring up all interfaces
501506
interfaces := []struct {
502-
netNS *os.File
503-
ifaceName string
507+
netNS *os.File
508+
ifaceName string
509+
defaultRoute string
504510
}{
505-
{network.ServerNetNS, network.ServerClient1VethPair.Outer},
506-
{network.ServerNetNS, network.ServerClient2VethPair.Outer},
507-
{network.Client1NetNS, network.ServerClient1VethPair.Inner},
508-
{network.Client1NetNS, network.Client1Client2VethPair.Outer},
509-
{network.Client2NetNS, network.ServerClient2VethPair.Inner},
510-
{network.Client2NetNS, network.Client1Client2VethPair.Inner},
511+
{network.ServerNetNS, network.ServerVethPair.Outer, ula + "3::1"},
512+
{network.Client1NetNS, network.Client1VethPair.Outer, ula + "1::1"},
513+
{network.Client2NetNS, network.Client2VethPair.Outer, ula + "2::1"},
514+
{network.RouterNetNS, network.ServerVethPair.Inner, ""},
515+
{network.RouterNetNS, network.Client1VethPair.Inner, ""},
516+
{network.RouterNetNS, network.Client2VethPair.Inner, ""},
511517
}
512518
for _, iface := range interfaces {
513519
err = setInterfaceUp(iface.netNS, iface.ifaceName)
514520
require.NoErrorf(t, err, "bring up interface %q", iface.ifaceName)
515-
// Note: routes are not needed as we are fully connected, so nothing needs to forward IP to a further
516-
// destination.
521+
522+
if iface.defaultRoute != "" {
523+
err = addRouteInNetNS(iface.netNS, []string{"default", "via", iface.defaultRoute, "dev", iface.ifaceName})
524+
require.NoErrorf(t, err, "add peer default route to %s", iface.defaultRoute)
525+
}
517526
}
518527

528+
// enable IP forwarding in the router
529+
_, err = commandInNetNS(network.RouterNetNS, "sysctl", []string{"-w", "net.ipv6.conf.all.forwarding=1"}).Output()
530+
require.NoError(t, wrapExitErr(err), "enable IPv6 forwarding in router NetNS")
531+
519532
return TestNetworking{
520533
Server: TestNetworkingServer{
521534
Process: TestNetworkingProcess{NetNS: network.ServerNetNS},
522535
ListenAddr: "[::]:8080", // Server listens on all IPs
523536
},
524537
Client1: TestNetworkingClient{
525538
Process: TestNetworkingProcess{NetNS: network.Client1NetNS},
526-
ServerAccessURL: "http://[" + ula + "2::3]:8080", // Client1 accesses server directly
539+
ServerAccessURL: "http://[" + ula + "3::2]:8080",
527540
},
528541
Client2: TestNetworkingClient{
529542
Process: TestNetworkingProcess{NetNS: network.Client2NetNS},
530-
ServerAccessURL: "http://[" + ula + "3::3]:8080", // Client2 accesses server directly
543+
ServerAccessURL: "http://[" + ula + "3::2]:8080",
531544
},
532545
}
533546
}

0 commit comments

Comments
 (0)