net/interfaces, net/netns: add node attributes to control default interface getting and binding

mihaip · mihaip · commit 62f4df325769 · 2023-02-08T13:15:10.000-08:00
With tailscale#6566 we started to more aggressively bind to the default interface on Darwin. We are seeing some reports of the wrong cellular interface being chosen on iOS. To help with the investigation, this adds to knobs to control the behavior changes: - CapabilityDebugDisableAlternateDefaultRouteInterface disables the alternate function that we use to get the default interface on macOS and iOS (implemented in tailscale/corp#8201). We still log what it would have returned so we can see if it gets things wrong. - CapabilityDebugDisableBindConnToInterface is a bigger hammer that disables binding of connections to the default interface altogether. Updates tailscale#7184 Updates tailscale#7188 Signed-off-by: Mihai Parparita <mihai@tailscale.com>
diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go
@@ -3840,6 +3840,8 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 
 	// See the netns package for documentation on what this capability does.
 	netns.SetBindToInterfaceByRoute(hasCapability(nm, tailcfg.CapabilityBindToInterfaceByRoute))
+	interfaces.SetDisableAlternateDefaultRouteInterface(hasCapability(nm, tailcfg.CapabilityDebugDisableAlternateDefaultRouteInterface))
+	netns.SetDisableBindConnToInterface(hasCapability(nm, tailcfg.CapabilityDebugDisableBindConnToInterface))
 
 	b.setTCPPortsInterceptedFromNetmapAndPrefsLocked(b.pm.CurrentPrefs())
 	if nm == nil {
diff --git a/net/interfaces/interfaces.go b/net/interfaces/interfaces.go
@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync/atomic"
 
 	"tailscale.com/hostinfo"
 	"tailscale.com/net/netaddr"
@@ -756,3 +757,14 @@ func HasCGNATInterface() (bool, error) {
 	}
 	return hasCGNATInterface, nil
 }
+
+var disableAlternateDefaultRouteInterface atomic.Bool
+
+// SetDisableAlternateDefaultRouteInterface disables the optional external/
+// alternate mechanism for getting the default route network interface.
+//
+// Currently, this only changes the behaviour on BSD-like sytems (FreeBSD and
+// Darwin).
+func SetDisableAlternateDefaultRouteInterface(v bool) {
+	disableAlternateDefaultRouteInterface.Store(v)
+}
diff --git a/net/interfaces/interfaces_bsd.go b/net/interfaces/interfaces_bsd.go
@@ -40,9 +40,15 @@ func defaultRoute() (d DefaultRouteDetails, err error) {
 // owns the default route. It returns the first IPv4 or IPv6 default route it
 // finds (it does not prefer one or the other).
 func DefaultRouteInterfaceIndex() (int, error) {
+	disabledAlternateDefaultRouteInterface := false
 	if f := defaultRouteInterfaceIndexFunc.Load(); f != nil {
 		if ifIndex := f(); ifIndex != 0 {
-			return ifIndex, nil
+			if !disableAlternateDefaultRouteInterface.Load() {
+				return ifIndex, nil
+			} else {
+				disabledAlternateDefaultRouteInterface = true
+				log.Printf("interfaces_bsd: alternate default route interface function disabled, would have returned interface %d", ifIndex)
+			}
 		}
 		// Fallthrough if we can't use the alternate implementation.
 	}
@@ -75,6 +81,9 @@ func DefaultRouteInterfaceIndex() (int, error) {
 			continue
 		}
 		if isDefaultGateway(rm) {
+			if disabledAlternateDefaultRouteInterface {
+				log.Printf("interfaces_bsd: alternate default route interface function disabled, default implementation returned %d", rm.Index)
+			}
 			return rm.Index, nil
 		}
 	}
diff --git a/net/netns/netns.go b/net/netns/netns.go
@@ -42,6 +42,16 @@ func SetBindToInterfaceByRoute(v bool) {
 	bindToInterfaceByRoute.Store(v)
 }
 
+var disableBindConnToInterface atomic.Bool
+
+// SetDisableBindConnToInterface disables the (normal) behavior of binding
+// connections to the default network interface.
+//
+// Currently, this only has an effect on Darwin.
+func SetDisableBindConnToInterface(v bool) {
+	disableBindConnToInterface.Store(v)
+}
+
 // Listener returns a new net.Listener with its Control hook func
 // initialized as necessary to run in logical network namespace that
 // doesn't route back into Tailscale.
diff --git a/net/netns/netns_darwin.go b/net/netns/netns_darwin.go
@@ -42,6 +42,11 @@ func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) e
 		return nil
 	}
 
+	if disableBindConnToInterface.Load() {
+		logf("netns_darwin: binding connection to interfaces disabled")
+		return nil
+	}
+
 	idx, err := getInterfaceIndex(logf, address)
 	if err != nil {
 		// callee logged
diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go
@@ -1765,6 +1765,17 @@ const (
 	// details on the behaviour of this capability.
 	CapabilityBindToInterfaceByRoute = "https://tailscale.com/cap/bind-to-interface-by-route"
 
+	// CapabilityDebugDisableAlternateDefaultRouteInterface changes how Darwin
+	// nodes get the default interface. There is an optional hook (used by the
+	// macOS and iOS clients) to override the default interface, this capability
+	// disables that and uses the default behavior (of parsing the routing
+	// table).
+	CapabilityDebugDisableAlternateDefaultRouteInterface = "https://tailscale.com/cap/debug-disable-alternate-default-route-interface"
+
+	// CapabilityDebugDisableBindConnToInterface disables the automatic binding
+	// of connections to the default network interface on Darwin nodes.
+	CapabilityDebugDisableBindConnToInterface = "https://tailscale.com/cap/debug-disable-bind-conn-to-interface"
+
 	// CapabilityTailnetLockAlpha indicates the node is in the tailnet lock alpha,
 	// and initialization of tailnet lock may proceed.
 	//

Original file line number	Diff line number	Diff line change
`@@ -40,9 +40,15 @@ func defaultRoute() (d DefaultRouteDetails, err error) {`
`40`	`40`	`// owns the default route. It returns the first IPv4 or IPv6 default route it`
`41`	`41`	`// finds (it does not prefer one or the other).`
`42`	`42`	`func DefaultRouteInterfaceIndex() (int, error) {`
	`43`	`+ disabledAlternateDefaultRouteInterface := false`
`43`	`44`	`if f := defaultRouteInterfaceIndexFunc.Load(); f != nil {`
`44`	`45`	`if ifIndex := f(); ifIndex != 0 {`
`45`		`- return ifIndex, nil`
	`46`	`+ if !disableAlternateDefaultRouteInterface.Load() {`
	`47`	`+ return ifIndex, nil`
	`48`	`+ } else {`
	`49`	`+ disabledAlternateDefaultRouteInterface = true`
	`50`	`+ log.Printf("interfaces_bsd: alternate default route interface function disabled, would have returned interface %d", ifIndex)`
	`51`	`+ }`
`46`	`52`	`}`
`47`	`53`	`// Fallthrough if we can't use the alternate implementation.`
`48`	`54`	`}`
`@@ -75,6 +81,9 @@ func DefaultRouteInterfaceIndex() (int, error) {`
`75`	`81`	`continue`
`76`	`82`	`}`
`77`	`83`	`if isDefaultGateway(rm) {`
	`84`	`+ if disabledAlternateDefaultRouteInterface {`
	`85`	`+ log.Printf("interfaces_bsd: alternate default route interface function disabled, default implementation returned %d", rm.Index)`
	`86`	`+ }`
`78`	`87`	`return rm.Index, nil`
`79`	`88`	`}`
`80`	`89`	`}`