Skip to content

Commit 482feef

Browse files
coadlercoadler
authored
feat(devtunnel): support geodistributed tunnels (#2711)
1 parent ae59f16 commit 482feef

File tree

4 files changed

+188
-36
lines changed

4 files changed

+188
-36
lines changed

‎coderd/devtunnel/servers.go

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package devtunnel
2+
3+
import (
4+
"runtime"
5+
"sync"
6+
"time"
7+
8+
"github.com/go-ping/ping"
9+
"golang.org/x/exp/slices"
10+
"golang.org/x/sync/errgroup"
11+
12+
"github.com/coder/coder/cryptorand"
13+
)
14+
15+
type Region struct {
16+
ID int
17+
LocationName string
18+
Nodes []Node
19+
}
20+
21+
type Node struct {
22+
ID int `json:"id"`
23+
HostnameHTTPS string `json:"hostname_https"`
24+
HostnameWireguard string `json:"hostname_wireguard"`
25+
WireguardPort uint16 `json:"wireguard_port"`
26+
27+
AvgLatency time.Duration `json:"avg_latency"`
28+
}
29+
30+
var Regions = []Region{
31+
{
32+
ID: 1,
33+
LocationName: "US East Pittsburgh",
34+
Nodes: []Node{
35+
{
36+
ID: 1,
37+
HostnameHTTPS: "pit-1.try.coder.app",
38+
HostnameWireguard: "pit-1.try.coder.app",
39+
WireguardPort: 55551,
40+
},
41+
},
42+
},
43+
}
44+
45+
func FindClosestNode() (Node, error) {
46+
nodes := []Node{}
47+
48+
for _, region := range Regions {
49+
// Pick a random node from each region.
50+
i, err := cryptorand.Intn(len(region.Nodes))
51+
if err != nil {
52+
return Node{}, err
53+
}
54+
nodes = append(nodes, region.Nodes[i])
55+
}
56+
57+
var (
58+
nodesMu sync.Mutex
59+
eg = errgroup.Group{}
60+
)
61+
for i, node := range nodes {
62+
i, node := i, node
63+
eg.Go(func() error {
64+
pinger, err := ping.NewPinger(node.HostnameHTTPS)
65+
if err != nil {
66+
return err
67+
}
68+
69+
if runtime.GOOS == "windows" {
70+
pinger.SetPrivileged(true)
71+
}
72+
73+
pinger.Count = 5
74+
err = pinger.Run()
75+
if err != nil {
76+
return err
77+
}
78+
79+
nodesMu.Lock()
80+
nodes[i].AvgLatency = pinger.Statistics().AvgRtt
81+
nodesMu.Unlock()
82+
return nil
83+
})
84+
}
85+
86+
err := eg.Wait()
87+
if err != nil {
88+
return Node{}, err
89+
}
90+
91+
slices.SortFunc(nodes, func(i, j Node) bool {
92+
return i.AvgLatency < j.AvgLatency
93+
})
94+
return nodes[0], nil
95+
}

‎coderd/devtunnel/tunnel.go

+90-36
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@ import (
2323
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
2424

2525
"cdr.dev/slog"
26+
"github.com/coder/coder/cryptorand"
2627
)
2728

28-
const (
29-
EndpointWireguard = "wg-tunnel-udp.coder.app"
30-
EndpointHTTPS = "wg-tunnel.coder.app"
29+
var (
30+
v0EndpointHTTPS = "wg-tunnel.coder.app"
3131

32-
ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
33-
ServerUUID = "fcad0000-0000-4000-8000-000000000001"
32+
v0ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
33+
v0ServerIP = netip.AddrFrom16(uuid.MustParse("fcad0000-0000-4000-8000-000000000001"))
3434
)
3535

3636
type Tunnel struct {
@@ -39,47 +39,54 @@ type Tunnel struct {
3939
}
4040

4141
type Config struct {
42+
Version int `json:"version"`
4243
ID uuid.UUID `json:"id"`
4344
PrivateKey device.NoisePrivateKey `json:"private_key"`
4445
PublicKey device.NoisePublicKey `json:"public_key"`
46+
47+
Tunnel Node `json:"tunnel"`
4548
}
4649
type configExt struct {
50+
Version int `json:"-"`
4751
ID uuid.UUID `json:"id"`
4852
PrivateKey device.NoisePrivateKey `json:"-"`
4953
PublicKey device.NoisePublicKey `json:"public_key"`
54+
55+
Tunnel Node `json:"-"`
5056
}
5157

5258
// NewWithConfig calls New with the given config. For documentation, see New.
5359
func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel, <-chan error, error) {
54-
routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
60+
server, routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
5561
if err != nil {
5662
return nil, nil, xerrors.Errorf("start update routine: %w", err)
5763
}
5864

5965
tun, tnet, err := netstack.CreateNetTUN(
60-
[]netip.Addr{netip.AddrFrom16(cfg.ID)},
66+
[]netip.Addr{server.ClientIP},
6167
[]netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
6268
1280,
6369
)
6470
if err != nil {
6571
return nil, nil, xerrors.Errorf("create net TUN: %w", err)
6672
}
6773

68-
wgip, err := net.ResolveIPAddr("ip", EndpointWireguard)
74+
wgip, err := net.ResolveIPAddr("ip", cfg.Tunnel.HostnameWireguard)
6975
if err != nil {
7076
return nil, nil, xerrors.Errorf("resolve endpoint: %w", err)
7177
}
7278

7379
dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelSilent, ""))
7480
err = dev.IpcSet(fmt.Sprintf(`private_key=%s
7581
public_key=%s
76-
endpoint=%s:55555
82+
endpoint=%s:%d
7783
persistent_keepalive_interval=21
7884
allowed_ip=%s/128`,
7985
hex.EncodeToString(cfg.PrivateKey[:]),
80-
encodeBase64ToHex(ServerPublicKey),
86+
server.ServerPublicKey,
8187
wgip.IP.String(),
82-
netip.AddrFrom16(uuid.MustParse(ServerUUID)).String(),
88+
cfg.Tunnel.WireguardPort,
89+
server.ServerIP.String(),
8390
))
8491
if err != nil {
8592
return nil, nil, xerrors.Errorf("configure wireguard ipc: %w", err)
@@ -110,7 +117,7 @@ allowed_ip=%s/128`,
110117
}()
111118

112119
return &Tunnel{
113-
URL: fmt.Sprintf("https://%s.%s", cfg.ID, EndpointHTTPS),
120+
URL: fmt.Sprintf("https://%s", server.Hostname),
114121
Listener: wgListen,
115122
}, ch, nil
116123
}
@@ -129,11 +136,11 @@ func New(ctx context.Context, logger slog.Logger) (*Tunnel, <-chan error, error)
129136
return NewWithConfig(ctx, logger, cfg)
130137
}
131138

132-
func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-chan struct{}, error) {
139+
func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (ServerResponse, <-chan struct{}, error) {
133140
// Ensure we send the first config before spawning in the background.
134-
_, err := sendConfigToServer(ctx, cfg)
141+
res, err := sendConfigToServer(ctx, cfg)
135142
if err != nil {
136-
return nil, xerrors.Errorf("send config to server: %w", err)
143+
return ServerResponse{}, nil, xerrors.Errorf("send config to server: %w", err)
137144
}
138145

139146
endCh := make(chan struct{})
@@ -156,29 +163,67 @@ func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-
156163
}
157164
}
158165
}()
159-
return endCh, nil
166+
return res, endCh, nil
167+
}
168+
169+
type ServerResponse struct {
170+
Hostname string `json:"hostname"`
171+
ServerIP netip.Addr `json:"server_ip"`
172+
ServerPublicKey string `json:"server_public_key"` // hex
173+
ClientIP netip.Addr `json:"client_ip"`
160174
}
161175

162-
func sendConfigToServer(ctx context.Context, cfg Config) (created bool, err error) {
176+
func sendConfigToServer(ctx context.Context, cfg Config) (ServerResponse, error) {
163177
raw, err := json.Marshal(configExt(cfg))
164178
if err != nil {
165-
return false, xerrors.Errorf("marshal config: %w", err)
179+
return ServerResponse{}, xerrors.Errorf("marshal config: %w", err)
166180
}
167181

168-
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+EndpointHTTPS+"/tun", bytes.NewReader(raw))
169-
if err != nil {
170-
return false, xerrors.Errorf("new request: %w", err)
182+
var req *http.Request
183+
switch cfg.Version {
184+
case 0:
185+
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+v0EndpointHTTPS+"/tun", bytes.NewReader(raw))
186+
if err != nil {
187+
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
188+
}
189+
190+
case 1:
191+
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+cfg.Tunnel.HostnameHTTPS+"/tun", bytes.NewReader(raw))
192+
if err != nil {
193+
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
194+
}
195+
196+
default:
197+
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
171198
}
172199

173200
res, err := http.DefaultClient.Do(req)
174201
if err != nil {
175-
return false, xerrors.Errorf("do request: %w", err)
202+
return ServerResponse{}, xerrors.Errorf("do request: %w", err)
176203
}
204+
defer res.Body.Close()
205+
206+
var resp ServerResponse
207+
switch cfg.Version {
208+
case 0:
209+
_, _ = io.Copy(io.Discard, res.Body)
210+
resp.Hostname = fmt.Sprintf("%s.%s", cfg.ID, v0EndpointHTTPS)
211+
resp.ServerIP = v0ServerIP
212+
resp.ServerPublicKey = encodeBase64ToHex(v0ServerPublicKey)
213+
resp.ClientIP = netip.AddrFrom16(cfg.ID)
214+
215+
case 1:
216+
err := json.NewDecoder(res.Body).Decode(&resp)
217+
if err != nil {
218+
return ServerResponse{}, xerrors.Errorf("decode response: %w", err)
219+
}
177220

178-
_, _ = io.Copy(io.Discard, res.Body)
179-
_ = res.Body.Close()
221+
default:
222+
_, _ = io.Copy(io.Discard, res.Body)
223+
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
224+
}
180225

181-
return res.StatusCode == http.StatusCreated, nil
226+
return resp, nil
182227
}
183228

184229
func cfgPath() (string, error) {
@@ -227,6 +272,15 @@ func readOrGenerateConfig() (Config, error) {
227272
return Config{}, xerrors.Errorf("unmarshal config: %w", err)
228273
}
229274

275+
if cfg.Version == 0 {
276+
cfg.Tunnel = Node{
277+
ID: 0,
278+
HostnameHTTPS: "wg-tunnel.coder.app",
279+
HostnameWireguard: "wg-tunnel-udp.coder.app",
280+
WireguardPort: 55555,
281+
}
282+
}
283+
230284
return cfg, nil
231285
}
232286

@@ -235,25 +289,25 @@ func GenerateConfig() (Config, error) {
235289
if err != nil {
236290
return Config{}, xerrors.Errorf("generate private key: %w", err)
237291
}
238-
239292
pub := priv.PublicKey()
240293

294+
node, err := FindClosestNode()
295+
if err != nil {
296+
region := Regions[0]
297+
n, _ := cryptorand.Intn(len(region.Nodes))
298+
node = region.Nodes[n]
299+
_, _ = fmt.Println("Error picking closest dev tunnel:", err)
300+
_, _ = fmt.Println("Defaulting to", Regions[0].LocationName)
301+
}
302+
241303
return Config{
242-
ID: newUUID(),
304+
Version: 1,
243305
PrivateKey: device.NoisePrivateKey(priv),
244306
PublicKey: device.NoisePublicKey(pub),
307+
Tunnel: node,
245308
}, nil
246309
}
247310

248-
func newUUID() uuid.UUID {
249-
u := uuid.New()
250-
// 0xfc is the IPV6 prefix for internal networks.
251-
u[0] = 0xfc
252-
u[1] = 0xca
253-
254-
return u
255-
}
256-
257311
func writeConfig(cfg Config) error {
258312
cfgFi, err := cfgPath()
259313
if err != nil {

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ require (
6767
github.com/go-chi/chi/v5 v5.0.7
6868
github.com/go-chi/httprate v0.5.3
6969
github.com/go-chi/render v1.0.1
70+
github.com/go-ping/ping v1.1.0
7071
github.com/go-playground/validator/v10 v10.11.0
7172
github.com/gofrs/flock v0.8.1
7273
github.com/gohugoio/hugo v0.101.0

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -689,6 +689,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
689689
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
690690
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
691691
github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
692+
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
693+
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
692694
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
693695
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
694696
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=

0 commit comments

Comments
 (0)