Skip to content

Commit d202f20

Browse files
kylecarbskylecarbs
authored
feat: Add TURN proxying to enable offline deployments (#1000)
* Add turnconn * Add option for passing ICE servers * Log TURN remote address * Add TURN server to coder start
1 parent e5a1c30 commit d202f20

25 files changed

+603
-112
lines changed

‎.github/workflows/coder.yaml

-4
Original file line numberDiff line numberDiff line change
@@ -169,10 +169,6 @@ jobs:
169169
terraform_version: 1.1.2
170170
terraform_wrapper: false
171171

172-
- name: Install socat
173-
if: runner.os == 'Linux'
174-
run: sudo apt-get install -y socat
175-
176172
- name: Test with Mock Database
177173
shell: bash
178174
env:

‎agent/agent.go

+20-21
Original file line numberDiff line numberDiff line change
@@ -32,23 +32,23 @@ type Options struct {
3232
Logger slog.Logger
3333
}
3434

35-
type Dialer func(ctx context.Context, options *peer.ConnOptions) (*peerbroker.Listener, error)
35+
type Dialer func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error)
3636

37-
func New(dialer Dialer, options *peer.ConnOptions) io.Closer {
37+
func New(dialer Dialer, logger slog.Logger) io.Closer {
3838
ctx, cancelFunc := context.WithCancel(context.Background())
3939
server := &agent{
40-
clientDialer: dialer,
41-
options: options,
42-
closeCancel: cancelFunc,
43-
closed: make(chan struct{}),
40+
dialer: dialer,
41+
logger: logger,
42+
closeCancel: cancelFunc,
43+
closed: make(chan struct{}),
4444
}
4545
server.init(ctx)
4646
return server
4747
}
4848

4949
type agent struct {
50-
clientDialer Dialer
51-
options *peer.ConnOptions
50+
dialer Dialer
51+
logger slog.Logger
5252

5353
connCloseWait sync.WaitGroup
5454
closeCancel context.CancelFunc
@@ -64,18 +64,18 @@ func (a *agent) run(ctx context.Context) {
6464
// An exponential back-off occurs when the connection is failing to dial.
6565
// This is to prevent server spam in case of a coderd outage.
6666
for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); {
67-
peerListener, err = a.clientDialer(ctx, a.options)
67+
peerListener, err = a.dialer(ctx, a.logger)
6868
if err != nil {
6969
if errors.Is(err, context.Canceled) {
7070
return
7171
}
7272
if a.isClosed() {
7373
return
7474
}
75-
a.options.Logger.Warn(context.Background(), "failed to dial", slog.Error(err))
75+
a.logger.Warn(context.Background(), "failed to dial", slog.Error(err))
7676
continue
7777
}
78-
a.options.Logger.Info(context.Background(), "connected")
78+
a.logger.Info(context.Background(), "connected")
7979
break
8080
}
8181
select {
@@ -90,7 +90,7 @@ func (a *agent) run(ctx context.Context) {
9090
if a.isClosed() {
9191
return
9292
}
93-
a.options.Logger.Debug(ctx, "peer listener accept exited; restarting connection", slog.Error(err))
93+
a.logger.Debug(ctx, "peer listener accept exited; restarting connection", slog.Error(err))
9494
a.run(ctx)
9595
return
9696
}
@@ -105,10 +105,9 @@ func (a *agent) handlePeerConn(ctx context.Context, conn *peer.Conn) {
105105
go func() {
106106
select {
107107
case <-a.closed:
108-
_ = conn.Close()
109108
case <-conn.Closed():
110109
}
111-
<-conn.Closed()
110+
_ = conn.Close()
112111
a.connCloseWait.Done()
113112
}()
114113
for {
@@ -117,15 +116,15 @@ func (a *agent) handlePeerConn(ctx context.Context, conn *peer.Conn) {
117116
if errors.Is(err, peer.ErrClosed) || a.isClosed() {
118117
return
119118
}
120-
a.options.Logger.Debug(ctx, "accept channel from peer connection", slog.Error(err))
119+
a.logger.Debug(ctx, "accept channel from peer connection", slog.Error(err))
121120
return
122121
}
123122

124123
switch channel.Protocol() {
125124
case "ssh":
126125
go a.sshServer.HandleConn(channel.NetConn())
127126
default:
128-
a.options.Logger.Warn(ctx, "unhandled protocol from channel",
127+
a.logger.Warn(ctx, "unhandled protocol from channel",
129128
slog.F("protocol", channel.Protocol()),
130129
slog.F("label", channel.Label()),
131130
)
@@ -145,7 +144,7 @@ func (a *agent) init(ctx context.Context) {
145144
if err != nil {
146145
panic(err)
147146
}
148-
sshLogger := a.options.Logger.Named("ssh-server")
147+
sshLogger := a.logger.Named("ssh-server")
149148
forwardHandler := &ssh.ForwardedTCPHandler{}
150149
a.sshServer = &ssh.Server{
151150
ChannelHandlers: map[string]ssh.ChannelHandler{
@@ -158,7 +157,7 @@ func (a *agent) init(ctx context.Context) {
158157
Handler: func(session ssh.Session) {
159158
err := a.handleSSHSession(session)
160159
if err != nil {
161-
a.options.Logger.Warn(ctx, "ssh session failed", slog.Error(err))
160+
a.logger.Warn(ctx, "ssh session failed", slog.Error(err))
162161
_ = session.Exit(1)
163162
return
164163
}
@@ -194,15 +193,15 @@ func (a *agent) init(ctx context.Context) {
194193
"sftp": func(session ssh.Session) {
195194
server, err := sftp.NewServer(session)
196195
if err != nil {
197-
a.options.Logger.Debug(session.Context(), "initialize sftp server", slog.Error(err))
196+
a.logger.Debug(session.Context(), "initialize sftp server", slog.Error(err))
198197
return
199198
}
200199
defer server.Close()
201200
err = server.Serve()
202201
if errors.Is(err, io.EOF) {
203202
return
204203
}
205-
a.options.Logger.Debug(session.Context(), "sftp server exited with error", slog.Error(err))
204+
a.logger.Debug(session.Context(), "sftp server exited with error", slog.Error(err))
206205
},
207206
},
208207
}
@@ -250,7 +249,7 @@ func (a *agent) handleSSHSession(session ssh.Session) error {
250249
for win := range windowSize {
251250
err = ptty.Resize(uint16(win.Width), uint16(win.Height))
252251
if err != nil {
253-
a.options.Logger.Warn(context.Background(), "failed to resize tty", slog.Error(err))
252+
a.logger.Warn(context.Background(), "failed to resize tty", slog.Error(err))
254253
}
255254
}
256255
}()

‎agent/agent_test.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -170,11 +170,9 @@ func setupSSHSession(t *testing.T) *ssh.Session {
170170

171171
func setupAgent(t *testing.T) *agent.Conn {
172172
client, server := provisionersdk.TransportPipe()
173-
closer := agent.New(func(ctx context.Context, opts *peer.ConnOptions) (*peerbroker.Listener, error) {
174-
return peerbroker.Listen(server, nil, opts)
175-
}, &peer.ConnOptions{
176-
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
177-
})
173+
closer := agent.New(func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) {
174+
return peerbroker.Listen(server, nil)
175+
}, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
178176
t.Cleanup(func() {
179177
_ = client.Close()
180178
_ = server.Close()

‎cli/agent.go

+1-4
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
"github.com/coder/coder/agent"
1717
"github.com/coder/coder/cli/cliflag"
1818
"github.com/coder/coder/codersdk"
19-
"github.com/coder/coder/peer"
2019
"github.com/coder/retry"
2120
)
2221

@@ -110,9 +109,7 @@ func workspaceAgent() *cobra.Command {
110109
return xerrors.Errorf("writing agent session token to config: %w", err)
111110
}
112111

113-
closer := agent.New(client.ListenWorkspaceAgent, &peer.ConnOptions{
114-
Logger: logger,
115-
})
112+
closer := agent.New(client.ListenWorkspaceAgent, logger)
116113
<-cmd.Context().Done()
117114
return closer.Close()
118115
},

‎cli/agent_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func TestWorkspaceAgent(t *testing.T) {
6161
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
6262
resources, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID)
6363
require.NoError(t, err)
64-
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil, nil)
64+
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
6565
require.NoError(t, err)
6666
defer dialer.Close()
6767
_, err = dialer.Ping()
@@ -115,7 +115,7 @@ func TestWorkspaceAgent(t *testing.T) {
115115
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
116116
resources, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID)
117117
require.NoError(t, err)
118-
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil, nil)
118+
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
119119
require.NoError(t, err)
120120
defer dialer.Close()
121121
_, err = dialer.Ping()

‎cli/configssh_test.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"github.com/coder/coder/cli/clitest"
2020
"github.com/coder/coder/coderd/coderdtest"
2121
"github.com/coder/coder/codersdk"
22-
"github.com/coder/coder/peer"
2322
"github.com/coder/coder/provisioner/echo"
2423
"github.com/coder/coder/provisionersdk/proto"
2524
"github.com/coder/coder/pty/ptytest"
@@ -72,17 +71,15 @@ func TestConfigSSH(t *testing.T) {
7271
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
7372
agentClient := codersdk.New(client.URL)
7473
agentClient.SessionToken = authToken
75-
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
76-
Logger: slogtest.Make(t, nil),
77-
})
74+
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil))
7875
t.Cleanup(func() {
7976
_ = agentCloser.Close()
8077
})
8178
tempFile, err := os.CreateTemp(t.TempDir(), "")
8279
require.NoError(t, err)
8380
_ = tempFile.Close()
8481
resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
85-
agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil, nil)
82+
agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil)
8683
require.NoError(t, err)
8784
defer agentConn.Close()
8885

‎cli/gitssh_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ func TestGitSSH(t *testing.T) {
7474
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
7575
resources, err := client.WorkspaceResourcesByBuild(context.Background(), workspace.LatestBuild.ID)
7676
require.NoError(t, err)
77-
dialer, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil, nil)
77+
dialer, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil)
7878
require.NoError(t, err)
7979
defer dialer.Close()
8080
_, err = dialer.Ping()

‎cli/ssh.go

+1-4
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010

1111
"github.com/google/uuid"
1212
"github.com/mattn/go-isatty"
13-
"github.com/pion/webrtc/v3"
1413
"github.com/spf13/cobra"
1514
gossh "golang.org/x/crypto/ssh"
1615
"golang.org/x/term"
@@ -99,9 +98,7 @@ func ssh() *cobra.Command {
9998
return xerrors.Errorf("await agent: %w", err)
10099
}
101100

102-
conn, err := client.DialWorkspaceAgent(cmd.Context(), agent.ID, []webrtc.ICEServer{{
103-
URLs: []string{"stun:stun.l.google.com:19302"},
104-
}}, nil)
101+
conn, err := client.DialWorkspaceAgent(cmd.Context(), agent.ID, nil)
105102
if err != nil {
106103
return err
107104
}

‎cli/ssh_test.go

+2-7
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
"github.com/coder/coder/cli/clitest"
1818
"github.com/coder/coder/coderd/coderdtest"
1919
"github.com/coder/coder/codersdk"
20-
"github.com/coder/coder/peer"
2120
"github.com/coder/coder/provisioner/echo"
2221
"github.com/coder/coder/provisionersdk/proto"
2322
"github.com/coder/coder/pty/ptytest"
@@ -70,9 +69,7 @@ func TestSSH(t *testing.T) {
7069
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
7170
agentClient := codersdk.New(client.URL)
7271
agentClient.SessionToken = agentToken
73-
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
74-
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
75-
})
72+
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
7673
t.Cleanup(func() {
7774
_ = agentCloser.Close()
7875
})
@@ -115,9 +112,7 @@ func TestSSH(t *testing.T) {
115112
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
116113
agentClient := codersdk.New(client.URL)
117114
agentClient.SessionToken = agentToken
118-
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
119-
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
120-
})
115+
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
121116
t.Cleanup(func() {
122117
_ = agentCloser.Close()
123118
})

‎cli/start.go

+15
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818

1919
"github.com/briandowns/spinner"
2020
"github.com/coreos/go-systemd/daemon"
21+
"github.com/pion/turn/v2"
2122
"github.com/spf13/cobra"
2223
"golang.org/x/xerrors"
2324
"google.golang.org/api/idtoken"
@@ -34,6 +35,7 @@ import (
3435
"github.com/coder/coder/coderd/database/databasefake"
3536
"github.com/coder/coder/coderd/devtunnel"
3637
"github.com/coder/coder/coderd/gitsshkey"
38+
"github.com/coder/coder/coderd/turnconn"
3739
"github.com/coder/coder/codersdk"
3840
"github.com/coder/coder/provisioner/terraform"
3941
"github.com/coder/coder/provisionerd"
@@ -56,11 +58,13 @@ func start() *cobra.Command {
5658
tlsEnable bool
5759
tlsKeyFile string
5860
tlsMinVersion string
61+
turnRelayAddress string
5962
skipTunnel bool
6063
traceDatadog bool
6164
secureAuthCookie bool
6265
sshKeygenAlgorithmRaw string
6366
)
67+
6468
root := &cobra.Command{
6569
Use: "start",
6670
RunE: func(cmd *cobra.Command, args []string) error {
@@ -156,6 +160,14 @@ func start() *cobra.Command {
156160
return xerrors.Errorf("parse ssh keygen algorithm %s: %w", sshKeygenAlgorithmRaw, err)
157161
}
158162

163+
turnServer, err := turnconn.New(&turn.RelayAddressGeneratorStatic{
164+
RelayAddress: net.ParseIP(turnRelayAddress),
165+
Address: turnRelayAddress,
166+
})
167+
if err != nil {
168+
return xerrors.Errorf("create turn server: %w", err)
169+
}
170+
159171
options := &coderd.Options{
160172
AccessURL: accessURLParsed,
161173
Logger: logger.Named("coderd"),
@@ -164,6 +176,7 @@ func start() *cobra.Command {
164176
GoogleTokenValidator: validator,
165177
SecureAuthCookie: secureAuthCookie,
166178
SSHKeygenAlgorithm: sshKeygenAlgorithm,
179+
TURNServer: turnServer,
167180
}
168181

169182
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "access-url: %s\n", accessURL)
@@ -376,6 +389,8 @@ func start() *cobra.Command {
376389
cliflag.BoolVarP(root.Flags(), &skipTunnel, "skip-tunnel", "", "CODER_DEV_SKIP_TUNNEL", false, "Skip serving dev mode through an exposed tunnel for simple setup.")
377390
_ = root.Flags().MarkHidden("skip-tunnel")
378391
cliflag.BoolVarP(root.Flags(), &traceDatadog, "trace-datadog", "", "CODER_TRACE_DATADOG", false, "Send tracing data to a datadog agent")
392+
cliflag.StringVarP(root.Flags(), &turnRelayAddress, "turn-relay-address", "", "CODER_TURN_RELAY_ADDRESS", "127.0.0.1",
393+
"Specifies the address to bind TURN connections.")
379394
cliflag.BoolVarP(root.Flags(), &secureAuthCookie, "secure-auth-cookie", "", "CODER_SECURE_AUTH_COOKIE", false, "Specifies if the 'Secure' property is set on browser session cookies")
380395
cliflag.StringVarP(root.Flags(), &sshKeygenAlgorithmRaw, "ssh-keygen-algorithm", "", "CODER_SSH_KEYGEN_ALGORITHM", "ed25519", "Specifies the algorithm to use for generating ssh keys. "+
381396
`Accepted values are "ed25519", "ecdsa", or "rsa4096"`)

0 commit comments

Comments
 (0)