diff --git a/agent/agent.go b/agent/agent.go
index 6da2db0098bf7..fda2958ab2db1 100644
--- a/agent/agent.go
+++ b/agent/agent.go
@@ -268,10 +268,13 @@ func (a *agent) run(ctx context.Context) error {
 
 		scriptDone := make(chan error, 1)
 		scriptStart := time.Now()
-		go func() {
+		err := a.trackConnGoroutine(func() {
 			defer close(scriptDone)
 			scriptDone <- a.runStartupScript(ctx, metadata.StartupScript)
-		}()
+		})
+		if err != nil {
+			return xerrors.Errorf("track startup script: %w", err)
+		}
 		go func() {
 			var timeout <-chan time.Time
 			// If timeout is zero, an older version of the coder
diff --git a/agent/agent_test.go b/agent/agent_test.go
index 3379c2dde250c..22290ba63b79f 100644
--- a/agent/agent_test.go
+++ b/agent/agent_test.go
@@ -305,7 +305,7 @@ func TestAgent_TCPLocalForwarding(t *testing.T) {
 		}
 	}()
 
-	cmd := setupSSHCommand(t, []string{"-L", fmt.Sprintf("%d:127.0.0.1:%d", randomPort, remotePort)}, []string{"sleep", "10"})
+	cmd := setupSSHCommand(t, []string{"-L", fmt.Sprintf("%d:127.0.0.1:%d", randomPort, remotePort)}, []string{"sleep", "5"})
 	err = cmd.Start()
 	require.NoError(t, err)
 
@@ -372,7 +372,7 @@ func TestAgent_TCPRemoteForwarding(t *testing.T) {
 		}
 	}()
 
-	cmd := setupSSHCommand(t, []string{"-R", fmt.Sprintf("127.0.0.1:%d:127.0.0.1:%d", randomPort, localPort)}, []string{"sleep", "10"})
+	cmd := setupSSHCommand(t, []string{"-R", fmt.Sprintf("127.0.0.1:%d:127.0.0.1:%d", randomPort, localPort)}, []string{"sleep", "5"})
 	err = cmd.Start()
 	require.NoError(t, err)
 
@@ -437,7 +437,7 @@ func TestAgent_UnixLocalForwarding(t *testing.T) {
 		}
 	}()
 
-	cmd := setupSSHCommand(t, []string{"-L", fmt.Sprintf("%s:%s", localSocketPath, remoteSocketPath)}, []string{"sleep", "10"})
+	cmd := setupSSHCommand(t, []string{"-L", fmt.Sprintf("%s:%s", localSocketPath, remoteSocketPath)}, []string{"sleep", "5"})
 	err = cmd.Start()
 	require.NoError(t, err)
 
@@ -495,7 +495,7 @@ func TestAgent_UnixRemoteForwarding(t *testing.T) {
 		}
 	}()
 
-	cmd := setupSSHCommand(t, []string{"-R", fmt.Sprintf("%s:%s", remoteSocketPath, localSocketPath)}, []string{"sleep", "10"})
+	cmd := setupSSHCommand(t, []string{"-R", fmt.Sprintf("%s:%s", remoteSocketPath, localSocketPath)}, []string{"sleep", "5"})
 	err = cmd.Start()
 	require.NoError(t, err)
 
@@ -703,7 +703,7 @@ func TestAgent_Lifecycle(t *testing.T) {
 		t.Parallel()
 
 		_, client, _, _ := setupAgent(t, agentsdk.Metadata{
-			StartupScript:        "sleep 10",
+			StartupScript:        "sleep 5",
 			StartupScriptTimeout: time.Nanosecond,
 		}, 0)
 
diff --git a/cli/ssh.go b/cli/ssh.go
index ed8baa7fc1152..5adeba63bbae6 100644
--- a/cli/ssh.go
+++ b/cli/ssh.go
@@ -458,7 +458,7 @@ func uploadGPGKeys(ctx context.Context, sshClient *gossh.Client) error {
 	//
 	// Note: we sleep after killing the agent because it doesn't always die
 	//       immediately.
-	agentSocketBytes, err := runRemoteSSH(sshClient, nil, `
+	agentSocketBytes, err := runRemoteSSH(sshClient, nil, `sh -c '
 set -eux
 agent_socket=$(gpgconf --list-dir agent-socket)
 echo "$agent_socket"
@@ -470,7 +470,7 @@ if [ -S "$agent_socket" ]; then
 fi
 
 test ! -S "$agent_socket"
-`)
+'`)
 	agentSocket := strings.TrimSpace(string(agentSocketBytes))
 	if err != nil {
 		return xerrors.Errorf("check if agent socket is running (check if %q exists): %w", agentSocket, err)