Skip to content

fix: use screen for reconnecting terminal sessions if available #8640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Aug 14, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
0b8499d
Add screen backend for reconnecting ptys
code-asher Jul 18, 2023
9cb043c
Fix leaking goroutine in wait
code-asher Aug 8, 2023
19633c9
Remove connection_id from reconnecting PTY
code-asher Aug 8, 2023
8f4956f
Remove error from close return
code-asher Aug 8, 2023
ee887b9
Refactor reconnecting PTY backends
code-asher Aug 8, 2023
1697070
Merge remote-tracking branch 'github/main' into asher/reconnection-wi…
code-asher Aug 8, 2023
a2149fc
Remove extra mutex unlock
code-asher Aug 8, 2023
e3c808b
Fix heartbeat typo in comment
code-asher Aug 8, 2023
369a36e
Tweak connection close
code-asher Aug 8, 2023
72d405c
Linter fixes
code-asher Aug 8, 2023
e37fc0f
Clear active conns on close
code-asher Aug 9, 2023
61a4253
Avoid useless buffer reset on close
code-asher Aug 9, 2023
c7978db
Move lifecycle after buffer and process are set
code-asher Aug 9, 2023
a083b31
Add info logs for starting, stopping, and attaching
code-asher Aug 9, 2023
bb40f78
Do not hold mutex while waiting for state in screen
code-asher Aug 9, 2023
089e1f9
Remove incorrect statement about closing on Attach
code-asher Aug 9, 2023
ee67045
Remove backend type from SDK/API
code-asher Aug 9, 2023
a6bcdd2
Use PATH to test buffered reconnecting pty
code-asher Aug 9, 2023
3ff1510
Do not hold mutex while waiting for state
code-asher Aug 9, 2023
a781173
Avoid clobbering attach error with close errors
code-asher Aug 10, 2023
7a8ec2e
Immediately read screen process
code-asher Aug 10, 2023
56ca7ac
Fix incorrect logger context on reconnecting PTY
code-asher Aug 10, 2023
9b88a68
Protect map and state with the same mutex
code-asher Aug 10, 2023
56e71c9
Fix incorrect test comment
code-asher Aug 11, 2023
d4170ca
Attempt fixing flake with 'echo test' command
code-asher Aug 11, 2023
34c5c1a
Avoid wait callback when context expires
code-asher Aug 11, 2023
88a6b96
Remove err from wait callback
code-asher Aug 11, 2023
1fd4e9a
Add state wait func where caller holds the lock
code-asher Aug 14, 2023
57f464a
Merge remote-tracking branch 'github/main' into asher/reconnection-wi…
code-asher Aug 14, 2023
968526d
Remove unused fn
code-asher Aug 14, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove backend type from SDK/API
Still need to figure out a way to test both, but we would rather not add
the ability to select the backend through the API.
  • Loading branch information
code-asher committed Aug 10, 2023
commit ee670454caf2159e56c6b5908eadd1f5638263c7
7 changes: 3 additions & 4 deletions agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -1108,7 +1108,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, logger slog.Logger, m
}
c <- rpty // Put it back for the next reconnect.
} else {
logger.Debug(ctx, "creating new reconnecting pty", slog.F("backend", msg.BackendType))
logger.Debug(ctx, "creating new reconnecting pty")

connected := false
defer func() {
Expand All @@ -1126,9 +1126,8 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, logger slog.Logger, m
}

rpty = reconnectingpty.New(ctx, cmd, &reconnectingpty.Options{
BackendType: msg.BackendType,
Timeout: a.reconnectingPTYTimeout,
Metrics: a.metrics.reconnectingPTYErrors,
Timeout: a.reconnectingPTYTimeout,
Metrics: a.metrics.reconnectingPTYErrors,
}, logger.With(slog.F("message_id", msg.ID)))

if err = a.trackConnGoroutine(func() {
Expand Down
17 changes: 7 additions & 10 deletions agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func TestAgent_Stats_ReconnectingPTY(t *testing.T) {
//nolint:dogsled
conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)

ptyConn, err := conn.ReconnectingPTY(ctx, uuid.New(), 128, 128, "bash", codersdk.ReconnectingPTYBackendTypeBuffered)
ptyConn, err := conn.ReconnectingPTY(ctx, uuid.New(), 128, 128, "bash")
require.NoError(t, err)
defer ptyConn.Close()

Expand Down Expand Up @@ -1601,18 +1601,15 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
t.Skip("ConPTY appears to be inconsistent on Windows.")
}

backends := []codersdk.ReconnectingPTYBackendType{
codersdk.ReconnectingPTYBackendTypeBuffered,
codersdk.ReconnectingPTYBackendTypeScreen,
}
backends := []string{"Buffered", "Screen"}

for _, backendType := range backends {
backendType := backendType
t.Run(string(backendType), func(t *testing.T) {
t.Run(backendType, func(t *testing.T) {
t.Parallel()
if runtime.GOOS == "darwin" {
t.Skip("`screen` is flaky on darwin")
} else if backendType == codersdk.ReconnectingPTYBackendTypeScreen {
} else if backendType == "Screen" {
_, err := exec.LookPath("screen")
if err != nil {
t.Skip("`screen` not found")
Expand All @@ -1625,14 +1622,14 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
//nolint:dogsled
conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
id := uuid.New()
netConn1, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash", backendType)
netConn1, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash")
require.NoError(t, err)
defer netConn1.Close()

scanner1 := bufio.NewScanner(netConn1)

// A second simultaneous connection.
netConn2, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash", backendType)
netConn2, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash")
require.NoError(t, err)
defer netConn2.Close()
scanner2 := bufio.NewScanner(netConn2)
Expand Down Expand Up @@ -1683,7 +1680,7 @@ func TestAgent_ReconnectingPTY(t *testing.T) {

_ = netConn1.Close()
_ = netConn2.Close()
netConn3, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash", backendType)
netConn3, err := conn.ReconnectingPTY(ctx, id, 100, 100, "bash")
require.NoError(t, err)
defer netConn3.Close()

Expand Down
21 changes: 8 additions & 13 deletions agent/reconnectingpty/reconnectingpty.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ type Options struct {
Timeout time.Duration
// Metrics tracks various error counters.
Metrics *prometheus.CounterVec
// BackendType indicates which backend to use for reconnections.
BackendType codersdk.ReconnectingPTYBackendType
}

// ReconnectingPTY is a pty that can be reconnected within a timeout and to
Expand Down Expand Up @@ -64,23 +62,20 @@ func New(ctx context.Context, cmd *pty.Cmd, options *Options, logger slog.Logger
}
// Screen seems flaky on Darwin. Locally the tests pass 100% of the time (100
// runs) but in CI screen often incorrectly claims the session name does not
// exist even though screen -list shows it.
if runtime.GOOS == "darwin" {
options.BackendType = codersdk.ReconnectingPTYBackendTypeBuffered
} else if options.BackendType == "" || options.BackendType == codersdk.ReconnectingPTYBackendTypeAuto {
// exist even though screen -list shows it. For now, restrict screen to
// Linux.
backendType := "buffered"
if runtime.GOOS == "linux" {
_, err := exec.LookPath("screen")
if err == nil {
options.BackendType = codersdk.ReconnectingPTYBackendTypeScreen
} else {
options.BackendType = codersdk.ReconnectingPTYBackendTypeBuffered
backendType = "screen"
}
logger.Debug(ctx, "auto backend selection", slog.F("backend", options.BackendType))
}

logger.Info(ctx, "start reconnecting pty")
logger.Info(ctx, "start reconnecting pty", slog.F("backend_type", backendType))

switch options.BackendType {
case codersdk.ReconnectingPTYBackendTypeScreen:
switch backendType {
case "screen":
return newScreen(ctx, cmd, options, logger)
default:
return newBuffered(ctx, cmd, options, logger)
Expand Down
9 changes: 4 additions & 5 deletions cli/exp_scaletest.go
Original file line number Diff line number Diff line change
Expand Up @@ -676,11 +676,10 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd {
config.ReconnectingPTY = &reconnectingpty.Config{
// AgentID is set by the test automatically.
Init: codersdk.WorkspaceAgentReconnectingPTYInit{
ID: uuid.Nil,
Height: 24,
Width: 80,
Command: runCommand,
BackendType: codersdk.ReconnectingPTYBackendTypeBuffered,
ID: uuid.Nil,
Height: 24,
Width: 80,
Command: runCommand,
},
Timeout: httpapi.Duration(runTimeout),
ExpectTimeout: runExpectTimeout,
Expand Down
2 changes: 1 addition & 1 deletion coderd/httpapi/queryparams.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func (p *QueryParamParser) Strings(vals url.Values, def []string, queryParam str
// ValidEnum parses enum query params. Add more to the list as needed.
type ValidEnum interface {
database.ResourceType | database.AuditAction | database.BuildReason | database.UserStatus |
database.WorkspaceStatus | codersdk.ReconnectingPTYBackendType
database.WorkspaceStatus

// Valid is required on the enum type to be used with ParseEnum.
Valid() bool
Expand Down
9 changes: 4 additions & 5 deletions coderd/insights_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -348,11 +348,10 @@ func TestTemplateInsights(t *testing.T) {

// Start an rpty session to generate rpty usage stats.
rpty, err := client.WorkspaceAgentReconnectingPTY(ctx, codersdk.WorkspaceAgentReconnectingPTYOpts{
AgentID: resources[0].Agents[0].ID,
Reconnect: uuid.New(),
Width: 80,
Height: 24,
BackendType: codersdk.ReconnectingPTYBackendTypeBuffered,
AgentID: resources[0].Agents[0].ID,
Reconnect: uuid.New(),
Width: 80,
Height: 24,
})
require.NoError(t, err)
defer rpty.Close()
Expand Down
113 changes: 45 additions & 68 deletions coderd/workspaceapps/apptest/apptest.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"net/http/cookiejar"
"net/http/httputil"
"net/url"
"os/exec"
"path"
"regexp"
"runtime"
Expand Down Expand Up @@ -57,81 +56,59 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) {
t.Skip("ConPTY appears to be inconsistent on Windows.")
}

backends := []codersdk.ReconnectingPTYBackendType{
codersdk.ReconnectingPTYBackendTypeBuffered,
codersdk.ReconnectingPTYBackendTypeScreen,
}

for _, backendType := range backends {
backendType := backendType
t.Run(string(backendType), func(t *testing.T) {
t.Parallel()
if runtime.GOOS == "darwin" {
t.Skip("`screen` is flaky on darwin")
} else if backendType == codersdk.ReconnectingPTYBackendTypeScreen {
_, err := exec.LookPath("screen")
if err != nil {
t.Skip("`screen` not found")
}
}

t.Run("OK", func(t *testing.T) {
t.Parallel()
appDetails := setupProxyTest(t, nil)
t.Run("OK", func(t *testing.T) {
t.Parallel()
appDetails := setupProxyTest(t, nil)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// Run the test against the path app hostname since that's where the
// reconnecting-pty proxy server we want to test is mounted.
client := appDetails.AppClient(t)
testReconnectingPTY(ctx, t, client, codersdk.WorkspaceAgentReconnectingPTYOpts{
AgentID: appDetails.Agent.ID,
Reconnect: uuid.New(),
Height: 80,
Width: 80,
Command: "bash",
BackendType: backendType,
})
})
// Run the test against the path app hostname since that's where the
// reconnecting-pty proxy server we want to test is mounted.
client := appDetails.AppClient(t)
testReconnectingPTY(ctx, t, client, codersdk.WorkspaceAgentReconnectingPTYOpts{
AgentID: appDetails.Agent.ID,
Reconnect: uuid.New(),
Height: 80,
Width: 80,
Command: "bash",
})
})

t.Run("SignedTokenQueryParameter", func(t *testing.T) {
t.Parallel()
if appHostIsPrimary {
t.Skip("Tickets are not used for terminal requests on the primary.")
}
t.Run("SignedTokenQueryParameter", func(t *testing.T) {
t.Parallel()
if appHostIsPrimary {
t.Skip("Tickets are not used for terminal requests on the primary.")
}

appDetails := setupProxyTest(t, nil)
appDetails := setupProxyTest(t, nil)

u := *appDetails.PathAppBaseURL
if u.Scheme == "http" {
u.Scheme = "ws"
} else {
u.Scheme = "wss"
}
u.Path = fmt.Sprintf("/api/v2/workspaceagents/%s/pty", appDetails.Agent.ID.String())
u := *appDetails.PathAppBaseURL
if u.Scheme == "http" {
u.Scheme = "ws"
} else {
u.Scheme = "wss"
}
u.Path = fmt.Sprintf("/api/v2/workspaceagents/%s/pty", appDetails.Agent.ID.String())

ctx := testutil.Context(t, testutil.WaitLong)
issueRes, err := appDetails.SDKClient.IssueReconnectingPTYSignedToken(ctx, codersdk.IssueReconnectingPTYSignedTokenRequest{
URL: u.String(),
AgentID: appDetails.Agent.ID,
})
require.NoError(t, err)
ctx := testutil.Context(t, testutil.WaitLong)
issueRes, err := appDetails.SDKClient.IssueReconnectingPTYSignedToken(ctx, codersdk.IssueReconnectingPTYSignedTokenRequest{
URL: u.String(),
AgentID: appDetails.Agent.ID,
})
require.NoError(t, err)

// Make an unauthenticated client.
unauthedAppClient := codersdk.New(appDetails.AppClient(t).URL)
testReconnectingPTY(ctx, t, unauthedAppClient, codersdk.WorkspaceAgentReconnectingPTYOpts{
AgentID: appDetails.Agent.ID,
Reconnect: uuid.New(),
Height: 80,
Width: 80,
Command: "bash",
SignedToken: issueRes.SignedToken,
BackendType: backendType,
})
})
// Make an unauthenticated client.
unauthedAppClient := codersdk.New(appDetails.AppClient(t).URL)
testReconnectingPTY(ctx, t, unauthedAppClient, codersdk.WorkspaceAgentReconnectingPTYOpts{
AgentID: appDetails.Agent.ID,
Reconnect: uuid.New(),
Height: 80,
Width: 80,
Command: "bash",
SignedToken: issueRes.SignedToken,
})
}
})
})

t.Run("WorkspaceAppsProxyPath", func(t *testing.T) {
Expand Down
3 changes: 1 addition & 2 deletions coderd/workspaceapps/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,7 +632,6 @@ func (s *Server) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
reconnect := parser.Required("reconnect").UUID(values, uuid.New(), "reconnect")
height := parser.UInt(values, 80, "height")
width := parser.UInt(values, 80, "width")
backendType := httpapi.ParseCustom(parser, values, codersdk.ReconnectingPTYBackendTypeAuto, "backend", httpapi.ParseEnum[codersdk.ReconnectingPTYBackendType])
if len(parser.Errors) > 0 {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Invalid query parameters.",
Expand Down Expand Up @@ -671,7 +670,7 @@ func (s *Server) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
}
defer release()
log.Debug(ctx, "dialed workspace agent")
ptNetConn, err := agentConn.ReconnectingPTY(ctx, reconnect, uint16(height), uint16(width), r.URL.Query().Get("command"), backendType)
ptNetConn, err := agentConn.ReconnectingPTY(ctx, reconnect, uint16(height), uint16(width), r.URL.Query().Get("command"))
if err != nil {
log.Debug(ctx, "dial reconnecting pty server in workspace agent", slog.Error(err))
_ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("dial: %s", err))
Expand Down
38 changes: 9 additions & 29 deletions codersdk/workspaceagentconn.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,32 +193,13 @@ func (c *WorkspaceAgentConn) Close() error {
return c.Conn.Close()
}

type ReconnectingPTYBackendType string

const (
ReconnectingPTYBackendTypeAuto ReconnectingPTYBackendType = "auto"
ReconnectingPTYBackendTypeScreen ReconnectingPTYBackendType = "screen"
ReconnectingPTYBackendTypeBuffered ReconnectingPTYBackendType = "buffered"
)

func (s ReconnectingPTYBackendType) Valid() bool {
switch s {
case ReconnectingPTYBackendTypeAuto, ReconnectingPTYBackendTypeBuffered,
ReconnectingPTYBackendTypeScreen:
return true
default:
return false
}
}

// WorkspaceAgentReconnectingPTYInit initializes a new reconnecting PTY session.
// @typescript-ignore WorkspaceAgentReconnectingPTYInit
type WorkspaceAgentReconnectingPTYInit struct {
ID uuid.UUID
Height uint16
Width uint16
Command string
BackendType ReconnectingPTYBackendType
ID uuid.UUID
Height uint16
Width uint16
Command string
}

// ReconnectingPTYRequest is sent from the client to the server
Expand All @@ -233,7 +214,7 @@ type ReconnectingPTYRequest struct {
// ReconnectingPTY spawns a new reconnecting terminal session.
// `ReconnectingPTYRequest` should be JSON marshaled and written to the returned net.Conn.
// Raw terminal output will be read from the returned net.Conn.
func (c *WorkspaceAgentConn) ReconnectingPTY(ctx context.Context, id uuid.UUID, height, width uint16, command string, backendType ReconnectingPTYBackendType) (net.Conn, error) {
func (c *WorkspaceAgentConn) ReconnectingPTY(ctx context.Context, id uuid.UUID, height, width uint16, command string) (net.Conn, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()

Expand All @@ -246,11 +227,10 @@ func (c *WorkspaceAgentConn) ReconnectingPTY(ctx context.Context, id uuid.UUID,
return nil, err
}
data, err := json.Marshal(WorkspaceAgentReconnectingPTYInit{
ID: id,
Height: height,
Width: width,
Command: command,
BackendType: backendType,
ID: id,
Height: height,
Width: width,
Command: command,
})
if err != nil {
_ = conn.Close()
Expand Down
12 changes: 5 additions & 7 deletions codersdk/workspaceagents.go
Original file line number Diff line number Diff line change
Expand Up @@ -552,12 +552,11 @@ func (c *Client) IssueReconnectingPTYSignedToken(ctx context.Context, req IssueR

// @typescript-ignore:WorkspaceAgentReconnectingPTYOpts
type WorkspaceAgentReconnectingPTYOpts struct {
AgentID uuid.UUID
Reconnect uuid.UUID
Width uint16
Height uint16
Command string
BackendType ReconnectingPTYBackendType
AgentID uuid.UUID
Reconnect uuid.UUID
Width uint16
Height uint16
Command string

// SignedToken is an optional signed token from the
// issue-reconnecting-pty-signed-token endpoint. If set, the session token
Expand All @@ -578,7 +577,6 @@ func (c *Client) WorkspaceAgentReconnectingPTY(ctx context.Context, opts Workspa
q.Set("width", strconv.Itoa(int(opts.Width)))
q.Set("height", strconv.Itoa(int(opts.Height)))
q.Set("command", opts.Command)
q.Set("backend", string(opts.BackendType))
// If we're using a signed token, set the query parameter.
if opts.SignedToken != "" {
q.Set(SignedAppTokenQueryParameter, opts.SignedToken)
Expand Down
Loading