Skip to content

Commit 08f4b19

Browse files
kylecarbskylecarbs
authored
fix: Elongate agent disconnect timeout in tests (coder#2687)
This will fix the flake seen here: https://github.com/coder/coder/runs/7071719863?check_suite_focus=true
1 parent 4a2d299 commit 08f4b19

File tree

5 files changed

+20
-12
lines changed

5 files changed

+20
-12
lines changed

‎coderd/coderd.go

+5
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ type Options struct {
4747
CacheDir string
4848

4949
AgentConnectionUpdateFrequency time.Duration
50+
AgentInactiveDisconnectTimeout time.Duration
5051
// APIRateLimit is the minutely throughput rate limit per user or ip.
5152
// Setting a rate limit <0 will disable the rate limiter across the entire
5253
// app. Specific routes may have their own limiters.
@@ -69,6 +70,10 @@ func New(options *Options) *API {
6970
if options.AgentConnectionUpdateFrequency == 0 {
7071
options.AgentConnectionUpdateFrequency = 3 * time.Second
7172
}
73+
if options.AgentInactiveDisconnectTimeout == 0 {
74+
// Multiply the update by two to allow for some lag-time.
75+
options.AgentInactiveDisconnectTimeout = options.AgentConnectionUpdateFrequency * 2
76+
}
7277
if options.APIRateLimit == 0 {
7378
options.APIRateLimit = 512
7479
}

‎coderd/coderdtest/coderdtest.go

+3
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,9 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API)
152152
// We set the handler after server creation for the access URL.
153153
coderAPI := coderd.New(&coderd.Options{
154154
AgentConnectionUpdateFrequency: 150 * time.Millisecond,
155+
// Force a long disconnection timeout to ensure
156+
// agents are not marked as disconnected during slow tests.
157+
AgentInactiveDisconnectTimeout: 5 * time.Second,
155158
AccessURL: serverURL,
156159
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
157160
Database: db,

‎coderd/provisionerjobs.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request,
258258
}
259259
}
260260

261-
apiAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentConnectionUpdateFrequency)
261+
apiAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
262262
if err != nil {
263263
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
264264
Message: "Internal error reading job agent.",

‎coderd/workspaceagents.go

+10-10
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) {
5050
})
5151
return
5252
}
53-
apiAgent, err := convertWorkspaceAgent(workspaceAgent, convertApps(dbApps), api.AgentConnectionUpdateFrequency)
53+
apiAgent, err := convertWorkspaceAgent(workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
5454
if err != nil {
5555
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
5656
Message: "Internal error reading workspace agent.",
@@ -74,7 +74,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) {
7474
httpapi.ResourceNotFound(rw)
7575
return
7676
}
77-
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentConnectionUpdateFrequency)
77+
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
7878
if err != nil {
7979
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
8080
Message: "Internal error reading workspace agent.",
@@ -121,7 +121,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) {
121121

122122
func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) {
123123
workspaceAgent := httpmw.WorkspaceAgent(r)
124-
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentConnectionUpdateFrequency)
124+
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
125125
if err != nil {
126126
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
127127
Message: "Internal error reading workspace agent.",
@@ -401,7 +401,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
401401
httpapi.ResourceNotFound(rw)
402402
return
403403
}
404-
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentConnectionUpdateFrequency)
404+
apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout)
405405
if err != nil {
406406
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
407407
Message: "Internal error reading workspace agent.",
@@ -689,7 +689,7 @@ func inetToNetaddr(inet pqtype.Inet) netaddr.IPPrefix {
689689
return ipp
690690
}
691691

692-
func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentUpdateFrequency time.Duration) (codersdk.WorkspaceAgent, error) {
692+
func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) {
693693
var envs map[string]string
694694
if dbAgent.EnvironmentVariables.Valid {
695695
err := json.Unmarshal(dbAgent.EnvironmentVariables.RawMessage, &envs)
@@ -734,13 +734,13 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work
734734
// If we've disconnected after our last connection, we know the
735735
// agent is no longer connected.
736736
workspaceAgent.Status = codersdk.WorkspaceAgentDisconnected
737-
case agentUpdateFrequency*2 >= database.Now().Sub(dbAgent.LastConnectedAt.Time):
738-
// The connection updated it's timestamp within the update frequency.
739-
// We multiply by two to allow for some lag.
740-
workspaceAgent.Status = codersdk.WorkspaceAgentConnected
741-
case database.Now().Sub(dbAgent.LastConnectedAt.Time) > agentUpdateFrequency*2:
737+
case database.Now().Sub(dbAgent.LastConnectedAt.Time) > agentInactiveDisconnectTimeout:
742738
// The connection died without updating the last connected.
743739
workspaceAgent.Status = codersdk.WorkspaceAgentDisconnected
740+
case dbAgent.LastConnectedAt.Valid:
741+
// The agent should be assumed connected if it's under inactivity timeouts
742+
// and last connected at has been properly set.
743+
workspaceAgent.Status = codersdk.WorkspaceAgentConnected
744744
}
745745

746746
return workspaceAgent, nil

‎coderd/workspaceresources.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (api *API) workspaceResource(rw http.ResponseWriter, r *http.Request) {
6969
}
7070
}
7171

72-
convertedAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentConnectionUpdateFrequency)
72+
convertedAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout)
7373
if err != nil {
7474
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
7575
Message: "Internal error reading workspace agent.",

0 commit comments

Comments
 (0)