Skip to content

fix: loadtest/reconnectingpty tweak timeout #5300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Dec 6, 2022
Merged

fix: loadtest/reconnectingpty tweak timeout #5300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
44408b3
flaky
mtojek Dec 6, 2022
fb22c21
Merge branch 'main' into flaky-test-1
mtojek Dec 6, 2022
cede0ac
fix: load test increase timeout
mtojek Dec 6, 2022
62ef96f
Remove flaky
mtojek Dec 6, 2022
fb34c4f
Improvement
mtojek Dec 6, 2022
46122f2
only Linux
mtojek Dec 6, 2022
0f8c211
WaitSuperLong
mtojek Dec 6, 2022
3620605
Fix
mtojek Dec 6, 2022
49f035c
Try longer
mtojek Dec 6, 2022
71f6e15
Merge branch 'main' into flaky-test-1
mtojek Dec 6, 2022
fd23ea6
Try: sleep 120
mtojek Dec 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 15 additions & 20 deletions loadtest/reconnectingpty/run_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package reconnectingpty_test
import (
"bytes"
"context"
"runtime"
"testing"
"time"

Expand All @@ -23,9 +22,6 @@ import (

func Test_Runner(t *testing.T) {
t.Parallel()
if runtime.GOOS != "linux" {
t.Skip("PTY is flakey on non-Linux platforms")
}

t.Run("OK", func(t *testing.T) {
t.Parallel()
Expand All @@ -40,7 +36,7 @@ func Test_Runner(t *testing.T) {
LogOutput: true,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -50,7 +46,9 @@ func Test_Runner(t *testing.T) {
require.NoError(t, err)

require.Contains(t, logStr, "Output:")
require.Contains(t, logStr, "\thello world")
// OSX: Output:\n\thello world\n
// Win: Output:\n\t\x1b[2J\x1b[m\x1b[H\x1b]0;Administrator: C:\\Program Files\\PowerShell\\7\\pwsh.exe\a\x1b[?25hhello world\n
require.Contains(t, logStr, "hello world\n")
})

t.Run("NoLogOutput", func(t *testing.T) {
Expand All @@ -66,7 +64,7 @@ func Test_Runner(t *testing.T) {
LogOutput: false,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -76,12 +74,10 @@ func Test_Runner(t *testing.T) {
require.NoError(t, err)

require.NotContains(t, logStr, "Output:")
require.NotContains(t, logStr, "\thello world")
})

t.Run("Timeout", func(t *testing.T) {
t.Parallel()
t.Skip("Flaky: https://github.com/coder/coder/issues/5187")

t.Run("NoTimeout", func(t *testing.T) {
t.Parallel()
Expand All @@ -93,11 +89,11 @@ func Test_Runner(t *testing.T) {
Init: codersdk.ReconnectingPTYInit{
Command: "echo 'hello world'",
},
Timeout: httpapi.Duration(5 * time.Second),
Timeout: httpapi.Duration(2 * testutil.WaitSuperLong),
LogOutput: true,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -115,13 +111,13 @@ func Test_Runner(t *testing.T) {
runner := reconnectingpty.NewRunner(client, reconnectingpty.Config{
AgentID: agentID,
Init: codersdk.ReconnectingPTYInit{
Command: "sleep 5",
Command: "sleep 120",
},
Timeout: httpapi.Duration(2 * time.Second),
LogOutput: true,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -144,14 +140,14 @@ func Test_Runner(t *testing.T) {
runner := reconnectingpty.NewRunner(client, reconnectingpty.Config{
AgentID: agentID,
Init: codersdk.ReconnectingPTYInit{
Command: "sleep 5",
Command: "sleep 120",
},
Timeout: httpapi.Duration(2 * time.Second),
ExpectTimeout: true,
LogOutput: true,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -171,12 +167,12 @@ func Test_Runner(t *testing.T) {
Init: codersdk.ReconnectingPTYInit{
Command: "echo 'hello world'",
},
Timeout: httpapi.Duration(5 * time.Second),
Timeout: httpapi.Duration(2 * testutil.WaitSuperLong),
ExpectTimeout: true,
LogOutput: true,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -190,7 +186,6 @@ func Test_Runner(t *testing.T) {

t.Run("ExpectOutput", func(t *testing.T) {
t.Parallel()
t.Skip("Flaky: https://github.com/coder/coder/issues/5187")

t.Run("Matches", func(t *testing.T) {
t.Parallel()
Expand All @@ -206,7 +201,7 @@ func Test_Runner(t *testing.T) {
LogOutput: false,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand All @@ -230,7 +225,7 @@ func Test_Runner(t *testing.T) {
LogOutput: false,
})

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong)
defer cancel()

logs := bytes.NewBuffer(nil)
Expand Down
7 changes: 4 additions & 3 deletions testutil/duration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import "time"
// Constants for timing out operations, usable for creating contexts
// that timeout or in require.Eventually.
const (
WaitShort = 5 * time.Second
WaitMedium = 10 * time.Second
WaitLong = 15 * time.Second
WaitShort = 5 * time.Second
WaitMedium = 10 * time.Second
WaitLong = 15 * time.Second
WaitSuperLong = 60 * time.Second
)

// Constants for delaying repeated operations, e.g. in
Expand Down
7 changes: 4 additions & 3 deletions testutil/duration_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import "time"
//
// Windows durations are adjusted for slow CI workers.
const (
WaitShort = 10 * time.Second
WaitMedium = 20 * time.Second
WaitLong = 30 * time.Second
WaitShort = 10 * time.Second
WaitMedium = 20 * time.Second
WaitLong = 30 * time.Second
WaitSuperLong = 60 * time.Second
)

// Constants for delaying repeated operations, e.g. in
Expand Down