From 44408b33f911330938be40470a9baa58f1bd7614 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 08:08:55 +0100 Subject: [PATCH 1/9] flaky --- flaky | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 flaky diff --git a/flaky b/flaky new file mode 100644 index 0000000000000..e69de29bb2d1d From cede0acf9d8d721e252f4efaf1f6595617bf1951 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 09:16:45 +0100 Subject: [PATCH 2/9] fix: load test increase timeout --- loadtest/reconnectingpty/run_test.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/loadtest/reconnectingpty/run_test.go b/loadtest/reconnectingpty/run_test.go index a6fe42847dc18..ef249eb9784da 100644 --- a/loadtest/reconnectingpty/run_test.go +++ b/loadtest/reconnectingpty/run_test.go @@ -3,7 +3,6 @@ package reconnectingpty_test import ( "bytes" "context" - "runtime" "testing" "time" @@ -23,9 +22,6 @@ import ( func Test_Runner(t *testing.T) { t.Parallel() - if runtime.GOOS != "linux" { - t.Skip("PTY is flakey on non-Linux platforms") - } t.Run("OK", func(t *testing.T) { t.Parallel() @@ -81,7 +77,6 @@ func Test_Runner(t *testing.T) { t.Run("Timeout", func(t *testing.T) { t.Parallel() - t.Skip("Flaky: https://github.com/coder/coder/issues/5187") t.Run("NoTimeout", func(t *testing.T) { t.Parallel() @@ -97,7 +92,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -121,7 +116,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -151,7 +146,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -176,7 +171,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -190,7 +185,6 @@ func Test_Runner(t *testing.T) { t.Run("ExpectOutput", func(t *testing.T) { t.Parallel() - t.Skip("Flaky: https://github.com/coder/coder/issues/5187") t.Run("Matches", func(t *testing.T) { t.Parallel() @@ -206,7 +200,7 @@ func Test_Runner(t *testing.T) { LogOutput: false, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -230,7 +224,7 @@ func Test_Runner(t *testing.T) { LogOutput: false, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() logs := bytes.NewBuffer(nil) From 62ef96f7da1fe65e16e3b0dc2c94bb7fcdce4e77 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 09:17:01 +0100 Subject: [PATCH 3/9] Remove flaky --- flaky | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 flaky diff --git a/flaky b/flaky deleted file mode 100644 index e69de29bb2d1d..0000000000000 From fb34c4fd641edec2a8b3ff98941fee4d0a3beafd Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 10:09:34 +0100 Subject: [PATCH 4/9] Improvement --- loadtest/reconnectingpty/run_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/loadtest/reconnectingpty/run_test.go b/loadtest/reconnectingpty/run_test.go index ef249eb9784da..4127a9e141ae3 100644 --- a/loadtest/reconnectingpty/run_test.go +++ b/loadtest/reconnectingpty/run_test.go @@ -46,7 +46,9 @@ func Test_Runner(t *testing.T) { require.NoError(t, err) require.Contains(t, logStr, "Output:") - require.Contains(t, logStr, "\thello world") + // OSX: Output:\n\thello world\n + // Win: Output:\n\t\x1b[2J\x1b[m\x1b[H\x1b]0;Administrator: C:\\Program Files\\PowerShell\\7\\pwsh.exe\a\x1b[?25hhello world\n + require.Contains(t, logStr, "hello world\n") }) t.Run("NoLogOutput", func(t *testing.T) { @@ -72,7 +74,6 @@ func Test_Runner(t *testing.T) { require.NoError(t, err) require.NotContains(t, logStr, "Output:") - require.NotContains(t, logStr, "\thello world") }) t.Run("Timeout", func(t *testing.T) { @@ -88,7 +89,7 @@ func Test_Runner(t *testing.T) { Init: codersdk.ReconnectingPTYInit{ Command: "echo 'hello world'", }, - Timeout: httpapi.Duration(5 * time.Second), + Timeout: httpapi.Duration(2 * testutil.WaitLong), LogOutput: true, }) @@ -166,7 +167,7 @@ func Test_Runner(t *testing.T) { Init: codersdk.ReconnectingPTYInit{ Command: "echo 'hello world'", }, - Timeout: httpapi.Duration(5 * time.Second), + Timeout: httpapi.Duration(2 * testutil.WaitLong), ExpectTimeout: true, LogOutput: true, }) From 46122f2ea63f33dc73d61b39b0e42386826f7abf Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 11:41:34 +0100 Subject: [PATCH 5/9] only Linux --- loadtest/reconnectingpty/run_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/loadtest/reconnectingpty/run_test.go b/loadtest/reconnectingpty/run_test.go index 4127a9e141ae3..6a9bd7abb04d4 100644 --- a/loadtest/reconnectingpty/run_test.go +++ b/loadtest/reconnectingpty/run_test.go @@ -3,6 +3,7 @@ package reconnectingpty_test import ( "bytes" "context" + "runtime" "testing" "time" @@ -22,6 +23,9 @@ import ( func Test_Runner(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("PTY is flakey on non-Linux platforms") + } t.Run("OK", func(t *testing.T) { t.Parallel() From 0f8c2119ba3f246984be4709182b62599b4cc7d9 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 11:58:51 +0100 Subject: [PATCH 6/9] WaitSuperLong --- loadtest/reconnectingpty/run_test.go | 24 ++++++++++-------------- testutil/duration.go | 7 ++++--- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/loadtest/reconnectingpty/run_test.go b/loadtest/reconnectingpty/run_test.go index 6a9bd7abb04d4..c7788185c6e91 100644 --- a/loadtest/reconnectingpty/run_test.go +++ b/loadtest/reconnectingpty/run_test.go @@ -3,7 +3,6 @@ package reconnectingpty_test import ( "bytes" "context" - "runtime" "testing" "time" @@ -23,9 +22,6 @@ import ( func Test_Runner(t *testing.T) { t.Parallel() - if runtime.GOOS != "linux" { - t.Skip("PTY is flakey on non-Linux platforms") - } t.Run("OK", func(t *testing.T) { t.Parallel() @@ -40,7 +36,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -68,7 +64,7 @@ func Test_Runner(t *testing.T) { LogOutput: false, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -93,11 +89,11 @@ func Test_Runner(t *testing.T) { Init: codersdk.ReconnectingPTYInit{ Command: "echo 'hello world'", }, - Timeout: httpapi.Duration(2 * testutil.WaitLong), + Timeout: httpapi.Duration(2 * testutil.WaitSuperLong), LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -121,7 +117,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -151,7 +147,7 @@ func Test_Runner(t *testing.T) { LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -171,12 +167,12 @@ func Test_Runner(t *testing.T) { Init: codersdk.ReconnectingPTYInit{ Command: "echo 'hello world'", }, - Timeout: httpapi.Duration(2 * testutil.WaitLong), + Timeout: httpapi.Duration(2 * testutil.WaitSuperLong), ExpectTimeout: true, LogOutput: true, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -205,7 +201,7 @@ func Test_Runner(t *testing.T) { LogOutput: false, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) @@ -229,7 +225,7 @@ func Test_Runner(t *testing.T) { LogOutput: false, }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitSuperLong) defer cancel() logs := bytes.NewBuffer(nil) diff --git a/testutil/duration.go b/testutil/duration.go index 8e3210f155178..3c87a2527a823 100644 --- a/testutil/duration.go +++ b/testutil/duration.go @@ -7,9 +7,10 @@ import "time" // Constants for timing out operations, usable for creating contexts // that timeout or in require.Eventually. const ( - WaitShort = 5 * time.Second - WaitMedium = 10 * time.Second - WaitLong = 15 * time.Second + WaitShort = 5 * time.Second + WaitMedium = 10 * time.Second + WaitLong = 15 * time.Second + WaitSuperLong = 30 * time.Second ) // Constants for delaying repeated operations, e.g. in From 362060577bc584a60bc9409eb4a7197a8616a2ff Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 12:06:45 +0100 Subject: [PATCH 7/9] Fix --- testutil/duration_windows.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/testutil/duration_windows.go b/testutil/duration_windows.go index 5707824800bbf..1037858a5cf95 100644 --- a/testutil/duration_windows.go +++ b/testutil/duration_windows.go @@ -7,9 +7,10 @@ import "time" // // Windows durations are adjusted for slow CI workers. const ( - WaitShort = 10 * time.Second - WaitMedium = 20 * time.Second - WaitLong = 30 * time.Second + WaitShort = 10 * time.Second + WaitMedium = 20 * time.Second + WaitLong = 30 * time.Second + WaitSuperLong = 60 * time.Second ) // Constants for delaying repeated operations, e.g. in From 49f035c0ecb4964536d96a98297f6f251eda71c6 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 12:17:06 +0100 Subject: [PATCH 8/9] Try longer --- testutil/duration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testutil/duration.go b/testutil/duration.go index 3c87a2527a823..5480e5f788528 100644 --- a/testutil/duration.go +++ b/testutil/duration.go @@ -10,7 +10,7 @@ const ( WaitShort = 5 * time.Second WaitMedium = 10 * time.Second WaitLong = 15 * time.Second - WaitSuperLong = 30 * time.Second + WaitSuperLong = 60 * time.Second ) // Constants for delaying repeated operations, e.g. in From fd23ea61bfd97c517a913b1075ed456ea25f5130 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 6 Dec 2022 13:44:20 +0100 Subject: [PATCH 9/9] Try: sleep 120 --- loadtest/reconnectingpty/run_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loadtest/reconnectingpty/run_test.go b/loadtest/reconnectingpty/run_test.go index c7788185c6e91..6d78f72e25823 100644 --- a/loadtest/reconnectingpty/run_test.go +++ b/loadtest/reconnectingpty/run_test.go @@ -111,7 +111,7 @@ func Test_Runner(t *testing.T) { runner := reconnectingpty.NewRunner(client, reconnectingpty.Config{ AgentID: agentID, Init: codersdk.ReconnectingPTYInit{ - Command: "sleep 5", + Command: "sleep 120", }, Timeout: httpapi.Duration(2 * time.Second), LogOutput: true, @@ -140,7 +140,7 @@ func Test_Runner(t *testing.T) { runner := reconnectingpty.NewRunner(client, reconnectingpty.Config{ AgentID: agentID, Init: codersdk.ReconnectingPTYInit{ - Command: "sleep 5", + Command: "sleep 120", }, Timeout: httpapi.Duration(2 * time.Second), ExpectTimeout: true,