feat: add load testing harness

coder · deansheather · Nov 2, 2022 · Nov 1, 2022 · Nov 1, 2022 · Nov 2, 2022
commit 02ec880bb6fb10663d19052e38f9dab757fb428b
diff --git a/loadtest/harness/harness.go b/loadtest/harness/harness.go
@@ -0,0 +1,101 @@
+package harness
+
+import (
+	"context"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/xerrors"
+)
+
+// ExecutionStrategy defines how a TestHarness should execute a set of runs. It
+// essentially defines the concurrency model for a given testing session.
+type ExecutionStrategy interface {
+	// Execute runs the given runs in whatever way the strategy wants. An error
+	// may only be returned if the strategy has a failure itself, not if any of
+	// the runs fail.
+	Execute(ctx context.Context, runs []*TestRun) error
+}
+
+// TestHarness runs a bunch of registered test runs using the given
+// ExecutionStrategy.
+type TestHarness struct {
+	execStrat ExecutionStrategy
+
+	mut     *sync.Mutex
+	runIDs  map[string]struct{}
+	runs    []*TestRun
+	started bool
+	done    chan struct{}
+}
+
+// NewTestHarness creates a new TestHarness with the given ExecutionStrategy.
+func NewTestHarness(execStrat ExecutionStrategy) *TestHarness {
+	return &TestHarness{
+		execStrat: execStrat,
+		mut:       new(sync.Mutex),
+		runIDs:    map[string]struct{}{},
+		runs:      []*TestRun{},
+		done:      make(chan struct{}),
+	}
+}
+
+// Run runs the registered tests using the given ExecutionStrategy. The provided
+// context can be used to cancel or set a deadline for the test run. Blocks
+// until the tests have finished and returns the test execution error (not
+// individual run errors).
+//
+// Panics if called more than once.
+func (h *TestHarness) Run(ctx context.Context) (err error) {
+	h.mut.Lock()
+	if h.started {
+		h.mut.Unlock()
+		panic("harness is already started")
+	}
+	h.started = true
+	h.mut.Unlock()
+
+	defer close(h.done)
+	defer func() {
+		e := recover()
+		if e != nil {
+			err = xerrors.Errorf("execution strategy panicked: %w", e)
+		}
+	}()
+
+	err = h.execStrat.Execute(ctx, h.runs)
+	//nolint:revive // we use named returns because we mutate it in a defer
+	return
+}
+
+// Cleanup should be called after the test run has finished and results have
+// been collected.
+func (h *TestHarness) Cleanup(ctx context.Context) (err error) {
+	h.mut.Lock()
+	defer h.mut.Unlock()
+	if !h.started {
+		panic("harness has not started")
+	}
+	select {
+	case <-h.done:
+	default:
+		panic("harness has not finished")
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			err = multierror.Append(err, xerrors.Errorf("panic in cleanup: %w", e))
+		}
+	}()
+
+	for _, run := range h.runs {
+		e := run.Cleanup(ctx)
+		if e != nil {
+			err = multierror.Append(err, xerrors.Errorf("cleanup for %s failed: %w", run.FullID(), e))
+		}
+	}
+
+	//nolint:revive // we use named returns because we mutate it in a defer
+	return
+}
diff --git a/loadtest/harness/harness_test.go b/loadtest/harness/harness_test.go
@@ -0,0 +1,262 @@
+package harness_test
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/loadtest/harness"
+)
+
+const testPanicMessage = "expected test panic"
+
+type panickingExecutionStrategy struct{}
+
+var _ harness.ExecutionStrategy = panickingExecutionStrategy{}
+
+func (panickingExecutionStrategy) Execute(_ context.Context, _ []*harness.TestRun) error {
+	panic(testPanicMessage)
+}
+
+type erroringExecutionStrategy struct {
+	err error
+}
+
+var _ harness.ExecutionStrategy = erroringExecutionStrategy{}
+
+func (e erroringExecutionStrategy) Execute(_ context.Context, _ []*harness.TestRun) error {
+	return e.err
+}
+
+func Test_TestHarness(t *testing.T) {
+	t.Parallel()
+
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		expectedErr := xerrors.New("expected error")
+
+		h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+		r1 := h.AddRun("test", "1", fakeTestFns(nil, nil))
+		r2 := h.AddRun("test", "2", fakeTestFns(expectedErr, nil))
+
+		err := h.Run(context.Background())
+		require.NoError(t, err)
+
+		res := h.Results()
+		require.Equal(t, 2, res.TotalRuns)
+		require.Equal(t, 1, res.TotalPass)
+		require.Equal(t, 1, res.TotalFail)
+		require.Equal(t, map[string]harness.RunResult{
+			r1.FullID(): r1.Result(),
+			r2.FullID(): r2.Result(),
+		}, res.Runs)
+
+		err = h.Cleanup(context.Background())
+		require.NoError(t, err)
+	})
+
+	t.Run("CatchesExecutionError", func(t *testing.T) {
+		t.Parallel()
+
+		expectedErr := xerrors.New("expected error")
+
+		h := harness.NewTestHarness(erroringExecutionStrategy{err: expectedErr})
+		_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
+
+		err := h.Run(context.Background())
+		require.Error(t, err)
+		require.ErrorIs(t, err, expectedErr)
+	})
+
+	t.Run("CatchesExecutionPanic", func(t *testing.T) {
+		t.Parallel()
+
+		h := harness.NewTestHarness(panickingExecutionStrategy{})
+		_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
+
+		err := h.Run(context.Background())
+		require.Error(t, err)
+		require.ErrorContains(t, err, "panic")
+		require.ErrorContains(t, err, testPanicMessage)
+	})
+
+	t.Run("Cleanup", func(t *testing.T) {
+		t.Parallel()
+
+		t.Run("Error", func(t *testing.T) {
+			t.Parallel()
+
+			expectedErr := xerrors.New("expected error")
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			_ = h.AddRun("test", "1", fakeTestFns(nil, expectedErr))
+
+			err := h.Run(context.Background())
+			require.NoError(t, err)
+
+			err = h.Cleanup(context.Background())
+			require.Error(t, err)
+			require.ErrorContains(t, err, expectedErr.Error())
+		})
+
+		t.Run("Panic", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			_ = h.AddRun("test", "1", testFns{
+				RunFn: func(_ context.Context, _ string, _ io.Writer) error {
+					return nil
+				},
+				CleanupFn: func(_ context.Context, _ string) error {
+					panic(testPanicMessage)
+				},
+			})
+
+			err := h.Run(context.Background())
+			require.NoError(t, err)
+
+			err = h.Cleanup(context.Background())
+			require.Error(t, err)
+			require.ErrorContains(t, err, "panic")
+			require.ErrorContains(t, err, testPanicMessage)
+		})
+	})
+
+	t.Run("Panics", func(t *testing.T) {
+		t.Parallel()
+
+		t.Run("RegisterAfterStart", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			_ = h.Run(context.Background())
+
+			require.Panics(t, func() {
+				_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
+			})
+		})
+
+		t.Run("DuplicateTestID", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+
+			name, id := "test", "1"
+			_ = h.AddRun(name, id, fakeTestFns(nil, nil))
+
+			require.Panics(t, func() {
+				_ = h.AddRun(name, id, fakeTestFns(nil, nil))
+			})
+		})
+
+		t.Run("StartedTwice", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			h.Run(context.Background())
+
+			require.Panics(t, func() {
+				h.Run(context.Background())
+			})
+		})
+
+		t.Run("ResultsBeforeStart", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+
+			require.Panics(t, func() {
+				h.Results()
+			})
+		})
+
+		t.Run("ResultsBeforeFinish", func(t *testing.T) {
+			t.Parallel()
+
+			var (
+				endRun     = make(chan struct{})
+				testsEnded = make(chan struct{})
+			)
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			_ = h.AddRun("test", "1", testFns{
+				RunFn: func(_ context.Context, _ string, _ io.Writer) error {
+					<-endRun
+					return nil
+				},
+			})
+			go func() {
+				defer close(testsEnded)
+				err := h.Run(context.Background())
+				assert.NoError(t, err)
+			}()
+
+			time.Sleep(100 * time.Millisecond)
+			require.Panics(t, func() {
+				h.Results()
+			})
+
+			close(endRun)
+			<-testsEnded
+			_ = h.Results()
+		})
+
+		t.Run("CleanupBeforeStart", func(t *testing.T) {
+			t.Parallel()
+
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+
+			require.Panics(t, func() {
+				h.Cleanup(context.Background())
+			})
+		})
+
+		t.Run("ClenaupBeforeFinish", func(t *testing.T) {
+			t.Parallel()
+
+			var (
+				endRun     = make(chan struct{})
+				testsEnded = make(chan struct{})
+			)
+			h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
+			_ = h.AddRun("test", "1", testFns{
+				RunFn: func(_ context.Context, _ string, _ io.Writer) error {
+					<-endRun
+					return nil
+				},
+			})
+			go func() {
+				defer close(testsEnded)
+				err := h.Run(context.Background())
+				assert.NoError(t, err)
+			}()
+
+			time.Sleep(100 * time.Millisecond)
+			require.Panics(t, func() {
+				h.Cleanup(context.Background())
+			})
+
+			close(endRun)
+			<-testsEnded
+
+			err := h.Cleanup(context.Background())
+			require.NoError(t, err)
+		})
+	})
+}
+
+func fakeTestFns(err, cleanupErr error) testFns {
+	return testFns{
+		RunFn: func(_ context.Context, _ string, _ io.Writer) error {
+			return err
+		},
+		CleanupFn: func(_ context.Context, _ string) error {
+			return cleanupErr
+		},
+	}
+}