From 2494de00799dec063952bcdec7569cd513b2bb4b Mon Sep 17 00:00:00 2001 From: Charlie Moog Date: Wed, 28 Oct 2020 01:08:59 -0500 Subject: [PATCH 1/2] Add env create|edit integration tests --- ci/integration/envs_test.go | 100 +++++++++++++++++++++-------- ci/integration/integration_test.go | 2 +- go.mod | 1 + 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/ci/integration/envs_test.go b/ci/integration/envs_test.go index a4f75480..af710fad 100644 --- a/ci/integration/envs_test.go +++ b/ci/integration/envs_test.go @@ -2,28 +2,23 @@ package integration import ( "context" + "fmt" + "math" "regexp" "testing" "cdr.dev/coder-cli/ci/tcli" + "cdr.dev/coder-cli/coder-sdk" + "cdr.dev/slog/sloggers/slogtest/assert" + "github.com/google/go-cmp/cmp" ) -// From Coder organization images -// const ubuntuImgID = "5f443b16-30652892427b955601330fa5" - func TestEnvsCLI(t *testing.T) { t.Parallel() run(t, "coder-cli-env-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { headlessLogin(ctx, t, c) - // Ensure binary is present. - c.Run(ctx, "which coder").Assert(t, - tcli.Success(), - tcli.StdoutMatches("/usr/sbin/coder"), - tcli.StderrEmpty(), - ) - // Minimum args not received. c.Run(ctx, "coder envs create").Assert(t, tcli.StderrMatches(regexp.QuoteMeta("accepts 1 arg(s), received 0")), @@ -49,21 +44,74 @@ func TestEnvsCLI(t *testing.T) { tcli.Error(), ) - // TODO(Faris) : uncomment this when we can safely purge the environments - // the integrations tests would create in the sidecar - // Successfully create environment. - // c.Run(ctx, "coder envs create --image "+ubuntuImgID+" test-ubuntu").Assert(t, - // tcli.Success(), - // // why does flog.Success write to stderr? - // tcli.StderrMatches(regexp.QuoteMeta("Successfully created environment \"test-ubuntu\"")), - // ) - - // TODO(Faris) : uncomment this when we can safely purge the environments - // the integrations tests would create in the sidecar - // Successfully provision environment with fractional resource amounts - // c.Run(ctx, fmt.Sprintf(`coder envs create -i %s -c 1.2 -m 1.4 non-whole-resource-amounts`, ubuntuImgID)).Assert(t, - // tcli.Success(), - // tcli.StderrMatches(regexp.QuoteMeta("Successfully created environment \"non-whole-resource-amounts\"")), - // ) + name := randString(10) + cpu := 2.3 + c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --cpu %f", name, cpu)).Assert(t, + tcli.Success(), + ) + + t.Cleanup(func() { + run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { + headlessLogin(ctx, t, c) + c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t) + }) + }) + + c.Run(ctx, "coder envs ls").Assert(t, + tcli.Success(), + tcli.StdoutMatches(regexp.QuoteMeta(name)), + ) + + var env coder.Environment + c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t, + tcli.Success(), + tcli.StdoutJSONUnmarshal(&env), + ) + assert.Equal(t, "environment cpu was correctly set", cpu, float64(env.CPUCores), floatComparer) + + c.Run(ctx, fmt.Sprintf("coder envs watch-build %s", name)).Assert(t, + tcli.Success(), + ) + + c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t, + tcli.Success(), + ) + }) + + run(t, "coder-cli-env-edit-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { + headlessLogin(ctx, t, c) + + name := randString(10) + c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --follow", name)).Assert(t, + tcli.Success(), + ) + t.Cleanup(func() { + run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { + headlessLogin(ctx, t, c) + c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t) + }) + }) + + cpu := 2.1 + c.Run(ctx, fmt.Sprintf(`coder envs edit %s --cpu %f --follow`, name, cpu)).Assert(t, + tcli.Success(), + ) + + var env coder.Environment + c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t, + tcli.Success(), + tcli.StdoutJSONUnmarshal(&env), + ) + assert.Equal(t, "cpu cores were updated", cpu, float64(env.CPUCores), floatComparer) + + c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t, + tcli.Success(), + ) }) } + +var floatComparer = cmp.Comparer(func(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.001 +}) diff --git a/ci/integration/integration_test.go b/ci/integration/integration_test.go index eccf68a2..7feb4f47 100644 --- a/ci/integration/integration_test.go +++ b/ci/integration/integration_test.go @@ -87,7 +87,7 @@ func TestCoderCLI(t *testing.T) { var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) func randString(length int) string { - const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + const charset = "abcdefghijklmnopqrstuvwxyz" b := make([]byte, length) for i := range b { b[i] = charset[seededRand.Intn(len(charset))] diff --git a/go.mod b/go.mod index 6c90ab40..cbc0fd04 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( cdr.dev/wsep v0.0.0-20200728013649-82316a09813f github.com/briandowns/spinner v1.11.1 github.com/fatih/color v1.9.0 + github.com/google/go-cmp v0.4.0 github.com/gorilla/websocket v1.4.2 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f github.com/klauspost/compress v1.10.8 // indirect From 84f51adcc6596650e48e09354e9f87feeef8f5a9 Mon Sep 17 00:00:00 2001 From: Charlie Moog Date: Thu, 29 Oct 2020 22:27:20 -0500 Subject: [PATCH 2/2] Use smaller image for integration tests --- .github/workflows/integration.yaml | 4 +-- ci/integration/Dockerfile | 3 ++ ci/integration/devurls_test.go | 1 - ci/integration/envs_test.go | 52 ++++++++++++++++++++++-------- ci/integration/integration_test.go | 3 +- ci/integration/setup_test.go | 7 ++-- ci/integration/users_test.go | 1 - ci/steps/integration.sh | 15 +++++++++ 8 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 ci/integration/Dockerfile create mode 100755 ci/steps/integration.sh diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index a8045e43..8ed56350 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -22,5 +22,5 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '^1.14' - - name: go test - run: go test -v ./ci/integration/... + - name: integration tests + run: ./ci/steps/integration.sh diff --git a/ci/integration/Dockerfile b/ci/integration/Dockerfile new file mode 100644 index 00000000..f81aa240 --- /dev/null +++ b/ci/integration/Dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install -y jq curl diff --git a/ci/integration/devurls_test.go b/ci/integration/devurls_test.go index 06553107..6dee8211 100644 --- a/ci/integration/devurls_test.go +++ b/ci/integration/devurls_test.go @@ -12,7 +12,6 @@ func TestDevURLCLI(t *testing.T) { run(t, "coder-cli-devurl-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { c.Run(ctx, "which coder").Assert(t, tcli.Success(), - tcli.StdoutMatches("/usr/sbin/coder"), tcli.StderrEmpty(), ) diff --git a/ci/integration/envs_test.go b/ci/integration/envs_test.go index af710fad..075f7222 100644 --- a/ci/integration/envs_test.go +++ b/ci/integration/envs_test.go @@ -4,20 +4,44 @@ import ( "context" "fmt" "math" + "net/url" "regexp" "testing" + "time" "cdr.dev/coder-cli/ci/tcli" "cdr.dev/coder-cli/coder-sdk" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" "cdr.dev/slog/sloggers/slogtest/assert" "github.com/google/go-cmp/cmp" ) +func cleanupClient(t *testing.T, ctx context.Context) *coder.Client { + creds := login(ctx, t) + + u, err := url.Parse(creds.url) + assert.Success(t, "parse base url", err) + + return &coder.Client{BaseURL: u, Token: creds.token} +} + +func cleanupEnv(t *testing.T, client *coder.Client, envID string) func() { + return func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + slogtest.Info(t, "cleanuping up environment", slog.F("env_id", envID)) + _ = client.DeleteEnvironment(ctx, envID) + } +} + func TestEnvsCLI(t *testing.T) { t.Parallel() run(t, "coder-cli-env-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { headlessLogin(ctx, t, c) + client := cleanupClient(t, ctx) // Minimum args not received. c.Run(ctx, "coder envs create").Assert(t, @@ -50,13 +74,6 @@ func TestEnvsCLI(t *testing.T) { tcli.Success(), ) - t.Cleanup(func() { - run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { - headlessLogin(ctx, t, c) - c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t) - }) - }) - c.Run(ctx, "coder envs ls").Assert(t, tcli.Success(), tcli.StdoutMatches(regexp.QuoteMeta(name)), @@ -67,6 +84,10 @@ func TestEnvsCLI(t *testing.T) { tcli.Success(), tcli.StdoutJSONUnmarshal(&env), ) + + // attempt to cleanup the environment even if tests fail + t.Cleanup(cleanupEnv(t, client, env.ID)) + assert.Equal(t, "environment cpu was correctly set", cpu, float64(env.CPUCores), floatComparer) c.Run(ctx, fmt.Sprintf("coder envs watch-build %s", name)).Assert(t, @@ -80,24 +101,27 @@ func TestEnvsCLI(t *testing.T) { run(t, "coder-cli-env-edit-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { headlessLogin(ctx, t, c) + client := cleanupClient(t, ctx) name := randString(10) c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --follow", name)).Assert(t, tcli.Success(), ) - t.Cleanup(func() { - run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { - headlessLogin(ctx, t, c) - c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t) - }) - }) + + var env coder.Environment + c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t, + tcli.Success(), + tcli.StdoutJSONUnmarshal(&env), + ) + + // attempt to cleanup the environment even if tests fail + t.Cleanup(cleanupEnv(t, client, env.ID)) cpu := 2.1 c.Run(ctx, fmt.Sprintf(`coder envs edit %s --cpu %f --follow`, name, cpu)).Assert(t, tcli.Success(), ) - var env coder.Environment c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t, tcli.Success(), tcli.StdoutJSONUnmarshal(&env), diff --git a/ci/integration/integration_test.go b/ci/integration/integration_test.go index 7feb4f47..52527839 100644 --- a/ci/integration/integration_test.go +++ b/ci/integration/integration_test.go @@ -18,7 +18,7 @@ func run(t *testing.T, container string, execute func(t *testing.T, ctx context. defer cancel() c, err := tcli.NewContainerRunner(ctx, &tcli.ContainerConfig{ - Image: "codercom/enterprise-dev", + Image: "coder-cli-integration:latest", Name: container, BindMounts: map[string]string{ binpath: "/bin/coder", @@ -36,7 +36,6 @@ func TestCoderCLI(t *testing.T) { run(t, "test-coder-cli", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { c.Run(ctx, "which coder").Assert(t, tcli.Success(), - tcli.StdoutMatches("/usr/sbin/coder"), tcli.StderrEmpty(), ) diff --git a/ci/integration/setup_test.go b/ci/integration/setup_test.go index 566f40e9..1919f6a0 100644 --- a/ci/integration/setup_test.go +++ b/ci/integration/setup_test.go @@ -51,14 +51,17 @@ func build(path string) error { // write session tokens to the given container runner func headlessLogin(ctx context.Context, t *testing.T, runner *tcli.ContainerRunner) { creds := login(ctx, t) - cmd := exec.CommandContext(ctx, "sh", "-c", "mkdir -p ~/.config/coder && cat > ~/.config/coder/session") + cmd := exec.CommandContext(ctx, "sh", "-c", "mkdir -p $HOME/.config/coder && cat > $HOME/.config/coder/session") // !IMPORTANT: be careful that this does not appear in logs cmd.Stdin = strings.NewReader(creds.token) runner.RunCmd(cmd).Assert(t, tcli.Success(), ) - runner.Run(ctx, fmt.Sprintf("echo -ne %s > ~/.config/coder/url", creds.url)).Assert(t, + + cmd = exec.CommandContext(ctx, "sh", "-c", "cat > $HOME/.config/coder/url") + cmd.Stdin = strings.NewReader(creds.url) + runner.RunCmd(cmd).Assert(t, tcli.Success(), ) } diff --git a/ci/integration/users_test.go b/ci/integration/users_test.go index 2138aeed..55d554f2 100644 --- a/ci/integration/users_test.go +++ b/ci/integration/users_test.go @@ -14,7 +14,6 @@ func TestUsers(t *testing.T) { run(t, "users-cli-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) { c.Run(ctx, "which coder").Assert(t, tcli.Success(), - tcli.StdoutMatches("/usr/sbin/coder"), tcli.StderrEmpty(), ) diff --git a/ci/steps/integration.sh b/ci/steps/integration.sh new file mode 100755 index 00000000..1ef04178 --- /dev/null +++ b/ci/steps/integration.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eo pipefail + +log() { + echo "--- $@" +} + +cd "$(git rev-parse --show-toplevel)" + +log "building integration test image" +docker build -f ./ci/integration/Dockerfile -t coder-cli-integration:latest . + +log "starting integration tests" +go test ./ci/integration -count=1