From 7f6b0f41d10e39c0403acc0ac5b489aca416d3bc Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 14:41:56 +0200 Subject: [PATCH 1/6] feat: Add golden files to test cli help output --- Makefile | 10 +- cli/root_test.go | 71 +++++++++ cli/testdata/coder_--help.golden | 67 ++++++++ cli/testdata/coder_server_--help.golden | 200 ++++++++++++++++++++++++ 4 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 cli/testdata/coder_--help.golden create mode 100644 cli/testdata/coder_server_--help.golden diff --git a/Makefile b/Makefile index 64995b7a2168a..a1cb39d08e88a 100644 --- a/Makefile +++ b/Makefile @@ -394,14 +394,16 @@ gen: \ coderd/database/querier.go \ provisionersdk/proto/provisioner.pb.go \ provisionerd/proto/provisionerd.pb.go \ - site/src/api/typesGenerated.ts + site/src/api/typesGenerated.ts \ + cli/testdata/golden .PHONY: gen # Mark all generated files as fresh so make thinks they're up-to-date. This is # used during releases so we don't run generation scripts. gen/mark-fresh: files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts" - for file in $$files; do + files=($$files cli/testdata/*.golden) + for file in "$${files[@]}"; do echo "$$file" if [ ! -f "$$file" ]; then echo "File '$$file' does not exist" @@ -443,6 +445,10 @@ site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk cd site yarn run format:types +cli/testdata/golden: $(wildcard cli/testdata/*.golden) + go test ./cli -run=TestCommandHelp -update +.PHONY: cli/testdata/golden + test: test-clean gotestsum -- -v -short ./... .PHONY: test diff --git a/cli/root_test.go b/cli/root_test.go index b95d9932491cf..8f437ea73ce2e 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -2,8 +2,12 @@ package cli_test import ( "bytes" + "flag" "net/http" "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" "github.com/spf13/cobra" @@ -16,8 +20,75 @@ import ( "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/clitest" "github.com/coder/coder/codersdk" + "github.com/coder/coder/testutil" ) +var updateGoldenFiles = flag.Bool("update", false, "update .golden files") + +//nolint:tparallel,paralleltest // These test sets env vars. +func TestCommandHelp(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cmd []string + env map[string]string + }{ + { + name: "coder --help", + cmd: []string{"--help"}, + env: map[string]string{ + "CODER_CONFIG_DIR": "/tmp/coder-cli-test-config", + }, + }, + { + name: "coder server --help", + cmd: []string{"server", "--help"}, + env: map[string]string{ + "CODER_CONFIG_DIR": "/tmp/coder-cli-test-config", + "CODER_CACHE_DIRECTORY": "/tmp/coder-cli-test-cache", + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // Unset all CODER_ environment variables for a clean slate. + for _, kv := range os.Environ() { + name := strings.Split(kv, "=")[0] + if _, ok := tt.env[name]; !ok && strings.HasPrefix(name, "CODER_") { + t.Setenv(name, "") + } + } + // Override environment variables for a reproducible test. + for k, v := range tt.env { + t.Setenv(k, v) + } + + ctx, _ := testutil.Context(t) + + var buf bytes.Buffer + root, _ := clitest.New(t, tt.cmd...) + root.SetOut(&buf) + err := root.ExecuteContext(ctx) + require.NoError(t, err) + + gf := filepath.Join("testdata", strings.Replace(tt.name, " ", "_", -1)+".golden") + if *updateGoldenFiles { + t.Logf("update golden file for: %q: %s", tt.name, gf) + err = os.WriteFile(gf, buf.Bytes(), 0o600) + require.NoError(t, err, "update golden file") + } + + want, err := os.ReadFile(gf) + require.NoError(t, err, "read golden file, run \"make gen\" and commit the changes") + got := buf.Bytes() + require.Equal(t, string(want), string(got), "golden file mismatch: %s, run \"make gen\", verify and commit the changes", gf) + }) + } +} + func TestRoot(t *testing.T) { t.Parallel() t.Run("FormatCobraError", func(t *testing.T) { diff --git a/cli/testdata/coder_--help.golden b/cli/testdata/coder_--help.golden new file mode 100644 index 0000000000000..567875fe1ce9c --- /dev/null +++ b/cli/testdata/coder_--help.golden @@ -0,0 +1,67 @@ +Coder v0.0.0-devel — A tool for provisioning self-hosted development environments with Terraform. + +Usage: + coder [flags] + + coder [command] + +Get Started: + - Start a Coder server: + + $ coder server + + - Get started by creating a template from an example: + + $ coder templates init + +Commands: + completion Generate the autocompletion script for the specified shell + dotfiles Checkout and install a dotfiles repository from a Git URL + help Help about any command + login Authenticate with Coder deployment + logout Unauthenticate your local session + port-forward Forward ports from machine to a workspace + publickey Output your Coder public key used for Git operations + reset-password Directly connect to the database to reset a user's password + server Start a Coder server + state Manually manage Terraform state to fix broken workspaces + templates Manage templates + tokens Manage personal access tokens + users Manage users + version Show coder version + +Workspace Commands: + config-ssh Add an SSH Host entry for your workspaces "ssh coder.workspace" + create Create a workspace + delete Delete a workspace + list List workspaces + schedule Schedule automated start and stop times for workspaces + show Display details of a workspace's resources and agents + speedtest Run upload and download tests from your machine to a workspace + ssh Start a shell into a workspace + start Start a workspace + stop Stop a workspace + update Update a workspace + +Flags: + --experimental Enable experimental features. Experimental features are not + ready for production. + Consumes $CODER_EXPERIMENTAL + --global-config coder Path to the global coder config directory. + Consumes $CODER_CONFIG_DIR (default "/tmp/coder-cli-test-config") + --header stringArray HTTP headers added to all requests. Provide as "Key=Value". + Consumes $CODER_HEADER + -h, --help help for coder + --no-feature-warning Suppress warnings about unlicensed features. + Consumes $CODER_NO_FEATURE_WARNING + --no-version-warning Suppress warning when client and server versions do not match. + Consumes $CODER_NO_VERSION_WARNING + --token string Specify an authentication token. For security reasons setting + CODER_SESSION_TOKEN is preferred. + Consumes $CODER_SESSION_TOKEN + --url string URL to a deployment. + Consumes $CODER_URL + -v, --verbose Enable verbose output. + Consumes $CODER_VERBOSE + +Use "coder [command] --help" for more information about a command. diff --git a/cli/testdata/coder_server_--help.golden b/cli/testdata/coder_server_--help.golden new file mode 100644 index 0000000000000..6690a0b8499e0 --- /dev/null +++ b/cli/testdata/coder_server_--help.golden @@ -0,0 +1,200 @@ +Start a Coder server + +Usage: + coder server [flags] + + coder server [command] + +Commands: + postgres-builtin-serve Run the built-in PostgreSQL deployment. + postgres-builtin-url Output the connection URL for the built-in PostgreSQL deployment. + +Flags: + --access-url string External URL to access your deployment. + This must be accessible by all provisioned + workspaces. + Consumes $CODER_ACCESS_URL + -a, --address string Bind address of the server. + Consumes $CODER_ADDRESS (default + "127.0.0.1:3000") + --cache-dir string The directory to cache temporary files. If + unspecified and $CACHE_DIRECTORY is set, it + will be used for compatibility with + systemd. + Consumes $CODER_CACHE_DIRECTORY (default + "/tmp/coder-cli-test-cache") + --derp-config-path string Path to read a DERP mapping from. See: + https://tailscale.com/kb/1118/custom-derp-servers/ + Consumes $CODER_DERP_CONFIG_PATH + --derp-config-url string URL to fetch a DERP mapping on startup. + See: + https://tailscale.com/kb/1118/custom-derp-servers/ + Consumes $CODER_DERP_CONFIG_URL + --derp-server-enable Whether to enable or disable the embedded + DERP relay server. + Consumes $CODER_DERP_SERVER_ENABLE (default + true) + --derp-server-region-code string Region code to use for the embedded DERP + server. + Consumes $CODER_DERP_SERVER_REGION_CODE + (default "coder") + --derp-server-region-id int Region ID to use for the embedded DERP + server. + Consumes $CODER_DERP_SERVER_REGION_ID + (default 999) + --derp-server-region-name string Region name that for the embedded DERP + server. + Consumes $CODER_DERP_SERVER_REGION_NAME + (default "Coder Embedded Relay") + --derp-server-stun-addresses strings Addresses for STUN servers to establish P2P + connections. Set empty to disable P2P + connections. + Consumes $CODER_DERP_SERVER_STUN_ADDRESSES + (default [stun.l.google.com:19302]) + -h, --help help for server + --oauth2-github-allow-signups Whether new users can sign up with GitHub. + Consumes $CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS + --oauth2-github-allowed-orgs strings Organizations the user must be a member of + to Login with GitHub. + Consumes $CODER_OAUTH2_GITHUB_ALLOWED_ORGS + --oauth2-github-allowed-teams strings Teams inside organizations the user must be + a member of to Login with GitHub. + Structured as: + /. + Consumes $CODER_OAUTH2_GITHUB_ALLOWED_TEAMS + --oauth2-github-client-id string Client ID for Login with GitHub. + Consumes $CODER_OAUTH2_GITHUB_CLIENT_ID + --oauth2-github-client-secret string Client secret for Login with GitHub. + Consumes $CODER_OAUTH2_GITHUB_CLIENT_SECRET + --oauth2-github-enterprise-base-url string Base URL of a GitHub Enterprise deployment + to use for Login with GitHub. + Consumes + $CODER_OAUTH2_GITHUB_ENTERPRISE_BASE_URL + --oidc-allow-signups Whether new users can sign up with OIDC. + Consumes $CODER_OIDC_ALLOW_SIGNUPS (default + true) + --oidc-client-id string Client ID to use for Login with OIDC. + Consumes $CODER_OIDC_CLIENT_ID + --oidc-client-secret string Client secret to use for Login with OIDC. + Consumes $CODER_OIDC_CLIENT_SECRET + --oidc-email-domain string Email domain that clients logging in with + OIDC must match. + Consumes $CODER_OIDC_EMAIL_DOMAIN + --oidc-issuer-url string Issuer URL to use for Login with OIDC. + Consumes $CODER_OIDC_ISSUER_URL + --oidc-scopes strings Scopes to grant when authenticating with + OIDC. + Consumes $CODER_OIDC_SCOPES (default + [openid,profile,email]) + --postgres-url string URL of a PostgreSQL database. If empty, + PostgreSQL binaries will be downloaded from + Maven (https://repo1.maven.org/maven2) and + store all data in the config root. Access + the built-in database with "coder server + postgres-builtin-url". + Consumes $CODER_PG_CONNECTION_URL + --pprof-address string The bind address to serve pprof. + Consumes $CODER_PPROF_ADDRESS (default + "127.0.0.1:6060") + --pprof-enable Serve pprof metrics on the address defined + by pprof address. + Consumes $CODER_PPROF_ENABLE + --prometheus-address string The bind address to serve prometheus + metrics. + Consumes $CODER_PROMETHEUS_ADDRESS (default + "127.0.0.1:2112") + --prometheus-enable Serve prometheus metrics on the address + defined by prometheus address. + Consumes $CODER_PROMETHEUS_ENABLE + --provisioner-daemons int Number of provisioner daemons to create on + start. If builds are stuck in queued state + for a long time, consider increasing this. + Consumes $CODER_PROVISIONER_DAEMONS (default 3) + --proxy-trusted-headers strings Headers to trust for forwarding IP + addresses. e.g. Cf-Connecting-Ip, + True-Client-Ip, X-Forwarded-For + Consumes $CODER_PROXY_TRUSTED_HEADERS + --proxy-trusted-origins strings Origin addresses to respect + "proxy-trusted-headers". e.g. + 192.168.1.0/24 + Consumes $CODER_PROXY_TRUSTED_ORIGINS + --secure-auth-cookie Controls if the 'Secure' property is set on + browser session cookies. + Consumes $CODER_SECURE_AUTH_COOKIE + --ssh-keygen-algorithm string The algorithm to use for generating ssh + keys. Accepted values are "ed25519", + "ecdsa", or "rsa4096". + Consumes $CODER_SSH_KEYGEN_ALGORITHM + (default "ed25519") + --telemetry Whether telemetry is enabled or not. Coder + collects anonymized usage data to help + improve our product. + Consumes $CODER_TELEMETRY_ENABLE + --telemetry-trace Whether Opentelemetry traces are sent to + Coder. Coder collects anonymized + application tracing to help improve our + product. Disabling telemetry also disables + this option. + Consumes $CODER_TELEMETRY_TRACE + --tls-cert-file strings Path to each certificate for TLS. It + requires a PEM-encoded file. To configure + the listener to use a CA certificate, + concatenate the primary certificate and the + CA certificate together. The primary + certificate should appear first in the + combined file. + Consumes $CODER_TLS_CERT_FILE + --tls-client-auth string Policy the server will follow for TLS + Client Authentication. Accepted values are + "none", "request", "require-any", + "verify-if-given", or "require-and-verify". + Consumes $CODER_TLS_CLIENT_AUTH (default + "request") + --tls-client-ca-file string PEM-encoded Certificate Authority file used + for checking the authenticity of client + Consumes $CODER_TLS_CLIENT_CA_FILE + --tls-enable Whether TLS will be enabled. + Consumes $CODER_TLS_ENABLE + --tls-key-file strings Paths to the private keys for each of the + certificates. It requires a PEM-encoded + file. + Consumes $CODER_TLS_KEY_FILE + --tls-min-version string Minimum supported version of TLS. Accepted + values are "tls10", "tls11", "tls12" or + "tls13" + Consumes $CODER_TLS_MIN_VERSION (default + "tls12") + --trace Whether application tracing data is + collected. It exports to a backend + configured by environment variables. See: + https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md + Consumes $CODER_TRACE_ENABLE + --trace-honeycomb-api-key string Enables trace exporting to Honeycomb.io + using the provided API Key. + Consumes $CODER_TRACE_HONEYCOMB_API_KEY + --wildcard-access-url string Specifies the wildcard hostname to use for + workspace applications in the form + "*.example.com". + Consumes $CODER_WILDCARD_ACCESS_URL + +Global Flags: + --experimental Enable experimental features. Experimental features are not + ready for production. + Consumes $CODER_EXPERIMENTAL + --global-config coder Path to the global coder config directory. + Consumes $CODER_CONFIG_DIR (default "/tmp/coder-cli-test-config") + --header stringArray HTTP headers added to all requests. Provide as "Key=Value". + Consumes $CODER_HEADER + --no-feature-warning Suppress warnings about unlicensed features. + Consumes $CODER_NO_FEATURE_WARNING + --no-version-warning Suppress warning when client and server versions do not match. + Consumes $CODER_NO_VERSION_WARNING + --token string Specify an authentication token. For security reasons setting + CODER_SESSION_TOKEN is preferred. + Consumes $CODER_SESSION_TOKEN + --url string URL to a deployment. + Consumes $CODER_URL + -v, --verbose Enable verbose output. + Consumes $CODER_VERBOSE + +Use "coder server [command] --help" for more information about a command. From 1b4dbe98d5a61125ee23dbca04ded0af7c9725fa Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 15:59:27 +0200 Subject: [PATCH 2/6] Separate make command -> make update-golden-files --- .gitignore | 3 +++ Makefile | 15 +++++++++------ cli/root_test.go | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 2df34372363aa..9a3c866c63ca9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ gotests.coverage .gitpod.yml .DS_Store +# Make target for updating golden files. +cli/testdata/.gen-golden + # Front-end ignore .next/ site/.eslintcache diff --git a/Makefile b/Makefile index a1cb39d08e88a..ea0b953dc1d3e 100644 --- a/Makefile +++ b/Makefile @@ -394,16 +394,14 @@ gen: \ coderd/database/querier.go \ provisionersdk/proto/provisioner.pb.go \ provisionerd/proto/provisionerd.pb.go \ - site/src/api/typesGenerated.ts \ - cli/testdata/golden + site/src/api/typesGenerated.ts .PHONY: gen # Mark all generated files as fresh so make thinks they're up-to-date. This is # used during releases so we don't run generation scripts. gen/mark-fresh: files="coderd/database/dump.sql coderd/database/querier.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts" - files=($$files cli/testdata/*.golden) - for file in "$${files[@]}"; do + for file in $$files; do echo "$$file" if [ ! -f "$$file" ]; then echo "File '$$file' does not exist" @@ -445,9 +443,14 @@ site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk cd site yarn run format:types -cli/testdata/golden: $(wildcard cli/testdata/*.golden) +update-golden-files: cli/testdata/.gen-golden +.PHONY: update-golden-files + +cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) \ + $(shell find . -not -path './vendor/*' -type f -name '*.go') + go test ./cli -run=TestCommandHelp -update -.PHONY: cli/testdata/golden + touch "$@" test: test-clean gotestsum -- -v -short ./... diff --git a/cli/root_test.go b/cli/root_test.go index 8f437ea73ce2e..dc223bf6dee80 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -82,9 +82,9 @@ func TestCommandHelp(t *testing.T) { } want, err := os.ReadFile(gf) - require.NoError(t, err, "read golden file, run \"make gen\" and commit the changes") + require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes") got := buf.Bytes() - require.Equal(t, string(want), string(got), "golden file mismatch: %s, run \"make gen\", verify and commit the changes", gf) + require.Equal(t, string(want), string(got), "golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes", gf) }) } } From 6450177accefa3606bad71a0d48ec04d01f359b3 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 16:15:43 +0200 Subject: [PATCH 3/6] Fix test on Windows with CRLF lines --- cli/root_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cli/root_test.go b/cli/root_test.go index dc223bf6dee80..eac53a11b5b76 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -74,16 +74,19 @@ func TestCommandHelp(t *testing.T) { err := root.ExecuteContext(ctx) require.NoError(t, err) + got := buf.Bytes() + // Remove CRLF newlines (Windows). + got = bytes.ReplaceAll(got, []byte{'\r', '\n'}, []byte{'\n'}) + gf := filepath.Join("testdata", strings.Replace(tt.name, " ", "_", -1)+".golden") if *updateGoldenFiles { t.Logf("update golden file for: %q: %s", tt.name, gf) - err = os.WriteFile(gf, buf.Bytes(), 0o600) + err = os.WriteFile(gf, got, 0o600) require.NoError(t, err, "update golden file") } want, err := os.ReadFile(gf) require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes") - got := buf.Bytes() require.Equal(t, string(want), string(got), "golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes", gf) }) } From 0928b8b8a59da7d286bca3b2988271bc469f65f1 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 16:43:47 +0200 Subject: [PATCH 4/6] Unset all envs --- cli/root_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/root_test.go b/cli/root_test.go index eac53a11b5b76..c6c0360f56dd4 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -54,12 +54,10 @@ func TestCommandHelp(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - // Unset all CODER_ environment variables for a clean slate. + // Unset all environment variables for a clean slate. for _, kv := range os.Environ() { name := strings.Split(kv, "=")[0] - if _, ok := tt.env[name]; !ok && strings.HasPrefix(name, "CODER_") { - t.Setenv(name, "") - } + t.Setenv(name, "") } // Override environment variables for a reproducible test. for k, v := range tt.env { From 91936c109403b824dd4f5a424de1c9f0e44cccd2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 17:35:47 +0200 Subject: [PATCH 5/6] Revert "Unset all envs" This reverts commit 0928b8b8a59da7d286bca3b2988271bc469f65f1. --- cli/root_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/root_test.go b/cli/root_test.go index c6c0360f56dd4..eac53a11b5b76 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -54,10 +54,12 @@ func TestCommandHelp(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - // Unset all environment variables for a clean slate. + // Unset all CODER_ environment variables for a clean slate. for _, kv := range os.Environ() { name := strings.Split(kv, "=")[0] - t.Setenv(name, "") + if _, ok := tt.env[name]; !ok && strings.HasPrefix(name, "CODER_") { + t.Setenv(name, "") + } } // Override environment variables for a reproducible test. for k, v := range tt.env { From 41e2660dbf92760c96aa319759acda4d5034934b Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Nov 2022 18:26:52 +0200 Subject: [PATCH 6/6] Remove CRLF when reading too --- cli/root_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/root_test.go b/cli/root_test.go index eac53a11b5b76..05768fcae75ac 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -87,6 +87,8 @@ func TestCommandHelp(t *testing.T) { want, err := os.ReadFile(gf) require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes") + // Remove CRLF newlines (Windows). + want = bytes.ReplaceAll(want, []byte{'\r', '\n'}, []byte{'\n'}) require.Equal(t, string(want), string(got), "golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes", gf) }) }