Skip to content

Commit de194a7

Browse files
elasticspoonelasticspoon
committed
feat(cli): add golden tests for errors (#11588)
Creates golden files from `coder/cli/errors.go`. Adds a unit test to test against golden files. Adds a make file command to regenerate golden files. Abstracts test against golden files.
1 parent 131d0bd commit de194a7

12 files changed

+154
-40
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -642,7 +642,7 @@ update-golden-files: \
642642
.PHONY: update-golden-files
643643

644644
cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
645-
go test ./cli -run="Test(CommandHelp|ServerYAML)" -update
645+
go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples)" -update
646646
touch "$@"
647647

648648
enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)

‎cli/clitest/golden.go

+29-24
Original file line numberDiff line numberDiff line change
@@ -87,34 +87,39 @@ ExtractCommandPathsLoop:
8787

8888
StartWithWaiter(t, inv.WithContext(ctx)).RequireSuccess()
8989

90-
actual := outBuf.Bytes()
91-
if len(actual) == 0 {
92-
t.Fatal("no output")
93-
}
94-
95-
for k, v := range replacements {
96-
actual = bytes.ReplaceAll(actual, []byte(k), []byte(v))
97-
}
90+
TestGoldenFile(t, tt.Name, outBuf.Bytes(), replacements)
91+
})
92+
}
93+
}
9894

99-
actual = NormalizeGoldenFile(t, actual)
100-
goldenPath := filepath.Join("testdata", strings.Replace(tt.Name, " ", "_", -1)+".golden")
101-
if *UpdateGoldenFiles {
102-
t.Logf("update golden file for: %q: %s", tt.Name, goldenPath)
103-
err := os.WriteFile(goldenPath, actual, 0o600)
104-
require.NoError(t, err, "update golden file")
105-
}
95+
// TestGoldenFile will test the given bytes slice input against the
96+
// golden file with the given file name, optionally using the given replacements.
97+
func TestGoldenFile(t *testing.T, fileName string, actual []byte, replacements map[string]string) {
98+
if len(actual) == 0 {
99+
t.Fatal("no output")
100+
}
106101

107-
expected, err := os.ReadFile(goldenPath)
108-
require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes")
102+
for k, v := range replacements {
103+
actual = bytes.ReplaceAll(actual, []byte(k), []byte(v))
104+
}
109105

110-
expected = NormalizeGoldenFile(t, expected)
111-
require.Equal(
112-
t, string(expected), string(actual),
113-
"golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes",
114-
goldenPath,
115-
)
116-
})
106+
actual = NormalizeGoldenFile(t, actual)
107+
goldenPath := filepath.Join("testdata", strings.ReplaceAll(fileName, " ", "_")+".golden")
108+
if *UpdateGoldenFiles {
109+
t.Logf("update golden file for: %q: %s", fileName, goldenPath)
110+
err := os.WriteFile(goldenPath, actual, 0o600)
111+
require.NoError(t, err, "update golden file")
117112
}
113+
114+
expected, err := os.ReadFile(goldenPath)
115+
require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes")
116+
117+
expected = NormalizeGoldenFile(t, expected)
118+
require.Equal(
119+
t, string(expected), string(actual),
120+
"golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes",
121+
goldenPath,
122+
)
118123
}
119124

120125
// NormalizeGoldenFile replaces any strings that are system or timing dependent

‎cli/errors_test.go

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package cli_test
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"os"
7+
"strings"
8+
"testing"
9+
10+
"github.com/stretchr/testify/require"
11+
12+
"github.com/coder/coder/v2/cli"
13+
"github.com/coder/coder/v2/cli/clitest"
14+
"github.com/coder/serpent"
15+
)
16+
17+
type commandErrorCase struct {
18+
Name string
19+
Cmd []string
20+
}
21+
22+
// TestErrorExamples will test the help output of the
23+
// coder exp example-error using golden files.
24+
func TestErrorExamples(t *testing.T) {
25+
t.Parallel()
26+
27+
var root cli.RootCmd
28+
rootCmd, err := root.Command(root.AGPL())
29+
require.NoError(t, err)
30+
31+
var cases []commandErrorCase
32+
33+
ExtractCommandPathsLoop:
34+
for _, cp := range extractCommandPaths(nil, rootCmd.Children) {
35+
name := fmt.Sprintf("coder %s", strings.Join(cp, " "))
36+
// space to end to not match base exp example-error
37+
if !strings.Contains(name, "exp example-error ") {
38+
continue
39+
}
40+
for _, tt := range cases {
41+
if tt.Name == name {
42+
continue ExtractCommandPathsLoop
43+
}
44+
}
45+
cases = append(cases, commandErrorCase{Name: name, Cmd: cp})
46+
}
47+
48+
for _, tt := range cases {
49+
tt := tt
50+
t.Run(tt.Name, func(t *testing.T) {
51+
t.Parallel()
52+
53+
var outBuf bytes.Buffer
54+
55+
rootCmd, err := root.Command(root.AGPL())
56+
require.NoError(t, err)
57+
58+
inv, _ := clitest.NewWithCommand(t, rootCmd, tt.Cmd...)
59+
inv.Stderr = &outBuf
60+
inv.Stdout = &outBuf
61+
62+
// This example expects to close stdin twice and joins
63+
// the error messages to create a multi-multi error.
64+
if tt.Name == "coder exp example-error multi-multi-error" {
65+
inv.Stdin = os.Stdin
66+
}
67+
68+
err = inv.Run()
69+
70+
errFormatter := cli.ExportNewPrettyErrorFormatter(&outBuf, false)
71+
cli.ExportFormat(errFormatter, err)
72+
73+
clitest.TestGoldenFile(t, tt.Name, outBuf.Bytes(), nil)
74+
})
75+
}
76+
}
77+
78+
func extractCommandPaths(cmdPath []string, cmds []*serpent.Command) [][]string {
79+
var cmdPaths [][]string
80+
for _, c := range cmds {
81+
cmdPath := append(cmdPath, c.Name())
82+
cmdPaths = append(cmdPaths, cmdPath)
83+
cmdPaths = append(cmdPaths, extractCommandPaths(cmdPath, c.Children)...)
84+
}
85+
return cmdPaths
86+
}

‎cli/export_cli_test.go

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
//nolint:testpackage // Exports needed to test internal functions without exposing.
2+
package cli
3+
4+
var (
5+
ExportNewPrettyErrorFormatter = newPrettyErrorFormatter
6+
ExportFormat = (*ExportPrettyErrorFormatter).format
7+
)
8+
9+
type ExportPrettyErrorFormatter = prettyErrorFormatter

‎cli/root.go

+7
Original file line numberDiff line numberDiff line change
@@ -1081,6 +1081,13 @@ func isConnectionError(err error) bool {
10811081
return xerrors.As(err, &dnsErr) || xerrors.As(err, &opErr)
10821082
}
10831083

1084+
func newPrettyErrorFormatter(w io.Writer, verbose bool) *prettyErrorFormatter {
1085+
return &prettyErrorFormatter{
1086+
w: w,
1087+
verbose: verbose,
1088+
}
1089+
}
1090+
10841091
type prettyErrorFormatter struct {
10851092
w io.Writer
10861093
// verbose turns on more detailed error logs, such as stack traces.

‎cli/server_test.go

+1-15
Original file line numberDiff line numberDiff line change
@@ -1773,21 +1773,7 @@ func TestServerYAMLConfig(t *testing.T) {
17731773
err = enc.Encode(n)
17741774
require.NoError(t, err)
17751775

1776-
wantByt := wantBuf.Bytes()
1777-
1778-
goldenPath := filepath.Join("testdata", "server-config.yaml.golden")
1779-
1780-
wantByt = clitest.NormalizeGoldenFile(t, wantByt)
1781-
if *clitest.UpdateGoldenFiles {
1782-
require.NoError(t, os.WriteFile(goldenPath, wantByt, 0o600))
1783-
return
1784-
}
1785-
1786-
got, err := os.ReadFile(goldenPath)
1787-
require.NoError(t, err)
1788-
got = clitest.NormalizeGoldenFile(t, got)
1789-
1790-
require.Equal(t, string(wantByt), string(got))
1776+
clitest.TestGoldenFile(t, "server-config.yaml", wantBuf.Bytes(), nil)
17911777
}
17921778

17931779
func TestConnectToPostgres(t *testing.T) {

‎cli/testdata/coder_exp_example-error_api.golden

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Encountered an error running "coder exp example-error api", see "coder exp example-error api --help" for more information
2+
error: Top level sdk error message.
3+
Have you tried turning it off and on again?

‎cli/testdata/coder_exp_example-error_arg-required.golden

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Encountered an error running "coder exp example-error arg-required", see "coder exp example-error arg-required --help" for more information
2+
error: wanted 1 args but got 0 []

‎cli/testdata/coder_exp_example-error_cmd.golden

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Encountered an error running "coder exp example-error cmd", see "coder exp example-error cmd --help" for more information
2+
error: some error: function decided not to work, and it never will

‎cli/testdata/coder_exp_example-error_multi-error.golden

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Encountered an error running "coder exp example-error multi-error", see "coder exp example-error multi-error --help" for more information
2+
error: 3 errors encountered: Trace=[wrapped: ])
3+
1. first error: function decided not to work, and it never will
4+
2. second error: function decided not to work, and it never will
5+
3. Trace=[wrapped api error: ]
6+
Top level sdk error message.
7+
magic dust unavailable, please try again later

‎cli/testdata/coder_exp_example-error_multi-multi-error.golden

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2 errors encountered:
2+
1. Encountered an error running "coder exp example-error multi-multi-error", see "coder exp example-error multi-multi-error --help" for more information
3+
error: 2 errors encountered:
4+
1. first error: function decided not to work, and it never will
5+
2. second error: function decided not to work, and it never will
6+
2. close /dev/stdin: file already closed

‎cli/testdata/coder_exp_example-error_validation.golden

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Missing values for the required flags: magic-word

0 commit comments

Comments
 (0)