Skip to content

Commit cfb9428

Browse files
elasticspoonelasticspoon
authored
feat(cli): add golden tests for errors (coder#11588) (coder#12698)
* feat(cli): add golden tests for errors (coder#11588) Creates golden files from `coder/cli/errors.go`. Adds a unit test to test against golden files. Adds a make file command to regenerate golden files. Abstracts test against golden files.
1 parent 75bf41b commit cfb9428

12 files changed

+166
-57
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -642,7 +642,7 @@ update-golden-files: \
642642
.PHONY: update-golden-files
643643

644644
cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
645-
go test ./cli -run="Test(CommandHelp|ServerYAML)" -update
645+
go test ./cli -run="Test(CommandHelp|ServerYAML|ErrorExamples)" -update
646646
touch "$@"
647647

648648
enterprise/cli/testdata/.gen-golden: $(wildcard enterprise/cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard enterprise/cli/*_test.go)

‎cli/clitest/golden.go

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -87,40 +87,45 @@ ExtractCommandPathsLoop:
8787

8888
StartWithWaiter(t, inv.WithContext(ctx)).RequireSuccess()
8989

90-
actual := outBuf.Bytes()
91-
if len(actual) == 0 {
92-
t.Fatal("no output")
93-
}
94-
95-
for k, v := range replacements {
96-
actual = bytes.ReplaceAll(actual, []byte(k), []byte(v))
97-
}
90+
TestGoldenFile(t, tt.Name, outBuf.Bytes(), replacements)
91+
})
92+
}
93+
}
9894

99-
actual = NormalizeGoldenFile(t, actual)
100-
goldenPath := filepath.Join("testdata", strings.Replace(tt.Name, " ", "_", -1)+".golden")
101-
if *UpdateGoldenFiles {
102-
t.Logf("update golden file for: %q: %s", tt.Name, goldenPath)
103-
err := os.WriteFile(goldenPath, actual, 0o600)
104-
require.NoError(t, err, "update golden file")
105-
}
95+
// TestGoldenFile will test the given bytes slice input against the
96+
// golden file with the given file name, optionally using the given replacements.
97+
func TestGoldenFile(t *testing.T, fileName string, actual []byte, replacements map[string]string) {
98+
if len(actual) == 0 {
99+
t.Fatal("no output")
100+
}
106101

107-
expected, err := os.ReadFile(goldenPath)
108-
require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes")
102+
for k, v := range replacements {
103+
actual = bytes.ReplaceAll(actual, []byte(k), []byte(v))
104+
}
109105

110-
expected = NormalizeGoldenFile(t, expected)
111-
require.Equal(
112-
t, string(expected), string(actual),
113-
"golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes",
114-
goldenPath,
115-
)
116-
})
106+
actual = normalizeGoldenFile(t, actual)
107+
goldenPath := filepath.Join("testdata", strings.ReplaceAll(fileName, " ", "_")+".golden")
108+
if *UpdateGoldenFiles {
109+
t.Logf("update golden file for: %q: %s", fileName, goldenPath)
110+
err := os.WriteFile(goldenPath, actual, 0o600)
111+
require.NoError(t, err, "update golden file")
117112
}
113+
114+
expected, err := os.ReadFile(goldenPath)
115+
require.NoError(t, err, "read golden file, run \"make update-golden-files\" and commit the changes")
116+
117+
expected = normalizeGoldenFile(t, expected)
118+
require.Equal(
119+
t, string(expected), string(actual),
120+
"golden file mismatch: %s, run \"make update-golden-files\", verify and commit the changes",
121+
goldenPath,
122+
)
118123
}
119124

120-
// NormalizeGoldenFile replaces any strings that are system or timing dependent
125+
// normalizeGoldenFile replaces any strings that are system or timing dependent
121126
// with a placeholder so that the golden files can be compared with a simple
122127
// equality check.
123-
func NormalizeGoldenFile(t *testing.T, byt []byte) []byte {
128+
func normalizeGoldenFile(t *testing.T, byt []byte) []byte {
124129
// Replace any timestamps with a placeholder.
125130
byt = timestampRegex.ReplaceAll(byt, []byte("[timestamp]"))
126131

‎cli/errors.go

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"fmt"
66
"net/http"
77
"net/http/httptest"
8-
"os"
98

109
"golang.org/x/xerrors"
1110

@@ -83,15 +82,12 @@ func (RootCmd) errorExample() *serpent.Command {
8382
Use: "multi-multi-error",
8483
Short: "This is a multi error inside a multi error",
8584
Handler: func(inv *serpent.Invocation) error {
86-
// Closing the stdin file descriptor will cause the next close
87-
// to fail. This is joined to the returned Command error.
88-
if f, ok := inv.Stdin.(*os.File); ok {
89-
_ = f.Close()
90-
}
91-
9285
return errors.Join(
93-
xerrors.Errorf("first error: %w", errorWithStackTrace()),
94-
xerrors.Errorf("second error: %w", errorWithStackTrace()),
86+
xerrors.Errorf("parent error: %w", errorWithStackTrace()),
87+
errors.Join(
88+
xerrors.Errorf("child first error: %w", errorWithStackTrace()),
89+
xerrors.Errorf("child second error: %w", errorWithStackTrace()),
90+
),
9591
)
9692
},
9793
},

‎cli/errors_test.go

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package cli_test
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"strings"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
11+
"github.com/coder/coder/v2/cli"
12+
"github.com/coder/coder/v2/cli/clitest"
13+
"github.com/coder/serpent"
14+
)
15+
16+
type commandErrorCase struct {
17+
Name string
18+
Cmd []string
19+
}
20+
21+
// TestErrorExamples will test the help output of the
22+
// coder exp example-error using golden files.
23+
func TestErrorExamples(t *testing.T) {
24+
t.Parallel()
25+
26+
coderRootCmd := getRoot(t)
27+
28+
var exampleErrorRootCmd *serpent.Command
29+
coderRootCmd.Walk(func(command *serpent.Command) {
30+
if command.Name() == "example-error" {
31+
// cannot abort early, but list is small
32+
exampleErrorRootCmd = command
33+
}
34+
})
35+
require.NotNil(t, exampleErrorRootCmd, "example-error command not found")
36+
37+
var cases []commandErrorCase
38+
39+
ExtractCommandPathsLoop:
40+
for _, cp := range extractCommandPaths(nil, exampleErrorRootCmd.Children) {
41+
cmd := append([]string{"exp", "example-error"}, cp...)
42+
name := fmt.Sprintf("coder %s", strings.Join(cmd, " "))
43+
for _, tt := range cases {
44+
if tt.Name == name {
45+
continue ExtractCommandPathsLoop
46+
}
47+
}
48+
cases = append(cases, commandErrorCase{Name: name, Cmd: cmd})
49+
}
50+
51+
for _, tt := range cases {
52+
tt := tt
53+
t.Run(tt.Name, func(t *testing.T) {
54+
t.Parallel()
55+
56+
var outBuf bytes.Buffer
57+
58+
coderRootCmd := getRoot(t)
59+
60+
inv, _ := clitest.NewWithCommand(t, coderRootCmd, tt.Cmd...)
61+
inv.Stderr = &outBuf
62+
inv.Stdout = &outBuf
63+
64+
err := inv.Run()
65+
66+
errFormatter := cli.NewPrettyErrorFormatter(&outBuf, false)
67+
errFormatter.Format(err)
68+
69+
clitest.TestGoldenFile(t, tt.Name, outBuf.Bytes(), nil)
70+
})
71+
}
72+
}
73+
74+
func extractCommandPaths(cmdPath []string, cmds []*serpent.Command) [][]string {
75+
var cmdPaths [][]string
76+
for _, c := range cmds {
77+
cmdPath := append(cmdPath, c.Name())
78+
cmdPaths = append(cmdPaths, cmdPath)
79+
cmdPaths = append(cmdPaths, extractCommandPaths(cmdPath, c.Children)...)
80+
}
81+
return cmdPaths
82+
}
83+
84+
// Must return a fresh instance of cmds each time.
85+
func getRoot(t *testing.T) *serpent.Command {
86+
t.Helper()
87+
88+
var root cli.RootCmd
89+
rootCmd, err := root.Command(root.AGPL())
90+
require.NoError(t, err)
91+
92+
return rootCmd
93+
}

‎cli/root.go

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,9 +167,9 @@ func (r *RootCmd) RunWithSubcommands(subcommands []*serpent.Command) {
167167
//nolint:revive
168168
os.Exit(code)
169169
}
170-
f := prettyErrorFormatter{w: os.Stderr, verbose: r.verbose}
170+
f := PrettyErrorFormatter{w: os.Stderr, verbose: r.verbose}
171171
if err != nil {
172-
f.format(err)
172+
f.Format(err)
173173
}
174174
//nolint:revive
175175
os.Exit(code)
@@ -909,15 +909,23 @@ func ExitError(code int, err error) error {
909909
return &exitError{code: code, err: err}
910910
}
911911

912-
type prettyErrorFormatter struct {
912+
// NewPrettyErrorFormatter creates a new PrettyErrorFormatter.
913+
func NewPrettyErrorFormatter(w io.Writer, verbose bool) *PrettyErrorFormatter {
914+
return &PrettyErrorFormatter{
915+
w: w,
916+
verbose: verbose,
917+
}
918+
}
919+
920+
type PrettyErrorFormatter struct {
913921
w io.Writer
914922
// verbose turns on more detailed error logs, such as stack traces.
915923
verbose bool
916924
}
917925

918-
// format formats the error to the console. This error should be human
919-
// readable.
920-
func (p *prettyErrorFormatter) format(err error) {
926+
// Format formats the error to the writer in PrettyErrorFormatter.
927+
// This error should be human readable.
928+
func (p *PrettyErrorFormatter) Format(err error) {
921929
output, _ := cliHumanFormatError("", err, &formatOpts{
922930
Verbose: p.verbose,
923931
})

‎cli/server_test.go

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1774,21 +1774,7 @@ func TestServerYAMLConfig(t *testing.T) {
17741774
err = enc.Encode(n)
17751775
require.NoError(t, err)
17761776

1777-
wantByt := wantBuf.Bytes()
1778-
1779-
goldenPath := filepath.Join("testdata", "server-config.yaml.golden")
1780-
1781-
wantByt = clitest.NormalizeGoldenFile(t, wantByt)
1782-
if *clitest.UpdateGoldenFiles {
1783-
require.NoError(t, os.WriteFile(goldenPath, wantByt, 0o600))
1784-
return
1785-
}
1786-
1787-
got, err := os.ReadFile(goldenPath)
1788-
require.NoError(t, err)
1789-
got = clitest.NormalizeGoldenFile(t, got)
1790-
1791-
require.Equal(t, string(wantByt), string(got))
1777+
clitest.TestGoldenFile(t, "server-config.yaml", wantBuf.Bytes(), nil)
17921778
}
17931779

17941780
func TestConnectToPostgres(t *testing.T) {

‎cli/testdata/coder_exp_example-error_api.golden

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Encountered an error running "coder exp example-error api", see "coder exp example-error api --help" for more information
2+
error: Top level sdk error message.
3+
Have you tried turning it off and on again?

‎cli/testdata/coder_exp_example-error_arg-required.golden

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Encountered an error running "coder exp example-error arg-required", see "coder exp example-error arg-required --help" for more information
2+
error: wanted 1 args but got 0 []

‎cli/testdata/coder_exp_example-error_cmd.golden

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Encountered an error running "coder exp example-error cmd", see "coder exp example-error cmd --help" for more information
2+
error: some error: function decided not to work, and it never will

‎cli/testdata/coder_exp_example-error_multi-error.golden

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Encountered an error running "coder exp example-error multi-error", see "coder exp example-error multi-error --help" for more information
2+
error: 3 errors encountered: Trace=[wrapped: ])
3+
1. first error: function decided not to work, and it never will
4+
2. second error: function decided not to work, and it never will
5+
3. Trace=[wrapped api error: ]
6+
Top level sdk error message.
7+
magic dust unavailable, please try again later

‎cli/testdata/coder_exp_example-error_multi-multi-error.golden

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Encountered an error running "coder exp example-error multi-multi-error", see "coder exp example-error multi-multi-error --help" for more information
2+
error: 2 errors encountered:
3+
1. parent error: function decided not to work, and it never will
4+
2. 2 errors encountered:
5+
1. child first error: function decided not to work, and it never will
6+
2. child second error: function decided not to work, and it never will

‎cli/testdata/coder_exp_example-error_validation.golden

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Missing values for the required flags: magic-word

0 commit comments

Comments
 (0)