Skip to content

Commit ac2f9bf

Browse files
EmyrkEmyrk
authored
test: unit test to document validation behavior of parameters (#387)
Documenting validation behavior with a unit test
1 parent f871a43 commit ac2f9bf

File tree

2 files changed

+302
-0
lines changed

2 files changed

+302
-0
lines changed

‎provider/parameter_test.go

+232
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package provider_test
22

33
import (
44
"fmt"
5+
"os"
56
"regexp"
7+
"strconv"
68
"strings"
79
"testing"
810

@@ -686,6 +688,217 @@ data "coder_parameter" "region" {
686688
}
687689
}
688690

691+
// TestParameterValidationEnforcement tests various parameter states and the
692+
// validation enforcement that should be applied to them. The table is described
693+
// by a markdown table. This is done so that the test cases can be more easily
694+
// edited and read.
695+
//
696+
// Copy and paste the table to https://www.tablesgenerator.com/markdown_tables for easier editing
697+
//
698+
//nolint:paralleltest,tparallel // Parameters load values from env vars
699+
func TestParameterValidationEnforcement(t *testing.T) {
700+
// Some interesting observations:
701+
// - Validation logic does not apply to the value of 'options'
702+
// - [NumDefInvOpt] So an invalid option can be present and selected, but would fail
703+
// - Validation logic does not apply to the default if a value is given
704+
// - [NumIns/DefInv] So the default can be invalid if an input value is valid.
705+
// The value is therefore not really optional, but it is marked as such.
706+
// - [NumInsNotOptsVal | NumsInsNotOpts] values do not need to be in the option set?
707+
table, err := os.ReadFile("testdata/parameter_table.md")
708+
require.NoError(t, err)
709+
710+
type row struct {
711+
Name string
712+
Types []string
713+
InputValue string
714+
Default string
715+
Options []string
716+
Validation *provider.Validation
717+
OutputValue string
718+
Optional bool
719+
Error *regexp.Regexp
720+
}
721+
722+
rows := make([]row, 0)
723+
lines := strings.Split(string(table), "\n")
724+
validMinMax := regexp.MustCompile("^[0-9]*-[0-9]*$")
725+
for _, line := range lines[2:] {
726+
columns := strings.Split(line, "|")
727+
columns = columns[1 : len(columns)-1]
728+
for i := range columns {
729+
// Trim the whitespace from all columns
730+
columns[i] = strings.TrimSpace(columns[i])
731+
}
732+
733+
if columns[0] == "" {
734+
continue // Skip rows with empty names
735+
}
736+
737+
optional, err := strconv.ParseBool(columns[8])
738+
if columns[8] != "" {
739+
// Value does not matter if not specified
740+
require.NoError(t, err)
741+
}
742+
743+
var rerr *regexp.Regexp
744+
if columns[9] != "" {
745+
rerr, err = regexp.Compile(columns[9])
746+
if err != nil {
747+
t.Fatalf("failed to parse error column %q: %v", columns[9], err)
748+
}
749+
}
750+
var options []string
751+
if columns[4] != "" {
752+
options = strings.Split(columns[4], ",")
753+
}
754+
755+
var validation *provider.Validation
756+
if columns[5] != "" {
757+
// Min-Max validation should look like:
758+
// 1-10 :: min=1, max=10
759+
// -10 :: max=10
760+
// 1- :: min=1
761+
if validMinMax.MatchString(columns[5]) {
762+
parts := strings.Split(columns[5], "-")
763+
min, _ := strconv.ParseInt(parts[0], 10, 64)
764+
max, _ := strconv.ParseInt(parts[1], 10, 64)
765+
validation = &provider.Validation{
766+
Min: int(min),
767+
MinDisabled: parts[0] == "",
768+
Max: int(max),
769+
MaxDisabled: parts[1] == "",
770+
Monotonic: "",
771+
Regex: "",
772+
Error: "{min} < {value} < {max}",
773+
}
774+
} else {
775+
validation = &provider.Validation{
776+
Min: 0,
777+
MinDisabled: true,
778+
Max: 0,
779+
MaxDisabled: true,
780+
Monotonic: "",
781+
Regex: columns[5],
782+
Error: "regex error",
783+
}
784+
}
785+
}
786+
787+
rows = append(rows, row{
788+
Name: columns[0],
789+
Types: strings.Split(columns[1], ","),
790+
InputValue: columns[2],
791+
Default: columns[3],
792+
Options: options,
793+
Validation: validation,
794+
OutputValue: columns[7],
795+
Optional: optional,
796+
Error: rerr,
797+
})
798+
}
799+
800+
stringLiteral := func(s string) string {
801+
if s == "" {
802+
return `""`
803+
}
804+
return fmt.Sprintf("%q", s)
805+
}
806+
807+
for rowIndex, row := range rows {
808+
for _, rt := range row.Types {
809+
//nolint:paralleltest,tparallel // Parameters load values from env vars
810+
t.Run(fmt.Sprintf("%d|%s:%s", rowIndex, row.Name, rt), func(t *testing.T) {
811+
if row.InputValue != "" {
812+
t.Setenv(provider.ParameterEnvironmentVariable("parameter"), row.InputValue)
813+
}
814+
815+
if row.Error != nil {
816+
if row.OutputValue != "" {
817+
t.Errorf("output value %q should not be set if error is set", row.OutputValue)
818+
}
819+
}
820+
821+
var cfg strings.Builder
822+
cfg.WriteString("data \"coder_parameter\" \"parameter\" {\n")
823+
cfg.WriteString("\tname = \"parameter\"\n")
824+
if rt == "multi-select" || rt == "tag-select" {
825+
cfg.WriteString(fmt.Sprintf("\ttype = \"%s\"\n", "list(string)"))
826+
cfg.WriteString(fmt.Sprintf("\tform_type = \"%s\"\n", rt))
827+
} else {
828+
cfg.WriteString(fmt.Sprintf("\ttype = \"%s\"\n", rt))
829+
}
830+
if row.Default != "" {
831+
cfg.WriteString(fmt.Sprintf("\tdefault = %s\n", stringLiteral(row.Default)))
832+
}
833+
834+
for _, opt := range row.Options {
835+
cfg.WriteString("\toption {\n")
836+
cfg.WriteString(fmt.Sprintf("\t\tname = %s\n", stringLiteral(opt)))
837+
cfg.WriteString(fmt.Sprintf("\t\tvalue = %s\n", stringLiteral(opt)))
838+
cfg.WriteString("\t}\n")
839+
}
840+
841+
if row.Validation != nil {
842+
cfg.WriteString("\tvalidation {\n")
843+
if !row.Validation.MinDisabled {
844+
cfg.WriteString(fmt.Sprintf("\t\tmin = %d\n", row.Validation.Min))
845+
}
846+
if !row.Validation.MaxDisabled {
847+
cfg.WriteString(fmt.Sprintf("\t\tmax = %d\n", row.Validation.Max))
848+
}
849+
if row.Validation.Monotonic != "" {
850+
cfg.WriteString(fmt.Sprintf("\t\tmonotonic = \"%s\"\n", row.Validation.Monotonic))
851+
}
852+
if row.Validation.Regex != "" {
853+
cfg.WriteString(fmt.Sprintf("\t\tregex = %q\n", row.Validation.Regex))
854+
}
855+
cfg.WriteString(fmt.Sprintf("\t\terror = %q\n", row.Validation.Error))
856+
cfg.WriteString("\t}\n")
857+
}
858+
859+
cfg.WriteString("}\n")
860+
861+
resource.Test(t, resource.TestCase{
862+
ProviderFactories: coderFactory(),
863+
IsUnitTest: true,
864+
Steps: []resource.TestStep{{
865+
Config: cfg.String(),
866+
ExpectError: row.Error,
867+
Check: func(state *terraform.State) error {
868+
require.Len(t, state.Modules, 1)
869+
require.Len(t, state.Modules[0].Resources, 1)
870+
param := state.Modules[0].Resources["data.coder_parameter.parameter"]
871+
require.NotNil(t, param)
872+
873+
if row.Default == "" {
874+
_, ok := param.Primary.Attributes["default"]
875+
require.False(t, ok, "default should not be set")
876+
} else {
877+
require.Equal(t, strings.Trim(row.Default, `"`), param.Primary.Attributes["default"])
878+
}
879+
880+
if row.OutputValue == "" {
881+
_, ok := param.Primary.Attributes["value"]
882+
require.False(t, ok, "output value should not be set")
883+
} else {
884+
require.Equal(t, strings.Trim(row.OutputValue, `"`), param.Primary.Attributes["value"])
885+
}
886+
887+
for key, expected := range map[string]string{
888+
"optional": strconv.FormatBool(row.Optional),
889+
} {
890+
require.Equal(t, expected, param.Primary.Attributes[key], "optional")
891+
}
892+
893+
return nil
894+
},
895+
}},
896+
})
897+
})
898+
}
899+
}
900+
}
901+
689902
func TestValueValidatesType(t *testing.T) {
690903
t.Parallel()
691904
for _, tc := range []struct {
@@ -798,6 +1011,25 @@ func TestValueValidatesType(t *testing.T) {
7981011
Value: `[]`,
7991012
MinDisabled: true,
8001013
MaxDisabled: true,
1014+
}, {
1015+
Name: "ValidListOfStrings",
1016+
Type: "list(string)",
1017+
Value: `["first","second","third"]`,
1018+
MinDisabled: true,
1019+
MaxDisabled: true,
1020+
}, {
1021+
Name: "InvalidListOfStrings",
1022+
Type: "list(string)",
1023+
Value: `["first","second","third"`,
1024+
MinDisabled: true,
1025+
MaxDisabled: true,
1026+
Error: regexp.MustCompile("is not valid list of strings"),
1027+
}, {
1028+
Name: "EmptyListOfStrings",
1029+
Type: "list(string)",
1030+
Value: `[]`,
1031+
MinDisabled: true,
1032+
MaxDisabled: true,
8011033
}} {
8021034
tc := tc
8031035
t.Run(tc.Name, func(t *testing.T) {

‎provider/testdata/parameter_table.md

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
| Name | Type | Input | Default | Options | Validation | -> | Output | Optional | Error |
2+
|----------------------|---------------|-----------|---------|-------------------|------------|----|--------|----------|--------------|
3+
| | Empty Vals | | | | | | | | |
4+
| Empty | string,number | | | | | | "" | false | |
5+
| EmptyList | list(string) | | | | | | "" | false | |
6+
| EmptyMulti | tag-select | | | | | | "" | false | |
7+
| EmptyOpts | string,number | | | 1,2,3 | | | "" | false | |
8+
| EmptyRegex | string | | | | world | | | | regex error |
9+
| EmptyMin | number | | | | 1-10 | | | | 1 < < 10 |
10+
| EmptyMinOpt | number | | | 1,2,3 | 2-5 | | | | 2 < < 5 |
11+
| EmptyRegexOpt | string | | | "hello","goodbye" | goodbye | | | | regex error |
12+
| EmptyRegexOk | string | | | | .* | | "" | false | |
13+
| | | | | | | | | | |
14+
| | Default Set | No inputs | | | | | | | |
15+
| NumDef | number | | 5 | | | | 5 | true | |
16+
| NumDefVal | number | | 5 | | 3-7 | | 5 | true | |
17+
| NumDefInv | number | | 5 | | 10- | | | | 10 < 5 < 0 |
18+
| NumDefOpts | number | | 5 | 1,3,5,7 | 2-6 | | 5 | true | |
19+
| NumDefNotOpts | number | | 5 | 1,3,7,9 | 2-6 | | | | valid option |
20+
| NumDefInvOpt | number | | 5 | 1,3,5,7 | 6-10 | | | | 6 < 5 < 10 |
21+
| | | | | | | | | | |
22+
| StrDef | string | | hello | | | | hello | true | |
23+
| StrDefInv | string | | hello | | world | | | | regex error |
24+
| StrDefOpts | string | | a | a,b,c | | | a | true | |
25+
| StrDefNotOpts | string | | a | b,c,d | | | | | valid option |
26+
| StrDefValOpts | string | | a | a,b,c,d,e,f | [a-c] | | a | true | |
27+
| StrDefInvOpt | string | | d | a,b,c,d,e,f | [a-c] | | | | regex error |
28+
| | | | | | | | | | |
29+
| LStrDef | list(string) | | ["a"] | | | | ["a"] | true | |
30+
| LStrDefOpts | list(string) | | ["a"] | ["a"], ["b"] | | | ["a"] | true | |
31+
| LStrDefNotOpts | list(string) | | ["a"] | ["b"], ["c"] | | | | | valid option |
32+
| | | | | | | | | | |
33+
| MulDef | tag-select | | ["a"] | | | | ["a"] | true | |
34+
| MulDefOpts | multi-select | | ["a"] | a,b | | | ["a"] | true | |
35+
| MulDefNotOpts | multi-select | | ["a"] | b,c | | | | | valid option |
36+
| | | | | | | | | | |
37+
| | Input Vals | | | | | | | | |
38+
| NumIns | number | 3 | | | | | 3 | false | |
39+
| NumInsDef | number | 3 | 5 | | | | 3 | true | |
40+
| NumIns/DefInv | number | 3 | 5 | | 1-3 | | 3 | true | |
41+
| NumIns=DefInv | number | 5 | 5 | | 1-3 | | | | 1 < 5 < 3 |
42+
| NumInsOpts | number | 3 | 5 | 1,2,3,4,5 | 1-3 | | 3 | true | |
43+
| NumInsNotOptsVal | number | 3 | 5 | 1,2,4,5 | 1-3 | | 3 | true | |
44+
| NumInsNotOptsInv | number | 3 | 5 | 1,2,4,5 | 1-2 | | | true | 1 < 3 < 2 |
45+
| NumInsNotOpts | number | 3 | 5 | 1,2,4,5 | | | 3 | true | |
46+
| NumInsNotOpts/NoDef | number | 3 | | 1,2,4,5 | | | 3 | false | |
47+
| | | | | | | | | | |
48+
| StrIns | string | c | | | | | c | false | |
49+
| StrInsDef | string | c | e | | | | c | true | |
50+
| StrIns/DefInv | string | c | e | | [a-c] | | c | true | |
51+
| StrIns=DefInv | string | e | e | | [a-c] | | | | regex error |
52+
| StrInsOpts | string | c | e | a,b,c,d,e | [a-c] | | c | true | |
53+
| StrInsNotOptsVal | string | c | e | a,b,d,e | [a-c] | | c | true | |
54+
| StrInsNotOptsInv | string | c | e | a,b,d,e | [a-b] | | | | regex error |
55+
| StrInsNotOpts | string | c | e | a,b,d,e | | | c | true | |
56+
| StrInsNotOpts/NoDef | string | c | | a,b,d,e | | | c | false | |
57+
| StrInsBadVal | string | c | | a,b,c,d,e | 1-10 | | | | min cannot |
58+
| | | | | | | | | | |
59+
| | list(string) | | | | | | | | |
60+
| LStrIns | list(string) | ["c"] | | | | | ["c"] | false | |
61+
| LStrInsDef | list(string) | ["c"] | ["e"] | | | | ["c"] | true | |
62+
| LStrIns/DefInv | list(string) | ["c"] | ["e"] | | [a-c] | | | | regex cannot |
63+
| LStrInsOpts | list(string) | ["c"] | ["e"] | ["c"],["d"],["e"] | | | ["c"] | true | |
64+
| LStrInsNotOpts | list(string) | ["c"] | ["e"] | ["d"],["e"] | | | ["c"] | true | |
65+
| LStrInsNotOpts/NoDef | list(string) | ["c"] | | ["d"],["e"] | | | ["c"] | false | |
66+
| | | | | | | | | | |
67+
| MulInsOpts | multi-select | ["c"] | ["e"] | c,d,e | | | ["c"] | true | |
68+
| MulInsNotOpts | multi-select | ["c"] | ["e"] | d,e | | | ["c"] | true | |
69+
| MulInsNotOpts/NoDef | multi-select | ["c"] | | d,e | | | ["c"] | false | |
70+
| MulInsInvOpts | multi-select | ["c"] | ["e"] | c,d,e | [a-c] | | | | regex cannot |

0 commit comments

Comments
 (0)