Skip to content

Commit 8c6fd63

Browse files
aslilacaslilac
committed
feat: load terraform modules when using dynamic parameters
1 parent 170f41a commit 8c6fd63

File tree

12 files changed

+374
-32
lines changed

12 files changed

+374
-32
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ site/stats/
5050
*.tfplan
5151
*.lock.hcl
5252
.terraform/
53+
!coderd/testdata/parameters/modules/.terraform/
5354
!provisioner/terraform/testdata/modules-source-caching/.terraform/
5455

5556
**/.coderv2/*

‎coderd/files/overlay.go

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package files
2+
3+
import (
4+
"io/fs"
5+
"path"
6+
"strings"
7+
8+
"golang.org/x/xerrors"
9+
)
10+
11+
// overlayFS allows you to "join" together the template files tar file fs.FS
12+
// with the Terraform modules tar file fs.FS. We could potentially turn this
13+
// into something more parameterized/configurable, but the requirements here are
14+
// a _bit_ odd, because every file in the modulesFS includes the
15+
// .terraform/modules/ folder at the beginning of it's path.
16+
type overlayFS struct {
17+
baseFS fs.FS
18+
overlays []Overlay
19+
}
20+
21+
type Overlay struct {
22+
Path string
23+
fs.FS
24+
}
25+
26+
func NewOverlayFS(baseFS fs.FS, overlays []Overlay) (fs.FS, error) {
27+
if err := valid(baseFS); err != nil {
28+
return nil, xerrors.Errorf("baseFS: %w", err)
29+
}
30+
31+
for _, overlay := range overlays {
32+
if err := valid(overlay.FS); err != nil {
33+
return nil, xerrors.Errorf("overlayFS: %w", err)
34+
}
35+
}
36+
37+
return overlayFS{
38+
baseFS: baseFS,
39+
overlays: overlays,
40+
}, nil
41+
}
42+
43+
func (f overlayFS) Open(p string) (fs.File, error) {
44+
for _, overlay := range f.overlays {
45+
if strings.HasPrefix(path.Clean(p), overlay.Path) {
46+
return overlay.FS.Open(p)
47+
}
48+
}
49+
return f.baseFS.Open(p)
50+
}
51+
52+
func (f overlayFS) ReadDir(p string) ([]fs.DirEntry, error) {
53+
for _, overlay := range f.overlays {
54+
if strings.HasPrefix(path.Clean(p), overlay.Path) {
55+
//nolint:forcetypeassert
56+
return overlay.FS.(fs.ReadDirFS).ReadDir(p)
57+
}
58+
}
59+
//nolint:forcetypeassert
60+
return f.baseFS.(fs.ReadDirFS).ReadDir(p)
61+
}
62+
63+
func (f overlayFS) ReadFile(p string) ([]byte, error) {
64+
for _, overlay := range f.overlays {
65+
if strings.HasPrefix(path.Clean(p), overlay.Path) {
66+
//nolint:forcetypeassert
67+
return overlay.FS.(fs.ReadFileFS).ReadFile(p)
68+
}
69+
}
70+
//nolint:forcetypeassert
71+
return f.baseFS.(fs.ReadFileFS).ReadFile(p)
72+
}
73+
74+
// valid checks that the fs.FS implements the required interfaces.
75+
// The fs.FS interface is not sufficient.
76+
func valid(fsys fs.FS) error {
77+
_, ok := fsys.(fs.ReadDirFS)
78+
if !ok {
79+
return xerrors.New("overlayFS does not implement ReadDirFS")
80+
}
81+
_, ok = fsys.(fs.ReadFileFS)
82+
if !ok {
83+
return xerrors.New("overlayFS does not implement ReadFileFS")
84+
}
85+
return nil
86+
}

‎coderd/files/overlay_test.go

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package files_test
2+
3+
import (
4+
"io/fs"
5+
"testing"
6+
7+
"github.com/spf13/afero"
8+
"github.com/stretchr/testify/require"
9+
10+
"github.com/coder/coder/v2/coderd/files"
11+
)
12+
13+
func TestOverlayFS(t *testing.T) {
14+
t.Parallel()
15+
16+
a := afero.NewMemMapFs()
17+
afero.WriteFile(a, "main.tf", []byte("terraform {}"), 0o644)
18+
afero.WriteFile(a, ".terraform/modules/example_module/main.tf", []byte("inaccessible"), 0o644)
19+
afero.WriteFile(a, ".terraform/modules/other_module/main.tf", []byte("inaccessible"), 0o644)
20+
b := afero.NewMemMapFs()
21+
afero.WriteFile(b, ".terraform/modules/modules.json", []byte("{}"), 0o644)
22+
afero.WriteFile(b, ".terraform/modules/example_module/main.tf", []byte("terraform {}"), 0o644)
23+
24+
it, err := files.NewOverlayFS(afero.NewIOFS(a), []files.Overlay{{
25+
Path: ".terraform/modules",
26+
FS: afero.NewIOFS(b),
27+
}})
28+
require.NoError(t, err)
29+
30+
content, err := fs.ReadFile(it, "main.tf")
31+
require.NoError(t, err)
32+
require.Equal(t, "terraform {}", string(content))
33+
34+
_, err = fs.ReadFile(it, ".terraform/modules/other_module/main.tf")
35+
require.Error(t, err)
36+
37+
content, err = fs.ReadFile(it, ".terraform/modules/modules.json")
38+
require.NoError(t, err)
39+
require.Equal(t, "{}", string(content))
40+
41+
content, err = fs.ReadFile(it, ".terraform/modules/example_module/main.tf")
42+
require.NoError(t, err)
43+
require.Equal(t, "terraform {}", string(content))
44+
}

‎coderd/parameters.go

+24-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313

1414
"github.com/coder/coder/v2/coderd/database"
1515
"github.com/coder/coder/v2/coderd/database/dbauthz"
16+
"github.com/coder/coder/v2/coderd/files"
1617
"github.com/coder/coder/v2/coderd/httpapi"
1718
"github.com/coder/coder/v2/coderd/httpmw"
1819
"github.com/coder/coder/v2/codersdk"
@@ -68,7 +69,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
6869
return
6970
}
7071

71-
fs, err := api.FileCache.Acquire(fileCtx, fileID)
72+
templateFS, err := api.FileCache.Acquire(fileCtx, fileID)
7273
if err != nil {
7374
httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
7475
Message: "Internal error fetching template version Terraform.",
@@ -85,6 +86,26 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
8586
tf, err := api.Database.GetTemplateVersionTerraformValues(ctx, templateVersion.ID)
8687
if err == nil {
8788
plan = tf.CachedPlan
89+
90+
if tf.CachedModuleFiles.Valid {
91+
moduleFilesFS, err := api.FileCache.Acquire(fileCtx, tf.CachedModuleFiles.UUID)
92+
if err != nil {
93+
httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
94+
Message: "Internal error fetching Terraform modules.",
95+
Detail: err.Error(),
96+
})
97+
return
98+
}
99+
defer api.FileCache.Release(tf.CachedModuleFiles.UUID)
100+
templateFS, err = files.NewOverlayFS(templateFS, []files.Overlay{{Path: ".terraform/modules", FS: moduleFilesFS}})
101+
if err != nil {
102+
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
103+
Message: "Internal error creating overlay filesystem.",
104+
Detail: err.Error(),
105+
})
106+
return
107+
}
108+
}
88109
} else if !xerrors.Is(err, sql.ErrNoRows) {
89110
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
90111
Message: "Failed to retrieve Terraform values for template version",
@@ -124,7 +145,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
124145
)
125146

126147
// Send an initial form state, computed without any user input.
127-
result, diagnostics := preview.Preview(ctx, input, fs)
148+
result, diagnostics := preview.Preview(ctx, input, templateFS)
128149
response := codersdk.DynamicParametersResponse{
129150
ID: -1,
130151
Diagnostics: previewtypes.Diagnostics(diagnostics),
@@ -152,7 +173,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
152173
return
153174
}
154175
input.ParameterValues = update.Inputs
155-
result, diagnostics := preview.Preview(ctx, input, fs)
176+
result, diagnostics := preview.Preview(ctx, input, templateFS)
156177
response := codersdk.DynamicParametersResponse{
157178
ID: update.ID,
158179
Diagnostics: previewtypes.Diagnostics(diagnostics),

‎coderd/parameters_test.go

+49
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/coder/coder/v2/coderd/rbac"
1111
"github.com/coder/coder/v2/codersdk"
1212
"github.com/coder/coder/v2/provisioner/echo"
13+
"github.com/coder/coder/v2/provisioner/terraform"
1314
"github.com/coder/coder/v2/provisionersdk/proto"
1415
"github.com/coder/coder/v2/testutil"
1516
"github.com/coder/websocket"
@@ -132,3 +133,51 @@ func TestDynamicParametersOwnerSSHPublicKey(t *testing.T) {
132133
require.True(t, preview.Parameters[0].Value.Valid())
133134
require.Equal(t, sshKey.PublicKey, preview.Parameters[0].Value.Value.AsString())
134135
}
136+
137+
func TestDynamicParametersWithTerraformModules(t *testing.T) {
138+
t.Parallel()
139+
140+
cfg := coderdtest.DeploymentValues(t)
141+
cfg.Experiments = []string{string(codersdk.ExperimentDynamicParameters)}
142+
ownerClient := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true, DeploymentValues: cfg})
143+
owner := coderdtest.CreateFirstUser(t, ownerClient)
144+
templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID, rbac.RoleTemplateAdmin())
145+
146+
dynamicParametersTerraformSource, err := os.ReadFile("testdata/parameters/modules/main.tf")
147+
require.NoError(t, err)
148+
modulesArchive, err := terraform.GetModulesArchive(os.DirFS("testdata/parameters/modules"))
149+
require.NoError(t, err)
150+
151+
files := echo.WithExtraFiles(map[string][]byte{
152+
"main.tf": dynamicParametersTerraformSource,
153+
})
154+
files.ProvisionPlan = []*proto.Response{{
155+
Type: &proto.Response_Plan{
156+
Plan: &proto.PlanComplete{
157+
Plan: []byte("{}"),
158+
ModuleFiles: modulesArchive,
159+
},
160+
},
161+
}}
162+
163+
version := coderdtest.CreateTemplateVersion(t, templateAdmin, owner.OrganizationID, files)
164+
coderdtest.AwaitTemplateVersionJobCompleted(t, templateAdmin, version.ID)
165+
_ = coderdtest.CreateTemplate(t, templateAdmin, owner.OrganizationID, version.ID)
166+
167+
ctx := testutil.Context(t, testutil.WaitShort)
168+
stream, err := templateAdmin.TemplateVersionDynamicParameters(ctx, templateAdminUser.ID, version.ID)
169+
require.NoError(t, err)
170+
defer stream.Close(websocket.StatusGoingAway)
171+
172+
previews := stream.Chan()
173+
174+
// Should see the output of the module represented
175+
preview := testutil.RequireReceive(ctx, t, previews)
176+
require.Equal(t, -1, preview.ID)
177+
require.Empty(t, preview.Diagnostics)
178+
179+
require.Len(t, preview.Parameters, 1)
180+
require.Equal(t, "jetbrains_ide", preview.Parameters[0].Name)
181+
require.True(t, preview.Parameters[0].Value.Valid())
182+
require.Equal(t, "CL", preview.Parameters[0].Value.AsString())
183+
}

‎coderd/testdata/parameters/modules/.terraform/modules/jetbrains_gateway/main.tf

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
terraform {
2+
required_version = ">= 1.0"
3+
4+
required_providers {
5+
coder = {
6+
source = "coder/coder"
7+
version = ">= 0.17"
8+
}
9+
}
10+
}
11+
12+
locals {
13+
jetbrains_ides = {
14+
"GO" = {
15+
icon = "/icon/goland.svg",
16+
name = "GoLand",
17+
identifier = "GO",
18+
},
19+
"WS" = {
20+
icon = "/icon/webstorm.svg",
21+
name = "WebStorm",
22+
identifier = "WS",
23+
},
24+
"IU" = {
25+
icon = "/icon/intellij.svg",
26+
name = "IntelliJ IDEA Ultimate",
27+
identifier = "IU",
28+
},
29+
"PY" = {
30+
icon = "/icon/pycharm.svg",
31+
name = "PyCharm Professional",
32+
identifier = "PY",
33+
},
34+
"CL" = {
35+
icon = "/icon/clion.svg",
36+
name = "CLion",
37+
identifier = "CL",
38+
},
39+
"PS" = {
40+
icon = "/icon/phpstorm.svg",
41+
name = "PhpStorm",
42+
identifier = "PS",
43+
},
44+
"RM" = {
45+
icon = "/icon/rubymine.svg",
46+
name = "RubyMine",
47+
identifier = "RM",
48+
},
49+
"RD" = {
50+
icon = "/icon/rider.svg",
51+
name = "Rider",
52+
identifier = "RD",
53+
},
54+
"RR" = {
55+
icon = "/icon/rustrover.svg",
56+
name = "RustRover",
57+
identifier = "RR"
58+
}
59+
}
60+
61+
icon = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].icon
62+
display_name = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].name
63+
identifier = data.coder_parameter.jetbrains_ide.value
64+
}
65+
66+
data "coder_parameter" "jetbrains_ide" {
67+
type = "string"
68+
name = "jetbrains_ide"
69+
display_name = "JetBrains IDE"
70+
icon = "/icon/gateway.svg"
71+
mutable = true
72+
default = sort(keys(local.jetbrains_ides))[0]
73+
74+
dynamic "option" {
75+
for_each = local.jetbrains_ides
76+
content {
77+
icon = option.value.icon
78+
name = option.value.name
79+
value = option.key
80+
}
81+
}
82+
}
83+
84+
output "identifier" {
85+
value = local.identifier
86+
}
87+
88+
output "display_name" {
89+
value = local.display_name
90+
}
91+
92+
output "icon" {
93+
value = local.icon
94+
}

‎coderd/testdata/parameters/modules/.terraform/modules/modules.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"jetbrains_gateway","Source":"jetbrains_gateway","Dir":".terraform/modules/jetbrains_gateway"}]}

‎coderd/testdata/parameters/modules/main.tf

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
terraform {}
2+
3+
module "jetbrains_gateway" {
4+
source = "jetbrains_gateway"
5+
}

‎provisioner/terraform/executor.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
309309

310310
graphTimings.ingest(createGraphTimingsEvent(timingGraphComplete))
311311

312-
moduleFiles, err := getModulesArchive(os.DirFS(e.workdir))
312+
moduleFiles, err := GetModulesArchive(os.DirFS(e.workdir))
313313
if err != nil {
314314
// TODO: we probably want to persist this error or make it louder eventually
315315
e.logger.Warn(ctx, "failed to archive terraform modules", slog.Error(err))

0 commit comments

Comments
 (0)