Skip to content

Commit 67fe3ae

Browse files
mtojekmtojek
authored
feat: clean stale provisioner files (#9545)
1 parent d055f93 commit 67fe3ae

13 files changed

+594
-9
lines changed

‎Makefile

+5-1
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,7 @@ coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS)
570570
./scripts/apidocgen/generate.sh
571571
pnpm run format:write:only ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
572572

573-
update-golden-files: cli/testdata/.gen-golden helm/coder/tests/testdata/.gen-golden helm/provisioner/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden coderd/.gen-golden
573+
update-golden-files: cli/testdata/.gen-golden helm/coder/tests/testdata/.gen-golden helm/provisioner/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden coderd/.gen-golden provisioner/terraform/testdata/.gen-golden
574574
.PHONY: update-golden-files
575575

576576
cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(wildcard cli/*.tpl) $(GO_SRC_FILES) $(wildcard cli/*_test.go)
@@ -593,6 +593,10 @@ coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wil
593593
go test ./coderd -run="Test.*Golden$$" -update
594594
touch "$@"
595595

596+
provisioner/terraform/testdata/.gen-golden: $(wildcard provisioner/terraform/testdata/*/*.golden) $(GO_SRC_FILES) $(wildcard provisioner/terraform/*_test.go)
597+
go test ./provisioner/terraform -run="Test.*Golden$$" -update
598+
touch "$@"
599+
596600
scripts/ci-report/testdata/.gen-golden: $(wildcard scripts/ci-report/testdata/*) $(wildcard scripts/ci-report/*.go)
597601
go test ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
598602
touch "$@"

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ require (
196196
tailscale.com v1.46.1
197197
)
198198

199+
require github.com/djherbis/times v1.5.0
200+
199201
require (
200202
cloud.google.com/go/compute v1.23.0 // indirect
201203
cloud.google.com/go/logging v1.8.1 // indirect

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
267267
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
268268
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
269269
github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
270+
github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU=
271+
github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0=
270272
github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
271273
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
272274
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=

‎provisioner/terraform/cleanup.go

+153
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package terraform
2+
3+
import (
4+
"context"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
"time"
9+
10+
"github.com/djherbis/times"
11+
"github.com/spf13/afero"
12+
"golang.org/x/xerrors"
13+
14+
"cdr.dev/slog"
15+
)
16+
17+
// CleanStaleTerraformPlugins browses the Terraform cache directory
18+
// and remove stale plugins that haven't been used for a while.
19+
// Additionally, it sweeps empty, old directory trees.
20+
//
21+
// Sample cachePath:
22+
//
23+
// /Users/john.doe/Library/Caches/coder/provisioner-1/tf
24+
// /tmp/coder/provisioner-0/tf
25+
func CleanStaleTerraformPlugins(ctx context.Context, cachePath string, fs afero.Fs, now time.Time, logger slog.Logger) error {
26+
cachePath, err := filepath.Abs(cachePath) // sanity check in case the path is e.g. ../../../cache
27+
if err != nil {
28+
return xerrors.Errorf("unable to determine absolute path %q: %w", cachePath, err)
29+
}
30+
31+
// Firstly, check if the cache path exists.
32+
_, err = fs.Stat(cachePath)
33+
if os.IsNotExist(err) {
34+
return nil
35+
} else if err != nil {
36+
return xerrors.Errorf("unable to stat cache path %q: %w", cachePath, err)
37+
}
38+
39+
logger.Info(ctx, "clean stale Terraform plugins", slog.F("cache_path", cachePath))
40+
41+
// Filter directory trees matching pattern: <repositoryURL>/<company>/<plugin>/<version>/<distribution>
42+
filterFunc := func(path string, info os.FileInfo) bool {
43+
if !info.IsDir() {
44+
return false
45+
}
46+
47+
relativePath, err := filepath.Rel(cachePath, path)
48+
if err != nil {
49+
logger.Error(ctx, "unable to evaluate a relative path", slog.F("base", cachePath), slog.F("target", path), slog.Error(err))
50+
return false
51+
}
52+
53+
parts := strings.Split(relativePath, string(filepath.Separator))
54+
return len(parts) == 5
55+
}
56+
57+
// Review cached Terraform plugins
58+
var pluginPaths []string
59+
err = afero.Walk(fs, cachePath, func(path string, info os.FileInfo, err error) error {
60+
if err != nil {
61+
return err
62+
}
63+
64+
if !filterFunc(path, info) {
65+
return nil
66+
}
67+
68+
logger.Debug(ctx, "plugin directory discovered", slog.F("path", path))
69+
pluginPaths = append(pluginPaths, path)
70+
return nil
71+
})
72+
if err != nil {
73+
return xerrors.Errorf("unable to walk through cache directory %q: %w", cachePath, err)
74+
}
75+
76+
// Identify stale plugins
77+
var stalePlugins []string
78+
for _, pluginPath := range pluginPaths {
79+
accessTime, err := latestAccessTime(fs, pluginPath)
80+
if err != nil {
81+
return xerrors.Errorf("unable to evaluate latest access time for directory %q: %w", pluginPath, err)
82+
}
83+
84+
if accessTime.Add(staleTerraformPluginRetention).Before(now) {
85+
logger.Info(ctx, "plugin directory is stale and will be removed", slog.F("plugin_path", pluginPath))
86+
stalePlugins = append(stalePlugins, pluginPath)
87+
} else {
88+
logger.Debug(ctx, "plugin directory is not stale", slog.F("plugin_path", pluginPath))
89+
}
90+
}
91+
92+
// Remove stale plugins
93+
for _, stalePluginPath := range stalePlugins {
94+
// Remove the plugin directory
95+
err = fs.RemoveAll(stalePluginPath)
96+
if err != nil {
97+
return xerrors.Errorf("unable to remove stale plugin %q: %w", stalePluginPath, err)
98+
}
99+
100+
// Compact the plugin structure by removing empty directories.
101+
wd := stalePluginPath
102+
level := 5 // <repositoryURL>/<company>/<plugin>/<version>/<distribution>
103+
for {
104+
level--
105+
if level == 0 {
106+
break // do not compact further
107+
}
108+
109+
wd = filepath.Dir(wd)
110+
111+
files, err := afero.ReadDir(fs, wd)
112+
if err != nil {
113+
return xerrors.Errorf("unable to read directory content %q: %w", wd, err)
114+
}
115+
116+
if len(files) > 0 {
117+
break // there are still other plugins
118+
}
119+
120+
logger.Debug(ctx, "remove empty directory", slog.F("path", wd))
121+
err = fs.Remove(wd)
122+
if err != nil {
123+
return xerrors.Errorf("unable to remove directory %q: %w", wd, err)
124+
}
125+
}
126+
}
127+
return nil
128+
}
129+
130+
// latestAccessTime walks recursively through the directory content, and locates
131+
// the last accessed file.
132+
func latestAccessTime(fs afero.Fs, pluginPath string) (time.Time, error) {
133+
var latest time.Time
134+
err := afero.Walk(fs, pluginPath, func(path string, info os.FileInfo, err error) error {
135+
if err != nil {
136+
return err
137+
}
138+
139+
accessTime := info.ModTime() // fallback to modTime if accessTime is not available (afero)
140+
if info.Sys() != nil {
141+
timeSpec := times.Get(info)
142+
accessTime = timeSpec.AccessTime()
143+
}
144+
if latest.Before(accessTime) {
145+
latest = accessTime
146+
}
147+
return nil
148+
})
149+
if err != nil {
150+
return time.Time{}, xerrors.Errorf("unable to walk the plugin path %q: %w", pluginPath, err)
151+
}
152+
return latest, nil
153+
}

‎provisioner/terraform/cleanup_test.go

+186
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
//go:build linux || darwin
2+
3+
package terraform_test
4+
5+
import (
6+
"bytes"
7+
"context"
8+
"flag"
9+
"os"
10+
"path/filepath"
11+
"strings"
12+
"testing"
13+
"time"
14+
15+
"github.com/google/go-cmp/cmp"
16+
"github.com/spf13/afero"
17+
"github.com/stretchr/testify/assert"
18+
"github.com/stretchr/testify/require"
19+
20+
"cdr.dev/slog"
21+
"cdr.dev/slog/sloggers/slogtest"
22+
"github.com/coder/coder/v2/provisioner/terraform"
23+
"github.com/coder/coder/v2/testutil"
24+
)
25+
26+
const cachePath = "/tmp/coder/provisioner-0/tf"
27+
28+
// updateGoldenFiles is a flag that can be set to update golden files.
29+
var updateGoldenFiles = flag.Bool("update", false, "Update golden files")
30+
31+
var (
32+
coderPluginPath = filepath.Join("registry.terraform.io", "coder", "coder", "0.11.1", "darwin_arm64")
33+
dockerPluginPath = filepath.Join("registry.terraform.io", "kreuzwerker", "docker", "2.25.0", "darwin_arm64")
34+
)
35+
36+
func TestPluginCache_Golden(t *testing.T) {
37+
t.Parallel()
38+
39+
prepare := func() (afero.Fs, time.Time, slog.Logger) {
40+
fs := afero.NewMemMapFs()
41+
now := time.Date(2023, time.June, 3, 4, 5, 6, 0, time.UTC)
42+
logger := slogtest.Make(t, nil).
43+
Leveled(slog.LevelDebug).
44+
Named("cleanup-test")
45+
return fs, now, logger
46+
}
47+
48+
t.Run("all plugins are stale", func(t *testing.T) {
49+
t.Parallel()
50+
51+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
52+
defer cancel()
53+
54+
fs, now, logger := prepare()
55+
56+
// given
57+
// This plugin is older than 30 days.
58+
addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-63*24*time.Hour))
59+
addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-33*24*time.Hour))
60+
addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-31*24*time.Hour))
61+
addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-31*24*time.Hour))
62+
addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-43*24*time.Hour))
63+
64+
// This plugin is older than 30 days.
65+
addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour))
66+
addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-32*24*time.Hour))
67+
addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour))
68+
69+
// when
70+
terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger)
71+
72+
// then
73+
diffFileSystem(t, fs)
74+
})
75+
76+
t.Run("one plugin is stale", func(t *testing.T) {
77+
t.Parallel()
78+
79+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
80+
defer cancel()
81+
82+
fs, now, logger := prepare()
83+
84+
// given
85+
addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-2*time.Hour))
86+
addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-3*time.Hour))
87+
addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-4*time.Hour))
88+
addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-5*time.Hour))
89+
addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-4*time.Hour))
90+
91+
// This plugin is older than 30 days.
92+
addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour))
93+
addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-32*24*time.Hour))
94+
addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour))
95+
96+
// when
97+
terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger)
98+
99+
// then
100+
diffFileSystem(t, fs)
101+
})
102+
103+
t.Run("one plugin file is touched", func(t *testing.T) {
104+
t.Parallel()
105+
106+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
107+
defer cancel()
108+
109+
fs, now, logger := prepare()
110+
111+
// given
112+
addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-63*24*time.Hour))
113+
addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-33*24*time.Hour))
114+
addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-31*24*time.Hour))
115+
addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-4*time.Hour)) // touched
116+
addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-43*24*time.Hour))
117+
118+
addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour))
119+
addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-2*time.Hour))
120+
addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour))
121+
122+
// when
123+
terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger)
124+
125+
// then
126+
diffFileSystem(t, fs)
127+
})
128+
}
129+
130+
func addPluginFile(t *testing.T, fs afero.Fs, pluginPath string, resourcePath string, accessTime time.Time) {
131+
err := fs.MkdirAll(filepath.Join(cachePath, pluginPath), 0o755)
132+
require.NoError(t, err, "can't create test folder for plugin file")
133+
134+
err = fs.Chtimes(filepath.Join(cachePath, pluginPath), accessTime, accessTime)
135+
require.NoError(t, err, "can't set times")
136+
137+
err = afero.WriteFile(fs, filepath.Join(cachePath, pluginPath, resourcePath), []byte("foo"), 0o644)
138+
require.NoError(t, err, "can't create test file")
139+
140+
err = fs.Chtimes(filepath.Join(cachePath, pluginPath, resourcePath), accessTime, accessTime)
141+
require.NoError(t, err, "can't set times")
142+
}
143+
144+
func addPluginFolder(t *testing.T, fs afero.Fs, pluginPath string, folderPath string, accessTime time.Time) {
145+
err := fs.MkdirAll(filepath.Join(cachePath, pluginPath, folderPath), 0o755)
146+
require.NoError(t, err, "can't create plugin folder")
147+
148+
err = fs.Chtimes(filepath.Join(cachePath, pluginPath, folderPath), accessTime, accessTime)
149+
require.NoError(t, err, "can't set times")
150+
}
151+
152+
func diffFileSystem(t *testing.T, fs afero.Fs) {
153+
actual := dumpFileSystem(t, fs)
154+
155+
partialName := strings.Join(strings.Split(t.Name(), "/")[1:], "_")
156+
goldenFile := filepath.Join("testdata", "cleanup-stale-plugins", partialName+".txt.golden")
157+
if *updateGoldenFiles {
158+
err := os.MkdirAll(filepath.Dir(goldenFile), 0o755)
159+
require.NoError(t, err, "want no error creating golden file directory")
160+
161+
err = os.WriteFile(goldenFile, actual, 0o600)
162+
require.NoError(t, err, "want no error creating golden file")
163+
return
164+
}
165+
166+
want, err := os.ReadFile(goldenFile)
167+
require.NoError(t, err, "open golden file, run \"make update-golden-files\" and commit the changes")
168+
assert.Empty(t, cmp.Diff(want, actual), "golden file mismatch (-want +got): %s, run \"make update-golden-files\", verify and commit the changes", goldenFile)
169+
}
170+
171+
func dumpFileSystem(t *testing.T, fs afero.Fs) []byte {
172+
var buffer bytes.Buffer
173+
err := afero.Walk(fs, "/", func(path string, info os.FileInfo, err error) error {
174+
_, _ = buffer.WriteString(path)
175+
_ = buffer.WriteByte(' ')
176+
if info.IsDir() {
177+
_ = buffer.WriteByte('d')
178+
} else {
179+
_ = buffer.WriteByte('f')
180+
}
181+
_ = buffer.WriteByte('\n')
182+
return nil
183+
})
184+
require.NoError(t, err, "can't dump the file system")
185+
return buffer.Bytes()
186+
}

0 commit comments

Comments
 (0)