fix: reduce size of terraform modules archive

coder · aslilac · May 12, 2025 · May 8, 2025 · May 9, 2025 · May 9, 2025
commit 144b1edc208130ba7d4c1076885eddf59c0b45fc
diff --git a/provisioner/terraform/modules.go b/provisioner/terraform/modules.go
@@ -24,12 +24,8 @@ type modulesFile struct {
 	Modules []*module `json:"Modules"`
 }
 
-func getModulesDirectory(workdir string) string {
-	return filepath.Join(workdir, ".terraform", "modules")
-}
-
 func getModulesFilePath(workdir string) string {
-	return filepath.Join(getModulesDirectory(workdir), "modules.json")
+	return filepath.Join(workdir, ".terraform", "modules", "modules.json")
 }
 
 func parseModulesFile(filePath string) ([]*proto.Module, error) {
@@ -72,52 +68,83 @@ func getModules(workdir string) ([]*proto.Module, error) {
 }
 
 func getModulesArchive(workdir string) ([]byte, error) {
-	modulesDir := getModulesDirectory(workdir)
-	if _, err := os.ReadDir(modulesDir); err != nil {
+	modulesFile, err := os.ReadFile(getModulesFilePath(workdir))
+	if err != nil {
 		if os.IsNotExist(err) {
 			return []byte{}, nil
 		}
-
-		return nil, err
+		return nil, xerrors.Errorf("failed to read modules.json: %w", err)
 	}
+	var m struct{ Modules []*proto.Module }
+	err = json.Unmarshal(modulesFile, &m)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse modules.json: %w", err)
+	}
+
 	empty := true
 	var b bytes.Buffer
 	w := tar.NewWriter(&b)
-	err := filepath.WalkDir(modulesDir, func(filePath string, info fs.DirEntry, err error) error {
-		if err != nil {
-			return xerrors.Errorf("failed to create modules archive: %w", err)
-		}
-		if info.IsDir() {
-			return nil
-		}
-		archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
-		if !found {
-			return xerrors.Errorf("walked invalid file path: %q", filePath)
-		}
 
-		content, err := os.ReadFile(filePath)
-		if err != nil {
-			return xerrors.Errorf("failed to read module file while archiving: %w", err)
+	for _, module := range m.Modules {
+		// Check to make sure that the module is a remote module fetched by
+		// Terraform. Any module that doesn't start with this path is already local,
+		// and should be part of the template files already.
+		if !strings.HasPrefix(module.Dir, ".terraform/modules/") {
+			continue
 		}
-		empty = false
-		err = w.WriteHeader(&tar.Header{
-			Name: archivePath,
-			Size: int64(len(content)),
-			Mode: 0o644,
-			Uid:  1000,
-			Gid:  1000,
+
+		err := filepath.WalkDir(filepath.Join(workdir, module.Dir), func(filePath string, info fs.DirEntry, err error) error {
+			if err != nil {
+				return xerrors.Errorf("failed to create modules archive: %w", err)
+			}
+			if info.IsDir() {
+				return nil
+			}
+			archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
+			if !found {
+				return xerrors.Errorf("walked invalid file path: %q", filePath)
+			}
+
+			content, err := os.ReadFile(filePath)
+			if err != nil {
+				return xerrors.Errorf("failed to read module file while archiving: %w", err)
+			}
+			empty = false
+			err = w.WriteHeader(&tar.Header{
+				Name: archivePath,
+				Size: int64(len(content)),
+				Mode: 0o644,
+				Uid:  1000,
+				Gid:  1000,
+			})
+			if err != nil {
+				return xerrors.Errorf("failed to add module file to archive: %w", err)
+			}
+			if _, err = w.Write(content); err != nil {
+				return xerrors.Errorf("failed to write module file to archive: %w", err)
+			}
+			return nil
 		})
 		if err != nil {
-			return xerrors.Errorf("failed to add module file to archive: %w", err)
-		}
-		if _, err = w.Write(content); err != nil {
-			return xerrors.Errorf("failed to write module file to archive: %w", err)
+			return nil, err
 		}
-		return nil
+	}
+
+	err = w.WriteHeader(&tar.Header{
+		Name: ".terraform/modules/modules.json",
+		Size: int64(len(modulesFile)),
+		Mode: 0o644,
+		Uid:  1000,
+		Gid:  1000,
 	})
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
+	}
+	_, err = w.Write(modulesFile)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
 	}
+
 	err = w.Close()
 	if err != nil {
 		return nil, xerrors.Errorf("failed to close module files archive: %w", err)
@@ -126,5 +153,9 @@ func getModulesArchive(workdir string) ([]byte, error) {
 	if empty {
 		return []byte{}, nil
 	}
+
+	if b.Len() > (10 << 20) {
+		return nil, xerrors.New("modules archive exceeds 10MB, modules will not be persisted")
+	}
 	return b.Bytes(), nil
 }
diff --git a/provisioner/terraform/modules_internal_test.go b/provisioner/terraform/modules_internal_test.go
@@ -25,9 +25,14 @@ func TestGetModulesArchive(t *testing.T) {
 	require.NoError(t, err)
 
 	// Check that all of the files it should contain are correct
-	r := bytes.NewBuffer(archive)
-	tarfs := archivefs.FromTarReader(r)
-	content, err := fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
+	b := bytes.NewBuffer(archive)
+	tarfs := archivefs.FromTarReader(b)
+
+	content, err := fs.ReadFile(tarfs, ".terraform/modules/modules.json")
+	require.NoError(t, err)
+	require.True(t, strings.HasPrefix(string(content), `{"Modules":[{"Key":"","Source":"","Dir":"."},`))
+
+	content, err = fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
 	require.NoError(t, err)
 	require.True(t, strings.HasPrefix(string(content), "terraform {"))
 	if runtime.GOOS != "windows" {
@@ -36,6 +41,9 @@ func TestGetModulesArchive(t *testing.T) {
 		require.Len(t, content, 3812)
 	}
 
+	_, err = fs.ReadFile(tarfs, ".terraform/modules/stuff_that_should_not_be_included/nothing.txt")
+	require.Error(t, err)
+
 	// It should always be byte-identical to optimize storage
 	hashBytes := sha256.Sum256(archive)
 	hash := hex.EncodeToString(hashBytes[:])

diff --git a/...a/modules-source-caching/.terraform/modules/stuff_that_should_not_be_included/nothing.txt b/...a/modules-source-caching/.terraform/modules/stuff_that_should_not_be_included/nothing.txt
@@ -0,0 +1 @@
+ここには何もありません