From fb35a81bb8bbe55622bb3ef8024569756b0e3add Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 25 Oct 2024 13:51:25 +0100 Subject: [PATCH 1/4] chore(provisioner/terraform): extract terraform parsing logic to package tfextract --- provisioner/terraform/parse.go | 176 +----------------- provisioner/terraform/tfextract/tfextract.go | 183 +++++++++++++++++++ 2 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 provisioner/terraform/tfextract/tfextract.go diff --git a/provisioner/terraform/parse.go b/provisioner/terraform/parse.go index ad55321f2e99a..fcaaffc143903 100644 --- a/provisioner/terraform/parse.go +++ b/provisioner/terraform/parse.go @@ -1,23 +1,15 @@ package terraform import ( - "context" - "encoding/json" "fmt" - "os" "path/filepath" - "slices" - "sort" "strings" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclparse" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/mitchellh/go-wordwrap" - "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/tracing" + "github.com/coder/coder/v2/provisioner/terraform/tfextract" "github.com/coder/coder/v2/provisionersdk" "github.com/coder/coder/v2/provisionersdk/proto" ) @@ -34,12 +26,12 @@ func (s *server) Parse(sess *provisionersdk.Session, _ *proto.ParseRequest, _ <- return provisionersdk.ParseErrorf("load module: %s", formatDiagnostics(sess.WorkDirectory, diags)) } - workspaceTags, err := s.loadWorkspaceTags(ctx, module) + workspaceTags, err := tfextract.WorkspaceTags(ctx, s.logger, module) if err != nil { return provisionersdk.ParseErrorf("can't load workspace tags: %v", err) } - templateVariables, err := loadTerraformVariables(module) + templateVariables, err := tfextract.LoadTerraformVariables(module) if err != nil { return provisionersdk.ParseErrorf("can't load template variables: %v", err) } @@ -50,160 +42,7 @@ func (s *server) Parse(sess *provisionersdk.Session, _ *proto.ParseRequest, _ <- } } -var rootTemplateSchema = &hcl.BodySchema{ - Blocks: []hcl.BlockHeaderSchema{ - { - Type: "data", - LabelNames: []string{"type", "name"}, - }, - }, -} - -var coderWorkspaceTagsSchema = &hcl.BodySchema{ - Attributes: []hcl.AttributeSchema{ - { - Name: "tags", - }, - }, -} - -func (s *server) loadWorkspaceTags(ctx context.Context, module *tfconfig.Module) (map[string]string, error) { - workspaceTags := map[string]string{} - - for _, dataResource := range module.DataResources { - if dataResource.Type != "coder_workspace_tags" { - s.logger.Debug(ctx, "skip resource as it is not a coder_workspace_tags", "resource_name", dataResource.Name, "resource_type", dataResource.Type) - continue - } - - var file *hcl.File - var diags hcl.Diagnostics - parser := hclparse.NewParser() - - if !strings.HasSuffix(dataResource.Pos.Filename, ".tf") { - s.logger.Debug(ctx, "only .tf files can be parsed", "filename", dataResource.Pos.Filename) - continue - } - // We know in which HCL file is the data resource defined. - file, diags = parser.ParseHCLFile(dataResource.Pos.Filename) - - if diags.HasErrors() { - return nil, xerrors.Errorf("can't parse the resource file: %s", diags.Error()) - } - - // Parse root to find "coder_workspace_tags". - content, _, diags := file.Body.PartialContent(rootTemplateSchema) - if diags.HasErrors() { - return nil, xerrors.Errorf("can't parse the resource file: %s", diags.Error()) - } - - // Iterate over blocks to locate the exact "coder_workspace_tags" data resource. - for _, block := range content.Blocks { - if !slices.Equal(block.Labels, []string{"coder_workspace_tags", dataResource.Name}) { - continue - } - - // Parse "coder_workspace_tags" to find all key-value tags. - resContent, _, diags := block.Body.PartialContent(coderWorkspaceTagsSchema) - if diags.HasErrors() { - return nil, xerrors.Errorf(`can't parse the resource coder_workspace_tags: %s`, diags.Error()) - } - - if resContent == nil { - continue // workspace tags are not present - } - - if _, ok := resContent.Attributes["tags"]; !ok { - return nil, xerrors.Errorf(`"tags" attribute is required by coder_workspace_tags`) - } - - expr := resContent.Attributes["tags"].Expr - tagsExpr, ok := expr.(*hclsyntax.ObjectConsExpr) - if !ok { - return nil, xerrors.Errorf(`"tags" attribute is expected to be a key-value map`) - } - - // Parse key-value entries in "coder_workspace_tags" - for _, tagItem := range tagsExpr.Items { - key, err := previewFileContent(tagItem.KeyExpr.Range()) - if err != nil { - return nil, xerrors.Errorf("can't preview the resource file: %v", err) - } - key = strings.Trim(key, `"`) - - value, err := previewFileContent(tagItem.ValueExpr.Range()) - if err != nil { - return nil, xerrors.Errorf("can't preview the resource file: %v", err) - } - - s.logger.Info(ctx, "workspace tag found", "key", key, "value", value) - - if _, ok := workspaceTags[key]; ok { - return nil, xerrors.Errorf(`workspace tag "%s" is defined multiple times`, key) - } - workspaceTags[key] = value - } - } - } - return workspaceTags, nil -} - -func previewFileContent(fileRange hcl.Range) (string, error) { - body, err := os.ReadFile(fileRange.Filename) - if err != nil { - return "", err - } - return string(fileRange.SliceBytes(body)), nil -} - -func loadTerraformVariables(module *tfconfig.Module) ([]*proto.TemplateVariable, error) { - // Sort variables by (filename, line) to make the ordering consistent - variables := make([]*tfconfig.Variable, 0, len(module.Variables)) - for _, v := range module.Variables { - variables = append(variables, v) - } - sort.Slice(variables, func(i, j int) bool { - return compareSourcePos(variables[i].Pos, variables[j].Pos) - }) - - var templateVariables []*proto.TemplateVariable - for _, v := range variables { - mv, err := convertTerraformVariable(v) - if err != nil { - return nil, err - } - templateVariables = append(templateVariables, mv) - } - return templateVariables, nil -} - -// Converts a Terraform variable to a template-wide variable, processed by Coder. -func convertTerraformVariable(variable *tfconfig.Variable) (*proto.TemplateVariable, error) { - var defaultData string - if variable.Default != nil { - var valid bool - defaultData, valid = variable.Default.(string) - if !valid { - defaultDataRaw, err := json.Marshal(variable.Default) - if err != nil { - return nil, xerrors.Errorf("parse variable %q default: %w", variable.Name, err) - } - defaultData = string(defaultDataRaw) - } - } - - return &proto.TemplateVariable{ - Name: variable.Name, - Description: variable.Description, - Type: variable.Type, - DefaultValue: defaultData, - // variable.Required is always false. Empty string is a valid default value, so it doesn't enforce required to be "true". - Required: variable.Default == nil, - Sensitive: variable.Sensitive, - }, nil -} - -// formatDiagnostics returns a nicely formatted string containing all of the +// FormatDiagnostics returns a nicely formatted string containing all of the // error details within the tfconfig.Diagnostics. We need to use this because // the default format doesn't provide much useful information. func formatDiagnostics(baseDir string, diags tfconfig.Diagnostics) string { @@ -246,10 +85,3 @@ func formatDiagnostics(baseDir string, diags tfconfig.Diagnostics) string { return spacer + strings.TrimSpace(msgs.String()) } - -func compareSourcePos(x, y tfconfig.SourcePos) bool { - if x.Filename != y.Filename { - return x.Filename < y.Filename - } - return x.Line < y.Line -} diff --git a/provisioner/terraform/tfextract/tfextract.go b/provisioner/terraform/tfextract/tfextract.go new file mode 100644 index 0000000000000..a4657ae079aca --- /dev/null +++ b/provisioner/terraform/tfextract/tfextract.go @@ -0,0 +1,183 @@ +package tfextract + +import ( + "context" + "encoding/json" + "os" + "slices" + "sort" + "strings" + + "github.com/coder/coder/v2/provisionersdk/proto" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/terraform-config-inspect/tfconfig" + "golang.org/x/xerrors" + + "cdr.dev/slog" +) + +// WorkspaceTags extracts tags from coder_workspace_tags data sources defined in module. +func WorkspaceTags(ctx context.Context, logger slog.Logger, module *tfconfig.Module) (map[string]string, error) { + workspaceTags := map[string]string{} + + for _, dataResource := range module.DataResources { + if dataResource.Type != "coder_workspace_tags" { + logger.Debug(ctx, "skip resource as it is not a coder_workspace_tags", "resource_name", dataResource.Name, "resource_type", dataResource.Type) + continue + } + + var file *hcl.File + var diags hcl.Diagnostics + parser := hclparse.NewParser() + + if !strings.HasSuffix(dataResource.Pos.Filename, ".tf") { + logger.Debug(ctx, "only .tf files can be parsed", "filename", dataResource.Pos.Filename) + continue + } + // We know in which HCL file is the data resource defined. + file, diags = parser.ParseHCLFile(dataResource.Pos.Filename) + + if diags.HasErrors() { + return nil, xerrors.Errorf("can't parse the resource file: %s", diags.Error()) + } + + // Parse root to find "coder_workspace_tags". + content, _, diags := file.Body.PartialContent(rootTemplateSchema) + if diags.HasErrors() { + return nil, xerrors.Errorf("can't parse the resource file: %s", diags.Error()) + } + + // Iterate over blocks to locate the exact "coder_workspace_tags" data resource. + for _, block := range content.Blocks { + if !slices.Equal(block.Labels, []string{"coder_workspace_tags", dataResource.Name}) { + continue + } + + // Parse "coder_workspace_tags" to find all key-value tags. + resContent, _, diags := block.Body.PartialContent(coderWorkspaceTagsSchema) + if diags.HasErrors() { + return nil, xerrors.Errorf(`can't parse the resource coder_workspace_tags: %s`, diags.Error()) + } + + if resContent == nil { + continue // workspace tags are not present + } + + if _, ok := resContent.Attributes["tags"]; !ok { + return nil, xerrors.Errorf(`"tags" attribute is required by coder_workspace_tags`) + } + + expr := resContent.Attributes["tags"].Expr + tagsExpr, ok := expr.(*hclsyntax.ObjectConsExpr) + if !ok { + return nil, xerrors.Errorf(`"tags" attribute is expected to be a key-value map`) + } + + // Parse key-value entries in "coder_workspace_tags" + for _, tagItem := range tagsExpr.Items { + key, err := previewFileContent(tagItem.KeyExpr.Range()) + if err != nil { + return nil, xerrors.Errorf("can't preview the resource file: %v", err) + } + key = strings.Trim(key, `"`) + + value, err := previewFileContent(tagItem.ValueExpr.Range()) + if err != nil { + return nil, xerrors.Errorf("can't preview the resource file: %v", err) + } + + logger.Info(ctx, "workspace tag found", "key", key, "value", value) + + if _, ok := workspaceTags[key]; ok { + return nil, xerrors.Errorf(`workspace tag "%s" is defined multiple times`, key) + } + workspaceTags[key] = value + } + } + } + return workspaceTags, nil +} + +var rootTemplateSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "data", + LabelNames: []string{"type", "name"}, + }, + }, +} + +var coderWorkspaceTagsSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "tags", + }, + }, +} + +func previewFileContent(fileRange hcl.Range) (string, error) { + body, err := os.ReadFile(fileRange.Filename) + if err != nil { + return "", err + } + return string(fileRange.SliceBytes(body)), nil +} + +// LoadTerraformVariables extracts all Terraform variables from module and converts them +// to template variables. The variables are sorted by source position. +func LoadTerraformVariables(module *tfconfig.Module) ([]*proto.TemplateVariable, error) { + // Sort variables by (filename, line) to make the ordering consistent + variables := make([]*tfconfig.Variable, 0, len(module.Variables)) + for _, v := range module.Variables { + variables = append(variables, v) + } + sort.Slice(variables, func(i, j int) bool { + return compareSourcePos(variables[i].Pos, variables[j].Pos) + }) + + var templateVariables []*proto.TemplateVariable + for _, v := range variables { + mv, err := convertTerraformVariable(v) + if err != nil { + return nil, err + } + templateVariables = append(templateVariables, mv) + } + return templateVariables, nil +} + +// /convertTerraformVariable converts a Terraform variable to a template-wide variable, processed by Coder. +func convertTerraformVariable(variable *tfconfig.Variable) (*proto.TemplateVariable, error) { + var defaultData string + if variable.Default != nil { + var valid bool + defaultData, valid = variable.Default.(string) + if !valid { + defaultDataRaw, err := json.Marshal(variable.Default) + if err != nil { + return nil, xerrors.Errorf("parse variable %q default: %w", variable.Name, err) + } + defaultData = string(defaultDataRaw) + } + } + + return &proto.TemplateVariable{ + Name: variable.Name, + Description: variable.Description, + Type: variable.Type, + DefaultValue: defaultData, + // variable.Required is always false. Empty string is a valid default value, so it doesn't enforce required to be "true". + Required: variable.Default == nil, + Sensitive: variable.Sensitive, + }, nil +} + +func compareSourcePos(x, y tfconfig.SourcePos) bool { + if x.Filename != y.Filename { + return x.Filename < y.Filename + } + return x.Line < y.Line +} From 4899bef8f5c5f08b47536109714775f17aa53134 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 25 Oct 2024 14:17:57 +0100 Subject: [PATCH 2/4] Apply suggestions from code review --- provisioner/terraform/tfextract/tfextract.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provisioner/terraform/tfextract/tfextract.go b/provisioner/terraform/tfextract/tfextract.go index a4657ae079aca..b9ab494e8361a 100644 --- a/provisioner/terraform/tfextract/tfextract.go +++ b/provisioner/terraform/tfextract/tfextract.go @@ -92,7 +92,7 @@ func WorkspaceTags(ctx context.Context, logger slog.Logger, module *tfconfig.Mod logger.Info(ctx, "workspace tag found", "key", key, "value", value) if _, ok := workspaceTags[key]; ok { - return nil, xerrors.Errorf(`workspace tag "%s" is defined multiple times`, key) + return nil, xerrors.Errorf(`workspace tag %q is defined multiple times`, key) } workspaceTags[key] = value } @@ -149,7 +149,7 @@ func LoadTerraformVariables(module *tfconfig.Module) ([]*proto.TemplateVariable, return templateVariables, nil } -// /convertTerraformVariable converts a Terraform variable to a template-wide variable, processed by Coder. +// convertTerraformVariable converts a Terraform variable to a template-wide variable, processed by Coder. func convertTerraformVariable(variable *tfconfig.Variable) (*proto.TemplateVariable, error) { var defaultData string if variable.Default != nil { From 2320dcc2e680eae0f494408278bb0044195393ad Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 25 Oct 2024 14:27:24 +0100 Subject: [PATCH 3/4] tfextract -> tfparse --- provisioner/terraform/parse.go | 6 +++--- provisioner/terraform/{tfextract => tfparse}/tfextract.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename provisioner/terraform/{tfextract => tfparse}/tfextract.go (99%) diff --git a/provisioner/terraform/parse.go b/provisioner/terraform/parse.go index fcaaffc143903..86dcec2e4cfeb 100644 --- a/provisioner/terraform/parse.go +++ b/provisioner/terraform/parse.go @@ -9,7 +9,7 @@ import ( "github.com/mitchellh/go-wordwrap" "github.com/coder/coder/v2/coderd/tracing" - "github.com/coder/coder/v2/provisioner/terraform/tfextract" + "github.com/coder/coder/v2/provisioner/terraform/tfparse" "github.com/coder/coder/v2/provisionersdk" "github.com/coder/coder/v2/provisionersdk/proto" ) @@ -26,12 +26,12 @@ func (s *server) Parse(sess *provisionersdk.Session, _ *proto.ParseRequest, _ <- return provisionersdk.ParseErrorf("load module: %s", formatDiagnostics(sess.WorkDirectory, diags)) } - workspaceTags, err := tfextract.WorkspaceTags(ctx, s.logger, module) + workspaceTags, err := tfparse.WorkspaceTags(ctx, s.logger, module) if err != nil { return provisionersdk.ParseErrorf("can't load workspace tags: %v", err) } - templateVariables, err := tfextract.LoadTerraformVariables(module) + templateVariables, err := tfparse.LoadTerraformVariables(module) if err != nil { return provisionersdk.ParseErrorf("can't load template variables: %v", err) } diff --git a/provisioner/terraform/tfextract/tfextract.go b/provisioner/terraform/tfparse/tfextract.go similarity index 99% rename from provisioner/terraform/tfextract/tfextract.go rename to provisioner/terraform/tfparse/tfextract.go index b9ab494e8361a..9f32391f4f900 100644 --- a/provisioner/terraform/tfextract/tfextract.go +++ b/provisioner/terraform/tfparse/tfextract.go @@ -1,4 +1,4 @@ -package tfextract +package tfparse import ( "context" From 561d03e99a236cddcb8cf38db6abef8cc4e084ee Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 25 Oct 2024 14:38:58 +0100 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Danielle Maywood --- provisioner/terraform/tfparse/tfextract.go | 1 - 1 file changed, 1 deletion(-) diff --git a/provisioner/terraform/tfparse/tfextract.go b/provisioner/terraform/tfparse/tfextract.go index 9f32391f4f900..ed85732e00d5e 100644 --- a/provisioner/terraform/tfparse/tfextract.go +++ b/provisioner/terraform/tfparse/tfextract.go @@ -39,7 +39,6 @@ func WorkspaceTags(ctx context.Context, logger slog.Logger, module *tfconfig.Mod } // We know in which HCL file is the data resource defined. file, diags = parser.ParseHCLFile(dataResource.Pos.Filename) - if diags.HasErrors() { return nil, xerrors.Errorf("can't parse the resource file: %s", diags.Error()) }