Skip to content

Commit 97ce44a

Browse files
hugodutkahugodutka
authored
chore: track terraform module source type in telemetry (coder#15590)
1 parent fbe2fa6 commit 97ce44a

File tree

2 files changed

+170
-2
lines changed

2 files changed

+170
-2
lines changed

‎coderd/telemetry/telemetry.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"net/http"
1212
"net/url"
1313
"os"
14+
"regexp"
1415
"runtime"
1516
"slices"
1617
"strings"
@@ -680,9 +681,95 @@ func shouldSendRawModuleSource(source string) bool {
680681
return strings.Contains(source, "registry.coder.com")
681682
}
682683

684+
// ModuleSourceType is the type of source for a module.
685+
// For reference, see https://developer.hashicorp.com/terraform/language/modules/sources
686+
type ModuleSourceType string
687+
688+
const (
689+
ModuleSourceTypeLocal ModuleSourceType = "local"
690+
ModuleSourceTypeLocalAbs ModuleSourceType = "local_absolute"
691+
ModuleSourceTypePublicRegistry ModuleSourceType = "public_registry"
692+
ModuleSourceTypePrivateRegistry ModuleSourceType = "private_registry"
693+
ModuleSourceTypeCoderRegistry ModuleSourceType = "coder_registry"
694+
ModuleSourceTypeGitHub ModuleSourceType = "github"
695+
ModuleSourceTypeBitbucket ModuleSourceType = "bitbucket"
696+
ModuleSourceTypeGit ModuleSourceType = "git"
697+
ModuleSourceTypeMercurial ModuleSourceType = "mercurial"
698+
ModuleSourceTypeHTTP ModuleSourceType = "http"
699+
ModuleSourceTypeS3 ModuleSourceType = "s3"
700+
ModuleSourceTypeGCS ModuleSourceType = "gcs"
701+
ModuleSourceTypeUnknown ModuleSourceType = "unknown"
702+
)
703+
704+
// Terraform supports a variety of module source types, like:
705+
// - local paths (./ or ../)
706+
// - absolute local paths (/)
707+
// - git URLs (git:: or git@)
708+
// - http URLs
709+
// - s3 URLs
710+
//
711+
// and more!
712+
//
713+
// See https://developer.hashicorp.com/terraform/language/modules/sources for an overview.
714+
//
715+
// This function attempts to classify the source type of a module. It's imperfect,
716+
// as checks that terraform actually does are pretty complicated.
717+
// See e.g. https://github.com/hashicorp/go-getter/blob/842d6c379e5e70d23905b8f6b5a25a80290acb66/detect.go#L47
718+
// if you're interested in the complexity.
719+
func GetModuleSourceType(source string) ModuleSourceType {
720+
source = strings.TrimSpace(source)
721+
source = strings.ToLower(source)
722+
if strings.HasPrefix(source, "./") || strings.HasPrefix(source, "../") {
723+
return ModuleSourceTypeLocal
724+
}
725+
if strings.HasPrefix(source, "/") {
726+
return ModuleSourceTypeLocalAbs
727+
}
728+
// Match public registry modules in the format <NAMESPACE>/<NAME>/<PROVIDER>
729+
// Sources can have a `//...` suffix, which signifies a subdirectory.
730+
// The allowed characters are based on
731+
// https://developer.hashicorp.com/terraform/cloud-docs/api-docs/private-registry/modules#request-body-1
732+
// because Hashicorp's documentation about module sources doesn't mention it.
733+
if matched, _ := regexp.MatchString(`^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+(//.*)?$`, source); matched {
734+
return ModuleSourceTypePublicRegistry
735+
}
736+
if strings.Contains(source, "github.com") {
737+
return ModuleSourceTypeGitHub
738+
}
739+
if strings.Contains(source, "bitbucket.org") {
740+
return ModuleSourceTypeBitbucket
741+
}
742+
if strings.HasPrefix(source, "git::") || strings.HasPrefix(source, "git@") {
743+
return ModuleSourceTypeGit
744+
}
745+
if strings.HasPrefix(source, "hg::") {
746+
return ModuleSourceTypeMercurial
747+
}
748+
if strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") {
749+
return ModuleSourceTypeHTTP
750+
}
751+
if strings.HasPrefix(source, "s3::") {
752+
return ModuleSourceTypeS3
753+
}
754+
if strings.HasPrefix(source, "gcs::") {
755+
return ModuleSourceTypeGCS
756+
}
757+
if strings.Contains(source, "registry.terraform.io") {
758+
return ModuleSourceTypePublicRegistry
759+
}
760+
if strings.Contains(source, "app.terraform.io") || strings.Contains(source, "localterraform.com") {
761+
return ModuleSourceTypePrivateRegistry
762+
}
763+
if strings.Contains(source, "registry.coder.com") {
764+
return ModuleSourceTypeCoderRegistry
765+
}
766+
return ModuleSourceTypeUnknown
767+
}
768+
683769
func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
684770
source := module.Source
685771
version := module.Version
772+
sourceType := GetModuleSourceType(source)
686773
if !shouldSendRawModuleSource(source) {
687774
source = fmt.Sprintf("%x", sha256.Sum256([]byte(source)))
688775
version = fmt.Sprintf("%x", sha256.Sum256([]byte(version)))
@@ -694,6 +781,7 @@ func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
694781
Transition: module.Transition,
695782
Source: source,
696783
Version: version,
784+
SourceType: sourceType,
697785
Key: module.Key,
698786
CreatedAt: module.CreatedAt,
699787
}
@@ -938,6 +1026,7 @@ type WorkspaceModule struct {
9381026
Key string `json:"key"`
9391027
Version string `json:"version"`
9401028
Source string `json:"source"`
1029+
SourceType ModuleSourceType `json:"source_type"`
9411030
}
9421031

9431032
type WorkspaceAgent struct {

‎coderd/telemetry/telemetry_test.go

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ func TestTelemetry(t *testing.T) {
133133
})
134134
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
135135
JobID: pj.ID,
136-
Source: "internal-url.com/some-module",
136+
Source: "https://internal-url.com/some-module",
137137
Version: "1.0.0",
138138
})
139139
_, snapshot := collectSnapshot(t, db, nil)
@@ -142,10 +142,89 @@ func TestTelemetry(t *testing.T) {
142142
sort.Slice(modules, func(i, j int) bool {
143143
return modules[i].Source < modules[j].Source
144144
})
145-
require.Equal(t, modules[0].Source, "921c61d6f3eef5118f3cae658d1518b378c5b02a4955a766c791440894d989c5")
145+
require.Equal(t, modules[0].Source, "ed662ec0396db67e77119f14afcb9253574cc925b04a51d4374bcb1eae299f5d")
146146
require.Equal(t, modules[0].Version, "92521fc3cbd964bdc9f584a991b89fddaa5754ed1cc96d6d42445338669c1305")
147+
require.Equal(t, modules[0].SourceType, telemetry.ModuleSourceTypeHTTP)
147148
require.Equal(t, modules[1].Source, "registry.coder.com/terraform/aws")
148149
require.Equal(t, modules[1].Version, "1.0.0")
150+
require.Equal(t, modules[1].SourceType, telemetry.ModuleSourceTypeCoderRegistry)
151+
})
152+
t.Run("ModuleSourceType", func(t *testing.T) {
153+
t.Parallel()
154+
cases := []struct {
155+
source string
156+
want telemetry.ModuleSourceType
157+
}{
158+
// Local relative paths
159+
{source: "./modules/terraform-aws-vpc", want: telemetry.ModuleSourceTypeLocal},
160+
{source: "../shared/modules/vpc", want: telemetry.ModuleSourceTypeLocal},
161+
{source: " ./my-module ", want: telemetry.ModuleSourceTypeLocal}, // with whitespace
162+
163+
// Local absolute paths
164+
{source: "/opt/terraform/modules/vpc", want: telemetry.ModuleSourceTypeLocalAbs},
165+
{source: "/Users/dev/modules/app", want: telemetry.ModuleSourceTypeLocalAbs},
166+
{source: "/etc/terraform/modules/network", want: telemetry.ModuleSourceTypeLocalAbs},
167+
168+
// Public registry
169+
{source: "hashicorp/consul/aws", want: telemetry.ModuleSourceTypePublicRegistry},
170+
{source: "registry.terraform.io/hashicorp/aws", want: telemetry.ModuleSourceTypePublicRegistry},
171+
{source: "terraform-aws-modules/vpc/aws", want: telemetry.ModuleSourceTypePublicRegistry},
172+
{source: "hashicorp/consul/aws//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
173+
{source: "hashicorp/co-nsul/aw_s//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
174+
175+
// Private registry
176+
{source: "app.terraform.io/company/vpc/aws", want: telemetry.ModuleSourceTypePrivateRegistry},
177+
{source: "localterraform.com/org/module", want: telemetry.ModuleSourceTypePrivateRegistry},
178+
{source: "APP.TERRAFORM.IO/test/module", want: telemetry.ModuleSourceTypePrivateRegistry}, // case insensitive
179+
180+
// Coder registry
181+
{source: "registry.coder.com/terraform/aws", want: telemetry.ModuleSourceTypeCoderRegistry},
182+
{source: "registry.coder.com/modules/base", want: telemetry.ModuleSourceTypeCoderRegistry},
183+
{source: "REGISTRY.CODER.COM/test/module", want: telemetry.ModuleSourceTypeCoderRegistry}, // case insensitive
184+
185+
// GitHub
186+
{source: "github.com/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeGitHub},
187+
{source: "git::https://github.com/org/repo.git", want: telemetry.ModuleSourceTypeGitHub},
188+
{source: "git::https://github.com/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeGitHub},
189+
190+
// Bitbucket
191+
{source: "bitbucket.org/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeBitbucket},
192+
{source: "git::https://bitbucket.org/org/repo.git", want: telemetry.ModuleSourceTypeBitbucket},
193+
{source: "https://bitbucket.org/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeBitbucket},
194+
195+
// Generic Git
196+
{source: "git::ssh://git.internal.com/repo.git", want: telemetry.ModuleSourceTypeGit},
197+
{source: "git@gitlab.com:org/repo.git", want: telemetry.ModuleSourceTypeGit},
198+
{source: "git::https://git.internal.com/repo.git?ref=v1.0.0", want: telemetry.ModuleSourceTypeGit},
199+
200+
// Mercurial
201+
{source: "hg::https://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
202+
{source: "hg::http://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
203+
{source: "hg::ssh://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
204+
205+
// HTTP
206+
{source: "https://example.com/vpc-module.zip", want: telemetry.ModuleSourceTypeHTTP},
207+
{source: "http://example.com/modules/vpc", want: telemetry.ModuleSourceTypeHTTP},
208+
{source: "https://internal.network/terraform/modules", want: telemetry.ModuleSourceTypeHTTP},
209+
210+
// S3
211+
{source: "s3::https://s3-eu-west-1.amazonaws.com/bucket/vpc", want: telemetry.ModuleSourceTypeS3},
212+
{source: "s3::https://bucket.s3.amazonaws.com/vpc", want: telemetry.ModuleSourceTypeS3},
213+
{source: "s3::http://bucket.s3.amazonaws.com/vpc?version=1", want: telemetry.ModuleSourceTypeS3},
214+
215+
// GCS
216+
{source: "gcs::https://www.googleapis.com/storage/v1/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
217+
{source: "gcs::https://storage.googleapis.com/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
218+
{source: "gcs::https://bucket.storage.googleapis.com/vpc", want: telemetry.ModuleSourceTypeGCS},
219+
220+
// Unknown
221+
{source: "custom://example.com/vpc", want: telemetry.ModuleSourceTypeUnknown},
222+
{source: "something-random", want: telemetry.ModuleSourceTypeUnknown},
223+
{source: "", want: telemetry.ModuleSourceTypeUnknown},
224+
}
225+
for _, c := range cases {
226+
require.Equal(t, c.want, telemetry.GetModuleSourceType(c.source))
227+
}
149228
})
150229
}
151230

0 commit comments

Comments
 (0)