Skip to content

chore: track terraform modules in telemetry #15450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 34 commits into from
Nov 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
0f713ed
add the workspace_modules table, add the module column to workspace_r…
hugodutka Nov 6, 2024
82c8a3c
insert workspace modules on provisioner job completion
hugodutka Nov 6, 2024
307a57e
add foreign key to job on workspace_modules
hugodutka Nov 6, 2024
2ae16fc
don't return the root module
hugodutka Nov 7, 2024
7f2f155
add and populate the module_path column to workspace_resources
hugodutka Nov 7, 2024
863235d
TestCompleteJob - Modules WIP
hugodutka Nov 7, 2024
993f0ab
TemplateImport CompleteJob Modules test
hugodutka Nov 8, 2024
e8fbd40
TestCompleteJob Modules WorkspaceBuild
hugodutka Nov 8, 2024
5c542e5
add a test for returning modules in provision_test.go, fix test in re…
hugodutka Nov 8, 2024
9ec5ee1
make gen
hugodutka Nov 8, 2024
119a686
fix dbauthz tests
hugodutka Nov 8, 2024
3a5a025
add workspace modules and workspace resources's ModulePath to telemetry
hugodutka Nov 8, 2024
3377d22
add a workspace_modules fixture
hugodutka Nov 8, 2024
422b112
add a workspace resource's modulePath default to the e2e helper
hugodutka Nov 8, 2024
46b9b36
add modules default on PlanComplete for e2e tests
hugodutka Nov 8, 2024
f3f4d5c
lint
hugodutka Nov 8, 2024
93ed136
set module path to sentinel value when parsing failed and log an error
hugodutka Nov 13, 2024
301a153
change error string
hugodutka Nov 13, 2024
212e7d5
obfuscate workspace module source and version
hugodutka Nov 13, 2024
e9b7c46
fixes after rebase
hugodutka Nov 14, 2024
25cba6d
change migration number
hugodutka Nov 15, 2024
780730f
change fixture number
hugodutka Nov 15, 2024
b47a018
add index on created_at
hugodutka Nov 15, 2024
03be52e
add a comment about using modules from plan instead of apply
hugodutka Nov 15, 2024
00b8131
convert sentinel string to an error
hugodutka Nov 15, 2024
4187fbf
remove unnecessary comment
hugodutka Nov 15, 2024
7e82c3d
don't fail if we can't get modules from disk
hugodutka Nov 15, 2024
42e57ca
add a test for a malformed module
hugodutka Nov 16, 2024
368fc25
add a comment
hugodutka Nov 16, 2024
73b22cf
combine assignment and conditional
hugodutka Nov 16, 2024
7b9d70a
make isCoderModule into a top level function
hugodutka Nov 16, 2024
ea82313
use dbtestutil.NewDB instead of dbmem
hugodutka Nov 16, 2024
8564e9a
make gen
hugodutka Nov 16, 2024
d11fc82
combine 2 more assignments and conditionals
hugodutka Nov 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -2666,6 +2666,20 @@ func (q *querier) GetWorkspaceByWorkspaceAppID(ctx context.Context, workspaceApp
return fetch(q.log, q.auth, q.db.GetWorkspaceByWorkspaceAppID)(ctx, workspaceAppID)
}

func (q *querier) GetWorkspaceModulesByJobID(ctx context.Context, jobID uuid.UUID) ([]database.WorkspaceModule, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
Copy link
Member

@johnstcn johnstcn Nov 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: rbac.ResourceSystem is fine just for telemetry, but we'll probably want to create a separate RBAC resource in future for some of the other use-cases here. Can you create a follow-up issue for this and add a comment referencing it?

return nil, err
}
return q.db.GetWorkspaceModulesByJobID(ctx, jobID)
}

func (q *querier) GetWorkspaceModulesCreatedAfter(ctx context.Context, createdAt time.Time) ([]database.WorkspaceModule, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
}
return q.db.GetWorkspaceModulesCreatedAfter(ctx, createdAt)
}

func (q *querier) GetWorkspaceProxies(ctx context.Context) ([]database.WorkspaceProxy, error) {
return fetchWithPostFilter(q.auth, policy.ActionRead, func(ctx context.Context, _ interface{}) ([]database.WorkspaceProxy, error) {
return q.db.GetWorkspaceProxies(ctx)
Expand Down Expand Up @@ -3222,6 +3236,13 @@ func (q *querier) InsertWorkspaceBuildParameters(ctx context.Context, arg databa
return q.db.InsertWorkspaceBuildParameters(ctx, arg)
}

func (q *querier) InsertWorkspaceModule(ctx context.Context, arg database.InsertWorkspaceModuleParams) (database.WorkspaceModule, error) {
if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
return database.WorkspaceModule{}, err
}
return q.db.InsertWorkspaceModule(ctx, arg)
}

func (q *querier) InsertWorkspaceProxy(ctx context.Context, arg database.InsertWorkspaceProxyParams) (database.WorkspaceProxy, error) {
return insert(q.log, q.auth, rbac.ResourceWorkspaceProxy, q.db.InsertWorkspaceProxy)(ctx, arg)
}
Expand Down
15 changes: 15 additions & 0 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2907,6 +2907,21 @@ func (s *MethodTestSuite) TestSystemFunctions() {
}
check.Args(build.ID).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(rows)
}))
s.Run("InsertWorkspaceModule", s.Subtest(func(db database.Store, check *expects) {
j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{
Type: database.ProvisionerJobTypeWorkspaceBuild,
})
check.Args(database.InsertWorkspaceModuleParams{
JobID: j.ID,
Transition: database.WorkspaceTransitionStart,
}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
}))
s.Run("GetWorkspaceModulesByJobID", s.Subtest(func(db database.Store, check *expects) {
check.Args(uuid.New()).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetWorkspaceModulesCreatedAfter", s.Subtest(func(db database.Store, check *expects) {
check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
}

func (s *MethodTestSuite) TestNotifications() {
Expand Down
18 changes: 18 additions & 0 deletions coderd/database/dbgen/dbgen.go
Original file line number Diff line number Diff line change
Expand Up @@ -657,11 +657,29 @@ func WorkspaceResource(t testing.TB, db database.Store, orig database.WorkspaceR
Valid: takeFirst(orig.InstanceType.Valid, false),
},
DailyCost: takeFirst(orig.DailyCost, 0),
ModulePath: sql.NullString{
String: takeFirst(orig.ModulePath.String, ""),
Valid: takeFirst(orig.ModulePath.Valid, true),
},
})
require.NoError(t, err, "insert resource")
return resource
}

func WorkspaceModule(t testing.TB, db database.Store, orig database.WorkspaceModule) database.WorkspaceModule {
module, err := db.InsertWorkspaceModule(genCtx, database.InsertWorkspaceModuleParams{
ID: takeFirst(orig.ID, uuid.New()),
JobID: takeFirst(orig.JobID, uuid.New()),
Transition: takeFirst(orig.Transition, database.WorkspaceTransitionStart),
Source: takeFirst(orig.Source, "test-source"),
Version: takeFirst(orig.Version, "v1.0.0"),
Key: takeFirst(orig.Key, "test-key"),
CreatedAt: takeFirst(orig.CreatedAt, dbtime.Now()),
})
require.NoError(t, err, "insert workspace module")
return module
}

func WorkspaceResourceMetadatums(t testing.TB, db database.Store, seed database.WorkspaceResourceMetadatum) []database.WorkspaceResourceMetadatum {
meta, err := db.InsertWorkspaceResourceMetadata(genCtx, database.InsertWorkspaceResourceMetadataParams{
WorkspaceResourceID: takeFirst(seed.WorkspaceResourceID, uuid.New()),
Expand Down
43 changes: 43 additions & 0 deletions coderd/database/dbmem/dbmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func New() database.Store {
workspaceAgents: make([]database.WorkspaceAgent, 0),
provisionerJobLogs: make([]database.ProvisionerJobLog, 0),
workspaceResources: make([]database.WorkspaceResource, 0),
workspaceModules: make([]database.WorkspaceModule, 0),
workspaceResourceMetadata: make([]database.WorkspaceResourceMetadatum, 0),
provisionerJobs: make([]database.ProvisionerJob, 0),
templateVersions: make([]database.TemplateVersionTable, 0),
Expand Down Expand Up @@ -232,6 +233,7 @@ type data struct {
workspaceBuildParameters []database.WorkspaceBuildParameter
workspaceResourceMetadata []database.WorkspaceResourceMetadatum
workspaceResources []database.WorkspaceResource
workspaceModules []database.WorkspaceModule
workspaces []database.WorkspaceTable
workspaceProxies []database.WorkspaceProxy
customRoles []database.CustomRole
Expand Down Expand Up @@ -6671,6 +6673,32 @@ func (q *FakeQuerier) GetWorkspaceByWorkspaceAppID(_ context.Context, workspaceA
return database.Workspace{}, sql.ErrNoRows
}

func (q *FakeQuerier) GetWorkspaceModulesByJobID(_ context.Context, jobID uuid.UUID) ([]database.WorkspaceModule, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()

modules := make([]database.WorkspaceModule, 0)
for _, module := range q.workspaceModules {
if module.JobID == jobID {
modules = append(modules, module)
}
}
return modules, nil
}

func (q *FakeQuerier) GetWorkspaceModulesCreatedAfter(_ context.Context, createdAt time.Time) ([]database.WorkspaceModule, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()

modules := make([]database.WorkspaceModule, 0)
for _, module := range q.workspaceModules {
if module.CreatedAt.After(createdAt) {
modules = append(modules, module)
}
}
return modules, nil
}

func (q *FakeQuerier) GetWorkspaceProxies(_ context.Context) ([]database.WorkspaceProxy, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
Expand Down Expand Up @@ -8233,6 +8261,20 @@ func (q *FakeQuerier) InsertWorkspaceBuildParameters(_ context.Context, arg data
return nil
}

func (q *FakeQuerier) InsertWorkspaceModule(_ context.Context, arg database.InsertWorkspaceModuleParams) (database.WorkspaceModule, error) {
err := validateDatabaseType(arg)
if err != nil {
return database.WorkspaceModule{}, err
}

q.mutex.Lock()
defer q.mutex.Unlock()

workspaceModule := database.WorkspaceModule(arg)
q.workspaceModules = append(q.workspaceModules, workspaceModule)
return workspaceModule, nil
}

func (q *FakeQuerier) InsertWorkspaceProxy(_ context.Context, arg database.InsertWorkspaceProxyParams) (database.WorkspaceProxy, error) {
q.mutex.Lock()
defer q.mutex.Unlock()
Expand Down Expand Up @@ -8283,6 +8325,7 @@ func (q *FakeQuerier) InsertWorkspaceResource(_ context.Context, arg database.In
Hide: arg.Hide,
Icon: arg.Icon,
DailyCost: arg.DailyCost,
ModulePath: arg.ModulePath,
}
q.workspaceResources = append(q.workspaceResources, resource)
return resource, nil
Expand Down
21 changes: 21 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 45 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 17 additions & 1 deletion coderd/database/dump.sql

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions coderd/database/foreign_key_constraint.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions coderd/database/migrations/000276_workspace_modules.down.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
DROP TABLE workspace_modules;

ALTER TABLE
workspace_resources
DROP COLUMN module_path;
16 changes: 16 additions & 0 deletions coderd/database/migrations/000276_workspace_modules.up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
ALTER TABLE
workspace_resources
ADD
COLUMN module_path TEXT;

CREATE TABLE workspace_modules (
id uuid NOT NULL,
job_id uuid NOT NULL REFERENCES provisioner_jobs (id) ON DELETE CASCADE,
transition workspace_transition NOT NULL,
source TEXT NOT NULL,
version TEXT NOT NULL,
key TEXT NOT NULL,
created_at timestamp with time zone NOT NULL
);

CREATE INDEX workspace_modules_created_at_idx ON workspace_modules (created_at);
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
INSERT INTO
public.workspace_modules (
id,
job_id,
transition,
source,
version,
key,
created_at
)
VALUES
(
'5b1a722c-b8a0-40b0-a3a0-d8078fff9f6c',
'424a58cb-61d6-4627-9907-613c396c4a38',
'start',
'test-source',
'v1.0.0',
'test-key',
'2024-11-08 10:00:00+00'
);
11 changes: 11 additions & 0 deletions coderd/database/models.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading