Skip to content

Commit 6de5937

Browse files
mtojekmtojek
authored
feat: notifications: report failed workspace builds (coder#14571)
1 parent 1e5438e commit 6de5937

29 files changed

+1545
-55
lines changed

‎cli/server.go

+5
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ import (
5656
"cdr.dev/slog"
5757
"cdr.dev/slog/sloggers/sloghuman"
5858
"github.com/coder/coder/v2/coderd/entitlements"
59+
"github.com/coder/coder/v2/coderd/notifications/reports"
5960
"github.com/coder/coder/v2/coderd/runtimeconfig"
6061
"github.com/coder/pretty"
6162
"github.com/coder/quartz"
@@ -1018,6 +1019,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
10181019

10191020
// nolint:gocritic // TODO: create own role.
10201021
notificationsManager.Run(dbauthz.AsSystemRestricted(ctx))
1022+
1023+
// Run report generator to distribute periodic reports.
1024+
notificationReportGenerator := reports.NewReportGenerator(ctx, logger, options.Database, options.NotificationsEnqueuer, quartz.NewReal())
1025+
defer notificationReportGenerator.Close()
10211026
}
10221027

10231028
// Wrap the server in middleware that redirects to the access URL if

‎coderd/database/dbauthz/dbauthz.go

+28
Original file line numberDiff line numberDiff line change
@@ -1459,6 +1459,13 @@ func (q *querier) GetExternalAuthLinksByUserID(ctx context.Context, userID uuid.
14591459
return fetchWithPostFilter(q.auth, policy.ActionReadPersonal, q.db.GetExternalAuthLinksByUserID)(ctx, userID)
14601460
}
14611461

1462+
func (q *querier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
1463+
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
1464+
return nil, err
1465+
}
1466+
return q.db.GetFailedWorkspaceBuildsByTemplateID(ctx, arg)
1467+
}
1468+
14621469
func (q *querier) GetFileByHashAndCreator(ctx context.Context, arg database.GetFileByHashAndCreatorParams) (database.File, error) {
14631470
file, err := q.db.GetFileByHashAndCreator(ctx, arg)
14641471
if err != nil {
@@ -1628,6 +1635,13 @@ func (q *querier) GetNotificationMessagesByStatus(ctx context.Context, arg datab
16281635
return q.db.GetNotificationMessagesByStatus(ctx, arg)
16291636
}
16301637

1638+
func (q *querier) GetNotificationReportGeneratorLogByTemplate(ctx context.Context, arg uuid.UUID) (database.NotificationReportGeneratorLog, error) {
1639+
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
1640+
return database.NotificationReportGeneratorLog{}, err
1641+
}
1642+
return q.db.GetNotificationReportGeneratorLogByTemplate(ctx, arg)
1643+
}
1644+
16311645
func (q *querier) GetNotificationTemplateByID(ctx context.Context, id uuid.UUID) (database.NotificationTemplate, error) {
16321646
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceNotificationTemplate); err != nil {
16331647
return database.NotificationTemplate{}, err
@@ -2510,6 +2524,13 @@ func (q *querier) GetWorkspaceBuildParameters(ctx context.Context, workspaceBuil
25102524
return q.db.GetWorkspaceBuildParameters(ctx, workspaceBuildID)
25112525
}
25122526

2527+
func (q *querier) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
2528+
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
2529+
return nil, err
2530+
}
2531+
return q.db.GetWorkspaceBuildStatsByTemplates(ctx, since)
2532+
}
2533+
25132534
func (q *querier) GetWorkspaceBuildsByWorkspaceID(ctx context.Context, arg database.GetWorkspaceBuildsByWorkspaceIDParams) ([]database.WorkspaceBuild, error) {
25142535
if _, err := q.GetWorkspaceByID(ctx, arg.WorkspaceID); err != nil {
25152536
return nil, err
@@ -3966,6 +3987,13 @@ func (q *querier) UpsertLogoURL(ctx context.Context, value string) error {
39663987
return q.db.UpsertLogoURL(ctx, value)
39673988
}
39683989

3990+
func (q *querier) UpsertNotificationReportGeneratorLog(ctx context.Context, arg database.UpsertNotificationReportGeneratorLogParams) error {
3991+
if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
3992+
return err
3993+
}
3994+
return q.db.UpsertNotificationReportGeneratorLog(ctx, arg)
3995+
}
3996+
39693997
func (q *querier) UpsertNotificationsSettings(ctx context.Context, value string) error {
39703998
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil {
39713999
return err

‎coderd/database/dbauthz/dbauthz_test.go

+22
Original file line numberDiff line numberDiff line change
@@ -2819,6 +2819,28 @@ func (s *MethodTestSuite) TestSystemFunctions() {
28192819
Value: "value",
28202820
}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
28212821
}))
2822+
s.Run("GetFailedWorkspaceBuildsByTemplateID", s.Subtest(func(db database.Store, check *expects) {
2823+
check.Args(database.GetFailedWorkspaceBuildsByTemplateIDParams{
2824+
TemplateID: uuid.New(),
2825+
Since: dbtime.Now(),
2826+
}).Asserts(rbac.ResourceSystem, policy.ActionRead)
2827+
}))
2828+
s.Run("GetNotificationReportGeneratorLogByTemplate", s.Subtest(func(db database.Store, check *expects) {
2829+
_ = db.UpsertNotificationReportGeneratorLog(context.Background(), database.UpsertNotificationReportGeneratorLogParams{
2830+
NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
2831+
LastGeneratedAt: dbtime.Now(),
2832+
})
2833+
check.Args(notifications.TemplateWorkspaceBuildsFailedReport).Asserts(rbac.ResourceSystem, policy.ActionRead)
2834+
}))
2835+
s.Run("GetWorkspaceBuildStatsByTemplates", s.Subtest(func(db database.Store, check *expects) {
2836+
check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead)
2837+
}))
2838+
s.Run("UpsertNotificationReportGeneratorLog", s.Subtest(func(db database.Store, check *expects) {
2839+
check.Args(database.UpsertNotificationReportGeneratorLogParams{
2840+
NotificationTemplateID: uuid.New(),
2841+
LastGeneratedAt: dbtime.Now(),
2842+
}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
2843+
}))
28222844
}
28232845

28242846
func (s *MethodTestSuite) TestNotifications() {

‎coderd/database/dbmem/dbmem.go

+211-47
Original file line numberDiff line numberDiff line change
@@ -187,53 +187,54 @@ type data struct {
187187
userLinks []database.UserLink
188188

189189
// New tables
190-
workspaceAgentStats []database.WorkspaceAgentStat
191-
auditLogs []database.AuditLog
192-
cryptoKeys []database.CryptoKey
193-
dbcryptKeys []database.DBCryptKey
194-
files []database.File
195-
externalAuthLinks []database.ExternalAuthLink
196-
gitSSHKey []database.GitSSHKey
197-
groupMembers []database.GroupMemberTable
198-
groups []database.Group
199-
jfrogXRayScans []database.JfrogXrayScan
200-
licenses []database.License
201-
notificationMessages []database.NotificationMessage
202-
notificationPreferences []database.NotificationPreference
203-
oauth2ProviderApps []database.OAuth2ProviderApp
204-
oauth2ProviderAppSecrets []database.OAuth2ProviderAppSecret
205-
oauth2ProviderAppCodes []database.OAuth2ProviderAppCode
206-
oauth2ProviderAppTokens []database.OAuth2ProviderAppToken
207-
parameterSchemas []database.ParameterSchema
208-
provisionerDaemons []database.ProvisionerDaemon
209-
provisionerJobLogs []database.ProvisionerJobLog
210-
provisionerJobs []database.ProvisionerJob
211-
provisionerKeys []database.ProvisionerKey
212-
replicas []database.Replica
213-
templateVersions []database.TemplateVersionTable
214-
templateVersionParameters []database.TemplateVersionParameter
215-
templateVersionVariables []database.TemplateVersionVariable
216-
templateVersionWorkspaceTags []database.TemplateVersionWorkspaceTag
217-
templates []database.TemplateTable
218-
templateUsageStats []database.TemplateUsageStat
219-
workspaceAgents []database.WorkspaceAgent
220-
workspaceAgentMetadata []database.WorkspaceAgentMetadatum
221-
workspaceAgentLogs []database.WorkspaceAgentLog
222-
workspaceAgentLogSources []database.WorkspaceAgentLogSource
223-
workspaceAgentScripts []database.WorkspaceAgentScript
224-
workspaceAgentPortShares []database.WorkspaceAgentPortShare
225-
workspaceApps []database.WorkspaceApp
226-
workspaceAppStatsLastInsertID int64
227-
workspaceAppStats []database.WorkspaceAppStat
228-
workspaceBuilds []database.WorkspaceBuild
229-
workspaceBuildParameters []database.WorkspaceBuildParameter
230-
workspaceResourceMetadata []database.WorkspaceResourceMetadatum
231-
workspaceResources []database.WorkspaceResource
232-
workspaces []database.Workspace
233-
workspaceProxies []database.WorkspaceProxy
234-
customRoles []database.CustomRole
235-
provisionerJobTimings []database.ProvisionerJobTiming
236-
runtimeConfig map[string]string
190+
auditLogs []database.AuditLog
191+
cryptoKeys []database.CryptoKey
192+
dbcryptKeys []database.DBCryptKey
193+
files []database.File
194+
externalAuthLinks []database.ExternalAuthLink
195+
gitSSHKey []database.GitSSHKey
196+
groupMembers []database.GroupMemberTable
197+
groups []database.Group
198+
jfrogXRayScans []database.JfrogXrayScan
199+
licenses []database.License
200+
notificationMessages []database.NotificationMessage
201+
notificationPreferences []database.NotificationPreference
202+
notificationReportGeneratorLogs []database.NotificationReportGeneratorLog
203+
oauth2ProviderApps []database.OAuth2ProviderApp
204+
oauth2ProviderAppSecrets []database.OAuth2ProviderAppSecret
205+
oauth2ProviderAppCodes []database.OAuth2ProviderAppCode
206+
oauth2ProviderAppTokens []database.OAuth2ProviderAppToken
207+
parameterSchemas []database.ParameterSchema
208+
provisionerDaemons []database.ProvisionerDaemon
209+
provisionerJobLogs []database.ProvisionerJobLog
210+
provisionerJobs []database.ProvisionerJob
211+
provisionerKeys []database.ProvisionerKey
212+
replicas []database.Replica
213+
templateVersions []database.TemplateVersionTable
214+
templateVersionParameters []database.TemplateVersionParameter
215+
templateVersionVariables []database.TemplateVersionVariable
216+
templateVersionWorkspaceTags []database.TemplateVersionWorkspaceTag
217+
templates []database.TemplateTable
218+
templateUsageStats []database.TemplateUsageStat
219+
workspaceAgents []database.WorkspaceAgent
220+
workspaceAgentMetadata []database.WorkspaceAgentMetadatum
221+
workspaceAgentLogs []database.WorkspaceAgentLog
222+
workspaceAgentLogSources []database.WorkspaceAgentLogSource
223+
workspaceAgentPortShares []database.WorkspaceAgentPortShare
224+
workspaceAgentScripts []database.WorkspaceAgentScript
225+
workspaceAgentStats []database.WorkspaceAgentStat
226+
workspaceApps []database.WorkspaceApp
227+
workspaceAppStatsLastInsertID int64
228+
workspaceAppStats []database.WorkspaceAppStat
229+
workspaceBuilds []database.WorkspaceBuild
230+
workspaceBuildParameters []database.WorkspaceBuildParameter
231+
workspaceResourceMetadata []database.WorkspaceResourceMetadatum
232+
workspaceResources []database.WorkspaceResource
233+
workspaces []database.Workspace
234+
workspaceProxies []database.WorkspaceProxy
235+
customRoles []database.CustomRole
236+
provisionerJobTimings []database.ProvisionerJobTiming
237+
runtimeConfig map[string]string
237238
// Locks is a map of lock names. Any keys within the map are currently
238239
// locked.
239240
locks map[int64]struct{}
@@ -2621,6 +2622,75 @@ func (q *FakeQuerier) GetExternalAuthLinksByUserID(_ context.Context, userID uui
26212622
return gals, nil
26222623
}
26232624

2625+
func (q *FakeQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
2626+
err := validateDatabaseType(arg)
2627+
if err != nil {
2628+
return nil, err
2629+
}
2630+
2631+
q.mutex.RLock()
2632+
defer q.mutex.RUnlock()
2633+
2634+
workspaceBuildStats := []database.GetFailedWorkspaceBuildsByTemplateIDRow{}
2635+
for _, wb := range q.workspaceBuilds {
2636+
job, err := q.getProvisionerJobByIDNoLock(ctx, wb.JobID)
2637+
if err != nil {
2638+
return nil, xerrors.Errorf("get provisioner job by ID: %w", err)
2639+
}
2640+
2641+
if job.JobStatus != database.ProvisionerJobStatusFailed {
2642+
continue
2643+
}
2644+
2645+
if !job.CompletedAt.Valid {
2646+
continue
2647+
}
2648+
2649+
if wb.CreatedAt.Before(arg.Since) {
2650+
continue
2651+
}
2652+
2653+
w, err := q.getWorkspaceByIDNoLock(ctx, wb.WorkspaceID)
2654+
if err != nil {
2655+
return nil, xerrors.Errorf("get workspace by ID: %w", err)
2656+
}
2657+
2658+
t, err := q.getTemplateByIDNoLock(ctx, w.TemplateID)
2659+
if err != nil {
2660+
return nil, xerrors.Errorf("get template by ID: %w", err)
2661+
}
2662+
2663+
if t.ID != arg.TemplateID {
2664+
continue
2665+
}
2666+
2667+
workspaceOwner, err := q.getUserByIDNoLock(w.OwnerID)
2668+
if err != nil {
2669+
return nil, xerrors.Errorf("get user by ID: %w", err)
2670+
}
2671+
2672+
templateVersion, err := q.getTemplateVersionByIDNoLock(ctx, wb.TemplateVersionID)
2673+
if err != nil {
2674+
return nil, xerrors.Errorf("get template version by ID: %w", err)
2675+
}
2676+
2677+
workspaceBuildStats = append(workspaceBuildStats, database.GetFailedWorkspaceBuildsByTemplateIDRow{
2678+
WorkspaceName: w.Name,
2679+
WorkspaceOwnerUsername: workspaceOwner.Username,
2680+
TemplateVersionName: templateVersion.Name,
2681+
WorkspaceBuildNumber: wb.BuildNumber,
2682+
})
2683+
}
2684+
2685+
sort.Slice(workspaceBuildStats, func(i, j int) bool {
2686+
if workspaceBuildStats[i].TemplateVersionName != workspaceBuildStats[j].TemplateVersionName {
2687+
return workspaceBuildStats[i].TemplateVersionName < workspaceBuildStats[j].TemplateVersionName
2688+
}
2689+
return workspaceBuildStats[i].WorkspaceBuildNumber > workspaceBuildStats[j].WorkspaceBuildNumber
2690+
})
2691+
return workspaceBuildStats, nil
2692+
}
2693+
26242694
func (q *FakeQuerier) GetFileByHashAndCreator(_ context.Context, arg database.GetFileByHashAndCreatorParams) (database.File, error) {
26252695
if err := validateDatabaseType(arg); err != nil {
26262696
return database.File{}, err
@@ -3044,6 +3114,23 @@ func (q *FakeQuerier) GetNotificationMessagesByStatus(_ context.Context, arg dat
30443114
return out, nil
30453115
}
30463116

3117+
func (q *FakeQuerier) GetNotificationReportGeneratorLogByTemplate(_ context.Context, templateID uuid.UUID) (database.NotificationReportGeneratorLog, error) {
3118+
err := validateDatabaseType(templateID)
3119+
if err != nil {
3120+
return database.NotificationReportGeneratorLog{}, err
3121+
}
3122+
3123+
q.mutex.RLock()
3124+
defer q.mutex.RUnlock()
3125+
3126+
for _, record := range q.notificationReportGeneratorLogs {
3127+
if record.NotificationTemplateID == templateID {
3128+
return record, nil
3129+
}
3130+
}
3131+
return database.NotificationReportGeneratorLog{}, sql.ErrNoRows
3132+
}
3133+
30473134
func (*FakeQuerier) GetNotificationTemplateByID(_ context.Context, _ uuid.UUID) (database.NotificationTemplate, error) {
30483135
// Not implementing this function because it relies on state in the database which is created with migrations.
30493136
// We could consider using code-generation to align the database state and dbmem, but it's not worth it right now.
@@ -5964,6 +6051,63 @@ func (q *FakeQuerier) GetWorkspaceBuildParameters(_ context.Context, workspaceBu
59646051
return params, nil
59656052
}
59666053

6054+
func (q *FakeQuerier) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
6055+
q.mutex.RLock()
6056+
defer q.mutex.RUnlock()
6057+
6058+
templateStats := map[uuid.UUID]database.GetWorkspaceBuildStatsByTemplatesRow{}
6059+
for _, wb := range q.workspaceBuilds {
6060+
job, err := q.getProvisionerJobByIDNoLock(ctx, wb.JobID)
6061+
if err != nil {
6062+
return nil, xerrors.Errorf("get provisioner job by ID: %w", err)
6063+
}
6064+
6065+
if !job.CompletedAt.Valid {
6066+
continue
6067+
}
6068+
6069+
if wb.CreatedAt.Before(since) {
6070+
continue
6071+
}
6072+
6073+
w, err := q.getWorkspaceByIDNoLock(ctx, wb.WorkspaceID)
6074+
if err != nil {
6075+
return nil, xerrors.Errorf("get workspace by ID: %w", err)
6076+
}
6077+
6078+
if _, ok := templateStats[w.TemplateID]; !ok {
6079+
t, err := q.getTemplateByIDNoLock(ctx, w.TemplateID)
6080+
if err != nil {
6081+
return nil, xerrors.Errorf("get template by ID: %w", err)
6082+
}
6083+
6084+
templateStats[w.TemplateID] = database.GetWorkspaceBuildStatsByTemplatesRow{
6085+
TemplateID: w.TemplateID,
6086+
TemplateName: t.Name,
6087+
TemplateDisplayName: t.DisplayName,
6088+
TemplateOrganizationID: w.OrganizationID,
6089+
}
6090+
}
6091+
6092+
s := templateStats[w.TemplateID]
6093+
s.TotalBuilds++
6094+
if job.JobStatus == database.ProvisionerJobStatusFailed {
6095+
s.FailedBuilds++
6096+
}
6097+
templateStats[w.TemplateID] = s
6098+
}
6099+
6100+
rows := make([]database.GetWorkspaceBuildStatsByTemplatesRow, 0, len(templateStats))
6101+
for _, ts := range templateStats {
6102+
rows = append(rows, ts)
6103+
}
6104+
6105+
sort.Slice(rows, func(i, j int) bool {
6106+
return rows[i].TemplateName < rows[j].TemplateName
6107+
})
6108+
return rows, nil
6109+
}
6110+
59676111
func (q *FakeQuerier) GetWorkspaceBuildsByWorkspaceID(_ context.Context,
59686112
params database.GetWorkspaceBuildsByWorkspaceIDParams,
59696113
) ([]database.WorkspaceBuild, error) {
@@ -9440,6 +9584,26 @@ func (q *FakeQuerier) UpsertLogoURL(_ context.Context, data string) error {
94409584
return nil
94419585
}
94429586

9587+
func (q *FakeQuerier) UpsertNotificationReportGeneratorLog(_ context.Context, arg database.UpsertNotificationReportGeneratorLogParams) error {
9588+
err := validateDatabaseType(arg)
9589+
if err != nil {
9590+
return err
9591+
}
9592+
9593+
q.mutex.Lock()
9594+
defer q.mutex.Unlock()
9595+
9596+
for i, record := range q.notificationReportGeneratorLogs {
9597+
if arg.NotificationTemplateID == record.NotificationTemplateID {
9598+
q.notificationReportGeneratorLogs[i].LastGeneratedAt = arg.LastGeneratedAt
9599+
return nil
9600+
}
9601+
}
9602+
9603+
q.notificationReportGeneratorLogs = append(q.notificationReportGeneratorLogs, database.NotificationReportGeneratorLog(arg))
9604+
return nil
9605+
}
9606+
94439607
func (q *FakeQuerier) UpsertNotificationsSettings(_ context.Context, data string) error {
94449608
q.mutex.Lock()
94459609
defer q.mutex.Unlock()

0 commit comments

Comments
 (0)