From 031d71e5ddfb99d9291bfcb6ee4639b06b0d047e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 10:28:52 +0200 Subject: [PATCH 1/8] WIP --- coderd/database/dbauthz/dbauthz.go | 38 +++--- coderd/database/dbfake/dbfake.go | 172 ++++++++++++------------ coderd/database/dbmetrics/dbmetrics.go | 14 +- coderd/database/dbmock/dbmock.go | 30 ++--- coderd/database/querier.go | 10 +- coderd/database/queries.sql.go | 174 +++++++++++++------------ coderd/database/queries/insights.sql | 18 +-- coderd/insights.go | 4 +- 8 files changed, 233 insertions(+), 227 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 6156329cf7ddd..74103e1ac462f 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -645,6 +645,25 @@ func authorizedTemplateVersionFromJob(ctx context.Context, q *querier, job datab } } +func (q *querier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + for _, templateID := range arg.TemplateIDs { + template, err := q.db.GetTemplateByID(ctx, templateID) + if err != nil { + return nil, err + } + + if err := q.authorizeContext(ctx, rbac.ActionUpdate, template); err != nil { + return nil, err + } + } + if len(arg.TemplateIDs) == 0 { + if err := q.authorizeContext(ctx, rbac.ActionUpdate, rbac.ResourceTemplate.All()); err != nil { + return nil, err + } + } + return q.db.GetTemplateInsightsByInterval(ctx, arg) +} + func (q *querier) AcquireLock(ctx context.Context, id int64) error { return q.db.AcquireLock(ctx, id) } @@ -1247,25 +1266,6 @@ func (q *querier) GetTemplateDAUs(ctx context.Context, arg database.GetTemplateD return q.db.GetTemplateDAUs(ctx, arg) } -func (q *querier) GetTemplateDailyInsights(ctx context.Context, arg database.GetTemplateDailyInsightsParams) ([]database.GetTemplateDailyInsightsRow, error) { - for _, templateID := range arg.TemplateIDs { - template, err := q.db.GetTemplateByID(ctx, templateID) - if err != nil { - return nil, err - } - - if err := q.authorizeContext(ctx, rbac.ActionUpdate, template); err != nil { - return nil, err - } - } - if len(arg.TemplateIDs) == 0 { - if err := q.authorizeContext(ctx, rbac.ActionUpdate, rbac.ResourceTemplate.All()); err != nil { - return nil, err - } - } - return q.db.GetTemplateDailyInsights(ctx, arg) -} - func (q *querier) GetTemplateInsights(ctx context.Context, arg database.GetTemplateInsightsParams) (database.GetTemplateInsightsRow, error) { for _, templateID := range arg.TemplateIDs { template, err := q.db.GetTemplateByID(ctx, templateID) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index ab7363b275a2f..82993e27f8b27 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -692,6 +692,92 @@ func minTime(t, u time.Time) time.Time { return u } +func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + err := validateDatabaseType(arg) + if err != nil { + return nil, err + } + + q.mutex.RLock() + defer q.mutex.RUnlock() + + type dailyStat struct { + startTime, endTime time.Time + userSet map[uuid.UUID]struct{} + templateIDSet map[uuid.UUID]struct{} + } + dailyStats := []dailyStat{{arg.StartTime, arg.StartTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + for dailyStats[len(dailyStats)-1].endTime.Before(arg.EndTime) { + dailyStats = append(dailyStats, dailyStat{dailyStats[len(dailyStats)-1].endTime, dailyStats[len(dailyStats)-1].endTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + } + if dailyStats[len(dailyStats)-1].endTime.After(arg.EndTime) { + dailyStats[len(dailyStats)-1].endTime = arg.EndTime + } + + for _, s := range q.workspaceAgentStats { + if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { + continue + } + if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, s.TemplateID) { + continue + } + if s.ConnectionCount == 0 { + continue + } + + for _, ds := range dailyStats { + if s.CreatedAt.Before(ds.startTime) || s.CreatedAt.Equal(ds.endTime) || s.CreatedAt.After(ds.endTime) { + continue + } + ds.userSet[s.UserID] = struct{}{} + ds.templateIDSet[s.TemplateID] = struct{}{} + } + } + + for _, s := range q.workspaceAppStats { + w, err := q.getWorkspaceByIDNoLock(ctx, s.WorkspaceID) + if err != nil { + return nil, err + } + + if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { + continue + } + + for _, ds := range dailyStats { + // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) + // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) + // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) + if !(((s.SessionStartedAt.After(ds.startTime) || s.SessionStartedAt.Equal(ds.startTime)) && s.SessionStartedAt.Before(ds.endTime)) || + (s.SessionEndedAt.After(ds.startTime) && s.SessionEndedAt.Before(ds.endTime)) || + (s.SessionStartedAt.Before(ds.startTime) && (s.SessionEndedAt.After(ds.endTime) || s.SessionEndedAt.Equal(ds.endTime)))) { + continue + } + + ds.userSet[s.UserID] = struct{}{} + ds.templateIDSet[w.TemplateID] = struct{}{} + } + } + + var result []database.GetTemplateInsightsByIntervalRow + for _, ds := range dailyStats { + templateIDs := make([]uuid.UUID, 0, len(ds.templateIDSet)) + for templateID := range ds.templateIDSet { + templateIDs = append(templateIDs, templateID) + } + slices.SortFunc(templateIDs, func(a, b uuid.UUID) int { + return slice.Ascending(a.String(), b.String()) + }) + result = append(result, database.GetTemplateInsightsByIntervalRow{ + StartTime: ds.startTime, + EndTime: ds.endTime, + TemplateIDs: templateIDs, + ActiveUsers: int64(len(ds.userSet)), + }) + } + return result, nil +} + func (*FakeQuerier) AcquireLock(_ context.Context, _ int64) error { return xerrors.New("AcquireLock must only be called within a transaction") } @@ -2340,92 +2426,6 @@ func (q *FakeQuerier) GetTemplateDAUs(_ context.Context, arg database.GetTemplat return rs, nil } -func (q *FakeQuerier) GetTemplateDailyInsights(ctx context.Context, arg database.GetTemplateDailyInsightsParams) ([]database.GetTemplateDailyInsightsRow, error) { - err := validateDatabaseType(arg) - if err != nil { - return nil, err - } - - q.mutex.RLock() - defer q.mutex.RUnlock() - - type dailyStat struct { - startTime, endTime time.Time - userSet map[uuid.UUID]struct{} - templateIDSet map[uuid.UUID]struct{} - } - dailyStats := []dailyStat{{arg.StartTime, arg.StartTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} - for dailyStats[len(dailyStats)-1].endTime.Before(arg.EndTime) { - dailyStats = append(dailyStats, dailyStat{dailyStats[len(dailyStats)-1].endTime, dailyStats[len(dailyStats)-1].endTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) - } - if dailyStats[len(dailyStats)-1].endTime.After(arg.EndTime) { - dailyStats[len(dailyStats)-1].endTime = arg.EndTime - } - - for _, s := range q.workspaceAgentStats { - if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { - continue - } - if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, s.TemplateID) { - continue - } - if s.ConnectionCount == 0 { - continue - } - - for _, ds := range dailyStats { - if s.CreatedAt.Before(ds.startTime) || s.CreatedAt.Equal(ds.endTime) || s.CreatedAt.After(ds.endTime) { - continue - } - ds.userSet[s.UserID] = struct{}{} - ds.templateIDSet[s.TemplateID] = struct{}{} - } - } - - for _, s := range q.workspaceAppStats { - w, err := q.getWorkspaceByIDNoLock(ctx, s.WorkspaceID) - if err != nil { - return nil, err - } - - if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { - continue - } - - for _, ds := range dailyStats { - // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) - // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) - // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) - if !(((s.SessionStartedAt.After(ds.startTime) || s.SessionStartedAt.Equal(ds.startTime)) && s.SessionStartedAt.Before(ds.endTime)) || - (s.SessionEndedAt.After(ds.startTime) && s.SessionEndedAt.Before(ds.endTime)) || - (s.SessionStartedAt.Before(ds.startTime) && (s.SessionEndedAt.After(ds.endTime) || s.SessionEndedAt.Equal(ds.endTime)))) { - continue - } - - ds.userSet[s.UserID] = struct{}{} - ds.templateIDSet[w.TemplateID] = struct{}{} - } - } - - var result []database.GetTemplateDailyInsightsRow - for _, ds := range dailyStats { - templateIDs := make([]uuid.UUID, 0, len(ds.templateIDSet)) - for templateID := range ds.templateIDSet { - templateIDs = append(templateIDs, templateID) - } - slices.SortFunc(templateIDs, func(a, b uuid.UUID) int { - return slice.Ascending(a.String(), b.String()) - }) - result = append(result, database.GetTemplateDailyInsightsRow{ - StartTime: ds.startTime, - EndTime: ds.endTime, - TemplateIDs: templateIDs, - ActiveUsers: int64(len(ds.userSet)), - }) - } - return result, nil -} - func (q *FakeQuerier) GetTemplateInsights(_ context.Context, arg database.GetTemplateInsightsParams) (database.GetTemplateInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go index 768c1d4adbcca..9e1efe72d716d 100644 --- a/coderd/database/dbmetrics/dbmetrics.go +++ b/coderd/database/dbmetrics/dbmetrics.go @@ -79,6 +79,13 @@ func (m metricsStore) InTx(f func(database.Store) error, options *sql.TxOptions) return err } +func (m metricsStore) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + start := time.Now() + r0, r1 := m.s.GetTemplateInsightsByInterval(ctx, arg) + m.queryLatencies.WithLabelValues("GetTemplateInsightsByInterval").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m metricsStore) AcquireLock(ctx context.Context, pgAdvisoryXactLock int64) error { start := time.Now() err := m.s.AcquireLock(ctx, pgAdvisoryXactLock) @@ -655,13 +662,6 @@ func (m metricsStore) GetTemplateDAUs(ctx context.Context, arg database.GetTempl return daus, err } -func (m metricsStore) GetTemplateDailyInsights(ctx context.Context, arg database.GetTemplateDailyInsightsParams) ([]database.GetTemplateDailyInsightsRow, error) { - start := time.Now() - r0, r1 := m.s.GetTemplateDailyInsights(ctx, arg) - m.queryLatencies.WithLabelValues("GetTemplateDailyInsights").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m metricsStore) GetTemplateInsights(ctx context.Context, arg database.GetTemplateInsightsParams) (database.GetTemplateInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateInsights(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 641dd7315b936..864b50f30f099 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -1315,21 +1315,6 @@ func (mr *MockStoreMockRecorder) GetTemplateDAUs(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateDAUs", reflect.TypeOf((*MockStore)(nil).GetTemplateDAUs), arg0, arg1) } -// GetTemplateDailyInsights mocks base method. -func (m *MockStore) GetTemplateDailyInsights(arg0 context.Context, arg1 database.GetTemplateDailyInsightsParams) ([]database.GetTemplateDailyInsightsRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTemplateDailyInsights", arg0, arg1) - ret0, _ := ret[0].([]database.GetTemplateDailyInsightsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTemplateDailyInsights indicates an expected call of GetTemplateDailyInsights. -func (mr *MockStoreMockRecorder) GetTemplateDailyInsights(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateDailyInsights", reflect.TypeOf((*MockStore)(nil).GetTemplateDailyInsights), arg0, arg1) -} - // GetTemplateGroupRoles mocks base method. func (m *MockStore) GetTemplateGroupRoles(arg0 context.Context, arg1 uuid.UUID) ([]database.TemplateGroup, error) { m.ctrl.T.Helper() @@ -1360,6 +1345,21 @@ func (mr *MockStoreMockRecorder) GetTemplateInsights(arg0, arg1 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateInsights", reflect.TypeOf((*MockStore)(nil).GetTemplateInsights), arg0, arg1) } +// GetTemplateInsightsByInterval mocks base method. +func (m *MockStore) GetTemplateInsightsByInterval(arg0 context.Context, arg1 database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTemplateInsightsByInterval", arg0, arg1) + ret0, _ := ret[0].([]database.GetTemplateInsightsByIntervalRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTemplateInsightsByInterval indicates an expected call of GetTemplateInsightsByInterval. +func (mr *MockStoreMockRecorder) GetTemplateInsightsByInterval(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateInsightsByInterval", reflect.TypeOf((*MockStore)(nil).GetTemplateInsightsByInterval), arg0, arg1) +} + // GetTemplateParameterInsights mocks base method. func (m *MockStore) GetTemplateParameterInsights(arg0 context.Context, arg1 database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 63c1f7321dc15..25e42dd5a669d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -124,15 +124,15 @@ type sqlcQuerier interface { GetTemplateByID(ctx context.Context, id uuid.UUID) (Template, error) GetTemplateByOrganizationAndName(ctx context.Context, arg GetTemplateByOrganizationAndNameParams) (Template, error) GetTemplateDAUs(ctx context.Context, arg GetTemplateDAUsParams) ([]GetTemplateDAUsRow, error) - // GetTemplateDailyInsights returns all daily intervals between start and end - // time, if end time is a partial day, it will be included in the results and - // that interval will be less than 24 hours. If there is no data for a selected - // interval/template, it will be included in the results with 0 active users. - GetTemplateDailyInsights(ctx context.Context, arg GetTemplateDailyInsightsParams) ([]GetTemplateDailyInsightsRow, error) // GetTemplateInsights has a granularity of 5 minutes where if a session/app was // in use during a minute, we will add 5 minutes to the total usage for that // session/app (per user). GetTemplateInsights(ctx context.Context, arg GetTemplateInsightsParams) (GetTemplateInsightsRow, error) + // GetTemplateInsightsByInterval returns all intervals between start and end + // time, if end time is a partial interval, it will be included in the results and + // that interval will be shorter than a full one. If there is no data for a selected + // interval/template, it will be included in the results with 0 active users. + GetTemplateInsightsByInterval(ctx context.Context, arg GetTemplateInsightsByIntervalParams) ([]GetTemplateInsightsByIntervalRow, error) // GetTemplateParameterInsights does for each template in a given timeframe, // look for the latest workspace build (for every workspace) that has been // created in the timeframe and return the aggregate usage counts of parameter diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 67b7b782b9e29..07f0091e33765 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1738,19 +1738,85 @@ func (q *sqlQuerier) GetTemplateAppInsights(ctx context.Context, arg GetTemplate return items, nil } -const getTemplateDailyInsights = `-- name: GetTemplateDailyInsights :many +const getTemplateInsights = `-- name: GetTemplateInsights :one +WITH agent_stats_by_interval_and_user AS ( + SELECT + date_trunc('minute', was.created_at), + was.user_id, + array_agg(was.template_id) AS template_ids, + CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds, + CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds, + CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds, + CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds + FROM workspace_agent_stats was + WHERE + was.created_at >= $1::timestamptz + AND was.created_at < $2::timestamptz + AND was.connection_count > 0 + AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN was.template_id = ANY($3::uuid[]) ELSE TRUE END + GROUP BY date_trunc('minute', was.created_at), was.user_id +), template_ids AS ( + SELECT array_agg(DISTINCT template_id) AS ids + FROM agent_stats_by_interval_and_user, unnest(template_ids) template_id + WHERE template_id IS NOT NULL +) + +SELECT + COALESCE((SELECT ids FROM template_ids), '{}')::uuid[] AS template_ids, + -- Return IDs so we can combine this with GetTemplateAppInsights. + COALESCE(array_agg(DISTINCT user_id), '{}')::uuid[] AS active_user_ids, + COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds, + COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds, + COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds, + COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds +FROM agent_stats_by_interval_and_user +` + +type GetTemplateInsightsParams struct { + StartTime time.Time `db:"start_time" json:"start_time"` + EndTime time.Time `db:"end_time" json:"end_time"` + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` +} + +type GetTemplateInsightsRow struct { + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` + ActiveUserIDs []uuid.UUID `db:"active_user_ids" json:"active_user_ids"` + UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"` + UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"` + UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"` + UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"` +} + +// GetTemplateInsights has a granularity of 5 minutes where if a session/app was +// in use during a minute, we will add 5 minutes to the total usage for that +// session/app (per user). +func (q *sqlQuerier) GetTemplateInsights(ctx context.Context, arg GetTemplateInsightsParams) (GetTemplateInsightsRow, error) { + row := q.db.QueryRowContext(ctx, getTemplateInsights, arg.StartTime, arg.EndTime, pq.Array(arg.TemplateIDs)) + var i GetTemplateInsightsRow + err := row.Scan( + pq.Array(&i.TemplateIDs), + pq.Array(&i.ActiveUserIDs), + &i.UsageVscodeSeconds, + &i.UsageJetbrainsSeconds, + &i.UsageReconnectingPtySeconds, + &i.UsageSshSeconds, + ) + return i, err +} + +const getTemplateInsightsByInterval = `-- name: GetTemplateInsightsByInterval :many WITH ts AS ( SELECT d::timestamptz AS from_, CASE - WHEN (d::timestamptz + '1 day'::interval) <= $1::timestamptz - THEN (d::timestamptz + '1 day'::interval) - ELSE $1::timestamptz + WHEN (d::timestamptz + $1::interval) <= $2::timestamptz + THEN (d::timestamptz + $1::interval) + ELSE $2::timestamptz END AS to_ FROM -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series($2::timestamptz, ($1::timestamptz) - '1 second'::interval, '1 day'::interval) AS d -), unflattened_usage_by_day AS ( + generate_series($3::timestamptz, ($2::timestamptz) - '1 second'::interval, $1::interval) AS d +), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by -- the combination of GetTemplateInsights and GetTemplateAppInsights. We use @@ -1767,7 +1833,7 @@ WITH ts AS ( was.created_at >= ts.from_ AND was.created_at < ts.to_ AND was.connection_count > 0 - AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN was.template_id = ANY($3::uuid[]) ELSE TRUE END + AND CASE WHEN COALESCE(array_length($4::uuid[], 1), 0) > 0 THEN was.template_id = ANY($4::uuid[]) ELSE TRUE END ) GROUP BY ts.from_, ts.to_, was.template_id, was.user_id @@ -1785,7 +1851,7 @@ WITH ts AS ( ) JOIN workspaces w ON ( w.id = was.workspace_id - AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN w.template_id = ANY($3::uuid[]) ELSE TRUE END + AND CASE WHEN COALESCE(array_length($4::uuid[], 1), 0) > 0 THEN w.template_id = ANY($4::uuid[]) ELSE TRUE END ) GROUP BY ts.from_, ts.to_, w.template_id, was.user_id ) @@ -1795,36 +1861,42 @@ SELECT to_ AS end_time, array_remove(array_agg(DISTINCT template_id), NULL)::uuid[] AS template_ids, COUNT(DISTINCT user_id) AS active_users -FROM unflattened_usage_by_day +FROM unflattened_usage_by_interval GROUP BY from_, to_ ` -type GetTemplateDailyInsightsParams struct { +type GetTemplateInsightsByIntervalParams struct { + Interval int64 `db:"interval" json:"interval"` EndTime time.Time `db:"end_time" json:"end_time"` StartTime time.Time `db:"start_time" json:"start_time"` TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` } -type GetTemplateDailyInsightsRow struct { +type GetTemplateInsightsByIntervalRow struct { StartTime time.Time `db:"start_time" json:"start_time"` EndTime time.Time `db:"end_time" json:"end_time"` TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` ActiveUsers int64 `db:"active_users" json:"active_users"` } -// GetTemplateDailyInsights returns all daily intervals between start and end -// time, if end time is a partial day, it will be included in the results and -// that interval will be less than 24 hours. If there is no data for a selected +// GetTemplateInsightsByInterval returns all intervals between start and end +// time, if end time is a partial interval, it will be included in the results and +// that interval will be shorter than a full one. If there is no data for a selected // interval/template, it will be included in the results with 0 active users. -func (q *sqlQuerier) GetTemplateDailyInsights(ctx context.Context, arg GetTemplateDailyInsightsParams) ([]GetTemplateDailyInsightsRow, error) { - rows, err := q.db.QueryContext(ctx, getTemplateDailyInsights, arg.EndTime, arg.StartTime, pq.Array(arg.TemplateIDs)) +func (q *sqlQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg GetTemplateInsightsByIntervalParams) ([]GetTemplateInsightsByIntervalRow, error) { + rows, err := q.db.QueryContext(ctx, getTemplateInsightsByInterval, + arg.Interval, + arg.EndTime, + arg.StartTime, + pq.Array(arg.TemplateIDs), + ) if err != nil { return nil, err } defer rows.Close() - var items []GetTemplateDailyInsightsRow + var items []GetTemplateInsightsByIntervalRow for rows.Next() { - var i GetTemplateDailyInsightsRow + var i GetTemplateInsightsByIntervalRow if err := rows.Scan( &i.StartTime, &i.EndTime, @@ -1844,72 +1916,6 @@ func (q *sqlQuerier) GetTemplateDailyInsights(ctx context.Context, arg GetTempla return items, nil } -const getTemplateInsights = `-- name: GetTemplateInsights :one -WITH agent_stats_by_interval_and_user AS ( - SELECT - date_trunc('minute', was.created_at), - was.user_id, - array_agg(was.template_id) AS template_ids, - CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds, - CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds, - CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds, - CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds - FROM workspace_agent_stats was - WHERE - was.created_at >= $1::timestamptz - AND was.created_at < $2::timestamptz - AND was.connection_count > 0 - AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN was.template_id = ANY($3::uuid[]) ELSE TRUE END - GROUP BY date_trunc('minute', was.created_at), was.user_id -), template_ids AS ( - SELECT array_agg(DISTINCT template_id) AS ids - FROM agent_stats_by_interval_and_user, unnest(template_ids) template_id - WHERE template_id IS NOT NULL -) - -SELECT - COALESCE((SELECT ids FROM template_ids), '{}')::uuid[] AS template_ids, - -- Return IDs so we can combine this with GetTemplateAppInsights. - COALESCE(array_agg(DISTINCT user_id), '{}')::uuid[] AS active_user_ids, - COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds, - COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds, - COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds, - COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds -FROM agent_stats_by_interval_and_user -` - -type GetTemplateInsightsParams struct { - StartTime time.Time `db:"start_time" json:"start_time"` - EndTime time.Time `db:"end_time" json:"end_time"` - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` -} - -type GetTemplateInsightsRow struct { - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` - ActiveUserIDs []uuid.UUID `db:"active_user_ids" json:"active_user_ids"` - UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"` - UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"` - UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"` - UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"` -} - -// GetTemplateInsights has a granularity of 5 minutes where if a session/app was -// in use during a minute, we will add 5 minutes to the total usage for that -// session/app (per user). -func (q *sqlQuerier) GetTemplateInsights(ctx context.Context, arg GetTemplateInsightsParams) (GetTemplateInsightsRow, error) { - row := q.db.QueryRowContext(ctx, getTemplateInsights, arg.StartTime, arg.EndTime, pq.Array(arg.TemplateIDs)) - var i GetTemplateInsightsRow - err := row.Scan( - pq.Array(&i.TemplateIDs), - pq.Array(&i.ActiveUserIDs), - &i.UsageVscodeSeconds, - &i.UsageJetbrainsSeconds, - &i.UsageReconnectingPtySeconds, - &i.UsageSshSeconds, - ) - return i, err -} - const getTemplateParameterInsights = `-- name: GetTemplateParameterInsights :many WITH latest_workspace_builds AS ( SELECT diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index d76d106edd5d1..53bc01a758dd4 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -113,23 +113,23 @@ SELECT FROM app_stats_by_user_and_agent GROUP BY access_method, slug_or_port, display_name, icon, is_app; --- name: GetTemplateDailyInsights :many --- GetTemplateDailyInsights returns all daily intervals between start and end --- time, if end time is a partial day, it will be included in the results and --- that interval will be less than 24 hours. If there is no data for a selected +-- name: GetTemplateInsightsByInterval :many +-- GetTemplateInsightsByInterval returns all intervals between start and end +-- time, if end time is a partial interval, it will be included in the results and +-- that interval will be shorter than a full one. If there is no data for a selected -- interval/template, it will be included in the results with 0 active users. WITH ts AS ( SELECT d::timestamptz AS from_, CASE - WHEN (d::timestamptz + '1 day'::interval) <= @end_time::timestamptz - THEN (d::timestamptz + '1 day'::interval) + WHEN (d::timestamptz + @interval::interval) <= @end_time::timestamptz + THEN (d::timestamptz + @interval::interval) ELSE @end_time::timestamptz END AS to_ FROM -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 second'::interval, '1 day'::interval) AS d -), unflattened_usage_by_day AS ( + generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 second'::interval, @interval::interval) AS d +), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by -- the combination of GetTemplateInsights and GetTemplateAppInsights. We use @@ -174,7 +174,7 @@ SELECT to_ AS end_time, array_remove(array_agg(DISTINCT template_id), NULL)::uuid[] AS template_ids, COUNT(DISTINCT user_id) AS active_users -FROM unflattened_usage_by_day +FROM unflattened_usage_by_interval GROUP BY from_, to_; -- name: GetTemplateParameterInsights :many diff --git a/coderd/insights.go b/coderd/insights.go index e19f95d40dc0c..13c4feff64b15 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -191,7 +191,7 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { var usage database.GetTemplateInsightsRow var appUsage []database.GetTemplateAppInsightsRow - var dailyUsage []database.GetTemplateDailyInsightsRow + var dailyUsage []database.GetTemplateInsightsByIntervalRow var parameterRows []database.GetTemplateParameterInsightsRow eg, egCtx := errgroup.WithContext(ctx) @@ -203,7 +203,7 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { eg.Go(func() error { var err error if interval != "" { - dailyUsage, err = api.Database.GetTemplateDailyInsights(egCtx, database.GetTemplateDailyInsightsParams{ + dailyUsage, err = api.Database.GetTemplateInsightsByInterval(egCtx, database.GetTemplateInsightsByIntervalParams{ StartTime: startTime, EndTime: endTime, TemplateIDs: templateIDs, From 0311f1eaf1e5e57e98d1041e245c6d8da5e8ef3e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 10:30:26 +0200 Subject: [PATCH 2/8] WIP format --- coderd/database/dbauthz/dbauthz.go | 38 +++--- coderd/database/dbfake/dbfake.go | 172 ++++++++++++------------- coderd/database/dbmetrics/dbmetrics.go | 14 +- 3 files changed, 112 insertions(+), 112 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 74103e1ac462f..b3e514c350faa 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -645,25 +645,6 @@ func authorizedTemplateVersionFromJob(ctx context.Context, q *querier, job datab } } -func (q *querier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { - for _, templateID := range arg.TemplateIDs { - template, err := q.db.GetTemplateByID(ctx, templateID) - if err != nil { - return nil, err - } - - if err := q.authorizeContext(ctx, rbac.ActionUpdate, template); err != nil { - return nil, err - } - } - if len(arg.TemplateIDs) == 0 { - if err := q.authorizeContext(ctx, rbac.ActionUpdate, rbac.ResourceTemplate.All()); err != nil { - return nil, err - } - } - return q.db.GetTemplateInsightsByInterval(ctx, arg) -} - func (q *querier) AcquireLock(ctx context.Context, id int64) error { return q.db.AcquireLock(ctx, id) } @@ -1285,6 +1266,25 @@ func (q *querier) GetTemplateInsights(ctx context.Context, arg database.GetTempl return q.db.GetTemplateInsights(ctx, arg) } +func (q *querier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + for _, templateID := range arg.TemplateIDs { + template, err := q.db.GetTemplateByID(ctx, templateID) + if err != nil { + return nil, err + } + + if err := q.authorizeContext(ctx, rbac.ActionUpdate, template); err != nil { + return nil, err + } + } + if len(arg.TemplateIDs) == 0 { + if err := q.authorizeContext(ctx, rbac.ActionUpdate, rbac.ResourceTemplate.All()); err != nil { + return nil, err + } + } + return q.db.GetTemplateInsightsByInterval(ctx, arg) +} + func (q *querier) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { for _, templateID := range arg.TemplateIDs { template, err := q.db.GetTemplateByID(ctx, templateID) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index 82993e27f8b27..36b3abcaf508d 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -692,92 +692,6 @@ func minTime(t, u time.Time) time.Time { return u } -func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { - err := validateDatabaseType(arg) - if err != nil { - return nil, err - } - - q.mutex.RLock() - defer q.mutex.RUnlock() - - type dailyStat struct { - startTime, endTime time.Time - userSet map[uuid.UUID]struct{} - templateIDSet map[uuid.UUID]struct{} - } - dailyStats := []dailyStat{{arg.StartTime, arg.StartTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} - for dailyStats[len(dailyStats)-1].endTime.Before(arg.EndTime) { - dailyStats = append(dailyStats, dailyStat{dailyStats[len(dailyStats)-1].endTime, dailyStats[len(dailyStats)-1].endTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) - } - if dailyStats[len(dailyStats)-1].endTime.After(arg.EndTime) { - dailyStats[len(dailyStats)-1].endTime = arg.EndTime - } - - for _, s := range q.workspaceAgentStats { - if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { - continue - } - if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, s.TemplateID) { - continue - } - if s.ConnectionCount == 0 { - continue - } - - for _, ds := range dailyStats { - if s.CreatedAt.Before(ds.startTime) || s.CreatedAt.Equal(ds.endTime) || s.CreatedAt.After(ds.endTime) { - continue - } - ds.userSet[s.UserID] = struct{}{} - ds.templateIDSet[s.TemplateID] = struct{}{} - } - } - - for _, s := range q.workspaceAppStats { - w, err := q.getWorkspaceByIDNoLock(ctx, s.WorkspaceID) - if err != nil { - return nil, err - } - - if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { - continue - } - - for _, ds := range dailyStats { - // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) - // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) - // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) - if !(((s.SessionStartedAt.After(ds.startTime) || s.SessionStartedAt.Equal(ds.startTime)) && s.SessionStartedAt.Before(ds.endTime)) || - (s.SessionEndedAt.After(ds.startTime) && s.SessionEndedAt.Before(ds.endTime)) || - (s.SessionStartedAt.Before(ds.startTime) && (s.SessionEndedAt.After(ds.endTime) || s.SessionEndedAt.Equal(ds.endTime)))) { - continue - } - - ds.userSet[s.UserID] = struct{}{} - ds.templateIDSet[w.TemplateID] = struct{}{} - } - } - - var result []database.GetTemplateInsightsByIntervalRow - for _, ds := range dailyStats { - templateIDs := make([]uuid.UUID, 0, len(ds.templateIDSet)) - for templateID := range ds.templateIDSet { - templateIDs = append(templateIDs, templateID) - } - slices.SortFunc(templateIDs, func(a, b uuid.UUID) int { - return slice.Ascending(a.String(), b.String()) - }) - result = append(result, database.GetTemplateInsightsByIntervalRow{ - StartTime: ds.startTime, - EndTime: ds.endTime, - TemplateIDs: templateIDs, - ActiveUsers: int64(len(ds.userSet)), - }) - } - return result, nil -} - func (*FakeQuerier) AcquireLock(_ context.Context, _ int64) error { return xerrors.New("AcquireLock must only be called within a transaction") } @@ -2495,6 +2409,92 @@ func (q *FakeQuerier) GetTemplateInsights(_ context.Context, arg database.GetTem return result, nil } +func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + err := validateDatabaseType(arg) + if err != nil { + return nil, err + } + + q.mutex.RLock() + defer q.mutex.RUnlock() + + type dailyStat struct { + startTime, endTime time.Time + userSet map[uuid.UUID]struct{} + templateIDSet map[uuid.UUID]struct{} + } + dailyStats := []dailyStat{{arg.StartTime, arg.StartTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + for dailyStats[len(dailyStats)-1].endTime.Before(arg.EndTime) { + dailyStats = append(dailyStats, dailyStat{dailyStats[len(dailyStats)-1].endTime, dailyStats[len(dailyStats)-1].endTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + } + if dailyStats[len(dailyStats)-1].endTime.After(arg.EndTime) { + dailyStats[len(dailyStats)-1].endTime = arg.EndTime + } + + for _, s := range q.workspaceAgentStats { + if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { + continue + } + if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, s.TemplateID) { + continue + } + if s.ConnectionCount == 0 { + continue + } + + for _, ds := range dailyStats { + if s.CreatedAt.Before(ds.startTime) || s.CreatedAt.Equal(ds.endTime) || s.CreatedAt.After(ds.endTime) { + continue + } + ds.userSet[s.UserID] = struct{}{} + ds.templateIDSet[s.TemplateID] = struct{}{} + } + } + + for _, s := range q.workspaceAppStats { + w, err := q.getWorkspaceByIDNoLock(ctx, s.WorkspaceID) + if err != nil { + return nil, err + } + + if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { + continue + } + + for _, ds := range dailyStats { + // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) + // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) + // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) + if !(((s.SessionStartedAt.After(ds.startTime) || s.SessionStartedAt.Equal(ds.startTime)) && s.SessionStartedAt.Before(ds.endTime)) || + (s.SessionEndedAt.After(ds.startTime) && s.SessionEndedAt.Before(ds.endTime)) || + (s.SessionStartedAt.Before(ds.startTime) && (s.SessionEndedAt.After(ds.endTime) || s.SessionEndedAt.Equal(ds.endTime)))) { + continue + } + + ds.userSet[s.UserID] = struct{}{} + ds.templateIDSet[w.TemplateID] = struct{}{} + } + } + + var result []database.GetTemplateInsightsByIntervalRow + for _, ds := range dailyStats { + templateIDs := make([]uuid.UUID, 0, len(ds.templateIDSet)) + for templateID := range ds.templateIDSet { + templateIDs = append(templateIDs, templateID) + } + slices.SortFunc(templateIDs, func(a, b uuid.UUID) int { + return slice.Ascending(a.String(), b.String()) + }) + result = append(result, database.GetTemplateInsightsByIntervalRow{ + StartTime: ds.startTime, + EndTime: ds.endTime, + TemplateIDs: templateIDs, + ActiveUsers: int64(len(ds.userSet)), + }) + } + return result, nil +} + func (q *FakeQuerier) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go index 9e1efe72d716d..13aa97d9af8ae 100644 --- a/coderd/database/dbmetrics/dbmetrics.go +++ b/coderd/database/dbmetrics/dbmetrics.go @@ -79,13 +79,6 @@ func (m metricsStore) InTx(f func(database.Store) error, options *sql.TxOptions) return err } -func (m metricsStore) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { - start := time.Now() - r0, r1 := m.s.GetTemplateInsightsByInterval(ctx, arg) - m.queryLatencies.WithLabelValues("GetTemplateInsightsByInterval").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m metricsStore) AcquireLock(ctx context.Context, pgAdvisoryXactLock int64) error { start := time.Now() err := m.s.AcquireLock(ctx, pgAdvisoryXactLock) @@ -669,6 +662,13 @@ func (m metricsStore) GetTemplateInsights(ctx context.Context, arg database.GetT return r0, r1 } +func (m metricsStore) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) { + start := time.Now() + r0, r1 := m.s.GetTemplateInsightsByInterval(ctx, arg) + m.queryLatencies.WithLabelValues("GetTemplateInsightsByInterval").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m metricsStore) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateParameterInsights(ctx, arg) From 45e6e95e94c853c25cfc161743255971c7606362 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 10:51:34 +0200 Subject: [PATCH 3/8] override interval --- coderd/database/queries.sql.go | 8 ++++---- coderd/database/sqlc.yaml | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 07f0091e33765..e3832eee7085b 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1866,10 +1866,10 @@ GROUP BY from_, to_ ` type GetTemplateInsightsByIntervalParams struct { - Interval int64 `db:"interval" json:"interval"` - EndTime time.Time `db:"end_time" json:"end_time"` - StartTime time.Time `db:"start_time" json:"start_time"` - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` + Interval time.Duration `db:"interval" json:"interval"` + EndTime time.Time `db:"end_time" json:"end_time"` + StartTime time.Time `db:"start_time" json:"start_time"` + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` } type GetTemplateInsightsByIntervalRow struct { diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 1bdc972927f6f..a31936c5322af 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -27,6 +27,11 @@ overrides: - column: "template_with_users.group_acl" go_type: type: "TemplateACL" + - db_type: "pg_catalog.interval" + engine: "postgresql" + go_type: + import: "time" + type: "Duration" rename: template: TemplateTable template_with_user: Template From 90def679fc0bf2f6402d539419039e8060cb2881 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 11:25:26 +0200 Subject: [PATCH 4/8] Use interval --- coderd/database/dbfake/dbfake.go | 19 ++++++++++--------- coderd/insights.go | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index 36b3abcaf508d..d8c18c023683a 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -2418,17 +2418,18 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat q.mutex.RLock() defer q.mutex.RUnlock() - type dailyStat struct { + type statByInterval struct { startTime, endTime time.Time userSet map[uuid.UUID]struct{} templateIDSet map[uuid.UUID]struct{} } - dailyStats := []dailyStat{{arg.StartTime, arg.StartTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} - for dailyStats[len(dailyStats)-1].endTime.Before(arg.EndTime) { - dailyStats = append(dailyStats, dailyStat{dailyStats[len(dailyStats)-1].endTime, dailyStats[len(dailyStats)-1].endTime.AddDate(0, 0, 1), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + + statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(arg.Interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + for statsByInterval[len(statsByInterval)-1].endTime.Before(arg.EndTime) { + statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(arg.Interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) } - if dailyStats[len(dailyStats)-1].endTime.After(arg.EndTime) { - dailyStats[len(dailyStats)-1].endTime = arg.EndTime + if statsByInterval[len(statsByInterval)-1].endTime.After(arg.EndTime) { + statsByInterval[len(statsByInterval)-1].endTime = arg.EndTime } for _, s := range q.workspaceAgentStats { @@ -2442,7 +2443,7 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat continue } - for _, ds := range dailyStats { + for _, ds := range statsByInterval { if s.CreatedAt.Before(ds.startTime) || s.CreatedAt.Equal(ds.endTime) || s.CreatedAt.After(ds.endTime) { continue } @@ -2461,7 +2462,7 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat continue } - for _, ds := range dailyStats { + for _, ds := range statsByInterval { // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) @@ -2477,7 +2478,7 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat } var result []database.GetTemplateInsightsByIntervalRow - for _, ds := range dailyStats { + for _, ds := range statsByInterval { templateIDs := make([]uuid.UUID, 0, len(ds.templateIDSet)) for templateID := range ds.templateIDSet { templateIDs = append(templateIDs, templateID) diff --git a/coderd/insights.go b/coderd/insights.go index 13c4feff64b15..c91d428d2ffd2 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -207,6 +207,7 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { StartTime: startTime, EndTime: endTime, TemplateIDs: templateIDs, + Interval: 24 * time.Hour, }) if err != nil { return xerrors.Errorf("get template daily insights: %w", err) From e957d16f22b2516fd256a4a8f8d28267a87257e7 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 11:57:46 +0200 Subject: [PATCH 5/8] Revert to int64 --- coderd/database/dbfake/dbfake.go | 4 ++-- coderd/database/queries.sql.go | 8 ++++---- coderd/database/sqlc.yaml | 5 ----- coderd/insights.go | 2 +- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index d8c18c023683a..b4b66dced68fb 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -2424,9 +2424,9 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat templateIDSet map[uuid.UUID]struct{} } - statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(arg.Interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(time.Duration(arg.Interval)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} for statsByInterval[len(statsByInterval)-1].endTime.Before(arg.EndTime) { - statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(arg.Interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(time.Duration(arg.Interval)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) } if statsByInterval[len(statsByInterval)-1].endTime.After(arg.EndTime) { statsByInterval[len(statsByInterval)-1].endTime = arg.EndTime diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index e3832eee7085b..07f0091e33765 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1866,10 +1866,10 @@ GROUP BY from_, to_ ` type GetTemplateInsightsByIntervalParams struct { - Interval time.Duration `db:"interval" json:"interval"` - EndTime time.Time `db:"end_time" json:"end_time"` - StartTime time.Time `db:"start_time" json:"start_time"` - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` + Interval int64 `db:"interval" json:"interval"` + EndTime time.Time `db:"end_time" json:"end_time"` + StartTime time.Time `db:"start_time" json:"start_time"` + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` } type GetTemplateInsightsByIntervalRow struct { diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index a31936c5322af..1bdc972927f6f 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -27,11 +27,6 @@ overrides: - column: "template_with_users.group_acl" go_type: type: "TemplateACL" - - db_type: "pg_catalog.interval" - engine: "postgresql" - go_type: - import: "time" - type: "Duration" rename: template: TemplateTable template_with_user: Template diff --git a/coderd/insights.go b/coderd/insights.go index c91d428d2ffd2..2ec1fa5af7e58 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -207,7 +207,7 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { StartTime: startTime, EndTime: endTime, TemplateIDs: templateIDs, - Interval: 24 * time.Hour, + Interval: int64(24 * time.Hour), }) if err != nil { return xerrors.Errorf("get template daily insights: %w", err) From 22a91a451fba64ca48262341cdaa5d37e01ddfcd Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 12:28:05 +0200 Subject: [PATCH 6/8] Use IntervalDays --- coderd/database/dbfake/dbfake.go | 6 ++++-- coderd/database/queries.sql.go | 16 ++++++++-------- coderd/database/queries/insights.sql | 6 +++--- coderd/insights.go | 8 ++++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index b4b66dced68fb..bb81a6057616d 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -2424,9 +2424,11 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat templateIDSet map[uuid.UUID]struct{} } - statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(time.Duration(arg.Interval)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + interval := time.Duration(arg.IntervalDays) * 24 * time.Hour + + statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} for statsByInterval[len(statsByInterval)-1].endTime.Before(arg.EndTime) { - statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(time.Duration(arg.Interval)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) } if statsByInterval[len(statsByInterval)-1].endTime.After(arg.EndTime) { statsByInterval[len(statsByInterval)-1].endTime = arg.EndTime diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 07f0091e33765..bb4869d25e9b8 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1809,13 +1809,13 @@ WITH ts AS ( SELECT d::timestamptz AS from_, CASE - WHEN (d::timestamptz + $1::interval) <= $2::timestamptz - THEN (d::timestamptz + $1::interval) + WHEN (d::timestamptz + ($1::int || ' day')::interval) <= $2::timestamptz + THEN (d::timestamptz + ($1::int || ' day')::interval) ELSE $2::timestamptz END AS to_ FROM -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series($3::timestamptz, ($2::timestamptz) - '1 second'::interval, $1::interval) AS d + generate_series($3::timestamptz, ($2::timestamptz) - '1 second'::interval, ($1::int || ' day')::interval) AS d ), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by @@ -1866,10 +1866,10 @@ GROUP BY from_, to_ ` type GetTemplateInsightsByIntervalParams struct { - Interval int64 `db:"interval" json:"interval"` - EndTime time.Time `db:"end_time" json:"end_time"` - StartTime time.Time `db:"start_time" json:"start_time"` - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` + IntervalDays int32 `db:"interval_days" json:"interval_days"` + EndTime time.Time `db:"end_time" json:"end_time"` + StartTime time.Time `db:"start_time" json:"start_time"` + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` } type GetTemplateInsightsByIntervalRow struct { @@ -1885,7 +1885,7 @@ type GetTemplateInsightsByIntervalRow struct { // interval/template, it will be included in the results with 0 active users. func (q *sqlQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg GetTemplateInsightsByIntervalParams) ([]GetTemplateInsightsByIntervalRow, error) { rows, err := q.db.QueryContext(ctx, getTemplateInsightsByInterval, - arg.Interval, + arg.IntervalDays, arg.EndTime, arg.StartTime, pq.Array(arg.TemplateIDs), diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 53bc01a758dd4..6bb5c3d9fcdf2 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -122,13 +122,13 @@ WITH ts AS ( SELECT d::timestamptz AS from_, CASE - WHEN (d::timestamptz + @interval::interval) <= @end_time::timestamptz - THEN (d::timestamptz + @interval::interval) + WHEN (d::timestamptz + (@interval_days::int || ' day')::interval) <= @end_time::timestamptz + THEN (d::timestamptz + (@interval_days::int || ' day')::interval) ELSE @end_time::timestamptz END AS to_ FROM -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 second'::interval, @interval::interval) AS d + generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 second'::interval, (@interval_days::int || ' day')::interval) AS d ), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by diff --git a/coderd/insights.go b/coderd/insights.go index 2ec1fa5af7e58..6b2c121033e8c 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -204,10 +204,10 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { var err error if interval != "" { dailyUsage, err = api.Database.GetTemplateInsightsByInterval(egCtx, database.GetTemplateInsightsByIntervalParams{ - StartTime: startTime, - EndTime: endTime, - TemplateIDs: templateIDs, - Interval: int64(24 * time.Hour), + StartTime: startTime, + EndTime: endTime, + TemplateIDs: templateIDs, + IntervalDays: 1, }) if err != nil { return xerrors.Errorf("get template daily insights: %w", err) From 68fe81f66f64f3798307ed1d3953925aba43b977 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 13:23:51 +0200 Subject: [PATCH 7/8] microsecond --- coderd/database/queries.sql.go | 4 ++-- coderd/database/queries/insights.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index bb4869d25e9b8..ffc468eb63862 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1814,8 +1814,8 @@ WITH ts AS ( ELSE $2::timestamptz END AS to_ FROM - -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series($3::timestamptz, ($2::timestamptz) - '1 second'::interval, ($1::int || ' day')::interval) AS d + -- Subtract 1 microsecond from end_time to avoid including the next interval in the results. + generate_series($3::timestamptz, ($2::timestamptz) - '1 microsecond'::interval, ($1::int || ' day')::interval) AS d ), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 6bb5c3d9fcdf2..dc69b69de2eef 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -127,8 +127,8 @@ WITH ts AS ( ELSE @end_time::timestamptz END AS to_ FROM - -- Subtract 1 second from end_time to avoid including the next interval in the results. - generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 second'::interval, (@interval_days::int || ' day')::interval) AS d + -- Subtract 1 microsecond from end_time to avoid including the next interval in the results. + generate_series(@start_time::timestamptz, (@end_time::timestamptz) - '1 microsecond'::interval, (@interval_days::int || ' day')::interval) AS d ), unflattened_usage_by_interval AS ( -- We select data from both workspace agent stats and workspace app stats to -- get a complete picture of usage. This matches how usage is calculated by From 2211e4970822205568886d148a04087e2fa69232 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 15 Sep 2023 13:46:58 +0200 Subject: [PATCH 8/8] fix: AddDate --- coderd/database/dbfake/dbfake.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index bb81a6057616d..252e8772b44ee 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -2424,11 +2424,9 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat templateIDSet map[uuid.UUID]struct{} } - interval := time.Duration(arg.IntervalDays) * 24 * time.Hour - - statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.Add(interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} + statsByInterval := []statByInterval{{arg.StartTime, arg.StartTime.AddDate(0, 0, int(arg.IntervalDays)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}} for statsByInterval[len(statsByInterval)-1].endTime.Before(arg.EndTime) { - statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.Add(interval), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) + statsByInterval = append(statsByInterval, statByInterval{statsByInterval[len(statsByInterval)-1].endTime, statsByInterval[len(statsByInterval)-1].endTime.AddDate(0, 0, int(arg.IntervalDays)), make(map[uuid.UUID]struct{}), make(map[uuid.UUID]struct{})}) } if statsByInterval[len(statsByInterval)-1].endTime.After(arg.EndTime) { statsByInterval[len(statsByInterval)-1].endTime = arg.EndTime