diff --git a/cli/server.go b/cli/server.go index 9f33ced438f84..2a33b5ae2e90d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -78,6 +78,7 @@ import ( "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/oauthpki" "github.com/coder/coder/v2/coderd/prometheusmetrics" + "github.com/coder/coder/v2/coderd/prometheusmetrics/insights" "github.com/coder/coder/v2/coderd/schedule" "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/coderd/tracing" @@ -198,6 +199,21 @@ func enablePrometheus( } afterCtx(ctx, closeWorkspacesFunc) + insightsMetricsCollector, err := insights.NewMetricsCollector(options.Database, options.Logger, 0) + if err != nil { + return nil, xerrors.Errorf("unable to initialize insights metrics collector: %w", err) + } + err = options.PrometheusRegistry.Register(insightsMetricsCollector) + if err != nil { + return nil, xerrors.Errorf("unable to register insights metrics collector: %w", err) + } + + closeInsightsMetricsCollector, err := insightsMetricsCollector.Run(ctx) + if err != nil { + return nil, xerrors.Errorf("unable to run insights metrics collector: %w", err) + } + afterCtx(ctx, closeInsightsMetricsCollector) + if vals.Prometheus.CollectAgentStats { closeAgentStatsFunc, err := prometheusmetrics.AgentStats(ctx, logger, options.PrometheusRegistry, options.Database, time.Now(), 0) if err != nil { diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 038f4e0c92807..9eaacab556ab9 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1324,6 +1324,10 @@ func (q *querier) GetTemplateInsightsByInterval(ctx context.Context, arg databas return q.db.GetTemplateInsightsByInterval(ctx, arg) } +func (q *querier) GetTemplateInsightsByTemplate(ctx context.Context, arg database.GetTemplateInsightsByTemplateParams) ([]database.GetTemplateInsightsByTemplateRow, error) { + panic("not implemented") +} + func (q *querier) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { for _, templateID := range arg.TemplateIDs { template, err := q.db.GetTemplateByID(ctx, templateID) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index bffd855da6b6b..9a135789f66c3 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -2500,6 +2500,10 @@ func (q *FakeQuerier) GetTemplateInsights(_ context.Context, arg database.GetTem templateIDSet := make(map[uuid.UUID]struct{}) appUsageIntervalsByUser := make(map[uuid.UUID]map[time.Time]*database.GetTemplateInsightsRow) + + q.mutex.RLock() + defer q.mutex.RUnlock() + for _, s := range q.workspaceAgentStats { if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { continue @@ -2648,6 +2652,101 @@ func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg dat return result, nil } +func (q *FakeQuerier) GetTemplateInsightsByTemplate(_ context.Context, arg database.GetTemplateInsightsByTemplateParams) ([]database.GetTemplateInsightsByTemplateRow, error) { + err := validateDatabaseType(arg) + if err != nil { + return nil, err + } + + q.mutex.RLock() + defer q.mutex.RUnlock() + + // map time.Time x TemplateID x UserID x + appUsageByTemplateAndUser := map[time.Time]map[uuid.UUID]map[uuid.UUID]database.GetTemplateInsightsByTemplateRow{} + + // Review agent stats in terms of usage + templateIDSet := make(map[uuid.UUID]struct{}) + + for _, s := range q.workspaceAgentStats { + if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) { + continue + } + if s.ConnectionCount == 0 { + continue + } + + t := s.CreatedAt.Truncate(time.Minute) + templateIDSet[s.TemplateID] = struct{}{} + + if _, ok := appUsageByTemplateAndUser[t]; !ok { + appUsageByTemplateAndUser[t] = make(map[uuid.UUID]map[uuid.UUID]database.GetTemplateInsightsByTemplateRow) + } + + if _, ok := appUsageByTemplateAndUser[t][s.TemplateID]; !ok { + appUsageByTemplateAndUser[t][s.TemplateID] = make(map[uuid.UUID]database.GetTemplateInsightsByTemplateRow) + } + + if _, ok := appUsageByTemplateAndUser[t][s.TemplateID][s.UserID]; !ok { + appUsageByTemplateAndUser[t][s.TemplateID][s.UserID] = database.GetTemplateInsightsByTemplateRow{} + } + + u := appUsageByTemplateAndUser[t][s.TemplateID][s.UserID] + if s.SessionCountJetBrains > 0 { + u.UsageJetbrainsSeconds = 60 + } + if s.SessionCountVSCode > 0 { + u.UsageVscodeSeconds = 60 + } + if s.SessionCountReconnectingPTY > 0 { + u.UsageReconnectingPtySeconds = 60 + } + if s.SessionCountSSH > 0 { + u.UsageSshSeconds = 60 + } + appUsageByTemplateAndUser[t][s.TemplateID][s.UserID] = u + } + + // Sort used templates + templateIDs := make([]uuid.UUID, 0, len(templateIDSet)) + for templateID := range templateIDSet { + templateIDs = append(templateIDs, templateID) + } + slices.SortFunc(templateIDs, func(a, b uuid.UUID) int { + return slice.Ascending(a.String(), b.String()) + }) + + // Build result + var result []database.GetTemplateInsightsByTemplateRow + for _, templateID := range templateIDs { + r := database.GetTemplateInsightsByTemplateRow{ + TemplateID: templateID, + } + + uniqueUsers := map[uuid.UUID]struct{}{} + + for _, mTemplateUserUsage := range appUsageByTemplateAndUser { + mUserUsage, ok := mTemplateUserUsage[templateID] + if !ok { + continue // template was not used in this time window + } + + for userID, usage := range mUserUsage { + uniqueUsers[userID] = struct{}{} + + r.UsageJetbrainsSeconds += usage.UsageJetbrainsSeconds + r.UsageVscodeSeconds += usage.UsageVscodeSeconds + r.UsageReconnectingPtySeconds += usage.UsageReconnectingPtySeconds + r.UsageSshSeconds += usage.UsageSshSeconds + } + } + + r.ActiveUsers = int64(len(uniqueUsers)) + + result = append(result, r) + } + return result, nil +} + func (q *FakeQuerier) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go index ece7020139b0f..50df68878e892 100644 --- a/coderd/database/dbmetrics/dbmetrics.go +++ b/coderd/database/dbmetrics/dbmetrics.go @@ -704,6 +704,13 @@ func (m metricsStore) GetTemplateInsightsByInterval(ctx context.Context, arg dat return r0, r1 } +func (m metricsStore) GetTemplateInsightsByTemplate(ctx context.Context, arg database.GetTemplateInsightsByTemplateParams) ([]database.GetTemplateInsightsByTemplateRow, error) { + start := time.Now() + r0, r1 := m.s.GetTemplateInsightsByTemplate(ctx, arg) + m.queryLatencies.WithLabelValues("GetTemplateInsightsByTemplate").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m metricsStore) GetTemplateParameterInsights(ctx context.Context, arg database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateParameterInsights(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 31614be3ae919..616230d2e0fc1 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -1433,6 +1433,21 @@ func (mr *MockStoreMockRecorder) GetTemplateInsightsByInterval(arg0, arg1 interf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateInsightsByInterval", reflect.TypeOf((*MockStore)(nil).GetTemplateInsightsByInterval), arg0, arg1) } +// GetTemplateInsightsByTemplate mocks base method. +func (m *MockStore) GetTemplateInsightsByTemplate(arg0 context.Context, arg1 database.GetTemplateInsightsByTemplateParams) ([]database.GetTemplateInsightsByTemplateRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTemplateInsightsByTemplate", arg0, arg1) + ret0, _ := ret[0].([]database.GetTemplateInsightsByTemplateRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTemplateInsightsByTemplate indicates an expected call of GetTemplateInsightsByTemplate. +func (mr *MockStoreMockRecorder) GetTemplateInsightsByTemplate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateInsightsByTemplate", reflect.TypeOf((*MockStore)(nil).GetTemplateInsightsByTemplate), arg0, arg1) +} + // GetTemplateParameterInsights mocks base method. func (m *MockStore) GetTemplateParameterInsights(arg0 context.Context, arg1 database.GetTemplateParameterInsightsParams) ([]database.GetTemplateParameterInsightsRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 99503ba40e3d6..5cbbd9c5bbfb4 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -145,6 +145,7 @@ type sqlcQuerier interface { // that interval will be shorter than a full one. If there is no data for a selected // interval/template, it will be included in the results with 0 active users. GetTemplateInsightsByInterval(ctx context.Context, arg GetTemplateInsightsByIntervalParams) ([]GetTemplateInsightsByIntervalRow, error) + GetTemplateInsightsByTemplate(ctx context.Context, arg GetTemplateInsightsByTemplateParams) ([]GetTemplateInsightsByTemplateRow, error) // GetTemplateParameterInsights does for each template in a given timeframe, // look for the latest workspace build (for every workspace) that has been // created in the timeframe and return the aggregate usage counts of parameter diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index fc301d427fa8d..2e480c8958553 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1937,6 +1937,79 @@ func (q *sqlQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg GetT return items, nil } +const getTemplateInsightsByTemplate = `-- name: GetTemplateInsightsByTemplate :many +WITH agent_stats_by_interval_and_user AS ( + SELECT + date_trunc('minute', was.created_at), + was.template_id, + was.user_id, + CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds, + CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds, + CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds, + CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds + FROM workspace_agent_stats was + WHERE + was.created_at >= $1::timestamptz + AND was.created_at < $2::timestamptz + AND was.connection_count > 0 + GROUP BY date_trunc('minute', was.created_at), was.template_id, was.user_id +) + +SELECT + template_id, + COALESCE(COUNT(DISTINCT user_id))::bigint AS active_users, + COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds, + COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds, + COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds, + COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds +FROM agent_stats_by_interval_and_user +GROUP BY template_id +` + +type GetTemplateInsightsByTemplateParams struct { + StartTime time.Time `db:"start_time" json:"start_time"` + EndTime time.Time `db:"end_time" json:"end_time"` +} + +type GetTemplateInsightsByTemplateRow struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + ActiveUsers int64 `db:"active_users" json:"active_users"` + UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"` + UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"` + UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"` + UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"` +} + +func (q *sqlQuerier) GetTemplateInsightsByTemplate(ctx context.Context, arg GetTemplateInsightsByTemplateParams) ([]GetTemplateInsightsByTemplateRow, error) { + rows, err := q.db.QueryContext(ctx, getTemplateInsightsByTemplate, arg.StartTime, arg.EndTime) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetTemplateInsightsByTemplateRow + for rows.Next() { + var i GetTemplateInsightsByTemplateRow + if err := rows.Scan( + &i.TemplateID, + &i.ActiveUsers, + &i.UsageVscodeSeconds, + &i.UsageJetbrainsSeconds, + &i.UsageReconnectingPtySeconds, + &i.UsageSshSeconds, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getTemplateParameterInsights = `-- name: GetTemplateParameterInsights :many WITH latest_workspace_builds AS ( SELECT diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 7fb48100d5d8a..49e6e84f90c72 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -134,6 +134,34 @@ SELECT COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds FROM agent_stats_by_interval_and_user; +-- name: GetTemplateInsightsByTemplate :many +WITH agent_stats_by_interval_and_user AS ( + SELECT + date_trunc('minute', was.created_at), + was.template_id, + was.user_id, + CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds, + CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds, + CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds, + CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds + FROM workspace_agent_stats was + WHERE + was.created_at >= @start_time::timestamptz + AND was.created_at < @end_time::timestamptz + AND was.connection_count > 0 + GROUP BY date_trunc('minute', was.created_at), was.template_id, was.user_id +) + +SELECT + template_id, + COALESCE(COUNT(DISTINCT user_id))::bigint AS active_users, + COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds, + COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds, + COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds, + COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds +FROM agent_stats_by_interval_and_user +GROUP BY template_id; + -- name: GetTemplateAppInsights :many -- GetTemplateAppInsights returns the aggregate usage of each app in a given -- timeframe. The result can be filtered on template_ids, meaning only user data diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go new file mode 100644 index 0000000000000..f4e4f6b8d1368 --- /dev/null +++ b/coderd/prometheusmetrics/insights/metricscollector.go @@ -0,0 +1,214 @@ +package insights + +import ( + "context" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" + + "cdr.dev/slog" + + "github.com/coder/coder/v2/coderd/database" +) + +var ( + activeUsersDesc = prometheus.NewDesc("coderd_insights_active_users", "The number of active users of the template.", []string{"template_name"}, nil) + applicationsUsageSecondsDesc = prometheus.NewDesc("coderd_insights_applications_usage_seconds", "The application usage per template.", []string{"application_name", "template_name"}, nil) + parametersDesc = prometheus.NewDesc("coderd_insights_parameters", "The parameter usage per template.", []string{"template_name", "name", "value"}, nil) +) + +type MetricsCollector struct { + database database.Store + logger slog.Logger + duration time.Duration + + data atomic.Pointer[insightsData] +} + +type insightsData struct { + templates []database.GetTemplateInsightsByTemplateRow + apps []database.GetTemplateAppInsightsRow + parameters []database.GetTemplateParameterInsightsRow + + templateNames map[uuid.UUID]string +} + +var _ prometheus.Collector = new(MetricsCollector) + +func NewMetricsCollector(db database.Store, logger slog.Logger, duration time.Duration) (*MetricsCollector, error) { + if duration == 0 { + duration = 5 * time.Minute + } + if duration < 5*time.Minute { + return nil, xerrors.Errorf("refresh interval must be at least 5 mins") + } + + return &MetricsCollector{ + database: db, + logger: logger.Named("insights_metrics_collector"), + duration: duration, + }, nil +} + +func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { + ctx, closeFunc := context.WithCancel(ctx) + done := make(chan struct{}) + + // Use time.Nanosecond to force an initial tick. It will be reset to the + // correct duration after executing once. + ticker := time.NewTicker(time.Nanosecond) + doTick := func() { + defer ticker.Reset(mc.duration) + + now := time.Now() + startTime := now.Add(-mc.duration) + endTime := now + + // TODO collect iteration time + + var templateInsights []database.GetTemplateInsightsByTemplateRow + var appInsights []database.GetTemplateAppInsightsRow + var parameterInsights []database.GetTemplateParameterInsightsRow + + // Phase I: Fetch insights from database + eg, egCtx := errgroup.WithContext(ctx) + eg.SetLimit(3) + + eg.Go(func() error { + var err error + templateInsights, err = mc.database.GetTemplateInsightsByTemplate(egCtx, database.GetTemplateInsightsByTemplateParams{ + StartTime: startTime, + EndTime: endTime, + }) + if err != nil { + mc.logger.Error(ctx, "unable to fetch template insights from database", slog.Error(err)) + } + return err + }) + eg.Go(func() error { + var err error + appInsights, err = mc.database.GetTemplateAppInsights(ctx, database.GetTemplateAppInsightsParams{ + StartTime: startTime, + EndTime: endTime, + }) + if err != nil { + mc.logger.Error(ctx, "unable to fetch app insights from database", slog.Error(err)) + } + return err + }) + eg.Go(func() error { + var err error + parameterInsights, err = mc.database.GetTemplateParameterInsights(ctx, database.GetTemplateParameterInsightsParams{ + StartTime: startTime, + EndTime: endTime, + }) + if err != nil { + mc.logger.Error(ctx, "unable to fetch parameter insights from database", slog.Error(err)) + } + return err + }) + + err := eg.Wait() + if err != nil { + return + } + + // Phase 2: Collect template IDs, and fetch relevant details + templateIDs := uniqueTemplateIDs(templateInsights, appInsights, parameterInsights) + templates, err := mc.database.GetTemplatesWithFilter(ctx, database.GetTemplatesWithFilterParams{ + IDs: templateIDs, + }) + if err != nil { + mc.logger.Error(ctx, "unable to fetch template details from database", slog.Error(err)) + return + } + + templateNames := onlyTemplateNames(templates) + + mc.data.Store(&insightsData{ + templates: templateInsights, + apps: appInsights, + parameters: parameterInsights, + + templateNames: templateNames, + }) + } + + go func() { + defer close(done) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + doTick() + } + } + }() + return func() { + closeFunc() + <-done + }, nil +} + +func (*MetricsCollector) Describe(descCh chan<- *prometheus.Desc) { + descCh <- activeUsersDesc + descCh <- applicationsUsageSecondsDesc + descCh <- parametersDesc +} + +func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { + // Phase 3: Collect metrics + + data := mc.data.Load() + if data == nil { + return // insights data not loaded yet + } + + for _, templateRow := range data.templates { + metricsCh <- prometheus.MustNewConstMetric(activeUsersDesc, prometheus.GaugeValue, float64(templateRow.ActiveUsers), data.templateNames[templateRow.TemplateID]) + + // TODO applicationsUsageSeconds, parameters + } +} + +// Helper functions below. + +func uniqueTemplateIDs(templateInsights []database.GetTemplateInsightsByTemplateRow, appInsights []database.GetTemplateAppInsightsRow, parameterInsights []database.GetTemplateParameterInsightsRow) []uuid.UUID { + tids := map[uuid.UUID]bool{} + for _, t := range templateInsights { + tids[t.TemplateID] = true + } + + for _, a := range appInsights { + for _, tid := range a.TemplateIDs { + tids[tid] = true + } + } + + for _, p := range parameterInsights { + for _, tid := range p.TemplateIDs { + tids[tid] = true + } + } + + uniqueUUIDs := make([]uuid.UUID, len(tids)) + var i int + for t := range tids { + uniqueUUIDs[i] = t + } + return uniqueUUIDs +} + +func onlyTemplateNames(templates []database.Template) map[uuid.UUID]string { + m := map[uuid.UUID]string{} + for _, t := range templates { + m[t.ID] = t.Name + } + return m +}