diff --git a/Makefile b/Makefile index 2cd40a7dabfa3..423402260c26b 100644 --- a/Makefile +++ b/Makefile @@ -521,6 +521,7 @@ lint/markdown: node_modules/.installed # All files generated by the database should be added here, and this can be used # as a target for jobs that need to run after the database is generated. DB_GEN_FILES := \ + coderd/database/dump.sql \ coderd/database/querier.go \ coderd/database/unique_constraint.go \ coderd/database/dbmem/dbmem.go \ @@ -540,8 +541,6 @@ GEN_FILES := \ provisionersdk/proto/provisioner.pb.go \ provisionerd/proto/provisionerd.pb.go \ vpn/vpn.pb.go \ - coderd/database/dump.sql \ - $(DB_GEN_FILES) \ site/src/api/typesGenerated.ts \ coderd/rbac/object_gen.go \ codersdk/rbacresources_gen.go \ @@ -559,7 +558,7 @@ GEN_FILES := \ coderd/database/pubsub/psmock/psmock.go # all gen targets should be added here and to gen/mark-fresh -gen: $(GEN_FILES) +gen: gen/db $(GEN_FILES) .PHONY: gen gen/db: $(DB_GEN_FILES) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index a8bfcb2af3b19..15da8b7eb5c36 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -1398,6 +1398,40 @@ const docTemplate = `{ } } }, + "/insights/user-status-counts": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Insights" + ], + "summary": "Get insights about user status counts", + "operationId": "get-insights-about-user-status-counts", + "parameters": [ + { + "type": "integer", + "description": "Time-zone offset (e.g. -2)", + "name": "tz_offset", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/codersdk.GetUserStatusCountsResponse" + } + } + } + } + }, "/integrations/jfrog/xray-scan": { "get": { "security": [ @@ -11207,6 +11241,20 @@ const docTemplate = `{ } } }, + "codersdk.GetUserStatusCountsResponse": { + "type": "object", + "properties": { + "status_counts": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.UserStatusChangeCount" + } + } + } + } + }, "codersdk.GetUsersResponse": { "type": "object", "properties": { @@ -14570,6 +14618,19 @@ const docTemplate = `{ "UserStatusSuspended" ] }, + "codersdk.UserStatusChangeCount": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "example": 10 + }, + "date": { + "type": "string", + "format": "date-time" + } + } + }, "codersdk.ValidateUserPasswordRequest": { "type": "object", "required": [ diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index d7c32d8a33a52..df288ed1876c8 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -1219,6 +1219,36 @@ } } }, + "/insights/user-status-counts": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": ["application/json"], + "tags": ["Insights"], + "summary": "Get insights about user status counts", + "operationId": "get-insights-about-user-status-counts", + "parameters": [ + { + "type": "integer", + "description": "Time-zone offset (e.g. -2)", + "name": "tz_offset", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/codersdk.GetUserStatusCountsResponse" + } + } + } + } + }, "/integrations/jfrog/xray-scan": { "get": { "security": [ @@ -10054,6 +10084,20 @@ } } }, + "codersdk.GetUserStatusCountsResponse": { + "type": "object", + "properties": { + "status_counts": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.UserStatusChangeCount" + } + } + } + } + }, "codersdk.GetUsersResponse": { "type": "object", "properties": { @@ -13244,6 +13288,19 @@ "UserStatusSuspended" ] }, + "codersdk.UserStatusChangeCount": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "example": 10 + }, + "date": { + "type": "string", + "format": "date-time" + } + } + }, "codersdk.ValidateUserPasswordRequest": { "type": "object", "required": ["password"], diff --git a/coderd/coderd.go b/coderd/coderd.go index fd8a10a44f140..7b8cde9dc6ae4 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -1281,6 +1281,7 @@ func New(options *Options) *API { r.Use(apiKeyMiddleware) r.Get("/daus", api.deploymentDAUs) r.Get("/user-activity", api.insightsUserActivity) + r.Get("/user-status-counts", api.insightsUserStatusCounts) r.Get("/user-latency", api.insightsUserLatency) r.Get("/templates", api.insightsTemplates) }) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 0a35667ed0178..a4c3208aa5e6d 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2421,6 +2421,13 @@ func (q *querier) GetUserNotificationPreferences(ctx context.Context, userID uui return q.db.GetUserNotificationPreferences(ctx, userID) } +func (q *querier) GetUserStatusCounts(ctx context.Context, arg database.GetUserStatusCountsParams) ([]database.GetUserStatusCountsRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceUser); err != nil { + return nil, err + } + return q.db.GetUserStatusCounts(ctx, arg) +} + func (q *querier) GetUserWorkspaceBuildParameters(ctx context.Context, params database.GetUserWorkspaceBuildParametersParams) ([]database.GetUserWorkspaceBuildParametersRow, error) { u, err := q.db.GetUserByID(ctx, params.OwnerID) if err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 93e9a4318d1ed..78500792933b5 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1708,6 +1708,12 @@ func (s *MethodTestSuite) TestUser() { rbac.ResourceTemplate.InOrg(orgID), policy.ActionRead, ) })) + s.Run("GetUserStatusCounts", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.GetUserStatusCountsParams{ + StartTime: time.Now().Add(-time.Hour * 24 * 30), + EndTime: time.Now(), + }).Asserts(rbac.ResourceUser, policy.ActionRead) + })) } func (s *MethodTestSuite) TestWorkspace() { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index d3b7b3fb35f5f..9e3c3621b3420 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -88,6 +88,7 @@ func New() database.Store { customRoles: make([]database.CustomRole, 0), locks: map[int64]struct{}{}, runtimeConfig: map[string]string{}, + userStatusChanges: make([]database.UserStatusChange, 0), }, } // Always start with a default org. Matching migration 198. @@ -256,6 +257,7 @@ type data struct { lastLicenseID int32 defaultProxyDisplayName string defaultProxyIconURL string + userStatusChanges []database.UserStatusChange } func tryPercentile(fs []float64, p float64) float64 { @@ -5669,6 +5671,42 @@ func (q *FakeQuerier) GetUserNotificationPreferences(_ context.Context, userID u return out, nil } +func (q *FakeQuerier) GetUserStatusCounts(_ context.Context, arg database.GetUserStatusCountsParams) ([]database.GetUserStatusCountsRow, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + err := validateDatabaseType(arg) + if err != nil { + return nil, err + } + + result := make([]database.GetUserStatusCountsRow, 0) + for _, change := range q.userStatusChanges { + if change.ChangedAt.Before(arg.StartTime) || change.ChangedAt.After(arg.EndTime) { + continue + } + date := time.Date(change.ChangedAt.Year(), change.ChangedAt.Month(), change.ChangedAt.Day(), 0, 0, 0, 0, time.UTC) + if !slices.ContainsFunc(result, func(r database.GetUserStatusCountsRow) bool { + return r.Status == change.NewStatus && r.Date.Equal(date) + }) { + result = append(result, database.GetUserStatusCountsRow{ + Status: change.NewStatus, + Date: date, + Count: 1, + }) + } else { + for i, r := range result { + if r.Status == change.NewStatus && r.Date.Equal(date) { + result[i].Count++ + break + } + } + } + } + + return result, nil +} + func (q *FakeQuerier) GetUserWorkspaceBuildParameters(_ context.Context, params database.GetUserWorkspaceBuildParametersParams) ([]database.GetUserWorkspaceBuildParametersRow, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -8021,6 +8059,12 @@ func (q *FakeQuerier) InsertUser(_ context.Context, arg database.InsertUserParam sort.Slice(q.users, func(i, j int) bool { return q.users[i].CreatedAt.Before(q.users[j].CreatedAt) }) + + q.userStatusChanges = append(q.userStatusChanges, database.UserStatusChange{ + UserID: user.ID, + NewStatus: user.Status, + ChangedAt: user.UpdatedAt, + }) return user, nil } @@ -9062,12 +9106,18 @@ func (q *FakeQuerier) UpdateInactiveUsersToDormant(_ context.Context, params dat Username: user.Username, LastSeenAt: user.LastSeenAt, }) + q.userStatusChanges = append(q.userStatusChanges, database.UserStatusChange{ + UserID: user.ID, + NewStatus: database.UserStatusDormant, + ChangedAt: params.UpdatedAt, + }) } } if len(updated) == 0 { return nil, sql.ErrNoRows } + return updated, nil } @@ -9868,6 +9918,12 @@ func (q *FakeQuerier) UpdateUserStatus(_ context.Context, arg database.UpdateUse user.Status = arg.Status user.UpdatedAt = arg.UpdatedAt q.users[index] = user + + q.userStatusChanges = append(q.userStatusChanges, database.UserStatusChange{ + UserID: user.ID, + NewStatus: user.Status, + ChangedAt: user.UpdatedAt, + }) return user, nil } return database.User{}, sql.ErrNoRows diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 5df5c547a20d6..599fad08779ac 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1344,6 +1344,13 @@ func (m queryMetricsStore) GetUserNotificationPreferences(ctx context.Context, u return r0, r1 } +func (m queryMetricsStore) GetUserStatusCounts(ctx context.Context, arg database.GetUserStatusCountsParams) ([]database.GetUserStatusCountsRow, error) { + start := time.Now() + r0, r1 := m.s.GetUserStatusCounts(ctx, arg) + m.queryLatencies.WithLabelValues("GetUserStatusCounts").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetUserWorkspaceBuildParameters(ctx context.Context, ownerID database.GetUserWorkspaceBuildParametersParams) ([]database.GetUserWorkspaceBuildParametersRow, error) { start := time.Now() r0, r1 := m.s.GetUserWorkspaceBuildParameters(ctx, ownerID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 6b552fe5060ff..51d0c59c1d879 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2825,6 +2825,21 @@ func (mr *MockStoreMockRecorder) GetUserNotificationPreferences(arg0, arg1 any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserNotificationPreferences", reflect.TypeOf((*MockStore)(nil).GetUserNotificationPreferences), arg0, arg1) } +// GetUserStatusCounts mocks base method. +func (m *MockStore) GetUserStatusCounts(arg0 context.Context, arg1 database.GetUserStatusCountsParams) ([]database.GetUserStatusCountsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserStatusCounts", arg0, arg1) + ret0, _ := ret[0].([]database.GetUserStatusCountsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUserStatusCounts indicates an expected call of GetUserStatusCounts. +func (mr *MockStoreMockRecorder) GetUserStatusCounts(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserStatusCounts", reflect.TypeOf((*MockStore)(nil).GetUserStatusCounts), arg0, arg1) +} + // GetUserWorkspaceBuildParameters mocks base method. func (m *MockStore) GetUserWorkspaceBuildParameters(arg0 context.Context, arg1 database.GetUserWorkspaceBuildParametersParams) ([]database.GetUserWorkspaceBuildParametersRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 50519485dc505..7812f6e8e4e5a 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -423,6 +423,36 @@ $$; COMMENT ON FUNCTION provisioner_tagset_contains(provisioner_tags tagset, job_tags tagset) IS 'Returns true if the provisioner_tags contains the job_tags, or if the job_tags represents an untagged provisioner and the superset is exactly equal to the subset.'; +CREATE FUNCTION record_user_status_change() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF TG_OP = 'INSERT' OR OLD.status IS DISTINCT FROM NEW.status THEN + INSERT INTO user_status_changes ( + user_id, + new_status, + changed_at + ) VALUES ( + NEW.id, + NEW.status, + NEW.updated_at + ); + END IF; + + IF OLD.deleted = FALSE AND NEW.deleted = TRUE THEN + INSERT INTO user_deleted ( + user_id, + deleted_at + ) VALUES ( + NEW.id, + NEW.updated_at + ); + END IF; + + RETURN NEW; +END; +$$; + CREATE FUNCTION remove_organization_member_role() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1377,6 +1407,14 @@ CREATE VIEW template_with_names AS COMMENT ON VIEW template_with_names IS 'Joins in the display name information such as username, avatar, and organization name.'; +CREATE TABLE user_deleted ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + user_id uuid NOT NULL, + deleted_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL +); + +COMMENT ON TABLE user_deleted IS 'Tracks when users were deleted'; + CREATE TABLE user_links ( user_id uuid NOT NULL, login_type login_type NOT NULL, @@ -1395,6 +1433,15 @@ COMMENT ON COLUMN user_links.oauth_refresh_token_key_id IS 'The ID of the key us COMMENT ON COLUMN user_links.claims IS 'Claims from the IDP for the linked user. Includes both id_token and userinfo claims. '; +CREATE TABLE user_status_changes ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + user_id uuid NOT NULL, + new_status user_status NOT NULL, + changed_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL +); + +COMMENT ON TABLE user_status_changes IS 'Tracks the history of user status changes'; + CREATE TABLE workspace_agent_log_sources ( workspace_agent_id uuid NOT NULL, id uuid NOT NULL, @@ -1980,9 +2027,15 @@ ALTER TABLE ONLY template_versions ALTER TABLE ONLY templates ADD CONSTRAINT templates_pkey PRIMARY KEY (id); +ALTER TABLE ONLY user_deleted + ADD CONSTRAINT user_deleted_pkey PRIMARY KEY (id); + ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_pkey PRIMARY KEY (user_id, login_type); +ALTER TABLE ONLY user_status_changes + ADD CONSTRAINT user_status_changes_pkey PRIMARY KEY (id); + ALTER TABLE ONLY users ADD CONSTRAINT users_pkey PRIMARY KEY (id); @@ -2093,6 +2146,10 @@ CREATE INDEX idx_tailnet_tunnels_dst_id ON tailnet_tunnels USING hash (dst_id); CREATE INDEX idx_tailnet_tunnels_src_id ON tailnet_tunnels USING hash (src_id); +CREATE INDEX idx_user_deleted_deleted_at ON user_deleted USING btree (deleted_at); + +CREATE INDEX idx_user_status_changes_changed_at ON user_status_changes USING btree (changed_at); + CREATE UNIQUE INDEX idx_users_email ON users USING btree (email) WHERE (deleted = false); CREATE UNIQUE INDEX idx_users_username ON users USING btree (username) WHERE (deleted = false); @@ -2235,6 +2292,8 @@ CREATE TRIGGER trigger_upsert_user_links BEFORE INSERT OR UPDATE ON user_links F CREATE TRIGGER update_notification_message_dedupe_hash BEFORE INSERT OR UPDATE ON notification_messages FOR EACH ROW EXECUTE FUNCTION compute_notification_message_dedupe_hash(); +CREATE TRIGGER user_status_change_trigger AFTER INSERT OR UPDATE ON users FOR EACH ROW EXECUTE FUNCTION record_user_status_change(); + ALTER TABLE ONLY api_keys ADD CONSTRAINT api_keys_user_id_uuid_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; @@ -2358,6 +2417,9 @@ ALTER TABLE ONLY templates ALTER TABLE ONLY templates ADD CONSTRAINT templates_organization_id_fkey FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; +ALTER TABLE ONLY user_deleted + ADD CONSTRAINT user_deleted_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id); + ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_oauth_access_token_key_id_fkey FOREIGN KEY (oauth_access_token_key_id) REFERENCES dbcrypt_keys(active_key_digest); @@ -2367,6 +2429,9 @@ ALTER TABLE ONLY user_links ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; +ALTER TABLE ONLY user_status_changes + ADD CONSTRAINT user_status_changes_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id); + ALTER TABLE ONLY workspace_agent_log_sources ADD CONSTRAINT workspace_agent_log_sources_workspace_agent_id_fkey FOREIGN KEY (workspace_agent_id) REFERENCES workspace_agents(id) ON DELETE CASCADE; diff --git a/coderd/database/foreign_key_constraint.go b/coderd/database/foreign_key_constraint.go index 669ab85f945bd..52f98a679a71b 100644 --- a/coderd/database/foreign_key_constraint.go +++ b/coderd/database/foreign_key_constraint.go @@ -47,9 +47,11 @@ const ( ForeignKeyTemplateVersionsTemplateID ForeignKeyConstraint = "template_versions_template_id_fkey" // ALTER TABLE ONLY template_versions ADD CONSTRAINT template_versions_template_id_fkey FOREIGN KEY (template_id) REFERENCES templates(id) ON DELETE CASCADE; ForeignKeyTemplatesCreatedBy ForeignKeyConstraint = "templates_created_by_fkey" // ALTER TABLE ONLY templates ADD CONSTRAINT templates_created_by_fkey FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE RESTRICT; ForeignKeyTemplatesOrganizationID ForeignKeyConstraint = "templates_organization_id_fkey" // ALTER TABLE ONLY templates ADD CONSTRAINT templates_organization_id_fkey FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; + ForeignKeyUserDeletedUserID ForeignKeyConstraint = "user_deleted_user_id_fkey" // ALTER TABLE ONLY user_deleted ADD CONSTRAINT user_deleted_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id); ForeignKeyUserLinksOauthAccessTokenKeyID ForeignKeyConstraint = "user_links_oauth_access_token_key_id_fkey" // ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_oauth_access_token_key_id_fkey FOREIGN KEY (oauth_access_token_key_id) REFERENCES dbcrypt_keys(active_key_digest); ForeignKeyUserLinksOauthRefreshTokenKeyID ForeignKeyConstraint = "user_links_oauth_refresh_token_key_id_fkey" // ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_oauth_refresh_token_key_id_fkey FOREIGN KEY (oauth_refresh_token_key_id) REFERENCES dbcrypt_keys(active_key_digest); ForeignKeyUserLinksUserID ForeignKeyConstraint = "user_links_user_id_fkey" // ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ForeignKeyUserStatusChangesUserID ForeignKeyConstraint = "user_status_changes_user_id_fkey" // ALTER TABLE ONLY user_status_changes ADD CONSTRAINT user_status_changes_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id); ForeignKeyWorkspaceAgentLogSourcesWorkspaceAgentID ForeignKeyConstraint = "workspace_agent_log_sources_workspace_agent_id_fkey" // ALTER TABLE ONLY workspace_agent_log_sources ADD CONSTRAINT workspace_agent_log_sources_workspace_agent_id_fkey FOREIGN KEY (workspace_agent_id) REFERENCES workspace_agents(id) ON DELETE CASCADE; ForeignKeyWorkspaceAgentMetadataWorkspaceAgentID ForeignKeyConstraint = "workspace_agent_metadata_workspace_agent_id_fkey" // ALTER TABLE ONLY workspace_agent_metadata ADD CONSTRAINT workspace_agent_metadata_workspace_agent_id_fkey FOREIGN KEY (workspace_agent_id) REFERENCES workspace_agents(id) ON DELETE CASCADE; ForeignKeyWorkspaceAgentPortShareWorkspaceID ForeignKeyConstraint = "workspace_agent_port_share_workspace_id_fkey" // ALTER TABLE ONLY workspace_agent_port_share ADD CONSTRAINT workspace_agent_port_share_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspaces(id) ON DELETE CASCADE; diff --git a/coderd/database/migrations/000283_user_status_changes.down.sql b/coderd/database/migrations/000283_user_status_changes.down.sql new file mode 100644 index 0000000000000..fbe85a6be0fe5 --- /dev/null +++ b/coderd/database/migrations/000283_user_status_changes.down.sql @@ -0,0 +1,9 @@ +DROP TRIGGER IF EXISTS user_status_change_trigger ON users; + +DROP FUNCTION IF EXISTS record_user_status_change(); + +DROP INDEX IF EXISTS idx_user_status_changes_changed_at; +DROP INDEX IF EXISTS idx_user_deleted_deleted_at; + +DROP TABLE IF EXISTS user_status_changes; +DROP TABLE IF EXISTS user_deleted; diff --git a/coderd/database/migrations/000283_user_status_changes.up.sql b/coderd/database/migrations/000283_user_status_changes.up.sql new file mode 100644 index 0000000000000..d712465851eff --- /dev/null +++ b/coderd/database/migrations/000283_user_status_changes.up.sql @@ -0,0 +1,75 @@ +CREATE TABLE user_status_changes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id), + new_status user_status NOT NULL, + changed_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +COMMENT ON TABLE user_status_changes IS 'Tracks the history of user status changes'; + +CREATE INDEX idx_user_status_changes_changed_at ON user_status_changes(changed_at); + +INSERT INTO user_status_changes ( + user_id, + new_status, + changed_at +) +SELECT + id, + status, + created_at +FROM users +WHERE NOT deleted; + +CREATE TABLE user_deleted ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id), + deleted_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +COMMENT ON TABLE user_deleted IS 'Tracks when users were deleted'; + +CREATE INDEX idx_user_deleted_deleted_at ON user_deleted(deleted_at); + +INSERT INTO user_deleted ( + user_id, + deleted_at +) +SELECT + id, + updated_at +FROM users +WHERE deleted; + +CREATE OR REPLACE FUNCTION record_user_status_change() RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'INSERT' OR OLD.status IS DISTINCT FROM NEW.status THEN + INSERT INTO user_status_changes ( + user_id, + new_status, + changed_at + ) VALUES ( + NEW.id, + NEW.status, + NEW.updated_at + ); + END IF; + + IF OLD.deleted = FALSE AND NEW.deleted = TRUE THEN + INSERT INTO user_deleted ( + user_id, + deleted_at + ) VALUES ( + NEW.id, + NEW.updated_at + ); + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER user_status_change_trigger + AFTER INSERT OR UPDATE ON users + FOR EACH ROW + EXECUTE FUNCTION record_user_status_change(); diff --git a/coderd/database/migrations/testdata/fixtures/000283_user_status_changes.up.sql b/coderd/database/migrations/testdata/fixtures/000283_user_status_changes.up.sql new file mode 100644 index 0000000000000..9559fa3ad0df8 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000283_user_status_changes.up.sql @@ -0,0 +1,42 @@ +INSERT INTO + users ( + id, + email, + username, + hashed_password, + created_at, + updated_at, + status, + rbac_roles, + login_type, + avatar_url, + last_seen_at, + quiet_hours_schedule, + theme_preference, + name, + github_com_user_id, + hashed_one_time_passcode, + one_time_passcode_expires_at + ) + VALUES ( + '5755e622-fadd-44ca-98da-5df070491844', -- uuid + 'test@example.com', + 'testuser', + 'hashed_password', + '2024-01-01 00:00:00', + '2024-01-01 00:00:00', + 'active', + '{}', + 'password', + '', + '2024-01-01 00:00:00', + '', + '', + '', + 123, + NULL, + NULL + ); + +UPDATE users SET status = 'dormant', updated_at = '2024-01-01 01:00:00' WHERE id = '5755e622-fadd-44ca-98da-5df070491844'; +UPDATE users SET deleted = true, updated_at = '2024-01-01 02:00:00' WHERE id = '5755e622-fadd-44ca-98da-5df070491844'; diff --git a/coderd/database/models.go b/coderd/database/models.go index 9ca80d119a502..35484c7856e14 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -2953,6 +2953,13 @@ type User struct { OneTimePasscodeExpiresAt sql.NullTime `db:"one_time_passcode_expires_at" json:"one_time_passcode_expires_at"` } +// Tracks when users were deleted +type UserDeleted struct { + ID uuid.UUID `db:"id" json:"id"` + UserID uuid.UUID `db:"user_id" json:"user_id"` + DeletedAt time.Time `db:"deleted_at" json:"deleted_at"` +} + type UserLink struct { UserID uuid.UUID `db:"user_id" json:"user_id"` LoginType LoginType `db:"login_type" json:"login_type"` @@ -2968,6 +2975,14 @@ type UserLink struct { Claims UserLinkClaims `db:"claims" json:"claims"` } +// Tracks the history of user status changes +type UserStatusChange struct { + ID uuid.UUID `db:"id" json:"id"` + UserID uuid.UUID `db:"user_id" json:"user_id"` + NewStatus UserStatus `db:"new_status" json:"new_status"` + ChangedAt time.Time `db:"changed_at" json:"changed_at"` +} + // Visible fields of users are allowed to be joined with other tables for including context of other resources. type VisibleUser struct { ID uuid.UUID `db:"id" json:"id"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 620cc14b3fd26..ba151e0e8abb0 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -289,6 +289,19 @@ type sqlcQuerier interface { GetUserLinkByUserIDLoginType(ctx context.Context, arg GetUserLinkByUserIDLoginTypeParams) (UserLink, error) GetUserLinksByUserID(ctx context.Context, userID uuid.UUID) ([]UserLink, error) GetUserNotificationPreferences(ctx context.Context, userID uuid.UUID) ([]NotificationPreference, error) + // GetUserStatusCounts returns the count of users in each status over time. + // The time range is inclusively defined by the start_time and end_time parameters. + // + // Bucketing: + // Between the start_time and end_time, we include each timestamp where a user's status changed or they were deleted. + // We do not bucket these results by day or some other time unit. This is because such bucketing would hide potentially + // important patterns. If a user was active for 23 hours and 59 minutes, and then suspended, a daily bucket would hide this. + // A daily bucket would also have required us to carefully manage the timezone of the bucket based on the timezone of the user. + // + // Accumulation: + // We do not start counting from 0 at the start_time. We check the last status change before the start_time for each user. As such, + // the result shows the total number of users in each status on any particular day. + GetUserStatusCounts(ctx context.Context, arg GetUserStatusCountsParams) ([]GetUserStatusCountsRow, error) GetUserWorkspaceBuildParameters(ctx context.Context, arg GetUserWorkspaceBuildParametersParams) ([]GetUserWorkspaceBuildParametersRow, error) // This will never return deleted users. GetUsers(ctx context.Context, arg GetUsersParams) ([]GetUsersRow, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 28d7108ae31ad..0c7a445ed7104 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/json" "fmt" + "maps" "sort" "testing" "time" @@ -2255,6 +2256,526 @@ func TestGroupRemovalTrigger(t *testing.T) { }, db2sdk.List(extraUserGroups, onlyGroupIDs)) } +func TestGetUserStatusCounts(t *testing.T) { + t.Parallel() + + if !dbtestutil.WillUsePostgres() { + t.SkipNow() + } + + timezones := []string{ + "Canada/Newfoundland", + "Africa/Johannesburg", + "America/New_York", + "Europe/London", + "Asia/Tokyo", + "Australia/Sydney", + } + + for _, tz := range timezones { + tz := tz + t.Run(tz, func(t *testing.T) { + t.Parallel() + + location, err := time.LoadLocation(tz) + if err != nil { + t.Fatalf("failed to load location: %v", err) + } + today := dbtime.Now().In(location) + createdAt := today.Add(-5 * 24 * time.Hour) + firstTransitionTime := createdAt.Add(2 * 24 * time.Hour) + secondTransitionTime := firstTransitionTime.Add(2 * 24 * time.Hour) + + t.Run("No Users", func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + end := dbtime.Now() + start := end.Add(-30 * 24 * time.Hour) + + counts, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: start, + EndTime: end, + }) + require.NoError(t, err) + require.Empty(t, counts, "should return no results when there are no users") + }) + + t.Run("One User/Creation Only", func(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + status database.UserStatus + }{ + { + name: "Active Only", + status: database.UserStatusActive, + }, + { + name: "Dormant Only", + status: database.UserStatusDormant, + }, + { + name: "Suspended Only", + status: database.UserStatusSuspended, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + // Create a user that's been in the specified status for the past 30 days + dbgen.User(t, db, database.User{ + Status: tc.status, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + // Query for the last 30 days + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: createdAt, + EndTime: today, + }) + require.NoError(t, err) + require.NotEmpty(t, userStatusChanges, "should return results") + + require.Len(t, userStatusChanges, 2, "should have 1 entry per status change plus and 1 entry for the end of the range = 2 entries") + + require.Equal(t, userStatusChanges[0].Status, tc.status, "should have the correct status") + require.Equal(t, userStatusChanges[0].Count, int64(1), "should have 1 user") + require.True(t, userStatusChanges[0].Date.Equal(createdAt), "should have the correct date") + + require.Equal(t, userStatusChanges[1].Status, tc.status, "should have the correct status") + require.Equal(t, userStatusChanges[1].Count, int64(1), "should have 1 user") + require.True(t, userStatusChanges[1].Date.Equal(today), "should have the correct date") + }) + } + }) + + t.Run("One User/One Transition", func(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + initialStatus database.UserStatus + targetStatus database.UserStatus + expectedCounts map[time.Time]map[database.UserStatus]int64 + }{ + { + name: "Active to Dormant", + initialStatus: database.UserStatusActive, + targetStatus: database.UserStatusDormant, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusActive: 1, + database.UserStatusDormant: 0, + }, + firstTransitionTime: { + database.UserStatusDormant: 1, + database.UserStatusActive: 0, + }, + today: { + database.UserStatusDormant: 1, + database.UserStatusActive: 0, + }, + }, + }, + { + name: "Active to Suspended", + initialStatus: database.UserStatusActive, + targetStatus: database.UserStatusSuspended, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusActive: 1, + database.UserStatusSuspended: 0, + }, + firstTransitionTime: { + database.UserStatusSuspended: 1, + database.UserStatusActive: 0, + }, + today: { + database.UserStatusSuspended: 1, + database.UserStatusActive: 0, + }, + }, + }, + { + name: "Dormant to Active", + initialStatus: database.UserStatusDormant, + targetStatus: database.UserStatusActive, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusDormant: 1, + database.UserStatusActive: 0, + }, + firstTransitionTime: { + database.UserStatusActive: 1, + database.UserStatusDormant: 0, + }, + today: { + database.UserStatusActive: 1, + database.UserStatusDormant: 0, + }, + }, + }, + { + name: "Dormant to Suspended", + initialStatus: database.UserStatusDormant, + targetStatus: database.UserStatusSuspended, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusDormant: 1, + database.UserStatusSuspended: 0, + }, + firstTransitionTime: { + database.UserStatusSuspended: 1, + database.UserStatusDormant: 0, + }, + today: { + database.UserStatusSuspended: 1, + database.UserStatusDormant: 0, + }, + }, + }, + { + name: "Suspended to Active", + initialStatus: database.UserStatusSuspended, + targetStatus: database.UserStatusActive, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusSuspended: 1, + database.UserStatusActive: 0, + }, + firstTransitionTime: { + database.UserStatusActive: 1, + database.UserStatusSuspended: 0, + }, + today: { + database.UserStatusActive: 1, + database.UserStatusSuspended: 0, + }, + }, + }, + { + name: "Suspended to Dormant", + initialStatus: database.UserStatusSuspended, + targetStatus: database.UserStatusDormant, + expectedCounts: map[time.Time]map[database.UserStatus]int64{ + createdAt: { + database.UserStatusSuspended: 1, + database.UserStatusDormant: 0, + }, + firstTransitionTime: { + database.UserStatusDormant: 1, + database.UserStatusSuspended: 0, + }, + today: { + database.UserStatusDormant: 1, + database.UserStatusSuspended: 0, + }, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + // Create a user that starts with initial status + user := dbgen.User(t, db, database.User{ + Status: tc.initialStatus, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + // After 2 days, change status to target status + user, err := db.UpdateUserStatus(ctx, database.UpdateUserStatusParams{ + ID: user.ID, + Status: tc.targetStatus, + UpdatedAt: firstTransitionTime, + }) + require.NoError(t, err) + + // Query for the last 5 days + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: createdAt, + EndTime: today, + }) + require.NoError(t, err) + require.NotEmpty(t, userStatusChanges, "should return results") + + gotCounts := map[time.Time]map[database.UserStatus]int64{} + for _, row := range userStatusChanges { + gotDateInLocation := row.Date.In(location) + if _, ok := gotCounts[gotDateInLocation]; !ok { + gotCounts[gotDateInLocation] = map[database.UserStatus]int64{} + } + if _, ok := gotCounts[gotDateInLocation][row.Status]; !ok { + gotCounts[gotDateInLocation][row.Status] = 0 + } + gotCounts[gotDateInLocation][row.Status] += row.Count + } + require.Equal(t, tc.expectedCounts, gotCounts) + }) + } + }) + + t.Run("Two Users/One Transition", func(t *testing.T) { + t.Parallel() + + type transition struct { + from database.UserStatus + to database.UserStatus + } + + type testCase struct { + name string + user1Transition transition + user2Transition transition + } + + testCases := []testCase{ + { + name: "Active->Dormant and Dormant->Suspended", + user1Transition: transition{ + from: database.UserStatusActive, + to: database.UserStatusDormant, + }, + user2Transition: transition{ + from: database.UserStatusDormant, + to: database.UserStatusSuspended, + }, + }, + { + name: "Suspended->Active and Active->Dormant", + user1Transition: transition{ + from: database.UserStatusSuspended, + to: database.UserStatusActive, + }, + user2Transition: transition{ + from: database.UserStatusActive, + to: database.UserStatusDormant, + }, + }, + { + name: "Dormant->Active and Suspended->Dormant", + user1Transition: transition{ + from: database.UserStatusDormant, + to: database.UserStatusActive, + }, + user2Transition: transition{ + from: database.UserStatusSuspended, + to: database.UserStatusDormant, + }, + }, + { + name: "Active->Suspended and Suspended->Active", + user1Transition: transition{ + from: database.UserStatusActive, + to: database.UserStatusSuspended, + }, + user2Transition: transition{ + from: database.UserStatusSuspended, + to: database.UserStatusActive, + }, + }, + { + name: "Dormant->Suspended and Dormant->Active", + user1Transition: transition{ + from: database.UserStatusDormant, + to: database.UserStatusSuspended, + }, + user2Transition: transition{ + from: database.UserStatusDormant, + to: database.UserStatusActive, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + user1 := dbgen.User(t, db, database.User{ + Status: tc.user1Transition.from, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + user2 := dbgen.User(t, db, database.User{ + Status: tc.user2Transition.from, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + // First transition at 2 days + user1, err := db.UpdateUserStatus(ctx, database.UpdateUserStatusParams{ + ID: user1.ID, + Status: tc.user1Transition.to, + UpdatedAt: firstTransitionTime, + }) + require.NoError(t, err) + + // Second transition at 4 days + user2, err = db.UpdateUserStatus(ctx, database.UpdateUserStatusParams{ + ID: user2.ID, + Status: tc.user2Transition.to, + UpdatedAt: secondTransitionTime, + }) + require.NoError(t, err) + + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: createdAt, + EndTime: today, + }) + require.NoError(t, err) + require.NotEmpty(t, userStatusChanges) + gotCounts := map[time.Time]map[database.UserStatus]int64{ + createdAt.In(location): {}, + firstTransitionTime.In(location): {}, + secondTransitionTime.In(location): {}, + today.In(location): {}, + } + for _, row := range userStatusChanges { + dateInLocation := row.Date.In(location) + switch { + case dateInLocation.Equal(createdAt.In(location)): + gotCounts[createdAt][row.Status] = row.Count + case dateInLocation.Equal(firstTransitionTime.In(location)): + gotCounts[firstTransitionTime][row.Status] = row.Count + case dateInLocation.Equal(secondTransitionTime.In(location)): + gotCounts[secondTransitionTime][row.Status] = row.Count + case dateInLocation.Equal(today.In(location)): + gotCounts[today][row.Status] = row.Count + default: + t.Fatalf("unexpected date %s", row.Date) + } + } + + expectedCounts := map[time.Time]map[database.UserStatus]int64{} + for _, status := range []database.UserStatus{ + tc.user1Transition.from, + tc.user1Transition.to, + tc.user2Transition.from, + tc.user2Transition.to, + } { + if _, ok := expectedCounts[createdAt]; !ok { + expectedCounts[createdAt] = map[database.UserStatus]int64{} + } + expectedCounts[createdAt][status] = 0 + } + + expectedCounts[createdAt][tc.user1Transition.from]++ + expectedCounts[createdAt][tc.user2Transition.from]++ + + expectedCounts[firstTransitionTime] = map[database.UserStatus]int64{} + maps.Copy(expectedCounts[firstTransitionTime], expectedCounts[createdAt]) + expectedCounts[firstTransitionTime][tc.user1Transition.from]-- + expectedCounts[firstTransitionTime][tc.user1Transition.to]++ + + expectedCounts[secondTransitionTime] = map[database.UserStatus]int64{} + maps.Copy(expectedCounts[secondTransitionTime], expectedCounts[firstTransitionTime]) + expectedCounts[secondTransitionTime][tc.user2Transition.from]-- + expectedCounts[secondTransitionTime][tc.user2Transition.to]++ + + expectedCounts[today] = map[database.UserStatus]int64{} + maps.Copy(expectedCounts[today], expectedCounts[secondTransitionTime]) + + require.Equal(t, expectedCounts[createdAt], gotCounts[createdAt]) + require.Equal(t, expectedCounts[firstTransitionTime], gotCounts[firstTransitionTime]) + require.Equal(t, expectedCounts[secondTransitionTime], gotCounts[secondTransitionTime]) + require.Equal(t, expectedCounts[today], gotCounts[today]) + }) + } + }) + + t.Run("User precedes and survives query range", func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + _ = dbgen.User(t, db, database.User{ + Status: database.UserStatusActive, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: createdAt.Add(time.Hour * 24), + EndTime: today, + }) + require.NoError(t, err) + + require.Len(t, userStatusChanges, 2) + require.Equal(t, userStatusChanges[0].Count, int64(1)) + require.Equal(t, userStatusChanges[0].Status, database.UserStatusActive) + require.Equal(t, userStatusChanges[1].Count, int64(1)) + require.Equal(t, userStatusChanges[1].Status, database.UserStatusActive) + }) + + t.Run("User deleted before query range", func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + user := dbgen.User(t, db, database.User{ + Status: database.UserStatusActive, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + err = db.UpdateUserDeletedByID(ctx, user.ID) + require.NoError(t, err) + + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: today.Add(time.Hour * 24), + EndTime: today.Add(time.Hour * 48), + }) + require.NoError(t, err) + require.Empty(t, userStatusChanges) + }) + + t.Run("User deleted during query range", func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + user := dbgen.User(t, db, database.User{ + Status: database.UserStatusActive, + CreatedAt: createdAt, + UpdatedAt: createdAt, + }) + + err := db.UpdateUserDeletedByID(ctx, user.ID) + require.NoError(t, err) + + userStatusChanges, err := db.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: createdAt, + EndTime: today.Add(time.Hour * 24), + }) + require.NoError(t, err) + require.Equal(t, userStatusChanges[0].Count, int64(1)) + require.Equal(t, userStatusChanges[0].Status, database.UserStatusActive) + require.Equal(t, userStatusChanges[1].Count, int64(0)) + require.Equal(t, userStatusChanges[1].Status, database.UserStatusActive) + require.Equal(t, userStatusChanges[2].Count, int64(0)) + require.Equal(t, userStatusChanges[2].Status, database.UserStatusActive) + }) + }) + } +} + func requireUsersMatch(t testing.TB, expected []database.User, found []database.GetUsersRow, msg string) { t.Helper() require.ElementsMatch(t, expected, database.ConvertUserRows(found), msg) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 8fbb7c0b5be6c..9301e4b6f725c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -3094,6 +3094,171 @@ func (q *sqlQuerier) GetUserLatencyInsights(ctx context.Context, arg GetUserLate return items, nil } +const getUserStatusCounts = `-- name: GetUserStatusCounts :many +WITH + -- dates_of_interest defines all points in time that are relevant to the query. + -- It includes the start_time, all status changes, all deletions, and the end_time. +dates_of_interest AS ( + SELECT $1::timestamptz AS date + + UNION + + SELECT DISTINCT changed_at AS date + FROM user_status_changes + WHERE changed_at > $1::timestamptz + AND changed_at < $2::timestamptz + + UNION + + SELECT DISTINCT deleted_at AS date + FROM user_deleted + WHERE deleted_at > $1::timestamptz + AND deleted_at < $2::timestamptz + + UNION + + SELECT $2::timestamptz AS date +), + -- latest_status_before_range defines the status of each user before the start_time. + -- We do not include users who were deleted before the start_time. We use this to ensure that + -- we correctly count users prior to the start_time for a complete graph. +latest_status_before_range AS ( + SELECT + DISTINCT usc.user_id, + usc.new_status, + usc.changed_at, + ud.deleted + FROM user_status_changes usc + LEFT JOIN LATERAL ( + SELECT COUNT(*) > 0 AS deleted + FROM user_deleted ud + WHERE ud.user_id = usc.user_id AND (ud.deleted_at < usc.changed_at OR ud.deleted_at < $1) + ) AS ud ON true + WHERE usc.changed_at < $1::timestamptz + ORDER BY usc.user_id, usc.changed_at DESC +), + -- status_changes_during_range defines the status of each user during the start_time and end_time. + -- If a user is deleted during the time range, we count status changes between the start_time and the deletion date. + -- Theoretically, it should probably not be possible to update the status of a deleted user, but we + -- need to ensure that this is enforced, so that a change in business logic later does not break this graph. +status_changes_during_range AS ( + SELECT + usc.user_id, + usc.new_status, + usc.changed_at, + ud.deleted + FROM user_status_changes usc + LEFT JOIN LATERAL ( + SELECT COUNT(*) > 0 AS deleted + FROM user_deleted ud + WHERE ud.user_id = usc.user_id AND ud.deleted_at < usc.changed_at + ) AS ud ON true + WHERE usc.changed_at >= $1::timestamptz + AND usc.changed_at <= $2::timestamptz +), + -- relevant_status_changes defines the status of each user at any point in time. + -- It includes the status of each user before the start_time, and the status of each user during the start_time and end_time. +relevant_status_changes AS ( + SELECT + user_id, + new_status, + changed_at + FROM latest_status_before_range + WHERE NOT deleted + + UNION ALL + + SELECT + user_id, + new_status, + changed_at + FROM status_changes_during_range + WHERE NOT deleted +), + -- statuses defines all the distinct statuses that were present just before and during the time range. + -- This is used to ensure that we have a series for every relevant status. +statuses AS ( + SELECT DISTINCT new_status FROM relevant_status_changes +), + -- We only want to count the latest status change for each user on each date and then filter them by the relevant status. + -- We use the row_number function to ensure that we only count the latest status change for each user on each date. + -- We then filter the status changes by the relevant status in the final select statement below. +ranked_status_change_per_user_per_date AS ( + SELECT + d.date, + rsc1.user_id, + ROW_NUMBER() OVER (PARTITION BY d.date, rsc1.user_id ORDER BY rsc1.changed_at DESC) AS rn, + rsc1.new_status + FROM dates_of_interest d + LEFT JOIN relevant_status_changes rsc1 ON rsc1.changed_at <= d.date +) +SELECT + rscpupd.date, + statuses.new_status AS status, + COUNT(rscpupd.user_id) FILTER ( + WHERE rscpupd.rn = 1 + AND ( + rscpupd.new_status = statuses.new_status + AND ( + -- Include users who haven't been deleted + NOT EXISTS (SELECT 1 FROM user_deleted WHERE user_id = rscpupd.user_id) + OR + -- Or users whose deletion date is after the current date we're looking at + rscpupd.date < (SELECT deleted_at FROM user_deleted WHERE user_id = rscpupd.user_id) + ) + ) + ) AS count +FROM ranked_status_change_per_user_per_date rscpupd +CROSS JOIN statuses +GROUP BY rscpupd.date, statuses.new_status +` + +type GetUserStatusCountsParams struct { + StartTime time.Time `db:"start_time" json:"start_time"` + EndTime time.Time `db:"end_time" json:"end_time"` +} + +type GetUserStatusCountsRow struct { + Date time.Time `db:"date" json:"date"` + Status UserStatus `db:"status" json:"status"` + Count int64 `db:"count" json:"count"` +} + +// GetUserStatusCounts returns the count of users in each status over time. +// The time range is inclusively defined by the start_time and end_time parameters. +// +// Bucketing: +// Between the start_time and end_time, we include each timestamp where a user's status changed or they were deleted. +// We do not bucket these results by day or some other time unit. This is because such bucketing would hide potentially +// important patterns. If a user was active for 23 hours and 59 minutes, and then suspended, a daily bucket would hide this. +// A daily bucket would also have required us to carefully manage the timezone of the bucket based on the timezone of the user. +// +// Accumulation: +// We do not start counting from 0 at the start_time. We check the last status change before the start_time for each user. As such, +// the result shows the total number of users in each status on any particular day. +func (q *sqlQuerier) GetUserStatusCounts(ctx context.Context, arg GetUserStatusCountsParams) ([]GetUserStatusCountsRow, error) { + rows, err := q.db.QueryContext(ctx, getUserStatusCounts, arg.StartTime, arg.EndTime) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetUserStatusCountsRow + for rows.Next() { + var i GetUserStatusCountsRow + if err := rows.Scan(&i.Date, &i.Status, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const upsertTemplateUsageStats = `-- name: UpsertTemplateUsageStats :exec WITH latest_start AS ( diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index de107bc0e80c7..c6c6a78fc4b73 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -771,3 +771,134 @@ SELECT FROM unique_template_params utp JOIN workspace_build_parameters wbp ON (utp.workspace_build_ids @> ARRAY[wbp.workspace_build_id] AND utp.name = wbp.name) GROUP BY utp.num, utp.template_ids, utp.name, utp.type, utp.display_name, utp.description, utp.options, wbp.value; + +-- name: GetUserStatusCounts :many +-- GetUserStatusCounts returns the count of users in each status over time. +-- The time range is inclusively defined by the start_time and end_time parameters. +-- +-- Bucketing: +-- Between the start_time and end_time, we include each timestamp where a user's status changed or they were deleted. +-- We do not bucket these results by day or some other time unit. This is because such bucketing would hide potentially +-- important patterns. If a user was active for 23 hours and 59 minutes, and then suspended, a daily bucket would hide this. +-- A daily bucket would also have required us to carefully manage the timezone of the bucket based on the timezone of the user. +-- +-- Accumulation: +-- We do not start counting from 0 at the start_time. We check the last status change before the start_time for each user. As such, +-- the result shows the total number of users in each status on any particular day. +WITH + -- dates_of_interest defines all points in time that are relevant to the query. + -- It includes the start_time, all status changes, all deletions, and the end_time. +dates_of_interest AS ( + SELECT @start_time::timestamptz AS date + + UNION + + SELECT DISTINCT changed_at AS date + FROM user_status_changes + WHERE changed_at > @start_time::timestamptz + AND changed_at < @end_time::timestamptz + + UNION + + SELECT DISTINCT deleted_at AS date + FROM user_deleted + WHERE deleted_at > @start_time::timestamptz + AND deleted_at < @end_time::timestamptz + + UNION + + SELECT @end_time::timestamptz AS date +), + -- latest_status_before_range defines the status of each user before the start_time. + -- We do not include users who were deleted before the start_time. We use this to ensure that + -- we correctly count users prior to the start_time for a complete graph. +latest_status_before_range AS ( + SELECT + DISTINCT usc.user_id, + usc.new_status, + usc.changed_at, + ud.deleted + FROM user_status_changes usc + LEFT JOIN LATERAL ( + SELECT COUNT(*) > 0 AS deleted + FROM user_deleted ud + WHERE ud.user_id = usc.user_id AND (ud.deleted_at < usc.changed_at OR ud.deleted_at < @start_time) + ) AS ud ON true + WHERE usc.changed_at < @start_time::timestamptz + ORDER BY usc.user_id, usc.changed_at DESC +), + -- status_changes_during_range defines the status of each user during the start_time and end_time. + -- If a user is deleted during the time range, we count status changes between the start_time and the deletion date. + -- Theoretically, it should probably not be possible to update the status of a deleted user, but we + -- need to ensure that this is enforced, so that a change in business logic later does not break this graph. +status_changes_during_range AS ( + SELECT + usc.user_id, + usc.new_status, + usc.changed_at, + ud.deleted + FROM user_status_changes usc + LEFT JOIN LATERAL ( + SELECT COUNT(*) > 0 AS deleted + FROM user_deleted ud + WHERE ud.user_id = usc.user_id AND ud.deleted_at < usc.changed_at + ) AS ud ON true + WHERE usc.changed_at >= @start_time::timestamptz + AND usc.changed_at <= @end_time::timestamptz +), + -- relevant_status_changes defines the status of each user at any point in time. + -- It includes the status of each user before the start_time, and the status of each user during the start_time and end_time. +relevant_status_changes AS ( + SELECT + user_id, + new_status, + changed_at + FROM latest_status_before_range + WHERE NOT deleted + + UNION ALL + + SELECT + user_id, + new_status, + changed_at + FROM status_changes_during_range + WHERE NOT deleted +), + -- statuses defines all the distinct statuses that were present just before and during the time range. + -- This is used to ensure that we have a series for every relevant status. +statuses AS ( + SELECT DISTINCT new_status FROM relevant_status_changes +), + -- We only want to count the latest status change for each user on each date and then filter them by the relevant status. + -- We use the row_number function to ensure that we only count the latest status change for each user on each date. + -- We then filter the status changes by the relevant status in the final select statement below. +ranked_status_change_per_user_per_date AS ( + SELECT + d.date, + rsc1.user_id, + ROW_NUMBER() OVER (PARTITION BY d.date, rsc1.user_id ORDER BY rsc1.changed_at DESC) AS rn, + rsc1.new_status + FROM dates_of_interest d + LEFT JOIN relevant_status_changes rsc1 ON rsc1.changed_at <= d.date +) +SELECT + rscpupd.date, + statuses.new_status AS status, + COUNT(rscpupd.user_id) FILTER ( + WHERE rscpupd.rn = 1 + AND ( + rscpupd.new_status = statuses.new_status + AND ( + -- Include users who haven't been deleted + NOT EXISTS (SELECT 1 FROM user_deleted WHERE user_id = rscpupd.user_id) + OR + -- Or users whose deletion date is after the current date we're looking at + rscpupd.date < (SELECT deleted_at FROM user_deleted WHERE user_id = rscpupd.user_id) + ) + ) + ) AS count +FROM ranked_status_change_per_user_per_date rscpupd +CROSS JOIN statuses +GROUP BY rscpupd.date, statuses.new_status; + diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index f4470c6546698..f253aa98ec266 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -62,7 +62,9 @@ const ( UniqueTemplateVersionsPkey UniqueConstraint = "template_versions_pkey" // ALTER TABLE ONLY template_versions ADD CONSTRAINT template_versions_pkey PRIMARY KEY (id); UniqueTemplateVersionsTemplateIDNameKey UniqueConstraint = "template_versions_template_id_name_key" // ALTER TABLE ONLY template_versions ADD CONSTRAINT template_versions_template_id_name_key UNIQUE (template_id, name); UniqueTemplatesPkey UniqueConstraint = "templates_pkey" // ALTER TABLE ONLY templates ADD CONSTRAINT templates_pkey PRIMARY KEY (id); + UniqueUserDeletedPkey UniqueConstraint = "user_deleted_pkey" // ALTER TABLE ONLY user_deleted ADD CONSTRAINT user_deleted_pkey PRIMARY KEY (id); UniqueUserLinksPkey UniqueConstraint = "user_links_pkey" // ALTER TABLE ONLY user_links ADD CONSTRAINT user_links_pkey PRIMARY KEY (user_id, login_type); + UniqueUserStatusChangesPkey UniqueConstraint = "user_status_changes_pkey" // ALTER TABLE ONLY user_status_changes ADD CONSTRAINT user_status_changes_pkey PRIMARY KEY (id); UniqueUsersPkey UniqueConstraint = "users_pkey" // ALTER TABLE ONLY users ADD CONSTRAINT users_pkey PRIMARY KEY (id); UniqueWorkspaceAgentLogSourcesPkey UniqueConstraint = "workspace_agent_log_sources_pkey" // ALTER TABLE ONLY workspace_agent_log_sources ADD CONSTRAINT workspace_agent_log_sources_pkey PRIMARY KEY (workspace_agent_id, id); UniqueWorkspaceAgentMetadataPkey UniqueConstraint = "workspace_agent_metadata_pkey" // ALTER TABLE ONLY workspace_agent_metadata ADD CONSTRAINT workspace_agent_metadata_pkey PRIMARY KEY (workspace_agent_id, key); diff --git a/coderd/insights.go b/coderd/insights.go index d5faacee90bd5..e4695f50495fb 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -292,6 +292,69 @@ func (api *API) insightsUserLatency(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, resp) } +// @Summary Get insights about user status counts +// @ID get-insights-about-user-status-counts +// @Security CoderSessionToken +// @Produce json +// @Tags Insights +// @Param tz_offset query int true "Time-zone offset (e.g. -2)" +// @Success 200 {object} codersdk.GetUserStatusCountsResponse +// @Router /insights/user-status-counts [get] +func (api *API) insightsUserStatusCounts(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + p := httpapi.NewQueryParamParser() + vals := r.URL.Query() + tzOffset := p.Int(vals, 0, "tz_offset") + p.ErrorExcessParams(vals) + + if len(p.Errors) > 0 { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Query parameters have invalid values.", + Validations: p.Errors, + }) + return + } + + loc := time.FixedZone("", tzOffset*3600) + // If the time is 14:01 or 14:31, we still want to include all the + // data between 14:00 and 15:00. Our rollups buckets are 30 minutes + // so this works nicely. It works just as well for 23:59 as well. + nextHourInLoc := time.Now().In(loc).Truncate(time.Hour).Add(time.Hour) + // Always return 60 days of data (2 months). + sixtyDaysAgo := nextHourInLoc.In(loc).Truncate(24*time.Hour).AddDate(0, 0, -60) + + rows, err := api.Database.GetUserStatusCounts(ctx, database.GetUserStatusCountsParams{ + StartTime: sixtyDaysAgo, + EndTime: nextHourInLoc, + }) + if err != nil { + if httpapi.IsUnauthorizedError(err) { + httpapi.Forbidden(rw) + return + } + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error fetching user status counts over time.", + Detail: err.Error(), + }) + return + } + + resp := codersdk.GetUserStatusCountsResponse{ + StatusCounts: make(map[codersdk.UserStatus][]codersdk.UserStatusChangeCount), + } + + for _, row := range rows { + status := codersdk.UserStatus(row.Status) + resp.StatusCounts[status] = append(resp.StatusCounts[status], codersdk.UserStatusChangeCount{ + Date: row.Date, + Count: row.Count, + }) + } + + httpapi.Write(ctx, rw, http.StatusOK, resp) +} + // @Summary Get insights about templates // @ID get-insights-about-templates // @Security CoderSessionToken diff --git a/codersdk/insights.go b/codersdk/insights.go index c9e708de8f34a..ef44b6b8d013e 100644 --- a/codersdk/insights.go +++ b/codersdk/insights.go @@ -282,3 +282,34 @@ func (c *Client) TemplateInsights(ctx context.Context, req TemplateInsightsReque var result TemplateInsightsResponse return result, json.NewDecoder(resp.Body).Decode(&result) } + +type GetUserStatusCountsResponse struct { + StatusCounts map[UserStatus][]UserStatusChangeCount `json:"status_counts"` +} + +type UserStatusChangeCount struct { + Date time.Time `json:"date" format:"date-time"` + Count int64 `json:"count" example:"10"` +} + +type GetUserStatusCountsRequest struct { + Offset time.Time `json:"offset" format:"date-time"` +} + +func (c *Client) GetUserStatusCounts(ctx context.Context, req GetUserStatusCountsRequest) (GetUserStatusCountsResponse, error) { + qp := url.Values{} + qp.Add("offset", req.Offset.Format(insightsTimeLayout)) + + reqURL := fmt.Sprintf("/api/v2/insights/user-status-counts?%s", qp.Encode()) + resp, err := c.Request(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return GetUserStatusCountsResponse{}, xerrors.Errorf("make request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return GetUserStatusCountsResponse{}, ReadBodyAsError(resp) + } + var result GetUserStatusCountsResponse + return result, json.NewDecoder(resp.Body).Decode(&result) +} diff --git a/docs/reference/api/insights.md b/docs/reference/api/insights.md index e59d74ec6c7f8..b8fcdbbb1e776 100644 --- a/docs/reference/api/insights.md +++ b/docs/reference/api/insights.md @@ -260,3 +260,53 @@ curl -X GET http://coder-server:8080/api/v2/insights/user-latency?start_time=201 | 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.UserLatencyInsightsResponse](schemas.md#codersdkuserlatencyinsightsresponse) | To perform this operation, you must be authenticated. [Learn more](authentication.md). + +## Get insights about user status counts + +### Code samples + +```shell +# Example request using curl +curl -X GET http://coder-server:8080/api/v2/insights/user-status-counts?tz_offset=0 \ + -H 'Accept: application/json' \ + -H 'Coder-Session-Token: API_KEY' +``` + +`GET /insights/user-status-counts` + +### Parameters + +| Name | In | Type | Required | Description | +|-------------|-------|---------|----------|----------------------------| +| `tz_offset` | query | integer | true | Time-zone offset (e.g. -2) | + +### Example responses + +> 200 Response + +```json +{ + "status_counts": { + "property1": [ + { + "count": 10, + "date": "2019-08-24T14:15:22Z" + } + ], + "property2": [ + { + "count": 10, + "date": "2019-08-24T14:15:22Z" + } + ] + } +} +``` + +### Responses + +| Status | Meaning | Description | Schema | +|--------|---------------------------------------------------------|-------------|----------------------------------------------------------------------------------------| +| 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.GetUserStatusCountsResponse](schemas.md#codersdkgetuserstatuscountsresponse) | + +To perform this operation, you must be authenticated. [Learn more](authentication.md). diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index b6874bc5b1bc9..542294a01fa37 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -3000,6 +3000,34 @@ Git clone makes use of this by parsing the URL from: 'Username for "https://gith |-------|--------|----------|--------------|-------------| | `key` | string | false | | | +## codersdk.GetUserStatusCountsResponse + +```json +{ + "status_counts": { + "property1": [ + { + "count": 10, + "date": "2019-08-24T14:15:22Z" + } + ], + "property2": [ + { + "count": 10, + "date": "2019-08-24T14:15:22Z" + } + ] + } +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|--------------------|---------------------------------------------------------------------------|----------|--------------|-------------| +| `status_counts` | object | false | | | +| ยป `[any property]` | array of [codersdk.UserStatusChangeCount](#codersdkuserstatuschangecount) | false | | | + ## codersdk.GetUsersResponse ```json @@ -6724,6 +6752,22 @@ If the schedule is empty, the user will be updated to use the default schedule.| | `dormant` | | `suspended` | +## codersdk.UserStatusChangeCount + +```json +{ + "count": 10, + "date": "2019-08-24T14:15:22Z" +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|---------|---------|----------|--------------|-------------| +| `count` | integer | false | | | +| `date` | string | false | | | + ## codersdk.ValidateUserPasswordRequest ```json diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 4956de8691ed7..acb5254a61a0a 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -882,6 +882,16 @@ export interface GenerateAPIKeyResponse { readonly key: string; } +// From codersdk/insights.go +export interface GetUserStatusCountsRequest { + readonly offset: string; +} + +// From codersdk/insights.go +export interface GetUserStatusCountsResponse { + readonly status_counts: Record; +} + // From codersdk/users.go export interface GetUsersResponse { readonly users: readonly User[]; @@ -2690,6 +2700,12 @@ export interface UserRoles { // From codersdk/users.go export type UserStatus = "active" | "dormant" | "suspended"; +// From codersdk/insights.go +export interface UserStatusChangeCount { + readonly date: string; + readonly count: number; +} + export const UserStatuses: UserStatus[] = ["active", "dormant", "suspended"]; // From codersdk/users.go