Skip to content

feat: add total users insight #15486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 23 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: add registered users insights
  • Loading branch information
BrunoQuaresma committed Nov 12, 2024
commit a20be9f58e75bc5c9d69cc9d3cae67f802dde264
1 change: 1 addition & 0 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -1260,6 +1260,7 @@ func New(options *Options) *API {
r.Get("/user-activity", api.insightsUserActivity)
r.Get("/user-latency", api.insightsUserLatency)
r.Get("/templates", api.insightsTemplates)
r.Get("/total-users", api.insightsTotalUsers)
})
r.Route("/debug", func(r chi.Router) {
r.Use(
Expand Down
7 changes: 7 additions & 0 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -2283,6 +2283,13 @@ func (q *querier) GetTemplatesWithFilter(ctx context.Context, arg database.GetTe
return q.db.GetAuthorizedTemplates(ctx, arg, prep)
}

func (q *querier) GetAccumulatedUsersInsights(ctx context.Context, arg database.GetAccumulatedUsersInsightsParams) ([]database.GetAccumulatedUsersInsightsRow, error) {
if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceUser); err != nil {
return nil, err
}
return q.db.GetAccumulatedUsersInsights(ctx, arg)
}

func (q *querier) GetUnexpiredLicenses(ctx context.Context) ([]database.License, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
Expand Down
42 changes: 42 additions & 0 deletions coderd/database/dbmem/dbmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -5169,6 +5169,48 @@ func (q *FakeQuerier) GetTemplatesWithFilter(ctx context.Context, arg database.G
return q.GetAuthorizedTemplates(ctx, arg, nil)
}

func (q *FakeQuerier) GetAccumulatedUsersInsights(_ context.Context, arg database.GetAccumulatedUsersInsightsParams) ([]database.GetAccumulatedUsersInsightsRow, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()

newUsersByDate := make(map[time.Time]int64)
for _, user := range q.users {
if user.CreatedAt.Before(arg.StartTime) || user.CreatedAt.After(arg.EndTime) {
continue
}
date := user.CreatedAt.Truncate(24 * time.Hour)
newUsersByDate[date]++
}

accumulatedByDate := make(map[time.Time]int64)
var accumulatedCount int64
sortedDates := make([]time.Time, 0, len(newUsersByDate))

for date := range newUsersByDate {
sortedDates = append(sortedDates, date)
}

// Sort dates in ascending order
sort.Slice(sortedDates, func(i, j int) bool {
return sortedDates[i].Before(sortedDates[j])
})

// Calculate accumulated count
for _, date := range sortedDates {
accumulatedCount += newUsersByDate[date]
accumulatedByDate[date] = accumulatedCount
}

var rows []database.GetAccumulatedUsersInsightsRow
for _, date := range sortedDates {
rows = append(rows, database.GetAccumulatedUsersInsightsRow{
Date: date,
Count: accumulatedByDate[date],
})
}
return rows, nil
}

func (q *FakeQuerier) GetUnexpiredLicenses(_ context.Context) ([]database.License, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
Expand Down
7 changes: 7 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

64 changes: 57 additions & 7 deletions coderd/database/queries.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions coderd/database/queries/insights.sql
Original file line number Diff line number Diff line change
Expand Up @@ -771,3 +771,18 @@ SELECT
FROM unique_template_params utp
JOIN workspace_build_parameters wbp ON (utp.workspace_build_ids @> ARRAY[wbp.workspace_build_id] AND utp.name = wbp.name)
GROUP BY utp.num, utp.template_ids, utp.name, utp.type, utp.display_name, utp.description, utp.options, wbp.value;

-- name: GetAccumulatedUsersInsights :many
-- GetAccumulatedUsersInsights returns the accumulated number of users created
-- in the given timeframe. It returns the accumulated number of users for each date
-- within the specified timeframe, providing a running total of user sign-ups.
SELECT
date_trunc('day', created_at)::date AS date,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might produce strange results if requested start time and end time are in a different timezone. Each day may be missing few/many entries and we might get ghost dates as well (+/-1 day outside start/end range).

Suggestion:

	date_trunc('day', (
		created_at AT TIME ZONE EXTRACT(TIMEZONE FROM @start_time::timestamptz)
	)) AT TIME ZONE EXTRACT(TIMEZONE FROM @start_time::timestamptz) AS date,

(Untested), why the double at timezone? One strips timezone and the other one adds it back.

coder=# select created_at from users;
          created_at
-------------------------------
 2024-04-08 11:41:35.096834+00
 2024-04-08 11:40:54.587231+00
(2 rows)

coder=# select created_at at time zone 'europe/helsinki' from users;
          timezone
----------------------------
 2024-04-08 14:41:35.096834
 2024-04-08 14:40:54.587231
(2 rows)

coder=# select created_at at time zone 'europe/helsinki' at time zone 'europe/helsinki' from users;
           timezone
-------------------------------
 2024-04-08 11:41:35.096834+00
 2024-04-08 11:40:54.587231+00
(2 rows)

Notice +00 and lack thereof. Now notice how we get a timezoneally correct truncation when we do it without timezone:

coder=# select date_trunc('day', created_at) from users;
       date_trunc
------------------------
 2024-04-08 00:00:00+00
 2024-04-08 00:00:00+00
(2 rows)

coder=# select date_trunc('day', created_at at time zone 'europe/helsinki') at time zone 'europe/helsinki' from users;
        timezone
------------------------
 2024-04-07 21:00:00+00
 2024-04-07 21:00:00+00
(2 rows)

You might want to use a subquery so you can count over the improved truncation without repeating all that noise.

COUNT(*) OVER (ORDER BY date_trunc('day', created_at)::date) AS accumulated_users
FROM
users
WHERE
created_at >= @start_time::timestamptz
AND created_at < @end_time::timestamptz
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More for confirmation than a request to change - is it expected to exclude the end_time from the query ?

If I say I want the insights from 01/01to 02/01- should we include 02/01 ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In-memory DB doesn't match this:

		if user.CreatedAt.Before(arg.StartTime) || user.CreatedAt.After(arg.EndTime) {

Missing || user.CreatedAt.Equal(arg.EndTime).

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@defelmnq I will double-check that.

@mafredri I didn't get your last comment. Sorry 😞

Copy link
Contributor

@SasSwart SasSwart Nov 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it expected to exclude the end_time from the query ?

No. Not quite, at least. The intention is to include it. We'll generate a series of dates starting from the start date with 24 hour increments. This means that subject to some offset (<24h), the date of the endtime param will be included.

ORDER BY
date_trunc('day', created_at)::date;
48 changes: 48 additions & 0 deletions coderd/insights.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,54 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) {
httpapi.Write(ctx, rw, http.StatusOK, resp)
}

func (api *API) insightsTotalUsers(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()

p := httpapi.NewQueryParamParser().
RequiredNotEmpty("start_time").
RequiredNotEmpty("end_time")
vals := r.URL.Query()
var (
// The QueryParamParser does not preserve timezone, so we need
// to parse the time ourselves.
startTimeString = p.String(vals, "", "start_time")
endTimeString = p.String(vals, "", "end_time")
)
p.ErrorExcessParams(vals)
if len(p.Errors) > 0 {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Query parameters have invalid values.",
Validations: p.Errors,
})
return
}

startTime, endTime, ok := parseInsightsStartAndEndTime(ctx, rw, time.Now(), startTimeString, endTimeString)
if !ok {
return
}

rows, err := api.Database.GetAccumulatedUsersInsights(ctx, database.GetAccumulatedUsersInsightsParams{
StartTime: startTime,
EndTime: endTime,
})
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error fetching total users.",
Detail: err.Error(),
})
return
}
var res codersdk.TotalUsersInsightResponse
for _, row := range rows {
res = append(res, codersdk.TotalUserByDate{
Date: row.Date.Format(time.DateOnly),
Total: uint64(row.Count),
})
}
httpapi.Write(r.Context(), rw, http.StatusOK, res)
}

// convertTemplateInsightsApps builds the list of builtin apps and template apps
// from the provided database rows, builtin apps are implicitly a part of all
// templates.
Expand Down
60 changes: 60 additions & 0 deletions coderd/insights_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2351,3 +2351,63 @@ func TestGenericInsights_RBAC(t *testing.T) {
})
}
}

func TestTotalUsersInsight(t *testing.T) {
t.Parallel()

t.Run("Success", func(t *testing.T) {
t.Parallel()

client, db := coderdtest.NewWithDatabase(t, nil)
coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
t.Cleanup(cancel)

// Given: a deployment with many users
dbgen.User(t, db, database.User{
Email: "user1@coder.com",
Username: "user1",
CreatedAt: time.Now().UTC().AddDate(0, 0, -2),
})
dbgen.User(t, db, database.User{
Email: "user2@coder.com",
Username: "user2",
CreatedAt: time.Now().UTC().AddDate(0, 0, -2),
})
dbgen.User(t, db, database.User{
Email: "user3@coder.com",
Username: "user3",
CreatedAt: time.Now().UTC().AddDate(0, 0, -1),
})

// When: requesting the total users by day
threeDaysAgo := time.Now().UTC().AddDate(0, 0, -2).Truncate(24 * time.Hour)
today := time.Now().UTC().Truncate(time.Hour).Add(time.Hour)
res, err := client.TotalUsersInsight(ctx, codersdk.TotalUsersInsightRequest{
StartTime: threeDaysAgo,
EndTime: today,
})
require.NoError(t, err)

// Then: expect to have the accumulated users count growing by day
require.Len(t, res, 4)
})

t.Run("EmptyTimes", func(t *testing.T) {
t.Parallel()

client, _ := coderdtest.NewWithDatabase(t, nil)
coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
t.Cleanup(cancel)

// When: requesting total users without a time range
_, err := client.TotalUsersInsight(ctx, codersdk.TotalUsersInsightRequest{})

// Then: expect a bad request error
require.Error(t, err)
var apiErr *codersdk.Error
require.ErrorAs(t, err, &apiErr)
require.Equal(t, http.StatusBadRequest, apiErr.StatusCode())
})
}
1 change: 1 addition & 0 deletions coderd/rbac/object_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions coderd/rbac/policy/policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ var RBACPermissions = map[string]PermissionDefinition{

ActionReadPersonal: actDef("read personal user data like user settings and auth links"),
ActionUpdatePersonal: actDef("update personal data"),

ActionViewInsights: actDef("view insights"),
},
},
"workspace": {
Expand Down
Loading