Skip to content

feat: expose workspace statuses (with details) as a prometheus metric #12762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Apr 2, 2024
Merged

feat: expose workspace statuses (with details) as a prometheus metric #12762

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
843d650
Add metric to show operators what statuses workspaces are in, with re…
dannykopping Mar 25, 2024
2ed42a3
Clear gauges if no db entries are found
dannykopping Mar 26, 2024
c31b498
Change default refresh rates of Agent and Workspaces calls which seem…
dannykopping Mar 26, 2024
2f5a948
Added tests for workspace detail metrics
dannykopping Mar 26, 2024
fc61d37
Refactor common funcs
dannykopping Mar 26, 2024
6f95371
Remove unnecessary metric from test
dannykopping Mar 26, 2024
f42af07
Use NoLock variety to fix hung tests
dannykopping Mar 26, 2024
2cb8ccc
Fix duration on windows builds
dannykopping Mar 26, 2024
b118044
Changing join type so that latest_build_status is non-nullable
dannykopping Mar 27, 2024
acd104c
Remove SuperShort duration as extraneous
dannykopping Mar 28, 2024
a333f98
Remove workspace cardinality; rename for clarity
dannykopping Mar 28, 2024
c920508
Fixing tests
dannykopping Mar 28, 2024
cf14b9d
Merge branch 'main' of github.com:/coder/coder into dk/workspace-metric
dannykopping Mar 28, 2024
a94914f
Merge branch 'main' of github.com:/coder/coder into dk/workspace-metric
dannykopping Mar 28, 2024
8e6cde9
Renaming for clarity
dannykopping Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Renaming for clarity 
Signed-off-by: Danny Kopping <danny@coder.com>
  • Loading branch information
@dannykopping
dannykopping committed Mar 28, 2024
commit 8e6cde9e7a2d94ca724e64bda44458126467dffb
26 changes: 13 additions & 13 deletions coderd/prometheusmetrics/prometheusmetrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,34 +79,34 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
duration = defaultRefreshRate
}

workspaceStatusTotals := prometheus.NewGaugeVec(prometheus.GaugeOpts{
workspaceLatestBuildTotals := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Subsystem: "api",
Name: "workspace_latest_build_total",
Help: "The current number of workspace builds by status.",
}, []string{"status"})
if err := registerer.Register(workspaceStatusTotals); err != nil {
if err := registerer.Register(workspaceLatestBuildTotals); err != nil {
return nil, err
}

workspaceStatuses := prometheus.NewGaugeVec(prometheus.GaugeOpts{
workspaceLatestBuildStatuses := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Name: "workspace_latest_build_status",
Help: "The current workspace statuses by template, transition, and owner.",
}, []string{"status", "template_name", "template_version", "workspace_owner", "workspace_transition"})
if err := registerer.Register(workspaceStatuses); err != nil {
if err := registerer.Register(workspaceLatestBuildStatuses); err != nil {
return nil, err
}

ctx, cancelFunc := context.WithCancel(ctx)
done := make(chan struct{})

updateWorkspaceStatuses := func() {
updateWorkspaceTotals := func() {
builds, err := db.GetLatestWorkspaceBuilds(ctx)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceStatusTotals.Reset()
workspaceLatestBuildTotals.Reset()
}

logger.Warn(ctx, "failed to load latest workspace builds", slog.Error(err))
Expand All @@ -127,31 +127,31 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
return
}

workspaceStatusTotals.Reset()
workspaceLatestBuildTotals.Reset()
for _, job := range jobs {
status := codersdk.ProvisionerJobStatus(job.JobStatus)
workspaceStatusTotals.WithLabelValues(string(status)).Add(1)
workspaceLatestBuildTotals.WithLabelValues(string(status)).Add(1)
}
}

updateWorkspaceDetails := func() {
updateWorkspaceStatuses := func() {
ws, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
Deleted: false,
WithSummary: false,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceStatuses.Reset()
workspaceLatestBuildStatuses.Reset()
}

logger.Warn(ctx, "failed to load active workspaces", slog.Error(err))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: maybe logger.Error?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wouldn't strictly call it an error because it could be an instance of sql.ErrNoRows which is valid but undesirable. To be the most correct I would make it warn in case of sql.ErrNoRows and error in all other cases, but that feels unnecessary.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

which is valid but undesirable

Will it also return sql.ErrNoRows if there are zero workspaces = fresh deployment?

Copy link
Contributor Author

@dannykopping dannykopping Mar 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Believe so, yes.
Or if all workspaces have been deleted.

return
}

workspaceStatuses.Reset()
workspaceLatestBuildStatuses.Reset()
for _, w := range ws {
workspaceStatuses.WithLabelValues(string(w.LatestBuildStatus), w.TemplateName, w.TemplateVersionName.String, w.Username, string(w.LatestBuildTransition)).Add(1)
workspaceLatestBuildStatuses.WithLabelValues(string(w.LatestBuildStatus), w.TemplateName, w.TemplateVersionName.String, w.Username, string(w.LatestBuildTransition)).Add(1)
}
}

Expand All @@ -161,8 +161,8 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
doTick := func() {
defer ticker.Reset(duration)

updateWorkspaceTotals()
updateWorkspaceStatuses()
updateWorkspaceDetails()
}

go func() {
Expand Down
4 changes: 2 additions & 2 deletions coderd/prometheusmetrics/prometheusmetrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestActiveUsers(t *testing.T) {
}
}

func TestWorkspaceStatuses(t *testing.T) {
func TestWorkspaceLatestBuildTotals(t *testing.T) {
t.Parallel()

for _, tc := range []struct {
Expand Down Expand Up @@ -184,7 +184,7 @@ func TestWorkspaceStatuses(t *testing.T) {
}
}

func TestWorkspaceDetails(t *testing.T) {
func TestWorkspaceLatestBuildStatuses(t *testing.T) {
t.Parallel()

for _, tc := range []struct {
Expand Down