Skip to content

feat: make agent stats' cardinality configurable #12535

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Mar 13, 2024
Merged

feat: make agent stats' cardinality configurable #12535

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
975caa5
Initial implementation
dannykopping Mar 7, 2024
fcaebb3
Refactoring
dannykopping Mar 8, 2024
70102f8
Drive-by change to add delve to nix flake so ./scripts/develop.sh --d…
dannykopping Mar 8, 2024
e3bfa0d
Update deployment config
dannykopping Mar 8, 2024
92e538a
Linting
dannykopping Mar 8, 2024
fd19896
Control cardinality of coderd metrics as well
dannykopping Mar 8, 2024
a682843
Remove default value
dannykopping Mar 8, 2024
48b68ad
make fmt && make gen
dannykopping Mar 8, 2024
661f5c2
Appeasing CI
dannykopping Mar 8, 2024
7d3ded4
Appeasing CI, again...
dannykopping Mar 8, 2024
defcf74
...
dannykopping Mar 8, 2024
97f2bd1
Review comments & CI appeasement
dannykopping Mar 11, 2024
8f2c505
More review comments & CI appeasement
dannykopping Mar 11, 2024
c0896a7
Fixing lints
dannykopping Mar 11, 2024
c11dc61
chore(dogfood): update keys
johnstcn Mar 11, 2024
cdd6734
Fix flaky label evaluation by sorting both sides
dannykopping Mar 11, 2024
06f2f87
Update node key
dannykopping Mar 11, 2024
3c1d016
Filter out unacceptable labels in AgentStats
dannykopping Mar 11, 2024
f8ebdf2
make fmt
dannykopping Mar 11, 2024
05e5cd0
Merge branch 'dk/configurable-cardinality' of github.com:dannykopping…
dannykopping Mar 12, 2024
b1ef0d6
Merge branch 'main' of github.com:/coder/coder into dk/configurable-c…
dannykopping Mar 12, 2024
a669411
Review comments
dannykopping Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Review comments 
Signed-off-by: Danny Kopping <danny@coder.com>
  • Loading branch information
@dannykopping
dannykopping committed Mar 12, 2024
commit a669411437bbee851e19a5c100f08863dc8b2bf8
4 changes: 4 additions & 0 deletions coderd/agentmetrics/labels_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ func TestValidateAggregationLabels(t *testing.T) {
labels: []string{""},
expectedErr: true,
},
{
name: "all valid entries",
labels: agentmetrics.LabelAll,
},
}

for _, tc := range tests {
Expand Down
20 changes: 8 additions & 12 deletions coderd/prometheusmetrics/prometheusmetrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,17 @@ import (
"sync/atomic"
"time"

"github.com/coder/coder/v2/coderd/agentmetrics"
"github.com/coder/coder/v2/codersdk"

"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"tailscale.com/tailcfg"

"cdr.dev/slog"

"github.com/coder/coder/v2/coderd/agentmetrics"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/tailnet"
)

Expand Down Expand Up @@ -338,7 +337,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
aggregateByLabels = agentmetrics.LabelAgentStats
}

aggregateByLabels = filterInvalidLabels(aggregateByLabels)
aggregateByLabels = filterAcceptableAgentLabels(aggregateByLabels)

metricsCollectorAgentStats := prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "coderd",
Expand Down Expand Up @@ -517,18 +516,15 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
}, nil
}

// filterInvalidLabels handles a slightly messy situation whereby `prometheus-aggregate-agent-stats-by` can control on
// filterAcceptableAgentLabels handles a slightly messy situation whereby `prometheus-aggregate-agent-stats-by` can control on
// which labels agent stats are aggregated, but for these specific metrics in this file there is no `template` label value,
// and therefore we have to exclude it from the list of acceptable labels.
func filterInvalidLabels(labels []string) []string {
var out []string

func filterAcceptableAgentLabels(labels []string) []string {
out := make([]string, 0, len(labels))
for _, label := range labels {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe optimize it this way?

validLabels := make([]string, 0, len(labels))

for _, label := range labels {
	if label != agentmetrics.LabelTemplateName {
		validLabels = append(validLabels, label)
	}
}

if label == agentmetrics.LabelTemplateName {
continue
if label != agentmetrics.LabelTemplateName {
out = append(out, label)
}

out = append(out, label)
}

return out
Expand Down
40 changes: 40 additions & 0 deletions coderd/prometheusmetrics/prometheusmetrics_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package prometheusmetrics

import (
"testing"

"github.com/stretchr/testify/require"

"github.com/coder/coder/v2/coderd/agentmetrics"
)

func TestFilterAcceptableAgentLabels(t *testing.T) {
t.Parallel()

tests := []struct {
name string
input []string
expected []string
}{
{
name: "template label is ignored",
input: []string{agentmetrics.LabelTemplateName},
expected: []string{},
},
{
name: "all other labels are returned",
input: agentmetrics.LabelAll,
expected: []string{agentmetrics.LabelAgentName, agentmetrics.LabelUsername, agentmetrics.LabelWorkspaceName},
},
}

for _, tc := range tests {
tc := tc

t.Run(tc.name, func(t *testing.T) {
t.Parallel()

require.Equal(t, tc.expected, filterAcceptableAgentLabels(tc.input))
})
}
}
10 changes: 4 additions & 6 deletions coderd/prometheusmetrics/prometheusmetrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,6 @@ import (
"testing"
"time"

"github.com/coder/coder/v2/coderd/agentmetrics"

"github.com/coder/coder/v2/coderd/batchstats"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"

"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
Expand All @@ -26,10 +20,14 @@ import (
"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"

"github.com/coder/coder/v2/coderd/agentmetrics"
"github.com/coder/coder/v2/coderd/batchstats"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/prometheusmetrics"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/agentsdk"
Expand Down
3 changes: 1 addition & 2 deletions codersdk/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,11 @@ import (
"golang.org/x/mod/semver"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/coderd/agentmetrics"

"github.com/coreos/go-oidc/v3/oidc"

"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/cli/clibase"
"github.com/coder/coder/v2/coderd/agentmetrics"
"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
)

Expand Down