From 3fa325886273ac6fc33f2c01543a3e16e61aa10a Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 09:38:13 +0200 Subject: [PATCH 01/21] WIP --- coderd/coderd.go | 3 + coderd/prometheusmetrics/license.go | 105 ++++++++++++++++++ coderd/prometheusmetrics/prometheusmetrics.go | 58 ++++++++++ enterprise/coderd/coderd.go | 6 + 4 files changed, 172 insertions(+) create mode 100644 coderd/prometheusmetrics/license.go diff --git a/coderd/coderd.go b/coderd/coderd.go index d46cb493dceee..c5ff22c00ad2b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -38,6 +38,7 @@ import ( // Used for swagger docs. _ "github.com/coder/coder/v2/coderd/apidoc" "github.com/coder/coder/v2/coderd/externalauth" + "github.com/coder/coder/v2/coderd/prometheusmetrics" "cdr.dev/slog" "github.com/coder/coder/v2/buildinfo" @@ -1033,6 +1034,8 @@ type API struct { statsBatcher *batchstats.Batcher Acquirer *provisionerdserver.Acquirer + + LicenseMetrics *prometheusmetrics.LicenseMetrics } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go new file mode 100644 index 0000000000000..88ca7498fa6fc --- /dev/null +++ b/coderd/prometheusmetrics/license.go @@ -0,0 +1,105 @@ +package prometheusmetrics + +import ( + "context" + "sync/atomic" + "time" + + "cdr.dev/slog" + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/codersdk" +) + +type LicenseMetrics struct { + db database.Store + interval time.Duration + logger slog.Logger + registry *prometheus.Registry + + Entitlements atomic.Pointer[codersdk.Entitlements] +} + +type LicenseMetricsOptions struct { + Interval time.Duration + Database database.Store + Logger slog.Logger + Registry *prometheus.Registry +} + +func NewLicenseMetrics(opts *LicenseMetricsOptions) (*LicenseMetrics, error) { + if opts.Interval == 0 { + opts.Interval = 1 * time.Minute + } + if opts.Database == nil { + return nil, xerrors.Errorf("database is required") + } + if opts.Registry == nil { + opts.Registry = prometheus.NewRegistry() + } + + return &LicenseMetrics{ + db: opts.Database, + interval: opts.Interval, + logger: opts.Logger, + registry: opts.Registry, + }, nil +} + +func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { + + licenseLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "user_limit", + Help: `The user seats limit based on the current license. "Zero" means unlimited or a disabled feature.`, + }) + err := registerer.Register(licenseLimitGauge) + if err != nil { + return nil, err + } + + activeUsersGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "active_users", + Help: "The number of active users.", + }) + err = registerer.Register(activeUsersGauge) + if err != nil { + return nil, err + } + + ctx, cancelFunc := context.WithCancel(ctx) + done := make(chan struct{}) + ticker := time.NewTicker(duration) + go func() { + defer close(done) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + apiKeys, err := db.GetAPIKeysLastUsedAfter(ctx, dbtime.Now().Add(-1*time.Hour)) + if err != nil { + continue + } + distinctUsers := map[uuid.UUID]struct{}{} + for _, apiKey := range apiKeys { + distinctUsers[apiKey.UserID] = struct{}{} + } + gauge.Set(float64(len(distinctUsers))) + } + }() + return func() { + cancelFunc() + <-done + }, nil +} diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go index 7145c2afa3b39..7d0579d2ee91f 100644 --- a/coderd/prometheusmetrics/prometheusmetrics.go +++ b/coderd/prometheusmetrics/prometheusmetrics.go @@ -502,3 +502,61 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R <-done }, nil } + +// License function tracks the number of occupied license seats, based on the entitlements. +func License(ctx context.Context, registerer prometheus.Registerer, db database.Store, entitlementsFetcher func() codersdk.Entitlements, duration time.Duration) (func(), error) { + if duration == 0 { + duration = 1 * time.Minute + } + + licenseLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "user_limit", + Help: `The user seats limit based on the current license. "Zero" means unlimited or a disabled feature.`, + }) + err := registerer.Register(licenseLimitGauge) + if err != nil { + return nil, err + } + + activeUsersGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "active_users", + Help: "The number of active users.", + }) + err = registerer.Register(activeUsersGauge) + if err != nil { + return nil, err + } + + ctx, cancelFunc := context.WithCancel(ctx) + done := make(chan struct{}) + ticker := time.NewTicker(duration) + go func() { + defer close(done) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + apiKeys, err := db.GetAPIKeysLastUsedAfter(ctx, dbtime.Now().Add(-1*time.Hour)) + if err != nil { + continue + } + distinctUsers := map[uuid.UUID]struct{}{} + for _, apiKey := range apiKeys { + distinctUsers[apiKey.UserID] = struct{}{} + } + gauge.Set(float64(len(distinctUsers))) + } + }() + return func() { + cancelFunc() + <-done + }, nil +} diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index eea08a488f567..19282f3695c6d 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -923,3 +923,9 @@ func (api *API) runEntitlementsLoop(ctx context.Context) { func (api *API) Authorize(r *http.Request, action rbac.Action, object rbac.Objecter) bool { return api.AGPL.HTTPAuth.Authorize(r, action, object) } + +func (api *API) fetchEntitlements() codersdk.Entitlements { + api.entitlementsMu.RLock() + defer api.entitlementsMu.RUnlock() + return api.entitlements +} From 404896564e91e6cc8425d2af04668ee1eb4bae49 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 11:31:37 +0200 Subject: [PATCH 02/21] WIP --- cli/server.go | 6 ++ coderd/coderd.go | 12 ++++ coderd/prometheusmetrics/license.go | 32 +++++----- coderd/prometheusmetrics/prometheusmetrics.go | 58 ------------------- enterprise/coderd/coderd.go | 1 + 5 files changed, 35 insertions(+), 74 deletions(-) diff --git a/cli/server.go b/cli/server.go index f9ef1aaa65c8c..2bf62faa988c5 100644 --- a/cli/server.go +++ b/cli/server.go @@ -814,6 +814,12 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("register agents prometheus metric: %w", err) } defer closeAgentsFunc() + + closeLicenseMetricsFunc, err := coderAPI.LicenseMetrics.Collect(ctx) + if err != nil { + return xerrors.Errorf("register license metric: %w", err) + } + defer closeLicenseMetricsFunc() } client := codersdk.New(localURL) diff --git a/coderd/coderd.go b/coderd/coderd.go index c5ff22c00ad2b..e42415de746e0 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -343,6 +343,16 @@ func New(options *Options) *API { if err != nil { panic(xerrors.Errorf("get deployment ID: %w", err)) } + + licenseMetrics, err := prometheusmetrics.NewLicenseMetrics(&prometheusmetrics.LicenseMetricsOptions{ + Database: options.Database, + Logger: options.Logger, + Registry: options.PrometheusRegistry, + }) + if err != nil { + panic(xerrors.Errorf("unable to initialize license metrics: %w", err)) + } + api := &API{ ctx: ctx, cancel: cancel, @@ -377,6 +387,8 @@ func New(options *Options) *API { options.Logger.Named("acquirer"), options.Database, options.Pubsub), + + LicenseMetrics: licenseMetrics, } if options.UpdateCheckOptions != nil { api.updateChecker = updatecheck.New( diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index 88ca7498fa6fc..75f62f9b110df 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -5,13 +5,12 @@ import ( "sync/atomic" "time" - "cdr.dev/slog" - "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "golang.org/x/xerrors" + "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/codersdk" ) @@ -51,14 +50,13 @@ func NewLicenseMetrics(opts *LicenseMetricsOptions) (*LicenseMetrics, error) { } func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { - licenseLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "license", Name: "user_limit", Help: `The user seats limit based on the current license. "Zero" means unlimited or a disabled feature.`, }) - err := registerer.Register(licenseLimitGauge) + err := lm.registry.Register(licenseLimitGauge) if err != nil { return nil, err } @@ -69,14 +67,25 @@ func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { Name: "active_users", Help: "The number of active users.", }) - err = registerer.Register(activeUsersGauge) + err = lm.registry.Register(activeUsersGauge) + if err != nil { + return nil, err + } + + userLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "user_limit", + Help: "The user seats limit based on the active Coder license.", + }) + err = lm.registry.Register(activeUsersGauge) if err != nil { return nil, err } ctx, cancelFunc := context.WithCancel(ctx) done := make(chan struct{}) - ticker := time.NewTicker(duration) + ticker := time.NewTicker(lm.interval) go func() { defer close(done) defer ticker.Stop() @@ -87,15 +96,6 @@ func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { case <-ticker.C: } - apiKeys, err := db.GetAPIKeysLastUsedAfter(ctx, dbtime.Now().Add(-1*time.Hour)) - if err != nil { - continue - } - distinctUsers := map[uuid.UUID]struct{}{} - for _, apiKey := range apiKeys { - distinctUsers[apiKey.UserID] = struct{}{} - } - gauge.Set(float64(len(distinctUsers))) } }() return func() { diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go index 7d0579d2ee91f..7145c2afa3b39 100644 --- a/coderd/prometheusmetrics/prometheusmetrics.go +++ b/coderd/prometheusmetrics/prometheusmetrics.go @@ -502,61 +502,3 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R <-done }, nil } - -// License function tracks the number of occupied license seats, based on the entitlements. -func License(ctx context.Context, registerer prometheus.Registerer, db database.Store, entitlementsFetcher func() codersdk.Entitlements, duration time.Duration) (func(), error) { - if duration == 0 { - duration = 1 * time.Minute - } - - licenseLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "user_limit", - Help: `The user seats limit based on the current license. "Zero" means unlimited or a disabled feature.`, - }) - err := registerer.Register(licenseLimitGauge) - if err != nil { - return nil, err - } - - activeUsersGauge := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "active_users", - Help: "The number of active users.", - }) - err = registerer.Register(activeUsersGauge) - if err != nil { - return nil, err - } - - ctx, cancelFunc := context.WithCancel(ctx) - done := make(chan struct{}) - ticker := time.NewTicker(duration) - go func() { - defer close(done) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - - apiKeys, err := db.GetAPIKeysLastUsedAfter(ctx, dbtime.Now().Add(-1*time.Hour)) - if err != nil { - continue - } - distinctUsers := map[uuid.UUID]struct{}{} - for _, apiKey := range apiKeys { - distinctUsers[apiKey.UserID] = struct{}{} - } - gauge.Set(float64(len(distinctUsers))) - } - }() - return func() { - cancelFunc() - <-done - }, nil -} diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 19282f3695c6d..bdccdd1780c43 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -661,6 +661,7 @@ func (api *API) updateEntitlements(ctx context.Context) error { defer api.entitlementsMu.Unlock() api.entitlements = entitlements api.AGPL.SiteHandler.Entitlements.Store(&entitlements) + api.AGPL.LicenseMetrics.Entitlements.Store(&entitlements) return nil } From fef09395d4a69cdbd1ae57d138dea8de82fd817b Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 13:21:41 +0200 Subject: [PATCH 03/21] doTick --- coderd/prometheusmetrics/license.go | 58 +++++++++++++++++++---------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index 75f62f9b110df..97a75d4a0b617 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -2,6 +2,7 @@ package prometheusmetrics import ( "context" + "fmt" "sync/atomic" "time" @@ -50,42 +51,59 @@ func NewLicenseMetrics(opts *LicenseMetricsOptions) (*LicenseMetrics, error) { } func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { - licenseLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "user_limit", - Help: `The user seats limit based on the current license. "Zero" means unlimited or a disabled feature.`, - }) - err := lm.registry.Register(licenseLimitGauge) - if err != nil { - return nil, err - } - - activeUsersGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + activeUsersGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "license", Name: "active_users", - Help: "The number of active users.", - }) - err = lm.registry.Register(activeUsersGauge) + Help: `The number of active users.`, + }, []string{"entitled"})) + err := lm.registry.Register(activeUsersGauge) if err != nil { return nil, err } - userLimitGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + userLimitGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "coderd", Subsystem: "license", Name: "user_limit", Help: "The user seats limit based on the active Coder license.", - }) - err = lm.registry.Register(activeUsersGauge) + }, []string{"entitled"})) + err = lm.registry.Register(userLimitGauge) if err != nil { return nil, err } ctx, cancelFunc := context.WithCancel(ctx) done := make(chan struct{}) - ticker := time.NewTicker(lm.interval) + ticker := time.NewTicker(time.Nanosecond) + + doTick := func() { + defer ticker.Reset(lm.interval) + + entitlements := lm.Entitlements.Load() + userLimitEntitlement, ok := entitlements.Features[codersdk.FeatureUserLimit] + if !ok { + lm.logger.Warn(ctx, `"user_limit" entitlement is not present`) + return + } + + enabled := fmt.Sprintf("%v", userLimitEntitlement.Enabled) + if userLimitEntitlement.Actual != nil { + activeUsersGauge.WithLabelValues(VectorOperationSet, float64(*userLimitEntitlement.Actual), enabled) + } else { + activeUsersGauge.WithLabelValues(VectorOperationSet, 0, enabled) + } + + if userLimitEntitlement.Limit != nil { + userLimitGauge.WithLabelValues(VectorOperationSet, float64(*userLimitEntitlement.Limit), enabled) + } else { + userLimitGauge.WithLabelValues(VectorOperationSet, 0, enabled) + } + + activeUsersGauge.Commit() + userLimitGauge.Commit() + } + go func() { defer close(done) defer ticker.Stop() @@ -94,8 +112,8 @@ func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { case <-ctx.Done(): return case <-ticker.C: + doTick() } - } }() return func() { From d1e1bab1ccac1d28ab331d10b8debb3ab1800809 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 13:32:06 +0200 Subject: [PATCH 04/21] Remove fetchEntitlements --- enterprise/coderd/coderd.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index bdccdd1780c43..c0eee3faca4c1 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -924,9 +924,3 @@ func (api *API) runEntitlementsLoop(ctx context.Context) { func (api *API) Authorize(r *http.Request, action rbac.Action, object rbac.Objecter) bool { return api.AGPL.HTTPAuth.Authorize(r, action, object) } - -func (api *API) fetchEntitlements() codersdk.Entitlements { - api.entitlementsMu.RLock() - defer api.entitlementsMu.RUnlock() - return api.entitlements -} From 395d64b09a39da68a6a0e7ce07fc497c093bcf0f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 13:45:07 +0200 Subject: [PATCH 05/21] Fix: nolint --- cli/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/server.go b/cli/server.go index 2bf62faa988c5..b305dfa9793cd 100644 --- a/cli/server.go +++ b/cli/server.go @@ -228,6 +228,7 @@ func enablePrometheus( ), nil } +// nolint:gocognit // This function makes more sense being large for now, until refactored. func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *clibase.Cmd { if newAPI == nil { newAPI = func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) { From 157cbe13813fbfdfdb0e9d24bb8b2e88591b2b03 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 14:15:35 +0200 Subject: [PATCH 06/21] Unit test --- coderd/coderd.go | 1 - coderd/prometheusmetrics/license.go | 8 -- coderd/prometheusmetrics/license_test.go | 87 +++++++++++++++++++ .../testdata/license-metrics.json | 4 + 4 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 coderd/prometheusmetrics/license_test.go create mode 100644 coderd/prometheusmetrics/testdata/license-metrics.json diff --git a/coderd/coderd.go b/coderd/coderd.go index e42415de746e0..2727f717df953 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -345,7 +345,6 @@ func New(options *Options) *API { } licenseMetrics, err := prometheusmetrics.NewLicenseMetrics(&prometheusmetrics.LicenseMetricsOptions{ - Database: options.Database, Logger: options.Logger, Registry: options.PrometheusRegistry, }) diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index 97a75d4a0b617..6cbb2bf3d93ba 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -7,16 +7,13 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "golang.org/x/xerrors" "cdr.dev/slog" - "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/codersdk" ) type LicenseMetrics struct { - db database.Store interval time.Duration logger slog.Logger registry *prometheus.Registry @@ -26,7 +23,6 @@ type LicenseMetrics struct { type LicenseMetricsOptions struct { Interval time.Duration - Database database.Store Logger slog.Logger Registry *prometheus.Registry } @@ -35,15 +31,11 @@ func NewLicenseMetrics(opts *LicenseMetricsOptions) (*LicenseMetrics, error) { if opts.Interval == 0 { opts.Interval = 1 * time.Minute } - if opts.Database == nil { - return nil, xerrors.Errorf("database is required") - } if opts.Registry == nil { opts.Registry = prometheus.NewRegistry() } return &LicenseMetrics{ - db: opts.Database, interval: opts.Interval, logger: opts.Logger, registry: opts.Registry, diff --git a/coderd/prometheusmetrics/license_test.go b/coderd/prometheusmetrics/license_test.go new file mode 100644 index 0000000000000..dffc2fcc09c4b --- /dev/null +++ b/coderd/prometheusmetrics/license_test.go @@ -0,0 +1,87 @@ +package prometheusmetrics + +import ( + "context" + "encoding/json" + "os" + "reflect" + "testing" + "time" + + "github.com/aws/smithy-go/ptr" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" +) + +func TestCollectLicenseMetrics(t *testing.T) { + t.Parallel() + + // Given + registry := prometheus.NewRegistry() + sut, err := NewLicenseMetrics(&LicenseMetricsOptions{ + Interval: time.Millisecond, + Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + Registry: registry, + }) + require.NoError(t, err) + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + const ( + actualUsers = 4 + userLimit = 7 + ) + sut.Entitlements.Store(&codersdk.Entitlements{ + Features: map[codersdk.FeatureName]codersdk.Feature{ + codersdk.FeatureUserLimit: { + Enabled: true, + Actual: ptr.Int64(actualUsers), + Limit: ptr.Int64(userLimit), + }, + }, + }) + + // When + closeFunc, err := sut.Collect(ctx) + require.NoError(t, err) + t.Cleanup(closeFunc) + + // Then + goldenFile, err := os.ReadFile("testdata/license-metrics.json") + require.NoError(t, err) + golden := map[string]int{} + err = json.Unmarshal(goldenFile, &golden) + require.NoError(t, err) + + collected := map[string]int{} + + assert.Eventually(t, func() bool { + metrics, err := registry.Gather() + assert.NoError(t, err) + + if len(metrics) < 1 { + return false + } + + for _, metric := range metrics { + switch metric.GetName() { + case "coderd_license_active_users", "coderd_license_user_limit": + for _, m := range metric.Metric { + collected[m.Label[0].GetValue()+":"+metric.GetName()] = int(m.Gauge.GetValue()) + } + default: + require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName()) + } + } + return reflect.DeepEqual(golden, collected) + }, testutil.WaitShort, testutil.IntervalFast) + + assert.EqualValues(t, golden, collected) +} diff --git a/coderd/prometheusmetrics/testdata/license-metrics.json b/coderd/prometheusmetrics/testdata/license-metrics.json new file mode 100644 index 0000000000000..d9f1cc1994d3c --- /dev/null +++ b/coderd/prometheusmetrics/testdata/license-metrics.json @@ -0,0 +1,4 @@ +{ + "true:coderd_license_active_users": 4, + "true:coderd_license_user_limit": 7 +} From 3e7cb9736c3c673a4975574b9b5ec9fcc562c233 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 14:17:00 +0200 Subject: [PATCH 07/21] imports --- coderd/prometheusmetrics/license.go | 3 +-- coderd/prometheusmetrics/license_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index 6cbb2bf3d93ba..cc359383ed235 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -6,9 +6,8 @@ import ( "sync/atomic" "time" - "github.com/prometheus/client_golang/prometheus" - "cdr.dev/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/coder/coder/v2/codersdk" ) diff --git a/coderd/prometheusmetrics/license_test.go b/coderd/prometheusmetrics/license_test.go index dffc2fcc09c4b..4c23bec947c50 100644 --- a/coderd/prometheusmetrics/license_test.go +++ b/coderd/prometheusmetrics/license_test.go @@ -8,13 +8,13 @@ import ( "testing" "time" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" "github.com/aws/smithy-go/ptr" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" ) From 4abcd001d69af2e9180c02e269f22aeeba67fff8 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 14:25:02 +0200 Subject: [PATCH 08/21] Fix: lint --- coderd/prometheusmetrics/license.go | 3 ++- coderd/prometheusmetrics/license_test.go | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index cc359383ed235..6cbb2bf3d93ba 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -6,9 +6,10 @@ import ( "sync/atomic" "time" - "cdr.dev/slog" "github.com/prometheus/client_golang/prometheus" + "cdr.dev/slog" + "github.com/coder/coder/v2/codersdk" ) diff --git a/coderd/prometheusmetrics/license_test.go b/coderd/prometheusmetrics/license_test.go index 4c23bec947c50..9ef010c1b61f2 100644 --- a/coderd/prometheusmetrics/license_test.go +++ b/coderd/prometheusmetrics/license_test.go @@ -1,4 +1,4 @@ -package prometheusmetrics +package prometheusmetrics_test import ( "context" @@ -8,13 +8,15 @@ import ( "testing" "time" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/slogtest" "github.com/aws/smithy-go/ptr" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/v2/coderd/prometheusmetrics" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" ) @@ -24,7 +26,7 @@ func TestCollectLicenseMetrics(t *testing.T) { // Given registry := prometheus.NewRegistry() - sut, err := NewLicenseMetrics(&LicenseMetricsOptions{ + sut, err := prometheusmetrics.NewLicenseMetrics(&prometheusmetrics.LicenseMetricsOptions{ Interval: time.Millisecond, Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), Registry: registry, From 9ee99d5c6c54c7248ce28bd20d62d7fc87a00c9f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 14:28:20 +0200 Subject: [PATCH 09/21] entitled --- coderd/prometheusmetrics/license_test.go | 2 +- coderd/prometheusmetrics/testdata/license-metrics.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/prometheusmetrics/license_test.go b/coderd/prometheusmetrics/license_test.go index 9ef010c1b61f2..617b79651276a 100644 --- a/coderd/prometheusmetrics/license_test.go +++ b/coderd/prometheusmetrics/license_test.go @@ -76,7 +76,7 @@ func TestCollectLicenseMetrics(t *testing.T) { switch metric.GetName() { case "coderd_license_active_users", "coderd_license_user_limit": for _, m := range metric.Metric { - collected[m.Label[0].GetValue()+":"+metric.GetName()] = int(m.Gauge.GetValue()) + collected[m.Label[0].GetName()+"="+m.Label[0].GetValue()+":"+metric.GetName()] = int(m.Gauge.GetValue()) } default: require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName()) diff --git a/coderd/prometheusmetrics/testdata/license-metrics.json b/coderd/prometheusmetrics/testdata/license-metrics.json index d9f1cc1994d3c..49c8a2e8f1c18 100644 --- a/coderd/prometheusmetrics/testdata/license-metrics.json +++ b/coderd/prometheusmetrics/testdata/license-metrics.json @@ -1,4 +1,4 @@ { - "true:coderd_license_active_users": 4, - "true:coderd_license_user_limit": 7 + "entitled=true:coderd_license_active_users": 4, + "entitled=true:coderd_license_user_limit": 7 } From b16f1c02a73d1e8f117549cc3fe918f1d67dd2b5 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 10 Oct 2023 15:06:00 +0200 Subject: [PATCH 10/21] fix: sigsegv --- cli/server.go | 3 +++ coderd/prometheusmetrics/license.go | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/cli/server.go b/cli/server.go index a2872f31f53d8..41fe4cf507dd2 100644 --- a/cli/server.go +++ b/cli/server.go @@ -816,6 +816,9 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. } defer closeAgentsFunc() + if coderAPI.LicenseMetrics == nil { + return xerrors.Errorf("developer error: LicenseMetrics should be not nil") + } closeLicenseMetricsFunc, err := coderAPI.LicenseMetrics.Collect(ctx) if err != nil { return xerrors.Errorf("register license metric: %w", err) diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go index 6cbb2bf3d93ba..16278476f09ed 100644 --- a/coderd/prometheusmetrics/license.go +++ b/coderd/prometheusmetrics/license.go @@ -73,6 +73,16 @@ func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { defer ticker.Reset(lm.interval) entitlements := lm.Entitlements.Load() + if entitlements == nil { + lm.logger.Warn(ctx, `entitlements have not been loaded yet`) + return + } + + if entitlements.Features == nil { + lm.logger.Warn(ctx, `entitlements features are undefined`) + return + } + userLimitEntitlement, ok := entitlements.Features[codersdk.FeatureUserLimit] if !ok { lm.logger.Warn(ctx, `"user_limit" entitlement is not present`) From 366f33a7ce7741f5d84e00c7fecc0b5f0e73207f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:15:43 +0200 Subject: [PATCH 11/21] WIP --- coderd/coderd.go | 13 -- coderd/database/dbmock/dbmock.go | 5 +- coderd/database/models.go | 50 +++---- coderd/externalauth.go | 3 +- coderd/prometheusmetrics/license.go | 125 ------------------ enterprise/coderd/coderd.go | 14 +- enterprise/coderd/license/metricscollector.go | 78 +++++++++++ .../coderd/license/metricscollector_test.go | 2 +- 8 files changed, 121 insertions(+), 169 deletions(-) delete mode 100644 coderd/prometheusmetrics/license.go create mode 100644 enterprise/coderd/license/metricscollector.go rename coderd/prometheusmetrics/license_test.go => enterprise/coderd/license/metricscollector_test.go (98%) diff --git a/coderd/coderd.go b/coderd/coderd.go index a14fab4a9dfba..6566faab99a41 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -38,7 +38,6 @@ import ( // Used for swagger docs. _ "github.com/coder/coder/v2/coderd/apidoc" "github.com/coder/coder/v2/coderd/externalauth" - "github.com/coder/coder/v2/coderd/prometheusmetrics" "cdr.dev/slog" "github.com/coder/coder/v2/buildinfo" @@ -344,14 +343,6 @@ func New(options *Options) *API { panic(xerrors.Errorf("get deployment ID: %w", err)) } - licenseMetrics, err := prometheusmetrics.NewLicenseMetrics(&prometheusmetrics.LicenseMetricsOptions{ - Logger: options.Logger, - Registry: options.PrometheusRegistry, - }) - if err != nil { - panic(xerrors.Errorf("unable to initialize license metrics: %w", err)) - } - api := &API{ ctx: ctx, cancel: cancel, @@ -386,8 +377,6 @@ func New(options *Options) *API { options.Logger.Named("acquirer"), options.Database, options.Pubsub), - - LicenseMetrics: licenseMetrics, } if options.UpdateCheckOptions != nil { api.updateChecker = updatecheck.New( @@ -1047,8 +1036,6 @@ type API struct { statsBatcher *batchstats.Batcher Acquirer *provisionerdserver.Acquirer - - LicenseMetrics *prometheusmetrics.LicenseMetrics } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index dfc0f6dad5694..254f2d73af406 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -10,10 +10,11 @@ import ( reflect "reflect" time "time" - database "github.com/coder/coder/v2/coderd/database" - rbac "github.com/coder/coder/v2/coderd/rbac" gomock "github.com/golang/mock/gomock" uuid "github.com/google/uuid" + + database "github.com/coder/coder/v2/coderd/database" + rbac "github.com/coder/coder/v2/coderd/rbac" ) // MockStore is a mock of Store interface. diff --git a/coderd/database/models.go b/coderd/database/models.go index 5f389f36be4b8..569fb29e9b11c 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -30,7 +30,7 @@ func (e *APIKeyScope) Scan(src interface{}) error { case string: *e = APIKeyScope(s) default: - return fmt.Errorf("unsupported scan type for APIKeyScope: %T", src) + return xerrors.Errorf("unsupported scan type for APIKeyScope: %T", src) } return nil } @@ -89,7 +89,7 @@ func (e *AppSharingLevel) Scan(src interface{}) error { case string: *e = AppSharingLevel(s) default: - return fmt.Errorf("unsupported scan type for AppSharingLevel: %T", src) + return xerrors.Errorf("unsupported scan type for AppSharingLevel: %T", src) } return nil } @@ -155,7 +155,7 @@ func (e *AuditAction) Scan(src interface{}) error { case string: *e = AuditAction(s) default: - return fmt.Errorf("unsupported scan type for AuditAction: %T", src) + return xerrors.Errorf("unsupported scan type for AuditAction: %T", src) } return nil } @@ -225,7 +225,7 @@ func (e *AutomaticUpdates) Scan(src interface{}) error { case string: *e = AutomaticUpdates(s) default: - return fmt.Errorf("unsupported scan type for AutomaticUpdates: %T", src) + return xerrors.Errorf("unsupported scan type for AutomaticUpdates: %T", src) } return nil } @@ -287,7 +287,7 @@ func (e *BuildReason) Scan(src interface{}) error { case string: *e = BuildReason(s) default: - return fmt.Errorf("unsupported scan type for BuildReason: %T", src) + return xerrors.Errorf("unsupported scan type for BuildReason: %T", src) } return nil } @@ -356,7 +356,7 @@ func (e *DisplayApp) Scan(src interface{}) error { case string: *e = DisplayApp(s) default: - return fmt.Errorf("unsupported scan type for DisplayApp: %T", src) + return xerrors.Errorf("unsupported scan type for DisplayApp: %T", src) } return nil } @@ -420,7 +420,7 @@ func (e *GroupSource) Scan(src interface{}) error { case string: *e = GroupSource(s) default: - return fmt.Errorf("unsupported scan type for GroupSource: %T", src) + return xerrors.Errorf("unsupported scan type for GroupSource: %T", src) } return nil } @@ -481,7 +481,7 @@ func (e *LogLevel) Scan(src interface{}) error { case string: *e = LogLevel(s) default: - return fmt.Errorf("unsupported scan type for LogLevel: %T", src) + return xerrors.Errorf("unsupported scan type for LogLevel: %T", src) } return nil } @@ -545,7 +545,7 @@ func (e *LogSource) Scan(src interface{}) error { case string: *e = LogSource(s) default: - return fmt.Errorf("unsupported scan type for LogSource: %T", src) + return xerrors.Errorf("unsupported scan type for LogSource: %T", src) } return nil } @@ -607,7 +607,7 @@ func (e *LoginType) Scan(src interface{}) error { case string: *e = LoginType(s) default: - return fmt.Errorf("unsupported scan type for LoginType: %T", src) + return xerrors.Errorf("unsupported scan type for LoginType: %T", src) } return nil } @@ -672,7 +672,7 @@ func (e *ParameterDestinationScheme) Scan(src interface{}) error { case string: *e = ParameterDestinationScheme(s) default: - return fmt.Errorf("unsupported scan type for ParameterDestinationScheme: %T", src) + return xerrors.Errorf("unsupported scan type for ParameterDestinationScheme: %T", src) } return nil } @@ -733,7 +733,7 @@ func (e *ParameterScope) Scan(src interface{}) error { case string: *e = ParameterScope(s) default: - return fmt.Errorf("unsupported scan type for ParameterScope: %T", src) + return xerrors.Errorf("unsupported scan type for ParameterScope: %T", src) } return nil } @@ -793,7 +793,7 @@ func (e *ParameterSourceScheme) Scan(src interface{}) error { case string: *e = ParameterSourceScheme(s) default: - return fmt.Errorf("unsupported scan type for ParameterSourceScheme: %T", src) + return xerrors.Errorf("unsupported scan type for ParameterSourceScheme: %T", src) } return nil } @@ -851,7 +851,7 @@ func (e *ParameterTypeSystem) Scan(src interface{}) error { case string: *e = ParameterTypeSystem(s) default: - return fmt.Errorf("unsupported scan type for ParameterTypeSystem: %T", src) + return xerrors.Errorf("unsupported scan type for ParameterTypeSystem: %T", src) } return nil } @@ -915,7 +915,7 @@ func (e *ProvisionerJobStatus) Scan(src interface{}) error { case string: *e = ProvisionerJobStatus(s) default: - return fmt.Errorf("unsupported scan type for ProvisionerJobStatus: %T", src) + return xerrors.Errorf("unsupported scan type for ProvisionerJobStatus: %T", src) } return nil } @@ -984,7 +984,7 @@ func (e *ProvisionerJobType) Scan(src interface{}) error { case string: *e = ProvisionerJobType(s) default: - return fmt.Errorf("unsupported scan type for ProvisionerJobType: %T", src) + return xerrors.Errorf("unsupported scan type for ProvisionerJobType: %T", src) } return nil } @@ -1043,7 +1043,7 @@ func (e *ProvisionerStorageMethod) Scan(src interface{}) error { case string: *e = ProvisionerStorageMethod(s) default: - return fmt.Errorf("unsupported scan type for ProvisionerStorageMethod: %T", src) + return xerrors.Errorf("unsupported scan type for ProvisionerStorageMethod: %T", src) } return nil } @@ -1099,7 +1099,7 @@ func (e *ProvisionerType) Scan(src interface{}) error { case string: *e = ProvisionerType(s) default: - return fmt.Errorf("unsupported scan type for ProvisionerType: %T", src) + return xerrors.Errorf("unsupported scan type for ProvisionerType: %T", src) } return nil } @@ -1167,7 +1167,7 @@ func (e *ResourceType) Scan(src interface{}) error { case string: *e = ResourceType(s) default: - return fmt.Errorf("unsupported scan type for ResourceType: %T", src) + return xerrors.Errorf("unsupported scan type for ResourceType: %T", src) } return nil } @@ -1245,7 +1245,7 @@ func (e *StartupScriptBehavior) Scan(src interface{}) error { case string: *e = StartupScriptBehavior(s) default: - return fmt.Errorf("unsupported scan type for StartupScriptBehavior: %T", src) + return xerrors.Errorf("unsupported scan type for StartupScriptBehavior: %T", src) } return nil } @@ -1305,7 +1305,7 @@ func (e *UserStatus) Scan(src interface{}) error { case string: *e = UserStatus(s) default: - return fmt.Errorf("unsupported scan type for UserStatus: %T", src) + return xerrors.Errorf("unsupported scan type for UserStatus: %T", src) } return nil } @@ -1372,7 +1372,7 @@ func (e *WorkspaceAgentLifecycleState) Scan(src interface{}) error { case string: *e = WorkspaceAgentLifecycleState(s) default: - return fmt.Errorf("unsupported scan type for WorkspaceAgentLifecycleState: %T", src) + return xerrors.Errorf("unsupported scan type for WorkspaceAgentLifecycleState: %T", src) } return nil } @@ -1446,7 +1446,7 @@ func (e *WorkspaceAgentSubsystem) Scan(src interface{}) error { case string: *e = WorkspaceAgentSubsystem(s) default: - return fmt.Errorf("unsupported scan type for WorkspaceAgentSubsystem: %T", src) + return xerrors.Errorf("unsupported scan type for WorkspaceAgentSubsystem: %T", src) } return nil } @@ -1510,7 +1510,7 @@ func (e *WorkspaceAppHealth) Scan(src interface{}) error { case string: *e = WorkspaceAppHealth(s) default: - return fmt.Errorf("unsupported scan type for WorkspaceAppHealth: %T", src) + return xerrors.Errorf("unsupported scan type for WorkspaceAppHealth: %T", src) } return nil } @@ -1573,7 +1573,7 @@ func (e *WorkspaceTransition) Scan(src interface{}) error { case string: *e = WorkspaceTransition(s) default: - return fmt.Errorf("unsupported scan type for WorkspaceTransition: %T", src) + return xerrors.Errorf("unsupported scan type for WorkspaceTransition: %T", src) } return nil } diff --git a/coderd/externalauth.go b/coderd/externalauth.go index 775ff5436284f..31dff667c28e7 100644 --- a/coderd/externalauth.go +++ b/coderd/externalauth.go @@ -8,13 +8,14 @@ import ( "golang.org/x/sync/errgroup" + "github.com/sqlc-dev/pqtype" + "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/externalauth" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/codersdk" - "github.com/sqlc-dev/pqtype" ) // @Summary Get external auth by ID diff --git a/coderd/prometheusmetrics/license.go b/coderd/prometheusmetrics/license.go deleted file mode 100644 index 16278476f09ed..0000000000000 --- a/coderd/prometheusmetrics/license.go +++ /dev/null @@ -1,125 +0,0 @@ -package prometheusmetrics - -import ( - "context" - "fmt" - "sync/atomic" - "time" - - "github.com/prometheus/client_golang/prometheus" - - "cdr.dev/slog" - - "github.com/coder/coder/v2/codersdk" -) - -type LicenseMetrics struct { - interval time.Duration - logger slog.Logger - registry *prometheus.Registry - - Entitlements atomic.Pointer[codersdk.Entitlements] -} - -type LicenseMetricsOptions struct { - Interval time.Duration - Logger slog.Logger - Registry *prometheus.Registry -} - -func NewLicenseMetrics(opts *LicenseMetricsOptions) (*LicenseMetrics, error) { - if opts.Interval == 0 { - opts.Interval = 1 * time.Minute - } - if opts.Registry == nil { - opts.Registry = prometheus.NewRegistry() - } - - return &LicenseMetrics{ - interval: opts.Interval, - logger: opts.Logger, - registry: opts.Registry, - }, nil -} - -func (lm *LicenseMetrics) Collect(ctx context.Context) (func(), error) { - activeUsersGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "active_users", - Help: `The number of active users.`, - }, []string{"entitled"})) - err := lm.registry.Register(activeUsersGauge) - if err != nil { - return nil, err - } - - userLimitGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "user_limit", - Help: "The user seats limit based on the active Coder license.", - }, []string{"entitled"})) - err = lm.registry.Register(userLimitGauge) - if err != nil { - return nil, err - } - - ctx, cancelFunc := context.WithCancel(ctx) - done := make(chan struct{}) - ticker := time.NewTicker(time.Nanosecond) - - doTick := func() { - defer ticker.Reset(lm.interval) - - entitlements := lm.Entitlements.Load() - if entitlements == nil { - lm.logger.Warn(ctx, `entitlements have not been loaded yet`) - return - } - - if entitlements.Features == nil { - lm.logger.Warn(ctx, `entitlements features are undefined`) - return - } - - userLimitEntitlement, ok := entitlements.Features[codersdk.FeatureUserLimit] - if !ok { - lm.logger.Warn(ctx, `"user_limit" entitlement is not present`) - return - } - - enabled := fmt.Sprintf("%v", userLimitEntitlement.Enabled) - if userLimitEntitlement.Actual != nil { - activeUsersGauge.WithLabelValues(VectorOperationSet, float64(*userLimitEntitlement.Actual), enabled) - } else { - activeUsersGauge.WithLabelValues(VectorOperationSet, 0, enabled) - } - - if userLimitEntitlement.Limit != nil { - userLimitGauge.WithLabelValues(VectorOperationSet, float64(*userLimitEntitlement.Limit), enabled) - } else { - userLimitGauge.WithLabelValues(VectorOperationSet, 0, enabled) - } - - activeUsersGauge.Commit() - userLimitGauge.Commit() - } - - go func() { - defer close(done) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - doTick() - } - } - }() - return func() { - cancelFunc() - <-done - }, nil -} diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 941418ae3e90d..e1e5b62b4c25a 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -91,6 +91,8 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { } options.Database = cryptDB + licenseMetricsCollector := new(license.MetricsCollector) + api := &API{ ctx: ctx, cancel: cancelFunc, @@ -100,6 +102,7 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { psk: options.ProvisionerDaemonPSK, authorizer: options.Authorizer, }, + licenseMetricsCollector: licenseMetricsCollector, } defer func() { if err != nil { @@ -365,6 +368,7 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { if err != nil { return nil, xerrors.Errorf("initialize proxy health: %w", err) } + go api.ProxyHealth.Run(ctx) // Force the initial loading of the cache. Do this in a go routine in case // the calls to the workspace proxies hang and this takes some time. @@ -375,6 +379,11 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { api.AGPL.WorkspaceProxyHostsFn.Store(&f) } + err = api.PrometheusRegistry.Register(api.licenseMetricsCollector) + if err != nil { + return nil, xerrors.Errorf("unable to register license metrics collector") + } + err = api.updateEntitlements(ctx) if err != nil { return nil, xerrors.Errorf("update entitlements: %w", err) @@ -434,6 +443,8 @@ type API struct { entitlements codersdk.Entitlements provisionerDaemonAuth *provisionerDaemonAuth + + licenseMetricsCollector *license.MetricsCollector } func (api *API) Close() error { @@ -660,9 +671,8 @@ func (api *API) updateEntitlements(ctx context.Context) error { api.entitlementsMu.Lock() defer api.entitlementsMu.Unlock() api.entitlements = entitlements + api.licenseMetricsCollector.Entitlements.Store(&entitlements) api.AGPL.SiteHandler.Entitlements.Store(&entitlements) - api.AGPL.LicenseMetrics.Entitlements.Store(&entitlements) - return nil } diff --git a/enterprise/coderd/license/metricscollector.go b/enterprise/coderd/license/metricscollector.go new file mode 100644 index 0000000000000..4f23aeba0eede --- /dev/null +++ b/enterprise/coderd/license/metricscollector.go @@ -0,0 +1,78 @@ +package license + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/coder/coder/v2/codersdk" +) + +type MetricsCollector struct { + Entitlements atomic.Pointer[codersdk.Entitlements] + + activeUsersGauge prometheus.Gauge + limitUsersGauge prometheus.Gauge + userLimitEnabledGauge prometheus.Gauge +} + +var _ prometheus.Collector = new(MetricsCollector) + +func NewMetricsCollector() *MetricsCollector { + return &MetricsCollector{ + activeUsersGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "active_users", + Help: `The number of active users.`, + }), + limitUsersGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "limit_users", + Help: "The user seats limit based on the active Coder license.", + }), + userLimitEnabledGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "license", + Name: "user_limit_enabled", + Help: "Returns 1 if the current license enforces the user limit.", + }), + } +} + +func (mc *MetricsCollector) Describe(descCh chan<- *prometheus.Desc) { + descCh <- mc.activeUsersGauge.Desc() + descCh <- mc.limitUsersGauge.Desc() + descCh <- mc.userLimitEnabledGauge.Desc() +} + +func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { + entitlements := mc.Entitlements.Load() + if entitlements == nil || entitlements.Features == nil { + return + } + + userLimitEntitlement, ok := entitlements.Features[codersdk.FeatureUserLimit] + if !ok { + return + } + + var enabled float64 + if userLimitEntitlement.Enabled { + enabled = 1 + } + mc.userLimitEnabledGauge.Set(enabled) + + if userLimitEntitlement.Actual != nil { + mc.activeUsersGauge.Set(float64(*userLimitEntitlement.Actual)) + } + + if userLimitEntitlement.Limit != nil { + mc.limitUsersGauge.Set(float64(*userLimitEntitlement.Limit)) + } + + metricsCh <- mc.activeUsersGauge + metricsCh <- mc.limitUsersGauge + metricsCh <- mc.userLimitEnabledGauge +} diff --git a/coderd/prometheusmetrics/license_test.go b/enterprise/coderd/license/metricscollector_test.go similarity index 98% rename from coderd/prometheusmetrics/license_test.go rename to enterprise/coderd/license/metricscollector_test.go index 617b79651276a..4e61eb7378415 100644 --- a/coderd/prometheusmetrics/license_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -1,4 +1,4 @@ -package prometheusmetrics_test +package license_test import ( "context" From f1f24be586233e335c98b97365eb919676f2650f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:22:37 +0200 Subject: [PATCH 12/21] WIP --- enterprise/coderd/license/metricscollector_test.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/enterprise/coderd/license/metricscollector_test.go b/enterprise/coderd/license/metricscollector_test.go index 4e61eb7378415..4bd9f163d21ad 100644 --- a/enterprise/coderd/license/metricscollector_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -6,18 +6,14 @@ import ( "os" "reflect" "testing" - "time" "github.com/aws/smithy-go/ptr" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/slogtest" - - "github.com/coder/coder/v2/coderd/prometheusmetrics" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/enterprise/coderd/license" "github.com/coder/coder/v2/testutil" ) @@ -26,12 +22,8 @@ func TestCollectLicenseMetrics(t *testing.T) { // Given registry := prometheus.NewRegistry() - sut, err := prometheusmetrics.NewLicenseMetrics(&prometheusmetrics.LicenseMetricsOptions{ - Interval: time.Millisecond, - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), - Registry: registry, - }) - require.NoError(t, err) + + sut := license.NewMetricsCollector() ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(cancelFunc) From 46d190c7c8b02a3e6afd920e03e3a2c49a58a87f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:30:17 +0200 Subject: [PATCH 13/21] WIP --- .../coderd/license}/testdata/license-metrics.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {coderd/prometheusmetrics => enterprise/coderd/license}/testdata/license-metrics.json (100%) diff --git a/coderd/prometheusmetrics/testdata/license-metrics.json b/enterprise/coderd/license/testdata/license-metrics.json similarity index 100% rename from coderd/prometheusmetrics/testdata/license-metrics.json rename to enterprise/coderd/license/testdata/license-metrics.json From 630a115c38bd8650d2d83f50b9da141ce4d41a1c Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:31:33 +0200 Subject: [PATCH 14/21] fix: db --- coderd/database/dbmock/dbmock.go | 5 ++-- coderd/database/models.go | 50 ++++++++++++++++---------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 560f4ca174ffa..8a4c3a298efb5 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -10,11 +10,10 @@ import ( reflect "reflect" time "time" - gomock "github.com/golang/mock/gomock" - uuid "github.com/google/uuid" - database "github.com/coder/coder/v2/coderd/database" rbac "github.com/coder/coder/v2/coderd/rbac" + gomock "github.com/golang/mock/gomock" + uuid "github.com/google/uuid" ) // MockStore is a mock of Store interface. diff --git a/coderd/database/models.go b/coderd/database/models.go index 8bd60de92499e..267bd1a7ce7c4 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -30,7 +30,7 @@ func (e *APIKeyScope) Scan(src interface{}) error { case string: *e = APIKeyScope(s) default: - return xerrors.Errorf("unsupported scan type for APIKeyScope: %T", src) + return fmt.Errorf("unsupported scan type for APIKeyScope: %T", src) } return nil } @@ -89,7 +89,7 @@ func (e *AppSharingLevel) Scan(src interface{}) error { case string: *e = AppSharingLevel(s) default: - return xerrors.Errorf("unsupported scan type for AppSharingLevel: %T", src) + return fmt.Errorf("unsupported scan type for AppSharingLevel: %T", src) } return nil } @@ -155,7 +155,7 @@ func (e *AuditAction) Scan(src interface{}) error { case string: *e = AuditAction(s) default: - return xerrors.Errorf("unsupported scan type for AuditAction: %T", src) + return fmt.Errorf("unsupported scan type for AuditAction: %T", src) } return nil } @@ -225,7 +225,7 @@ func (e *AutomaticUpdates) Scan(src interface{}) error { case string: *e = AutomaticUpdates(s) default: - return xerrors.Errorf("unsupported scan type for AutomaticUpdates: %T", src) + return fmt.Errorf("unsupported scan type for AutomaticUpdates: %T", src) } return nil } @@ -287,7 +287,7 @@ func (e *BuildReason) Scan(src interface{}) error { case string: *e = BuildReason(s) default: - return xerrors.Errorf("unsupported scan type for BuildReason: %T", src) + return fmt.Errorf("unsupported scan type for BuildReason: %T", src) } return nil } @@ -356,7 +356,7 @@ func (e *DisplayApp) Scan(src interface{}) error { case string: *e = DisplayApp(s) default: - return xerrors.Errorf("unsupported scan type for DisplayApp: %T", src) + return fmt.Errorf("unsupported scan type for DisplayApp: %T", src) } return nil } @@ -420,7 +420,7 @@ func (e *GroupSource) Scan(src interface{}) error { case string: *e = GroupSource(s) default: - return xerrors.Errorf("unsupported scan type for GroupSource: %T", src) + return fmt.Errorf("unsupported scan type for GroupSource: %T", src) } return nil } @@ -481,7 +481,7 @@ func (e *LogLevel) Scan(src interface{}) error { case string: *e = LogLevel(s) default: - return xerrors.Errorf("unsupported scan type for LogLevel: %T", src) + return fmt.Errorf("unsupported scan type for LogLevel: %T", src) } return nil } @@ -545,7 +545,7 @@ func (e *LogSource) Scan(src interface{}) error { case string: *e = LogSource(s) default: - return xerrors.Errorf("unsupported scan type for LogSource: %T", src) + return fmt.Errorf("unsupported scan type for LogSource: %T", src) } return nil } @@ -607,7 +607,7 @@ func (e *LoginType) Scan(src interface{}) error { case string: *e = LoginType(s) default: - return xerrors.Errorf("unsupported scan type for LoginType: %T", src) + return fmt.Errorf("unsupported scan type for LoginType: %T", src) } return nil } @@ -672,7 +672,7 @@ func (e *ParameterDestinationScheme) Scan(src interface{}) error { case string: *e = ParameterDestinationScheme(s) default: - return xerrors.Errorf("unsupported scan type for ParameterDestinationScheme: %T", src) + return fmt.Errorf("unsupported scan type for ParameterDestinationScheme: %T", src) } return nil } @@ -733,7 +733,7 @@ func (e *ParameterScope) Scan(src interface{}) error { case string: *e = ParameterScope(s) default: - return xerrors.Errorf("unsupported scan type for ParameterScope: %T", src) + return fmt.Errorf("unsupported scan type for ParameterScope: %T", src) } return nil } @@ -793,7 +793,7 @@ func (e *ParameterSourceScheme) Scan(src interface{}) error { case string: *e = ParameterSourceScheme(s) default: - return xerrors.Errorf("unsupported scan type for ParameterSourceScheme: %T", src) + return fmt.Errorf("unsupported scan type for ParameterSourceScheme: %T", src) } return nil } @@ -851,7 +851,7 @@ func (e *ParameterTypeSystem) Scan(src interface{}) error { case string: *e = ParameterTypeSystem(s) default: - return xerrors.Errorf("unsupported scan type for ParameterTypeSystem: %T", src) + return fmt.Errorf("unsupported scan type for ParameterTypeSystem: %T", src) } return nil } @@ -915,7 +915,7 @@ func (e *ProvisionerJobStatus) Scan(src interface{}) error { case string: *e = ProvisionerJobStatus(s) default: - return xerrors.Errorf("unsupported scan type for ProvisionerJobStatus: %T", src) + return fmt.Errorf("unsupported scan type for ProvisionerJobStatus: %T", src) } return nil } @@ -984,7 +984,7 @@ func (e *ProvisionerJobType) Scan(src interface{}) error { case string: *e = ProvisionerJobType(s) default: - return xerrors.Errorf("unsupported scan type for ProvisionerJobType: %T", src) + return fmt.Errorf("unsupported scan type for ProvisionerJobType: %T", src) } return nil } @@ -1043,7 +1043,7 @@ func (e *ProvisionerStorageMethod) Scan(src interface{}) error { case string: *e = ProvisionerStorageMethod(s) default: - return xerrors.Errorf("unsupported scan type for ProvisionerStorageMethod: %T", src) + return fmt.Errorf("unsupported scan type for ProvisionerStorageMethod: %T", src) } return nil } @@ -1099,7 +1099,7 @@ func (e *ProvisionerType) Scan(src interface{}) error { case string: *e = ProvisionerType(s) default: - return xerrors.Errorf("unsupported scan type for ProvisionerType: %T", src) + return fmt.Errorf("unsupported scan type for ProvisionerType: %T", src) } return nil } @@ -1167,7 +1167,7 @@ func (e *ResourceType) Scan(src interface{}) error { case string: *e = ResourceType(s) default: - return xerrors.Errorf("unsupported scan type for ResourceType: %T", src) + return fmt.Errorf("unsupported scan type for ResourceType: %T", src) } return nil } @@ -1245,7 +1245,7 @@ func (e *StartupScriptBehavior) Scan(src interface{}) error { case string: *e = StartupScriptBehavior(s) default: - return xerrors.Errorf("unsupported scan type for StartupScriptBehavior: %T", src) + return fmt.Errorf("unsupported scan type for StartupScriptBehavior: %T", src) } return nil } @@ -1305,7 +1305,7 @@ func (e *UserStatus) Scan(src interface{}) error { case string: *e = UserStatus(s) default: - return xerrors.Errorf("unsupported scan type for UserStatus: %T", src) + return fmt.Errorf("unsupported scan type for UserStatus: %T", src) } return nil } @@ -1372,7 +1372,7 @@ func (e *WorkspaceAgentLifecycleState) Scan(src interface{}) error { case string: *e = WorkspaceAgentLifecycleState(s) default: - return xerrors.Errorf("unsupported scan type for WorkspaceAgentLifecycleState: %T", src) + return fmt.Errorf("unsupported scan type for WorkspaceAgentLifecycleState: %T", src) } return nil } @@ -1446,7 +1446,7 @@ func (e *WorkspaceAgentSubsystem) Scan(src interface{}) error { case string: *e = WorkspaceAgentSubsystem(s) default: - return xerrors.Errorf("unsupported scan type for WorkspaceAgentSubsystem: %T", src) + return fmt.Errorf("unsupported scan type for WorkspaceAgentSubsystem: %T", src) } return nil } @@ -1510,7 +1510,7 @@ func (e *WorkspaceAppHealth) Scan(src interface{}) error { case string: *e = WorkspaceAppHealth(s) default: - return xerrors.Errorf("unsupported scan type for WorkspaceAppHealth: %T", src) + return fmt.Errorf("unsupported scan type for WorkspaceAppHealth: %T", src) } return nil } @@ -1573,7 +1573,7 @@ func (e *WorkspaceTransition) Scan(src interface{}) error { case string: *e = WorkspaceTransition(s) default: - return xerrors.Errorf("unsupported scan type for WorkspaceTransition: %T", src) + return fmt.Errorf("unsupported scan type for WorkspaceTransition: %T", src) } return nil } From cadbaa835ba6696c940e1105642c97b6b37a3a95 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:35:34 +0200 Subject: [PATCH 15/21] fix --- cli/server.go | 9 --------- enterprise/coderd/license/metricscollector_test.go | 4 ---- 2 files changed, 13 deletions(-) diff --git a/cli/server.go b/cli/server.go index 41fe4cf507dd2..98f0468c32e18 100644 --- a/cli/server.go +++ b/cli/server.go @@ -815,15 +815,6 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("register agents prometheus metric: %w", err) } defer closeAgentsFunc() - - if coderAPI.LicenseMetrics == nil { - return xerrors.Errorf("developer error: LicenseMetrics should be not nil") - } - closeLicenseMetricsFunc, err := coderAPI.LicenseMetrics.Collect(ctx) - if err != nil { - return xerrors.Errorf("register license metric: %w", err) - } - defer closeLicenseMetricsFunc() } client := codersdk.New(localURL) diff --git a/enterprise/coderd/license/metricscollector_test.go b/enterprise/coderd/license/metricscollector_test.go index 4bd9f163d21ad..c9d7f4d42dfee 100644 --- a/enterprise/coderd/license/metricscollector_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -1,7 +1,6 @@ package license_test import ( - "context" "encoding/json" "os" "reflect" @@ -25,9 +24,6 @@ func TestCollectLicenseMetrics(t *testing.T) { sut := license.NewMetricsCollector() - ctx, cancelFunc := context.WithCancel(context.Background()) - t.Cleanup(cancelFunc) - const ( actualUsers = 4 userLimit = 7 From d221c464cfdf3f7b6cd9fc756743f6137364182d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:37:10 +0200 Subject: [PATCH 16/21] cleanup --- coderd/coderd.go | 1 - enterprise/coderd/coderd.go | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index b5f6594a07d3d..d0aaee5b1ce98 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -342,7 +342,6 @@ func New(options *Options) *API { if err != nil { panic(xerrors.Errorf("get deployment ID: %w", err)) } - api := &API{ ctx: ctx, cancel: cancel, diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index e1e5b62b4c25a..2ffa134b5db42 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -91,8 +91,6 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { } options.Database = cryptDB - licenseMetricsCollector := new(license.MetricsCollector) - api := &API{ ctx: ctx, cancel: cancelFunc, @@ -102,7 +100,7 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { psk: options.ProvisionerDaemonPSK, authorizer: options.Authorizer, }, - licenseMetricsCollector: licenseMetricsCollector, + licenseMetricsCollector: new(license.MetricsCollector), } defer func() { if err != nil { @@ -368,7 +366,6 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { if err != nil { return nil, xerrors.Errorf("initialize proxy health: %w", err) } - go api.ProxyHealth.Run(ctx) // Force the initial loading of the cache. Do this in a go routine in case // the calls to the workspace proxies hang and this takes some time. From 073dce97e9f81e7e500c862f632b0a81312415e5 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:39:10 +0200 Subject: [PATCH 17/21] fix --- cli/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/server.go b/cli/server.go index 98f0468c32e18..9f33ced438f84 100644 --- a/cli/server.go +++ b/cli/server.go @@ -228,7 +228,6 @@ func enablePrometheus( ), nil } -// nolint:gocognit // This function makes more sense being large for now, until refactored. func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *clibase.Cmd { if newAPI == nil { newAPI = func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) { From 0c6d00e4819f18b315a00b66434b7066c2622a75 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 12:49:17 +0200 Subject: [PATCH 18/21] Collector adjustments --- enterprise/coderd/coderd.go | 2 +- enterprise/coderd/license/metricscollector.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 2ffa134b5db42..0531a8a865333 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -100,7 +100,7 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { psk: options.ProvisionerDaemonPSK, authorizer: options.Authorizer, }, - licenseMetricsCollector: new(license.MetricsCollector), + licenseMetricsCollector: license.NewMetricsCollector(), } defer func() { if err != nil { diff --git a/enterprise/coderd/license/metricscollector.go b/enterprise/coderd/license/metricscollector.go index 4f23aeba0eede..bc49be4d7b841 100644 --- a/enterprise/coderd/license/metricscollector.go +++ b/enterprise/coderd/license/metricscollector.go @@ -63,16 +63,15 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { enabled = 1 } mc.userLimitEnabledGauge.Set(enabled) + metricsCh <- mc.userLimitEnabledGauge if userLimitEntitlement.Actual != nil { mc.activeUsersGauge.Set(float64(*userLimitEntitlement.Actual)) + metricsCh <- mc.activeUsersGauge } if userLimitEntitlement.Limit != nil { mc.limitUsersGauge.Set(float64(*userLimitEntitlement.Limit)) + metricsCh <- mc.limitUsersGauge } - - metricsCh <- mc.activeUsersGauge - metricsCh <- mc.limitUsersGauge - metricsCh <- mc.userLimitEnabledGauge } From b8a969df354624a56e8408675916dd30fcda127b Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 13:14:04 +0200 Subject: [PATCH 19/21] Fix tests --- .../coderd/license/metricscollector_test.go | 36 +++++++------------ .../license/testdata/license-metrics.json | 5 +-- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/enterprise/coderd/license/metricscollector_test.go b/enterprise/coderd/license/metricscollector_test.go index c9d7f4d42dfee..9f9fa88fadbce 100644 --- a/enterprise/coderd/license/metricscollector_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -3,17 +3,14 @@ package license_test import ( "encoding/json" "os" - "reflect" "testing" "github.com/aws/smithy-go/ptr" "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/enterprise/coderd/license" - "github.com/coder/coder/v2/testutil" ) func TestCollectLicenseMetrics(t *testing.T) { @@ -39,9 +36,7 @@ func TestCollectLicenseMetrics(t *testing.T) { }) // When - closeFunc, err := sut.Collect(ctx) - require.NoError(t, err) - t.Cleanup(closeFunc) + registry.Register(sut) // Then goldenFile, err := os.ReadFile("testdata/license-metrics.json") @@ -52,26 +47,19 @@ func TestCollectLicenseMetrics(t *testing.T) { collected := map[string]int{} - assert.Eventually(t, func() bool { - metrics, err := registry.Gather() - assert.NoError(t, err) - - if len(metrics) < 1 { - return false - } + metrics, err := registry.Gather() + require.NoError(t, err) - for _, metric := range metrics { - switch metric.GetName() { - case "coderd_license_active_users", "coderd_license_user_limit": - for _, m := range metric.Metric { - collected[m.Label[0].GetName()+"="+m.Label[0].GetValue()+":"+metric.GetName()] = int(m.Gauge.GetValue()) - } - default: - require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName()) + for _, metric := range metrics { + switch metric.GetName() { + case "coderd_license_active_users", "coderd_license_limit_users", "coderd_license_user_limit_enabled": + for _, m := range metric.Metric { + collected[metric.GetName()] = int(m.Gauge.GetValue()) } + default: + require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName()) } - return reflect.DeepEqual(golden, collected) - }, testutil.WaitShort, testutil.IntervalFast) + } - assert.EqualValues(t, golden, collected) + require.EqualValues(t, golden, collected) } diff --git a/enterprise/coderd/license/testdata/license-metrics.json b/enterprise/coderd/license/testdata/license-metrics.json index 49c8a2e8f1c18..7326b47da7a35 100644 --- a/enterprise/coderd/license/testdata/license-metrics.json +++ b/enterprise/coderd/license/testdata/license-metrics.json @@ -1,4 +1,5 @@ { - "entitled=true:coderd_license_active_users": 4, - "entitled=true:coderd_license_user_limit": 7 + "coderd_license_active_users": 4, + "coderd_license_limit_users": 7, + "coderd_license_user_limit_enabled": 1 } From c4c434e3a052b6c243fd7ff278986eaa872bf154 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 12 Oct 2023 13:48:52 +0200 Subject: [PATCH 20/21] Given-When-Then --- enterprise/coderd/license/metricscollector_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/enterprise/coderd/license/metricscollector_test.go b/enterprise/coderd/license/metricscollector_test.go index 9f9fa88fadbce..16fe61c46f110 100644 --- a/enterprise/coderd/license/metricscollector_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -35,9 +35,12 @@ func TestCollectLicenseMetrics(t *testing.T) { }, }) - // When registry.Register(sut) + // When + metrics, err := registry.Gather() + require.NoError(t, err) + // Then goldenFile, err := os.ReadFile("testdata/license-metrics.json") require.NoError(t, err) @@ -46,10 +49,6 @@ func TestCollectLicenseMetrics(t *testing.T) { require.NoError(t, err) collected := map[string]int{} - - metrics, err := registry.Gather() - require.NoError(t, err) - for _, metric := range metrics { switch metric.GetName() { case "coderd_license_active_users", "coderd_license_limit_users", "coderd_license_user_limit_enabled": @@ -60,6 +59,5 @@ func TestCollectLicenseMetrics(t *testing.T) { require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName()) } } - require.EqualValues(t, golden, collected) } From 8845fb3825a71eb58bd362456454266b1f10947d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 13 Oct 2023 09:56:03 +0200 Subject: [PATCH 21/21] Use const metrics --- enterprise/coderd/coderd.go | 5 +- enterprise/coderd/license/metricscollector.go | 50 +++++-------------- .../coderd/license/metricscollector_test.go | 4 +- 3 files changed, 17 insertions(+), 42 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 0531a8a865333..b49890589a6c3 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -100,7 +100,6 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { psk: options.ProvisionerDaemonPSK, authorizer: options.Authorizer, }, - licenseMetricsCollector: license.NewMetricsCollector(), } defer func() { if err != nil { @@ -376,7 +375,7 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { api.AGPL.WorkspaceProxyHostsFn.Store(&f) } - err = api.PrometheusRegistry.Register(api.licenseMetricsCollector) + err = api.PrometheusRegistry.Register(&api.licenseMetricsCollector) if err != nil { return nil, xerrors.Errorf("unable to register license metrics collector") } @@ -441,7 +440,7 @@ type API struct { provisionerDaemonAuth *provisionerDaemonAuth - licenseMetricsCollector *license.MetricsCollector + licenseMetricsCollector license.MetricsCollector } func (api *API) Close() error { diff --git a/enterprise/coderd/license/metricscollector.go b/enterprise/coderd/license/metricscollector.go index bc49be4d7b841..85aac23b2fcaf 100644 --- a/enterprise/coderd/license/metricscollector.go +++ b/enterprise/coderd/license/metricscollector.go @@ -8,43 +8,22 @@ import ( "github.com/coder/coder/v2/codersdk" ) +var ( + activeUsersDesc = prometheus.NewDesc("coderd_license_active_users", "The number of active users.", nil, nil) + limitUsersDesc = prometheus.NewDesc("coderd_license_limit_users", "The user seats limit based on the active Coder license.", nil, nil) + userLimitEnabledDesc = prometheus.NewDesc("coderd_license_user_limit_enabled", "Returns 1 if the current license enforces the user limit.", nil, nil) +) + type MetricsCollector struct { Entitlements atomic.Pointer[codersdk.Entitlements] - - activeUsersGauge prometheus.Gauge - limitUsersGauge prometheus.Gauge - userLimitEnabledGauge prometheus.Gauge } var _ prometheus.Collector = new(MetricsCollector) -func NewMetricsCollector() *MetricsCollector { - return &MetricsCollector{ - activeUsersGauge: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "active_users", - Help: `The number of active users.`, - }), - limitUsersGauge: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "limit_users", - Help: "The user seats limit based on the active Coder license.", - }), - userLimitEnabledGauge: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "coderd", - Subsystem: "license", - Name: "user_limit_enabled", - Help: "Returns 1 if the current license enforces the user limit.", - }), - } -} - -func (mc *MetricsCollector) Describe(descCh chan<- *prometheus.Desc) { - descCh <- mc.activeUsersGauge.Desc() - descCh <- mc.limitUsersGauge.Desc() - descCh <- mc.userLimitEnabledGauge.Desc() +func (*MetricsCollector) Describe(descCh chan<- *prometheus.Desc) { + descCh <- activeUsersDesc + descCh <- limitUsersDesc + descCh <- userLimitEnabledDesc } func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { @@ -62,16 +41,13 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { if userLimitEntitlement.Enabled { enabled = 1 } - mc.userLimitEnabledGauge.Set(enabled) - metricsCh <- mc.userLimitEnabledGauge + metricsCh <- prometheus.MustNewConstMetric(userLimitEnabledDesc, prometheus.GaugeValue, enabled) if userLimitEntitlement.Actual != nil { - mc.activeUsersGauge.Set(float64(*userLimitEntitlement.Actual)) - metricsCh <- mc.activeUsersGauge + metricsCh <- prometheus.MustNewConstMetric(activeUsersDesc, prometheus.GaugeValue, float64(*userLimitEntitlement.Actual)) } if userLimitEntitlement.Limit != nil { - mc.limitUsersGauge.Set(float64(*userLimitEntitlement.Limit)) - metricsCh <- mc.limitUsersGauge + metricsCh <- prometheus.MustNewConstMetric(limitUsersDesc, prometheus.GaugeValue, float64(*userLimitEntitlement.Limit)) } } diff --git a/enterprise/coderd/license/metricscollector_test.go b/enterprise/coderd/license/metricscollector_test.go index 16fe61c46f110..36661c8cdb364 100644 --- a/enterprise/coderd/license/metricscollector_test.go +++ b/enterprise/coderd/license/metricscollector_test.go @@ -19,7 +19,7 @@ func TestCollectLicenseMetrics(t *testing.T) { // Given registry := prometheus.NewRegistry() - sut := license.NewMetricsCollector() + var sut license.MetricsCollector const ( actualUsers = 4 @@ -35,7 +35,7 @@ func TestCollectLicenseMetrics(t *testing.T) { }, }) - registry.Register(sut) + registry.Register(&sut) // When metrics, err := registry.Gather()