Add monitoring package

coder · code-asher · May 13, 2022 · May 13, 2022 · May 13, 2022 · May 16, 2022
commit e614ce56bccd4672274b0aaddeca1990546b9538
diff --git a/cli/server.go b/cli/server.go
@@ -44,6 +44,7 @@ import (
 	"github.com/coder/coder/coderd/database/databasefake"
 	"github.com/coder/coder/coderd/devtunnel"
 	"github.com/coder/coder/coderd/gitsshkey"
+	"github.com/coder/coder/coderd/monitoring"
 	"github.com/coder/coder/coderd/turnconn"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/cryptorand"
@@ -73,6 +74,7 @@ func server() *cobra.Command {
 		oauth2GithubClientSecret         string
 		oauth2GithubAllowedOrganizations []string
 		oauth2GithubAllowSignups         bool
+		telemetryLevelRaw                string
 		tlsCertFile                      string
 		tlsClientCAFile                  string
 		tlsClientAuth                    string
@@ -192,6 +194,11 @@ func server() *cobra.Command {
 				return xerrors.Errorf("parse ssh keygen algorithm %s: %w", sshKeygenAlgorithmRaw, err)
 			}
 
+			telemetryLevel, err := monitoring.ParseTelemetryLevel(telemetryLevelRaw)
+			if err != nil {
+				return xerrors.Errorf("parse telemetry level %s: %w", telemetryLevelRaw, err)
+			}
+
 			turnServer, err := turnconn.New(&turn.RelayAddressGeneratorStatic{
 				RelayAddress: net.ParseIP(turnRelayAddress),
 				Address:      turnRelayAddress,
@@ -249,6 +256,13 @@ func server() *cobra.Command {
 				}
 			}
 
+			options.Monitor = monitoring.New(cmd.Context(), &monitoring.Options{
+				Database:        options.Database,
+				Logger:          options.Logger,
+				RefreshInterval: time.Hour,
+				TelemetryLevel:  telemetryLevel,
+			})
+
 			handler, closeCoderd := coderd.New(options)
 			client := codersdk.New(localURL)
 			if tlsEnable {
@@ -461,6 +475,8 @@ func server() *cobra.Command {
 		"Specifies organizations the user must be a member of to authenticate with GitHub.")
 	cliflag.BoolVarP(root.Flags(), &oauth2GithubAllowSignups, "oauth2-github-allow-signups", "", "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS", false,
 		"Specifies whether new users can sign up with GitHub.")
+	cliflag.StringVarP(root.Flags(), &telemetryLevelRaw, "telemetry", "", "CODER_TELEMETRY", "all", "The level of telemetry to send. "+
+		`Accepted values are "all", "core", or "none"`)
 	cliflag.BoolVarP(root.Flags(), &tlsEnable, "tls-enable", "", "CODER_TLS_ENABLE", false, "Specifies if TLS will be enabled")
 	cliflag.StringVarP(root.Flags(), &tlsCertFile, "tls-cert-file", "", "CODER_TLS_CERT_FILE", "",
 		"Specifies the path to the certificate for TLS. It requires a PEM-encoded file. "+
@@ -569,16 +585,16 @@ func newProvisionerDaemon(ctx context.Context, client *codersdk.Client, logger s
 func printLogo(cmd *cobra.Command, spooky bool) {
 	if spooky {
 		_, _ = fmt.Fprintf(cmd.OutOrStdout(), `
-		▄████▄   ▒█████  ▓█████▄ ▓█████  ██▀███  
+		▄████▄   ▒█████  ▓█████▄ ▓█████  ██▀███
 		▒██▀ ▀█  ▒██▒  ██▒▒██▀ ██▌▓█   ▀ ▓██ ▒ ██▒
 		▒▓█    ▄ ▒██░  ██▒░██   █▌▒███   ▓██ ░▄█ ▒
-		▒▓▓▄ ▄██▒▒██   ██░░▓█▄   ▌▒▓█  ▄ ▒██▀▀█▄  
+		▒▓▓▄ ▄██▒▒██   ██░░▓█▄   ▌▒▓█  ▄ ▒██▀▀█▄
 		▒ ▓███▀ ░░ ████▓▒░░▒████▓ ░▒████▒░██▓ ▒██▒
 		░ ░▒ ▒  ░░ ▒░▒░▒░  ▒▒▓  ▒ ░░ ▒░ ░░ ▒▓ ░▒▓░
 		  ░  ▒     ░ ▒ ▒░  ░ ▒  ▒  ░ ░  ░  ░▒ ░ ▒░
-		░        ░ ░ ░ ▒   ░ ░  ░    ░     ░░   ░ 
-		░ ░          ░ ░     ░       ░  ░   ░     
-		░                  ░                      		
+		░        ░ ░ ░ ▒   ░ ░  ░    ░     ░░   ░
+		░ ░          ░ ░     ░       ░  ░   ░
+		░                  ░
 
 `)
 		return

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -24,6 +24,7 @@ import (
 	"github.com/coder/coder/coderd/gitsshkey"
 	"github.com/coder/coder/coderd/httpapi"
 	"github.com/coder/coder/coderd/httpmw"
+	"github.com/coder/coder/coderd/monitoring"
 	"github.com/coder/coder/coderd/rbac"
 	"github.com/coder/coder/coderd/turnconn"
 	"github.com/coder/coder/codersdk"
@@ -47,6 +48,7 @@ type Options struct {
 	GoogleTokenValidator *idtoken.Validator
 	GithubOAuth2Config   *GithubOAuth2Config
 	ICEServers           []webrtc.ICEServer
+	Monitor              *monitoring.Monitor
 	SecureAuthCookie     bool
 	SSHKeygenAlgorithm   gitsshkey.Algorithm
 	TURNServer           *turnconn.Server
@@ -91,7 +93,7 @@ func New(options *Options) (http.Handler, func()) {
 				next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r)
 			})
 		},
-		httpmw.Prometheus,
+		httpmw.Prometheus(options.Monitor),
 		chitrace.Middleware(),
 	)
 

diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go
@@ -45,6 +45,7 @@ import (
 	"github.com/coder/coder/coderd/database/databasefake"
 	"github.com/coder/coder/coderd/database/postgres"
 	"github.com/coder/coder/coderd/gitsshkey"
+	"github.com/coder/coder/coderd/monitoring"
 	"github.com/coder/coder/coderd/turnconn"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/cryptorand"
@@ -145,10 +146,16 @@ func NewMemoryCoderd(t *testing.T, options *Options) (*httptest.Server, *codersd
 		AzureCertificates:    options.AzureCertificates,
 		GithubOAuth2Config:   options.GithubOAuth2Config,
 		GoogleTokenValidator: options.GoogleTokenValidator,
-		SSHKeygenAlgorithm:   options.SSHKeygenAlgorithm,
-		TURNServer:           turnServer,
-		APIRateLimit:         options.APIRateLimit,
-		Authorizer:           options.Authorizer,
+		Monitor: monitoring.New(ctx, &monitoring.Options{
+			Database:        db,
+			Logger:          slogtest.Make(t, nil),
+			RefreshInterval: time.Minute,
+			TelemetryLevel:  monitoring.TelemetryLevelNone,
+		}),
+		SSHKeygenAlgorithm: options.SSHKeygenAlgorithm,
+		TURNServer:         turnServer,
+		APIRateLimit:       options.APIRateLimit,
+		Authorizer:         options.Authorizer,
 	})
 	t.Cleanup(func() {
 		cancelFunc()

diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go
@@ -1081,6 +1081,7 @@ func (q *fakeQuerier) GetWorkspaceResourcesByJobID(_ context.Context, jobID uuid
 	return resources, nil
 }
 
+// revive:disable-next-line:flag-parameter
 func (q *fakeQuerier) GetWorkspaces(_ context.Context, deleted bool) ([]database.Workspace, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()

diff --git a/coderd/httpmw/prometheus.go b/coderd/httpmw/prometheus.go
@@ -9,29 +9,30 @@ import (
 	chimw "github.com/go-chi/chi/v5/middleware"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/coder/coder/coderd/monitoring"
 )
 
 var (
-	requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+	requestsProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "requests_processed_total",
 		Help:      "The total number of processed API requests",
 	}, []string{"code", "method", "path"})
-	requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
+	requestsConcurrent = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "concurrent_requests",
 		Help:      "The number of concurrent API requests",
 	})
-	websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
+	websocketsConcurrent = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "concurrent_websockets",
 		Help:      "The total number of concurrent API websockets",
 	})
-	websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	websocketsDist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "websocket_durations_ms",
@@ -45,7 +46,7 @@ var (
 			durationToFloatMs(30 * time.Hour),
 		},
 	}, []string{"path"})
-	requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	requestsDist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "request_latencies_ms",
@@ -58,45 +59,55 @@ func durationToFloatMs(d time.Duration) float64 {
 	return float64(d.Milliseconds())
 }
 
-func Prometheus(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		var (
-			start  = time.Now()
-			method = r.Method
-			rctx   = chi.RouteContext(r.Context())
-		)
-		sw, ok := w.(chimw.WrapResponseWriter)
-		if !ok {
-			panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter")
-		}
+func Prometheus(monitor *monitoring.Monitor) func(http.Handler) http.Handler {
+	monitor.MustRegister(
+		monitoring.TelemetryLevelNone,
+		requestsProcessed,
+		requestsConcurrent,
+		websocketsConcurrent,
+		requestsDist,
+	)
 
-		var (
-			dist     *prometheus.HistogramVec
-			distOpts []string
-		)
-		// We want to count websockets separately.
-		if isWebsocketUpgrade(r) {
-			websocketsConcurrent.Inc()
-			defer websocketsConcurrent.Dec()
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var (
+				start  = time.Now()
+				method = r.Method
+				rctx   = chi.RouteContext(r.Context())
+			)
+			sw, ok := w.(chimw.WrapResponseWriter)
+			if !ok {
+				panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter")
+			}
 
-			dist = websocketsDist
-		} else {
-			requestsConcurrent.Inc()
-			defer requestsConcurrent.Dec()
+			var (
+				dist     *prometheus.HistogramVec
+				distOpts []string
+			)
+			// We want to count websockets separately.
+			if isWebsocketUpgrade(r) {
+				websocketsConcurrent.Inc()
+				defer websocketsConcurrent.Dec()
 
-			dist = requestsDist
-			distOpts = []string{method}
-		}
+				dist = websocketsDist
+			} else {
+				requestsConcurrent.Inc()
+				defer requestsConcurrent.Dec()
 
-		next.ServeHTTP(w, r)
+				dist = requestsDist
+				distOpts = []string{method}
+			}
 
-		path := rctx.RoutePattern()
-		distOpts = append(distOpts, path)
-		statusStr := strconv.Itoa(sw.Status())
+			next.ServeHTTP(w, r)
 
-		requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
-		dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6)
-	})
+			path := rctx.RoutePattern()
+			distOpts = append(distOpts, path)
+			statusStr := strconv.Itoa(sw.Status())
+
+			requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
+			dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6)
+		})
+	}
 }
 
 func isWebsocketUpgrade(r *http.Request) bool {