Use MustNewConstMetric

coder · mtojek · Apr 27, 2023 · Apr 24, 2023 · Apr 24, 2023 · Apr 24, 2023
commit df80e9b5692644e02adafa810e5a0e35d2c00e49
diff --git a/agent/metrics.go b/agent/metrics.go
@@ -33,7 +33,8 @@ func isIgnoredMetric(metricName string) bool {
 	if strings.HasPrefix(metricName, "dns_") ||
 		strings.HasPrefix(metricName, "controlclient_") ||
 		strings.HasPrefix(metricName, "peerapi_") ||
-		strings.HasPrefix(metricName, "profiles_") {
+		strings.HasPrefix(metricName, "profiles_") ||
+		strings.HasPrefix(metricName, "tstun_") {
 		return true
 	}
 	return false

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -149,7 +149,7 @@ type Options struct {
 
 	HTTPClient *http.Client
 
-	UpdateAgentMetrics func(ctx context.Context, workspaceID uuid.UUID, agentID uuid.UUID, metrics []agentsdk.AgentMetric)
+	UpdateAgentMetrics func(ctx context.Context, username, workspaceName, agentName string, metrics []agentsdk.AgentMetric)
 }
 
 // @title Coder API

diff --git a/coderd/prometheusmetrics/aggregator.go b/coderd/prometheusmetrics/aggregator.go
@@ -2,15 +2,37 @@ package prometheusmetrics
 
 import (
 	"context"
-	"log"
+	"sync"
 
-	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
 
 	"github.com/coder/coder/codersdk/agentsdk"
 )
 
-type MetricsAggregator struct{}
+const (
+	// MetricHelpForAgent is a help string that replaces all agent metric help
+	// messages. This is because a registry cannot have conflicting
+	// help messages for the same metric in a "gather". If our coder agents are
+	// on different versions, this is a possible scenario.
+	metricHelpForAgent = "Metric is forwarded from workspace agent connected to this instance of coderd."
-	metricHelpForAgent = "Metric is forwarded from workspace agent connected to this instance of coderd."
+	metricHelpForAgent = "Metrics are forwarded from workspace agents connected to this instance of coderd."
-	metricHelpForAgent = "Metric is forwarded from workspace agent connected to this instance of coderd."
+	metricHelpForAgent = "Metrics are forwarded from workspace agents connected to this instance of coderd."
+)
+
+type MetricsAggregator struct {
+	m     sync.Mutex
+	log   slog.Logger
+	queue []annotatedMetrics
+}
+
+type annotatedMetrics struct {
+	username      string
+	workspaceName string
+	agentName     string
+
+	metrics []agentsdk.AgentMetric
+}
 
 var _ prometheus.Collector = new(MetricsAggregator)
 
@@ -19,11 +41,47 @@ var _ prometheus.Collector = new(MetricsAggregator)
 func (*MetricsAggregator) Describe(_ chan<- *prometheus.Desc) {
 }
 
+var agentMetricsLabels = []string{usernameLabel, workspaceNameLabel, agentNameLabel}
+
 func (ma *MetricsAggregator) Collect(ch chan<- prometheus.Metric) {
+	ma.m.Lock()
+	defer ma.m.Unlock()
+
+	for _, annotated := range ma.queue {
+		for _, m := range annotated.metrics {
+			desc := prometheus.NewDesc(m.Name, metricHelpForAgent, agentMetricsLabels, nil)
+			valueType, err := asPrometheusValueType(m.Type)
+			if err != nil {
+				ma.log.Error(context.Background(), "can't convert Prometheus value type", slog.F("value_type", m.Type), slog.Error(err))
+			}
+			constMetric := prometheus.MustNewConstMetric(desc, valueType, m.Value, annotated.username, annotated.workspaceName, annotated.agentName)
+			ch <- constMetric
+		}
+	}
 }
 
 // TODO Run function with done channel
 
-func (ma *MetricsAggregator) Update(ctx context.Context, workspaceID uuid.UUID, agentID uuid.UUID, metrics []agentsdk.AgentMetric) {
-	log.Printf("Workspace: %s, Agent: %s, Metrics: %v", workspaceID, agentID, metrics) // FIXME
+func (ma *MetricsAggregator) Update(_ context.Context, username, workspaceName, agentName string, metrics []agentsdk.AgentMetric) {
+	ma.m.Lock()
+	defer ma.m.Unlock()
+
+	ma.queue = append(ma.queue, annotatedMetrics{
+		username:      username,
+		workspaceName: workspaceName,
+		agentName:     agentName,
+
+		metrics: metrics,
+	})
+}
+
+func asPrometheusValueType(metricType agentsdk.AgentMetricType) (prometheus.ValueType, error) {
+	switch metricType {
+	case agentsdk.AgentMetricTypeGauge:
+		return prometheus.GaugeValue, nil
+	case agentsdk.AgentMetricTypeCounter:
+		return prometheus.CounterValue, nil
+	default:
+		return -1, xerrors.Errorf("unsupported value type: %s", metricType)
+	}
 }
diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go
@@ -22,6 +22,12 @@ import (
 	"github.com/coder/coder/tailnet"
 )
 
+const (
+	agentNameLabel     = "agent_name"
+	usernameLabel      = "username"
+	workspaceNameLabel = "workspace_name"
+)
+
 // ActiveUsers tracks the number of users that have authenticated within the past hour.
 func ActiveUsers(ctx context.Context, registerer prometheus.Registerer, db database.Store, duration time.Duration) (func(), error) {
 	if duration == 0 {
@@ -140,7 +146,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
 		Subsystem: "agents",
 		Name:      "up",
 		Help:      "The number of active agents per workspace.",
-	}, []string{"username", "workspace_name"}))
+	}, []string{usernameLabel, workspaceNameLabel}))
 	err := registerer.Register(agentsGauge)
 	if err != nil {
 		return nil, err
@@ -151,7 +157,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
 		Subsystem: "agents",
 		Name:      "connections",
 		Help:      "Agent connections with statuses.",
-	}, []string{"agent_name", "username", "workspace_name", "status", "lifecycle_state", "tailnet_node"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel, "status", "lifecycle_state", "tailnet_node"}))
 	err = registerer.Register(agentsConnectionsGauge)
 	if err != nil {
 		return nil, err
@@ -162,7 +168,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
 		Subsystem: "agents",
 		Name:      "connection_latencies_seconds",
 		Help:      "Agent connection latencies in seconds.",
-	}, []string{"agent_name", "username", "workspace_name", "derp_region", "preferred"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel, "derp_region", "preferred"}))
 	err = registerer.Register(agentsConnectionLatenciesGauge)
 	if err != nil {
 		return nil, err
@@ -173,7 +179,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
 		Subsystem: "agents",
 		Name:      "apps",
 		Help:      "Agent applications with statuses.",
-	}, []string{"agent_name", "username", "workspace_name", "app_name", "health"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel, "app_name", "health"}))
 	err = registerer.Register(agentsAppsGauge)
 	if err != nil {
 		return nil, err
@@ -333,7 +339,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "tx_bytes",
 		Help:      "Agent Tx bytes",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsTxBytesGauge)
 	if err != nil {
 		return nil, err
@@ -344,7 +350,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "rx_bytes",
 		Help:      "Agent Rx bytes",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsRxBytesGauge)
 	if err != nil {
 		return nil, err
@@ -355,7 +361,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "connection_count",
 		Help:      "The number of established connections by agent",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsConnectionCountGauge)
 	if err != nil {
 		return nil, err
@@ -366,7 +372,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "connection_median_latency_seconds",
 		Help:      "The median agent connection latency in seconds",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsConnectionMedianLatencyGauge)
 	if err != nil {
 		return nil, err
@@ -377,7 +383,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "session_count_jetbrains",
 		Help:      "The number of session established by JetBrains",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsSessionCountJetBrainsGauge)
 	if err != nil {
 		return nil, err
@@ -388,7 +394,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "session_count_reconnecting_pty",
 		Help:      "The number of session established by reconnecting PTY",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsSessionCountReconnectingPTYGauge)
 	if err != nil {
 		return nil, err
@@ -399,7 +405,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "session_count_ssh",
 		Help:      "The number of session established by SSH",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsSessionCountSSHGauge)
 	if err != nil {
 		return nil, err
@@ -410,7 +416,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		Subsystem: "agentstats",
 		Name:      "session_count_vscode",
 		Help:      "The number of session established by VSCode",
-	}, []string{"agent_name", "username", "workspace_name"}))
+	}, []string{agentNameLabel, usernameLabel, workspaceNameLabel}))
 	err = registerer.Register(agentStatsSessionCountVSCodeGauge)
 	if err != nil {
 		return nil, err

diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go
@@ -1246,7 +1246,12 @@ func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Reques
 	})
 	if api.Options.UpdateAgentMetrics != nil {
 		errGroup.Go(func() error {
-			api.Options.UpdateAgentMetrics(ctx, workspace.ID, workspaceAgent.ID, req.Metrics)
+			user, err := api.Database.GetUserByID(ctx, workspace.OwnerID)
+			if err != nil {
+				return err
+			}
+
+			api.Options.UpdateAgentMetrics(ctx, user.Username, workspace.Name, workspaceAgent.Name, req.Metrics)
 			return nil
 		})
 	}