gauges

coder · mtojek · Apr 14, 2023 · Apr 3, 2023 · Apr 4, 2023 · Apr 4, 2023
commit 8c6f96bbe3b64b223fb89d86441bb49b8ec4dd45
diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go
@@ -330,7 +330,73 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 		return nil, err
 	}
 
-	createdAfter := database.Now().Add(-duration)
+	agentStatsConnectionCountGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "connection_count",
+		Help:      "The number of established connections by agent",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsConnectionCountGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	agentStatsConnectionMedianLatencyGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "connection_median_latency",
+		Help:      "The median agent connection latency",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsConnectionMedianLatencyGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	agentStatsSessionCountJetBrainsGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "session_count_jetbrains",
+		Help:      "The number of session established by JetBrains",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsSessionCountJetBrainsGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	agentStatsSessionCountReconnectingPTYGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "session_count_reconnecting_pty",
+		Help:      "The number of session established by reconnecting PTY",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsSessionCountReconnectingPTYGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	agentStatsSessionCountSSHGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "session_count_ssh",
+		Help:      "The number of session established by SSH",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsSessionCountSSHGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	agentStatsSessionCountVSCodeGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: "agentstats",
+		Name:      "session_count_vscode",
+		Help:      "The number of session established by VSCode",
+	}, []string{"agent_name", "username", "workspace_name"}))
+	err = registerer.Register(agentStatsSessionCountVSCodeGauge)
+	if err != nil {
+		return nil, err
+	}
+
+	createdAfter := time.Now()
 	ctx, cancelFunc := context.WithCancel(ctx)
 	ticker := time.NewTicker(duration)
 	go func() {
@@ -354,14 +420,32 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R
 			for _, agentStat := range stats {
 				agentStatsRxBytesGauge.WithLabelValues(VectorOperationAdd, float64(agentStat.WorkspaceTxBytes), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
 				agentStatsTxBytesGauge.WithLabelValues(VectorOperationAdd, float64(agentStat.WorkspaceRxBytes), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+
+				agentStatsConnectionCountGauge.WithLabelValues(VectorOperationSet, float64(agentStat.ConnectionCount), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+				agentStatsConnectionMedianLatencyGauge.WithLabelValues(VectorOperationSet, agentStat.ConnectionMedianLatencyMS/1000.0 /* (to seconds) */, agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+
+				agentStatsSessionCountJetBrainsGauge.WithLabelValues(VectorOperationSet, float64(agentStat.SessionCountJetBrains), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+				agentStatsSessionCountReconnectingPTYGauge.WithLabelValues(VectorOperationSet, float64(agentStat.SessionCountReconnectingPTY), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+				agentStatsSessionCountSSHGauge.WithLabelValues(VectorOperationSet, float64(agentStat.SessionCountSSH), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
+				agentStatsSessionCountVSCodeGauge.WithLabelValues(VectorOperationSet, float64(agentStat.SessionCountVSCode), agentStat.AgentName, agentStat.Username, agentStat.WorkspaceName)
 			}
 
 			agentStatsRxBytesGauge.Commit()
 			agentStatsTxBytesGauge.Commit()
 
+			agentStatsConnectionCountGauge.Commit()
+			agentStatsConnectionMedianLatencyGauge.Commit()
+
+			agentStatsSessionCountJetBrainsGauge.Commit()
+			agentStatsSessionCountReconnectingPTYGauge.Commit()
+			agentStatsSessionCountSSHGauge.Commit()
+			agentStatsSessionCountVSCodeGauge.Commit()
+
 		done:
 			logger.Debug(ctx, "Agent metrics collection is done")
 			metricsCollectorAgentStats.Observe(timer.ObserveDuration().Seconds())
+
+			createdAfter = time.Now()
 		}
 	}()
 	return cancelFunc, nil

diff --git a/docs/admin/prometheus.md b/docs/admin/prometheus.md
@@ -32,9 +32,17 @@ The environment variable `CODER_PROMETHEUS_ENABLE` will be enabled automatically
 | Name                                                | Type      | Description                                                        | Labels                                                                              |
 | --------------------------------------------------- | --------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------- |
 | `coderd_agents_apps`                                | gauge     | Agent applications with statuses.                                  | `agent_name` `app_name` `health` `username` `workspace_name`                        |
-| `coderd_agents_connection_latencies_seconds`        | gauge     | Agent connection latencies in seconds.                             | `agent_id` `derp_region` `preferred` `username` `workspace_name`                    |
+| `coderd_agents_connection_latencies_seconds`        | gauge     | Agent connection latencies in seconds.                             | `agent_name` `derp_region` `preferred` `username` `workspace_name`                  |
 | `coderd_agents_connections`                         | gauge     | Agent connections with statuses.                                   | `agent_name` `lifecycle_state` `status` `tailnet_node` `username` `workspace_name`  |
 | `coderd_agents_up`                                  | gauge     | The number of active agents per workspace.                         | `username` `workspace_name`                                                         |
+| `coderd_agentstats_connection_count`                | gauge     | The number of established connections by agent                     | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_connection_median_latency`       | gauge     | The median agent connection latency                                | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_rx_bytes`                        | gauge     | Agent Rx bytes                                                     | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_session_count_jetbrains`         | gauge     | The number of session established by JetBrains                     | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_session_count_reconnecting_pty`  | gauge     | The number of session established by reconnecting PTY              | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_session_count_ssh`               | gauge     | The number of session established by SSH                           | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_session_count_vscode`            | gauge     | The number of session established by VSCode                        | `agent_name` `username` `workspace_name`                                            |
+| `coderd_agentstats_tx_bytes`                        | gauge     | Agent Tx bytes                                                     | `agent_name` `username` `workspace_name`                                            |
 | `coderd_api_active_users_duration_hour`             | gauge     | The number of users that have been active within the last hour.    |                                                                                     |
 | `coderd_api_concurrent_requests`                    | gauge     | The number of concurrent API requests.                             |                                                                                     |
 | `coderd_api_concurrent_websockets`                  | gauge     | The total number of concurrent API websockets.                     |                                                                                     |

diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics
@@ -5,9 +5,9 @@ coderd_agents_apps{agent_name="main",app_name="code-server",health="healthy",use
 coderd_agents_apps{agent_name="main",app_name="code-server",health="healthy",username="admin",workspace_name="workspace-3"} 1
 # HELP coderd_agents_connection_latencies_seconds Agent connection latencies in seconds.
 # TYPE coderd_agents_connection_latencies_seconds gauge
-coderd_agents_connection_latencies_seconds{agent_id="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-1"} 0.03018125
-coderd_agents_connection_latencies_seconds{agent_id="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-2"} 0.028658416
-coderd_agents_connection_latencies_seconds{agent_id="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-3"} 0.028041416
+coderd_agents_connection_latencies_seconds{agent_name="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-1"} 0.03018125
+coderd_agents_connection_latencies_seconds{agent_name="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-2"} 0.028658416
+coderd_agents_connection_latencies_seconds{agent_name="main",derp_region="Coder Embedded Relay",preferred="true",username="admin",workspace_name="workspace-3"} 0.028041416
 # HELP coderd_agents_connections Agent connections with statuses.
 # TYPE coderd_agents_connections gauge
 coderd_agents_connections{agent_name="main",lifecycle_state="ready",status="connected",tailnet_node="nodeid:16966f7df70d8cc5",username="admin",workspace_name="workspace-3"} 1
@@ -18,6 +18,30 @@ coderd_agents_connections{agent_name="main",lifecycle_state="start_timeout",stat
 coderd_agents_up{username="admin",workspace_name="workspace-1"} 1
 coderd_agents_up{username="admin",workspace_name="workspace-2"} 1
 coderd_agents_up{username="admin",workspace_name="workspace-3"} 1
+# HELP coderd_agentstats_connection_count The number of established connections by agent
+# TYPE coderd_agentstats_connection_count gauge
+coderd_agentstats_connection_count{agent_name="main",username="admin",workspace_name="workspace1"} 2
+# HELP coderd_agentstats_connection_median_latency The median agent connection latency
+# TYPE coderd_agentstats_connection_median_latency gauge
+coderd_agentstats_connection_median_latency{agent_name="main",username="admin",workspace_name="workspace1"} 0.001784
+# HELP coderd_agentstats_rx_bytes Agent Rx bytes
+# TYPE coderd_agentstats_rx_bytes gauge
+coderd_agentstats_rx_bytes{agent_name="main",username="admin",workspace_name="workspace1"} 7731
+# HELP coderd_agentstats_session_count_jetbrains The number of session established by JetBrains
+# TYPE coderd_agentstats_session_count_jetbrains gauge
+coderd_agentstats_session_count_jetbrains{agent_name="main",username="admin",workspace_name="workspace1"} 0
+# HELP coderd_agentstats_session_count_reconnecting_pty The number of session established by reconnecting PTY
+# TYPE coderd_agentstats_session_count_reconnecting_pty gauge
+coderd_agentstats_session_count_reconnecting_pty{agent_name="main",username="admin",workspace_name="workspace1"} 1
+# HELP coderd_agentstats_session_count_ssh The number of session established by SSH
+# TYPE coderd_agentstats_session_count_ssh gauge
+coderd_agentstats_session_count_ssh{agent_name="main",username="admin",workspace_name="workspace1"} 0
+# HELP coderd_agentstats_session_count_vscode The number of session established by VSCode
+# TYPE coderd_agentstats_session_count_vscode gauge
+coderd_agentstats_session_count_vscode{agent_name="main",username="admin",workspace_name="workspace1"} 0
+# HELP coderd_agentstats_tx_bytes Agent Tx bytes
+# TYPE coderd_agentstats_tx_bytes gauge
+coderd_agentstats_tx_bytes{agent_name="main",username="admin",workspace_name="workspace1"} 6643
 # HELP coderd_api_websocket_durations_seconds Websocket duration distribution of requests in seconds.
 # TYPE coderd_api_websocket_durations_seconds histogram
 coderd_api_websocket_durations_seconds_bucket{path="/api/v2/workspaceagents/me/coordinate",le="0.001"} 0