From f89a49e0310438fa9e1dd55ecff990f1a15438e2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 16 Jun 2023 13:12:12 +0100 Subject: [PATCH 1/3] add benchmark for prom metrics aggregator --- coderd/prometheusmetrics/aggregator_test.go | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/coderd/prometheusmetrics/aggregator_test.go b/coderd/prometheusmetrics/aggregator_test.go index d7a15a547e6d3..de8bdc948ef90 100644 --- a/coderd/prometheusmetrics/aggregator_test.go +++ b/coderd/prometheusmetrics/aggregator_test.go @@ -13,6 +13,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/prometheusmetrics" "github.com/coder/coder/codersdk/agentsdk" + "github.com/coder/coder/cryptorand" "github.com/coder/coder/testutil" ) @@ -20,6 +21,14 @@ const ( testWorkspaceName = "yogi-workspace" testUsername = "yogi-bear" testAgentName = "main-agent" + + labelAgentName = "agent_name" + labelUsername = "username" + labelWorkspaceName = "workspace_name" +) + +var ( + commonLabelNames = []string{labelAgentName, labelUsername, labelWorkspaceName} ) func TestUpdateMetrics_MetricsDoNotExpire(t *testing.T) { @@ -175,3 +184,66 @@ func TestUpdateMetrics_MetricsExpire(t *testing.T) { return len(actual) == 0 }, testutil.WaitShort, testutil.IntervalFast) } + +func Benchmark_MetricsAggregator_Run(b *testing.B) { + b.StopTimer() + // given + registry := prometheus.NewRegistry() + metricsAggregator := must(prometheusmetrics.NewMetricsAggregator( + slogtest.Make(b, &slogtest.Options{IgnoreErrors: true}), + registry, + time.Hour, + )) + + ctx, cancelFunc := context.WithCancel(context.Background()) + b.Cleanup(cancelFunc) + + closeFunc := metricsAggregator.Run(ctx) + b.Cleanup(closeFunc) + + metrics := make([]agentsdk.AgentMetric, 0, b.N) + for i := 0; i < b.N; i++ { + metrics = append(metrics, genAgentMetric(b, commonLabelNames...)) + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + metricsAggregator.Update(ctx, testUsername, testWorkspaceName, testAgentName, metrics[i:i+1]) + } + b.StopTimer() +} + +func genAgentMetric(t testing.TB, labelNames ...string) agentsdk.AgentMetric { + t.Helper() + + var metricType agentsdk.AgentMetricType + if must(cryptorand.Float64()) >= 0.5 { + metricType = agentsdk.AgentMetricTypeCounter + } else { + metricType = agentsdk.AgentMetricTypeGauge + } + + // Ensure that metric name does not start or end with underscore, as it is not allowed by Prometheus. + metricName := "metric_" + must(cryptorand.StringCharset("a-zA-Z", 80)) + "_gen" + // Generate random metric value between 0 and 1000. + metricValue := must(cryptorand.Float64()) * float64(must(cryptorand.Intn(1000))) + + lvs := make([]agentsdk.AgentMetricLabel, 0, len(labelNames)) + for _, labelName := range labelNames { + lvs = append(lvs, agentsdk.AgentMetricLabel{ + Name: labelName, + Value: must(cryptorand.StringCharset("a-zA-Z", 80)), + }) + } + + return agentsdk.AgentMetric{ + Name: metricName, Type: metricType, Value: metricValue, Labels: lvs, + } +} + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + return t +} From 83d609a8b91bc493ce0c6ebdb8f72f0037eaa77b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 16 Jun 2023 13:49:51 +0100 Subject: [PATCH 2/3] fixup! add benchmark for prom metrics aggregator --- coderd/prometheusmetrics/aggregator_test.go | 67 ++++++++++++--------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/coderd/prometheusmetrics/aggregator_test.go b/coderd/prometheusmetrics/aggregator_test.go index de8bdc948ef90..9298ea4c4cc7d 100644 --- a/coderd/prometheusmetrics/aggregator_test.go +++ b/coderd/prometheusmetrics/aggregator_test.go @@ -2,6 +2,7 @@ package prometheusmetrics_test import ( "context" + "sync/atomic" "testing" "time" @@ -21,14 +22,6 @@ const ( testWorkspaceName = "yogi-workspace" testUsername = "yogi-bear" testAgentName = "main-agent" - - labelAgentName = "agent_name" - labelUsername = "username" - labelWorkspaceName = "workspace_name" -) - -var ( - commonLabelNames = []string{labelAgentName, labelUsername, labelWorkspaceName} ) func TestUpdateMetrics_MetricsDoNotExpire(t *testing.T) { @@ -186,7 +179,10 @@ func TestUpdateMetrics_MetricsExpire(t *testing.T) { } func Benchmark_MetricsAggregator_Run(b *testing.B) { - b.StopTimer() + // Number of metrics to generate and send in each iteration. + // Hard-coded to 1024 to avoid overflowing the queue in the metrics aggregator. + var numMetrics = 1024 + // given registry := prometheus.NewRegistry() metricsAggregator := must(prometheusmetrics.NewMetricsAggregator( @@ -201,19 +197,44 @@ func Benchmark_MetricsAggregator_Run(b *testing.B) { closeFunc := metricsAggregator.Run(ctx) b.Cleanup(closeFunc) - metrics := make([]agentsdk.AgentMetric, 0, b.N) - for i := 0; i < b.N; i++ { - metrics = append(metrics, genAgentMetric(b, commonLabelNames...)) - } + ch := make(chan prometheus.Metric) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + metricsAggregator.Collect(ch) + } + } + }() - b.StartTimer() for i := 0; i < b.N; i++ { - metricsAggregator.Update(ctx, testUsername, testWorkspaceName, testAgentName, metrics[i:i+1]) + b.StopTimer() + b.Logf("N=%d generating %d metrics", b.N, numMetrics) + metrics := make([]agentsdk.AgentMetric, 0, numMetrics) + for i := 0; i < numMetrics; i++ { + metrics = append(metrics, genAgentMetric(b)) + } + + b.Logf("N=%d sending %d metrics", b.N, numMetrics) + var nGot atomic.Int64 + b.StartTimer() + metricsAggregator.Update(ctx, testUsername, testWorkspaceName, testAgentName, metrics) + for i := 0; i < numMetrics; i++ { + select { + case <-ctx.Done(): + b.FailNow() + case <-ch: + nGot.Add(1) + } + } + b.StopTimer() + b.Logf("N=%d got %d metrics", b.N, nGot.Load()) } - b.StopTimer() } -func genAgentMetric(t testing.TB, labelNames ...string) agentsdk.AgentMetric { +func genAgentMetric(t testing.TB) agentsdk.AgentMetric { t.Helper() var metricType agentsdk.AgentMetricType @@ -224,20 +245,12 @@ func genAgentMetric(t testing.TB, labelNames ...string) agentsdk.AgentMetric { } // Ensure that metric name does not start or end with underscore, as it is not allowed by Prometheus. - metricName := "metric_" + must(cryptorand.StringCharset("a-zA-Z", 80)) + "_gen" + metricName := "metric_" + must(cryptorand.StringCharset(cryptorand.Alpha, 80)) + "_gen" // Generate random metric value between 0 and 1000. metricValue := must(cryptorand.Float64()) * float64(must(cryptorand.Intn(1000))) - lvs := make([]agentsdk.AgentMetricLabel, 0, len(labelNames)) - for _, labelName := range labelNames { - lvs = append(lvs, agentsdk.AgentMetricLabel{ - Name: labelName, - Value: must(cryptorand.StringCharset("a-zA-Z", 80)), - }) - } - return agentsdk.AgentMetric{ - Name: metricName, Type: metricType, Value: metricValue, Labels: lvs, + Name: metricName, Type: metricType, Value: metricValue, Labels: []agentsdk.AgentMetricLabel{}, } } From f449e2eebf36e0ddcf65761d5e9c484bc10d8710 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 16 Jun 2023 13:52:36 +0100 Subject: [PATCH 3/3] make fmt --- coderd/prometheusmetrics/aggregator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/prometheusmetrics/aggregator_test.go b/coderd/prometheusmetrics/aggregator_test.go index 9298ea4c4cc7d..a5e5b0f11eaef 100644 --- a/coderd/prometheusmetrics/aggregator_test.go +++ b/coderd/prometheusmetrics/aggregator_test.go @@ -181,7 +181,7 @@ func TestUpdateMetrics_MetricsExpire(t *testing.T) { func Benchmark_MetricsAggregator_Run(b *testing.B) { // Number of metrics to generate and send in each iteration. // Hard-coded to 1024 to avoid overflowing the queue in the metrics aggregator. - var numMetrics = 1024 + numMetrics := 1024 // given registry := prometheus.NewRegistry()