Skip to content

Commit a6e6b54

Browse files
mafredrimafredri
committed
rewrite test
1 parent 4889f67 commit a6e6b54

File tree

1 file changed

+80
-142
lines changed

1 file changed

+80
-142
lines changed

‎coderd/prometheusmetrics/insights/metricscollector_test.go

Lines changed: 80 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package insights_test
33
import (
44
"context"
55
"encoding/json"
6-
"io"
6+
"fmt"
77
"os"
88
"strings"
99
"testing"
@@ -18,34 +18,33 @@ import (
1818

1919
"cdr.dev/slog"
2020
"cdr.dev/slog/sloggers/slogtest"
21-
"github.com/coder/coder/v2/agent"
22-
"github.com/coder/coder/v2/agent/agenttest"
2321
"github.com/coder/coder/v2/coderd/coderdtest"
22+
"github.com/coder/coder/v2/coderd/database"
2423
"github.com/coder/coder/v2/coderd/database/dbauthz"
24+
"github.com/coder/coder/v2/coderd/database/dbgen"
2525
"github.com/coder/coder/v2/coderd/database/dbtestutil"
2626
"github.com/coder/coder/v2/coderd/prometheusmetrics/insights"
2727
"github.com/coder/coder/v2/coderd/workspaceapps"
28-
"github.com/coder/coder/v2/codersdk"
2928
"github.com/coder/coder/v2/codersdk/agentsdk"
30-
"github.com/coder/coder/v2/provisioner/echo"
31-
"github.com/coder/coder/v2/provisionersdk/proto"
3229
"github.com/coder/coder/v2/testutil"
3330
)
3431

3532
func TestCollectInsights(t *testing.T) {
3633
t.Parallel()
3734

3835
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
39-
db, ps := dbtestutil.NewDB(t)
36+
db, ps := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
4037

4138
options := &coderdtest.Options{
4239
IncludeProvisionerDaemon: true,
4340
AgentStatsRefreshInterval: time.Millisecond * 100,
4441
Database: db,
4542
Pubsub: ps,
4643
}
47-
client := coderdtest.New(t, options)
48-
client.SetLogger(logger.Named("client").Leveled(slog.LevelDebug))
44+
ownerClient := coderdtest.New(t, options)
45+
ownerClient.SetLogger(logger.Named("ownerClient").Leveled(slog.LevelDebug))
46+
owner := coderdtest.CreateFirstUser(t, ownerClient)
47+
client, user := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
4948

5049
// Given
5150
// Initialize metrics collector
@@ -55,47 +54,53 @@ func TestCollectInsights(t *testing.T) {
5554
registry := prometheus.NewRegistry()
5655
registry.Register(mc)
5756

58-
// Create two users, one that will appear in the report and another that
59-
// won't (due to not having/using a workspace).
60-
user := coderdtest.CreateFirstUser(t, client)
61-
_, _ = coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
62-
authToken := uuid.NewString()
63-
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
64-
Parse: echo.ParseComplete,
65-
ProvisionPlan: provisionPlanWithParameters(),
66-
ProvisionApply: provisionApplyWithAgentAndApp(authToken),
67-
})
68-
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
69-
ctr.Name = "golden-template"
70-
})
71-
require.Empty(t, template.BuildTimeStats[codersdk.WorkspaceTransitionStart])
72-
73-
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
74-
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
75-
cwr.RichParameterValues = []codersdk.WorkspaceBuildParameter{
76-
{Name: "first_parameter", Value: "Foobar"},
77-
{Name: "second_parameter", Value: "true"},
78-
{Name: "third_parameter", Value: "789"},
79-
}
80-
})
81-
coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
57+
var (
58+
orgID = owner.OrganizationID
59+
tpl = dbgen.Template(t, db, database.Template{OrganizationID: orgID, CreatedBy: user.ID, Name: "golden-template"})
60+
ver = dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: orgID, CreatedBy: user.ID, TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true}})
61+
param1 = dbgen.TemplateVersionParameter(t, db, database.TemplateVersionParameter{TemplateVersionID: ver.ID, Name: "first_parameter"})
62+
param2 = dbgen.TemplateVersionParameter(t, db, database.TemplateVersionParameter{TemplateVersionID: ver.ID, Name: "second_parameter", Type: "bool"})
63+
param3 = dbgen.TemplateVersionParameter(t, db, database.TemplateVersionParameter{TemplateVersionID: ver.ID, Name: "third_parameter", Type: "number"})
64+
workspace1 = dbgen.Workspace(t, db, database.Workspace{OrganizationID: orgID, TemplateID: tpl.ID, OwnerID: user.ID})
65+
workspace2 = dbgen.Workspace(t, db, database.Workspace{OrganizationID: orgID, TemplateID: tpl.ID, OwnerID: user.ID})
66+
job1 = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: orgID})
67+
job2 = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: orgID})
68+
build1 = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{TemplateVersionID: ver.ID, WorkspaceID: workspace1.ID, JobID: job1.ID})
69+
build2 = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{TemplateVersionID: ver.ID, WorkspaceID: workspace2.ID, JobID: job2.ID})
70+
res1 = dbgen.WorkspaceResource(t, db, database.WorkspaceResource{JobID: build1.JobID})
71+
res2 = dbgen.WorkspaceResource(t, db, database.WorkspaceResource{JobID: build2.JobID})
72+
agent1 = dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ResourceID: res1.ID})
73+
agent2 = dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ResourceID: res2.ID})
74+
app1 = dbgen.WorkspaceApp(t, db, database.WorkspaceApp{AgentID: agent1.ID, Slug: "golden-slug", DisplayName: "Golden Slug"})
75+
app2 = dbgen.WorkspaceApp(t, db, database.WorkspaceApp{AgentID: agent2.ID, Slug: "golden-slug", DisplayName: "Golden Slug"})
76+
_ = dbgen.WorkspaceBuildParameters(t, db, []database.WorkspaceBuildParameter{
77+
{WorkspaceBuildID: build1.ID, Name: param1.Name, Value: "Foobar"},
78+
{WorkspaceBuildID: build1.ID, Name: param2.Name, Value: "true"},
79+
{WorkspaceBuildID: build1.ID, Name: param3.Name, Value: "789"},
80+
})
81+
// _ = dbgen.WorkspaceBuildParameters(t, db, []database.WorkspaceBuildParameter{
82+
// {WorkspaceBuildID: build2.ID, Name: param1.Name, Value: "Baz"},
83+
// {WorkspaceBuildID: build2.ID, Name: param2.Name, Value: "false"},
84+
// {WorkspaceBuildID: build2.ID, Name: param3.Name, Value: "999"},
85+
// })
86+
)
8287

8388
// Start an agent so that we can generate stats.
84-
agentClient := agentsdk.New(client.URL)
85-
agentClient.SetSessionToken(authToken)
86-
agentClient.SDK.SetLogger(logger.Leveled(slog.LevelDebug).Named("agent"))
87-
88-
_ = agenttest.New(t, client.URL, authToken, func(o *agent.Options) {
89-
o.Client = agentClient
90-
})
91-
resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
89+
var agentClients []*agentsdk.Client
90+
for i, agent := range []database.WorkspaceAgent{agent1, agent2} {
91+
agentClient := agentsdk.New(client.URL)
92+
agentClient.SetSessionToken(agent.AuthToken.String())
93+
agentClient.SDK.SetLogger(logger.Leveled(slog.LevelDebug).Named(fmt.Sprintf("agent%d", i+1)))
94+
agentClients = append(agentClients, agentClient)
95+
}
9296

9397
// Fake app stats
94-
_, err = agentClient.PostStats(context.Background(), &agentsdk.Stats{
98+
_, err = agentClients[0].PostStats(context.Background(), &agentsdk.Stats{
9599
// ConnectionsByProto can't be nil, otherwise stats get rejected
96100
ConnectionsByProto: map[string]int64{"TCP": 1},
97101
// ConnectionCount must be positive as database query ignores stats with no active connections at the time frame
98-
ConnectionCount: 74,
102+
ConnectionCount: 1,
103+
SessionCountSSH: 99,
99104
// SessionCountJetBrains, SessionCountVSCode must be positive, but the exact value is ignored.
100105
// Database query approximates it to 60s of usage.
101106
SessionCountJetBrains: 47,
@@ -105,19 +110,44 @@ func TestCollectInsights(t *testing.T) {
105110

106111
// Fake app usage
107112
reporter := workspaceapps.NewStatsDBReporter(db, workspaceapps.DefaultStatsDBReporterBatchSize)
113+
refTime := time.Now().Add(-3 * time.Minute).Truncate(time.Minute)
108114
//nolint:gocritic // This is a test.
109115
err = reporter.Report(dbauthz.AsSystemRestricted(context.Background()), []workspaceapps.StatsReport{
110116
{
111-
UserID: user.UserID,
112-
WorkspaceID: workspace.ID,
113-
AgentID: resources[0].Agents[0].ID,
117+
UserID: user.ID,
118+
WorkspaceID: workspace1.ID,
119+
AgentID: agent1.ID,
120+
AccessMethod: "path",
121+
SlugOrPort: app1.Slug,
122+
SessionID: uuid.New(),
123+
SessionStartedAt: refTime,
124+
SessionEndedAt: refTime.Add(2 * time.Minute).Add(-time.Second),
125+
Requests: 1,
126+
},
127+
// Same usage on differrent workspace/agent in same template,
128+
// should not be counted as extra.
129+
{
130+
UserID: user.ID,
131+
WorkspaceID: workspace2.ID,
132+
AgentID: agent2.ID,
114133
AccessMethod: "path",
115-
SlugOrPort: "golden-slug",
134+
SlugOrPort: app2.Slug,
116135
SessionID: uuid.New(),
117-
SessionStartedAt: time.Now().Add(-3 * time.Minute),
118-
SessionEndedAt: time.Now().Add(-time.Minute).Add(-time.Second),
136+
SessionStartedAt: refTime,
137+
SessionEndedAt: refTime.Add(2 * time.Minute).Add(-time.Second),
119138
Requests: 1,
120139
},
140+
// {
141+
// UserID: user.ID,
142+
// WorkspaceID: workspace2.ID,
143+
// AgentID: agent2.ID,
144+
// AccessMethod: "path",
145+
// SlugOrPort: app2.Slug,
146+
// SessionID: uuid.New(),
147+
// SessionStartedAt: time.Now().Add(-time.Minute),
148+
// SessionEndedAt: time.Now().Add(-time.Minute).Add(30 * time.Second),
149+
// Requests: 1,
150+
// },
121151
})
122152
require.NoError(t, err, "want no error inserting app stats")
123153

@@ -129,34 +159,6 @@ func TestCollectInsights(t *testing.T) {
129159
require.NoError(t, err)
130160
defer closeFunc()
131161

132-
// Connect to the agent to generate usage/latency stats.
133-
conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, &codersdk.DialWorkspaceAgentOptions{
134-
Logger: logger.Named("client"),
135-
})
136-
require.NoError(t, err)
137-
defer conn.Close()
138-
139-
sshConn, err := conn.SSHClient(ctx)
140-
require.NoError(t, err)
141-
defer sshConn.Close()
142-
143-
sess, err := sshConn.NewSession()
144-
require.NoError(t, err)
145-
defer sess.Close()
146-
147-
r, w := io.Pipe()
148-
defer r.Close()
149-
defer w.Close()
150-
sess.Stdin = r
151-
sess.Stdout = io.Discard
152-
err = sess.Start("cat")
153-
require.NoError(t, err)
154-
155-
defer func() {
156-
_ = sess.Close()
157-
_ = sshConn.Close()
158-
}()
159-
160162
goldenFile, err := os.ReadFile("testdata/insights-metrics.json")
161163
require.NoError(t, err)
162164
golden := map[string]int{}
@@ -188,7 +190,7 @@ func TestCollectInsights(t *testing.T) {
188190
}
189191
}
190192

191-
return insightsMetricsAreEqual(golden, collected)
193+
return assert.ObjectsAreEqualValues(golden, collected)
192194
}, testutil.WaitMedium, testutil.IntervalFast, "template insights are inconsistent with golden files")
193195
if !ok {
194196
diff := cmp.Diff(golden, collected)
@@ -203,67 +205,3 @@ func metricLabelAsString(m *io_prometheus_client.Metric) string {
203205
}
204206
return strings.Join(labels, ",")
205207
}
206-
207-
func provisionPlanWithParameters() []*proto.Response {
208-
return []*proto.Response{
209-
{
210-
Type: &proto.Response_Plan{
211-
Plan: &proto.PlanComplete{
212-
Parameters: []*proto.RichParameter{
213-
{Name: "first_parameter", Type: "string", Mutable: true},
214-
{Name: "second_parameter", Type: "bool", Mutable: true},
215-
{Name: "third_parameter", Type: "number", Mutable: true},
216-
},
217-
},
218-
},
219-
},
220-
}
221-
}
222-
223-
func provisionApplyWithAgentAndApp(authToken string) []*proto.Response {
224-
return []*proto.Response{{
225-
Type: &proto.Response_Apply{
226-
Apply: &proto.ApplyComplete{
227-
Resources: []*proto.Resource{{
228-
Name: "example",
229-
Type: "aws_instance",
230-
Agents: []*proto.Agent{{
231-
Id: uuid.NewString(),
232-
Name: "example",
233-
Auth: &proto.Agent_Token{
234-
Token: authToken,
235-
},
236-
Apps: []*proto.App{
237-
{
238-
Slug: "golden-slug",
239-
DisplayName: "Golden Slug",
240-
SharingLevel: proto.AppSharingLevel_OWNER,
241-
Url: "http://localhost:1234",
242-
},
243-
},
244-
}},
245-
}},
246-
},
247-
},
248-
}}
249-
}
250-
251-
// insightsMetricsAreEqual patches collected metrics to be used
252-
// in comparison with golden metrics using `assert.ObjectsAreEqualValues`.
253-
// Collected metrics must be patched as sometimes they may slip
254-
// due to timestamp truncation.
255-
// See:
256-
// https://github.com/coder/coder/blob/92ef0baff3b632c52c2335aae1d643a3cc49e26a/coderd/database/dbmem/dbmem.go#L2463
257-
// https://github.com/coder/coder/blob/9b6433e3a7c788b7e87b7d8f539ea111957a0cf1/coderd/database/queries/insights.sql#L246
258-
func insightsMetricsAreEqual(golden, collected map[string]int) bool {
259-
greaterOrEqualKeys := []string{
260-
"coderd_insights_applications_usage_seconds[application_name=Golden Slug,slug=golden-slug,template_name=golden-template]",
261-
"coderd_insights_applications_usage_seconds[application_name=SSH,slug=,template_name=golden-template]",
262-
}
263-
for _, key := range greaterOrEqualKeys {
264-
if v, ok := collected[key]; ok && v > golden[key] {
265-
collected[key] = golden[key]
266-
}
267-
}
268-
return assert.ObjectsAreEqualValues(golden, collected)
269-
}

0 commit comments

Comments
 (0)