Skip to content

Commit 8bd9f9c

Browse files
coadlercoadler
authored
feat: unified tracing between coderd<->provisionerd (coder#7370)
1 parent 3368b8b commit 8bd9f9c

28 files changed

+590
-329
lines changed

‎cli/server.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1185,6 +1185,7 @@ func newProvisionerDaemon(
11851185
return nil, xerrors.Errorf("mkdir %q: %w", cacheDir, err)
11861186
}
11871187

1188+
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
11881189
terraformClient, terraformServer := provisionersdk.MemTransportPipe()
11891190
wg.Add(1)
11901191
go func() {
@@ -1204,6 +1205,7 @@ func newProvisionerDaemon(
12041205
},
12051206
CachePath: cacheDir,
12061207
Logger: logger,
1208+
Tracer: tracer,
12071209
})
12081210
if err != nil && !xerrors.Is(err, context.Canceled) {
12091211
select {

‎coderd/coderd.go

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,9 @@ func New(options *Options) *API {
236236
if options.SSHConfig.HostnamePrefix == "" {
237237
options.SSHConfig.HostnamePrefix = "coder."
238238
}
239+
if options.TracerProvider == nil {
240+
options.TracerProvider = trace.NewNoopTracerProvider()
241+
}
239242
if options.SetUserGroups == nil {
240243
options.SetUserGroups = func(ctx context.Context, _ database.Store, id uuid.UUID, groups []string) error {
241244
options.Logger.Warn(ctx, "attempted to assign OIDC groups without enterprise license",
@@ -898,6 +901,7 @@ func compressHandler(h http.Handler) http.Handler {
898901
// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd.
899902
// Useful when starting coderd and provisionerd in the same process.
900903
func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce time.Duration) (client proto.DRPCProvisionerDaemonClient, err error) {
904+
tracer := api.TracerProvider.Tracer(tracing.TracerName)
901905
clientSession, serverSession := provisionersdk.MemTransportPipe()
902906
defer func() {
903907
if err != nil {
@@ -937,6 +941,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
937941
Provisioners: daemon.Provisioners,
938942
GitAuthConfigs: api.GitAuthConfigs,
939943
Telemetry: api.Telemetry,
944+
Tracer: tracer,
940945
Tags: tags,
941946
QuotaCommitter: &api.QuotaCommitter,
942947
Auditor: &api.Auditor,
@@ -947,14 +952,16 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
947952
if err != nil {
948953
return nil, err
949954
}
950-
server := drpcserver.NewWithOptions(mux, drpcserver.Options{
951-
Log: func(err error) {
952-
if xerrors.Is(err, io.EOF) {
953-
return
954-
}
955-
api.Logger.Debug(ctx, "drpc server error", slog.Error(err))
955+
server := drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
956+
drpcserver.Options{
957+
Log: func(err error) {
958+
if xerrors.Is(err, io.EOF) {
959+
return
960+
}
961+
api.Logger.Debug(ctx, "drpc server error", slog.Error(err))
962+
},
956963
},
957-
})
964+
)
958965
go func() {
959966
err := server.Serve(ctx, serverSession)
960967
if err != nil && !xerrors.Is(err, io.EOF) {

‎coderd/database/dump.sql

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/migrations/000121_provisioner_job_tracing.down.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE provisioner_jobs DROP COLUMN trace_metadata;

‎coderd/database/migrations/000121_provisioner_job_tracing.up.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE provisioner_jobs ADD COLUMN trace_metadata jsonb;

‎coderd/database/models.go

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries.sql.go

Lines changed: 14 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries/provisionerjobs.sql

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,11 @@ INSERT INTO
6363
file_id,
6464
"type",
6565
"input",
66-
tags
66+
tags,
67+
trace_metadata
6768
)
6869
VALUES
69-
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *;
70+
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *;
7071

7172
-- name: UpdateProvisionerJobByID :exec
7273
UPDATE

‎coderd/provisionerdserver/provisionerdserver.go

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717

1818
"github.com/google/uuid"
1919
"github.com/tabbed/pqtype"
20+
semconv "go.opentelemetry.io/otel/semconv/v1.14.0"
21+
"go.opentelemetry.io/otel/trace"
2022
"golang.org/x/exp/maps"
2123
"golang.org/x/exp/slices"
2224
"golang.org/x/oauth2"
@@ -33,6 +35,7 @@ import (
3335
"github.com/coder/coder/coderd/parameter"
3436
"github.com/coder/coder/coderd/schedule"
3537
"github.com/coder/coder/coderd/telemetry"
38+
"github.com/coder/coder/coderd/tracing"
3639
"github.com/coder/coder/codersdk"
3740
"github.com/coder/coder/provisioner"
3841
"github.com/coder/coder/provisionerd/proto"
@@ -55,6 +58,7 @@ type Server struct {
5558
Database database.Store
5659
Pubsub database.Pubsub
5760
Telemetry telemetry.Reporter
61+
Tracer trace.Tracer
5862
QuotaCommitter *atomic.Pointer[proto.QuotaCommitter]
5963
Auditor *atomic.Pointer[audit.Auditor]
6064
TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore]
@@ -129,12 +133,22 @@ func (server *Server) AcquireJob(ctx context.Context, _ *proto.Empty) (*proto.Ac
129133
return nil, failJob(fmt.Sprintf("get user: %s", err))
130134
}
131135

136+
jobTraceMetadata := map[string]string{}
137+
if job.TraceMetadata.Valid {
138+
err := json.Unmarshal(job.TraceMetadata.RawMessage, &jobTraceMetadata)
139+
if err != nil {
140+
return nil, failJob(fmt.Sprintf("unmarshal metadata: %s", err))
141+
}
142+
}
143+
132144
protoJob := &proto.AcquiredJob{
133-
JobId: job.ID.String(),
134-
CreatedAt: job.CreatedAt.UnixMilli(),
135-
Provisioner: string(job.Provisioner),
136-
UserName: user.Username,
145+
JobId: job.ID.String(),
146+
CreatedAt: job.CreatedAt.UnixMilli(),
147+
Provisioner: string(job.Provisioner),
148+
UserName: user.Username,
149+
TraceMetadata: jobTraceMetadata,
137150
}
151+
138152
switch job.Type {
139153
case database.ProvisionerJobTypeWorkspaceBuild:
140154
var input WorkspaceProvisionJob
@@ -411,6 +425,9 @@ func (server *Server) includeLastVariableValues(ctx context.Context, templateVer
411425
}
412426

413427
func (server *Server) CommitQuota(ctx context.Context, request *proto.CommitQuotaRequest) (*proto.CommitQuotaResponse, error) {
428+
ctx, span := server.startTrace(ctx, tracing.FuncName())
429+
defer span.End()
430+
414431
//nolint:gocritic // Provisionerd has specific authz rules.
415432
ctx = dbauthz.AsProvisionerd(ctx)
416433
jobID, err := uuid.Parse(request.JobId)
@@ -442,6 +459,9 @@ func (server *Server) CommitQuota(ctx context.Context, request *proto.CommitQuot
442459
}
443460

444461
func (server *Server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) (*proto.UpdateJobResponse, error) {
462+
ctx, span := server.startTrace(ctx, tracing.FuncName())
463+
defer span.End()
464+
445465
//nolint:gocritic // Provisionerd has specific authz rules.
446466
ctx = dbauthz.AsProvisionerd(ctx)
447467
parsedID, err := uuid.Parse(request.JobId)
@@ -671,6 +691,9 @@ func (server *Server) UpdateJob(ctx context.Context, request *proto.UpdateJobReq
671691
}
672692

673693
func (server *Server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*proto.Empty, error) {
694+
ctx, span := server.startTrace(ctx, tracing.FuncName())
695+
defer span.End()
696+
674697
//nolint:gocritic // Provisionerd has specific authz rules.
675698
ctx = dbauthz.AsProvisionerd(ctx)
676699
jobID, err := uuid.Parse(failJob.JobId)
@@ -822,6 +845,9 @@ func (server *Server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*p
822845
//
823846
//nolint:gocyclo
824847
func (server *Server) CompleteJob(ctx context.Context, completed *proto.CompletedJob) (*proto.Empty, error) {
848+
ctx, span := server.startTrace(ctx, tracing.FuncName())
849+
defer span.End()
850+
825851
//nolint:gocritic // Provisionerd has specific authz rules.
826852
ctx = dbauthz.AsProvisionerd(ctx)
827853
jobID, err := uuid.Parse(completed.JobId)
@@ -1192,6 +1218,12 @@ func (server *Server) CompleteJob(ctx context.Context, completed *proto.Complete
11921218
return &proto.Empty{}, nil
11931219
}
11941220

1221+
func (server *Server) startTrace(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
1222+
return server.Tracer.Start(ctx, name, append(opts, trace.WithAttributes(
1223+
semconv.ServiceNameKey.String("coderd.provisionerd"),
1224+
))...)
1225+
}
1226+
11951227
func InsertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource, snapshot *telemetry.Snapshot) error {
11961228
resource, err := db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{
11971229
ID: uuid.New(),

‎coderd/provisionerdserver/provisionerdserver_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/google/uuid"
1313
"github.com/stretchr/testify/require"
14+
"go.opentelemetry.io/otel/trace"
1415
"golang.org/x/oauth2"
1516

1617
"cdr.dev/slog/sloggers/slogtest"
@@ -59,6 +60,7 @@ func TestAcquireJob(t *testing.T) {
5960
AcquireJobDebounce: time.Hour,
6061
Auditor: mockAuditor(),
6162
TemplateScheduleStore: testTemplateScheduleStore(),
63+
Tracer: trace.NewNoopTracerProvider().Tracer("noop"),
6264
}
6365
job, err := srv.AcquireJob(context.Background(), nil)
6466
require.NoError(t, err)
@@ -1202,6 +1204,7 @@ func setup(t *testing.T, ignoreLogErrors bool) *provisionerdserver.Server {
12021204
Telemetry: telemetry.NewNoop(),
12031205
Auditor: mockAuditor(),
12041206
TemplateScheduleStore: testTemplateScheduleStore(),
1207+
Tracer: trace.NewNoopTracerProvider().Tracer("noop"),
12051208
}
12061209
}
12071210

0 commit comments

Comments
 (0)