coder · coadler · May 3, 2023 · May 3, 2023 · May 3, 2023 · May 3, 2023
diff --git a/cli/server.go b/cli/server.go
@@ -1185,6 +1185,7 @@ func newProvisionerDaemon(
 		return nil, xerrors.Errorf("mkdir %q: %w", cacheDir, err)
 	}
 
+	tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
 	terraformClient, terraformServer := provisionersdk.MemTransportPipe()
 	wg.Add(1)
 	go func() {
@@ -1204,6 +1205,7 @@ func newProvisionerDaemon(
 			},
 			CachePath: cacheDir,
 			Logger:    logger,
+			Tracer:    tracer,
 		})
 		if err != nil && !xerrors.Is(err, context.Canceled) {
 			select {

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -236,6 +236,9 @@ func New(options *Options) *API {
 	if options.SSHConfig.HostnamePrefix == "" {
 		options.SSHConfig.HostnamePrefix = "coder."
 	}
+	if options.TracerProvider == nil {
+		options.TracerProvider = trace.NewNoopTracerProvider()
+	}
 	if options.SetUserGroups == nil {
 		options.SetUserGroups = func(ctx context.Context, _ database.Store, id uuid.UUID, groups []string) error {
 			options.Logger.Warn(ctx, "attempted to assign OIDC groups without enterprise license",
@@ -898,6 +901,7 @@ func compressHandler(h http.Handler) http.Handler {
 // CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd.
 // Useful when starting coderd and provisionerd in the same process.
 func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce time.Duration) (client proto.DRPCProvisionerDaemonClient, err error) {
+	tracer := api.TracerProvider.Tracer(tracing.TracerName)
 	clientSession, serverSession := provisionersdk.MemTransportPipe()
 	defer func() {
 		if err != nil {
@@ -937,6 +941,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
 		Provisioners:          daemon.Provisioners,
 		GitAuthConfigs:        api.GitAuthConfigs,
 		Telemetry:             api.Telemetry,
+		Tracer:                tracer,
 		Tags:                  tags,
 		QuotaCommitter:        &api.QuotaCommitter,
 		Auditor:               &api.Auditor,
@@ -947,14 +952,16 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti
 	if err != nil {
 		return nil, err
 	}
-	server := drpcserver.NewWithOptions(mux, drpcserver.Options{
-		Log: func(err error) {
-			if xerrors.Is(err, io.EOF) {
-				return
-			}
-			api.Logger.Debug(ctx, "drpc server error", slog.Error(err))
+	server := drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
+		drpcserver.Options{
+			Log: func(err error) {
+				if xerrors.Is(err, io.EOF) {
+					return
+				}
+				api.Logger.Debug(ctx, "drpc server error", slog.Error(err))
+			},
 		},
-	})
+	)
 	go func() {
 		err := server.Serve(ctx, serverSession)
 		if err != nil && !xerrors.Is(err, io.EOF) {

diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql
diff --git a/coderd/database/migrations/000121_provisioner_job_tracing.down.sql b/coderd/database/migrations/000121_provisioner_job_tracing.down.sql
@@ -0,0 +1 @@
+ALTER TABLE provisioner_jobs DROP COLUMN trace_metadata;
diff --git a/coderd/database/migrations/000121_provisioner_job_tracing.up.sql b/coderd/database/migrations/000121_provisioner_job_tracing.up.sql
@@ -0,0 +1 @@
+ALTER TABLE provisioner_jobs ADD COLUMN trace_metadata jsonb;
diff --git a/coderd/database/models.go b/coderd/database/models.go
diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go
diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql
@@ -63,10 +63,11 @@ INSERT INTO
 		file_id,
 		"type",
 		"input",
-		tags
+		tags,
+		trace_metadata
 	)
 VALUES
-	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *;
+	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *;
 
 -- name: UpdateProvisionerJobByID :exec
 UPDATE

diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -17,6 +17,8 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/tabbed/pqtype"
+	semconv "go.opentelemetry.io/otel/semconv/v1.14.0"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
@@ -33,6 +35,7 @@ import (
 	"github.com/coder/coder/coderd/parameter"
 	"github.com/coder/coder/coderd/schedule"
 	"github.com/coder/coder/coderd/telemetry"
+	"github.com/coder/coder/coderd/tracing"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/provisioner"
 	"github.com/coder/coder/provisionerd/proto"
@@ -55,6 +58,7 @@ type Server struct {
 	Database              database.Store
 	Pubsub                database.Pubsub
 	Telemetry             telemetry.Reporter
+	Tracer                trace.Tracer
 	QuotaCommitter        *atomic.Pointer[proto.QuotaCommitter]
 	Auditor               *atomic.Pointer[audit.Auditor]
 	TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore]
@@ -129,12 +133,22 @@ func (server *Server) AcquireJob(ctx context.Context, _ *proto.Empty) (*proto.Ac
 		return nil, failJob(fmt.Sprintf("get user: %s", err))
 	}
 
+	jobTraceMetadata := map[string]string{}
+	if job.TraceMetadata.Valid {
+		err := json.Unmarshal(job.TraceMetadata.RawMessage, &jobTraceMetadata)
+		if err != nil {
+			return nil, failJob(fmt.Sprintf("unmarshal metadata: %s", err))
+		}
+	}
+
 	protoJob := &proto.AcquiredJob{
-		JobId:       job.ID.String(),
-		CreatedAt:   job.CreatedAt.UnixMilli(),
-		Provisioner: string(job.Provisioner),
-		UserName:    user.Username,
+		JobId:         job.ID.String(),
+		CreatedAt:     job.CreatedAt.UnixMilli(),
+		Provisioner:   string(job.Provisioner),
+		UserName:      user.Username,
+		TraceMetadata: jobTraceMetadata,
 	}
+
 	switch job.Type {
 	case database.ProvisionerJobTypeWorkspaceBuild:
 		var input WorkspaceProvisionJob
@@ -411,6 +425,9 @@ func (server *Server) includeLastVariableValues(ctx context.Context, templateVer
 }
 
 func (server *Server) CommitQuota(ctx context.Context, request *proto.CommitQuotaRequest) (*proto.CommitQuotaResponse, error) {
+	ctx, span := server.startTrace(ctx, tracing.FuncName())
+	defer span.End()
+
 	//nolint:gocritic // Provisionerd has specific authz rules.
 	ctx = dbauthz.AsProvisionerd(ctx)
 	jobID, err := uuid.Parse(request.JobId)
@@ -442,6 +459,9 @@ func (server *Server) CommitQuota(ctx context.Context, request *proto.CommitQuot
 }
 
 func (server *Server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) (*proto.UpdateJobResponse, error) {
+	ctx, span := server.startTrace(ctx, tracing.FuncName())
+	defer span.End()
+
 	//nolint:gocritic // Provisionerd has specific authz rules.
 	ctx = dbauthz.AsProvisionerd(ctx)
 	parsedID, err := uuid.Parse(request.JobId)
@@ -671,6 +691,9 @@ func (server *Server) UpdateJob(ctx context.Context, request *proto.UpdateJobReq
 }
 
 func (server *Server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*proto.Empty, error) {
+	ctx, span := server.startTrace(ctx, tracing.FuncName())
+	defer span.End()
+
 	//nolint:gocritic // Provisionerd has specific authz rules.
 	ctx = dbauthz.AsProvisionerd(ctx)
 	jobID, err := uuid.Parse(failJob.JobId)
@@ -822,6 +845,9 @@ func (server *Server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*p
 //
 //nolint:gocyclo
 func (server *Server) CompleteJob(ctx context.Context, completed *proto.CompletedJob) (*proto.Empty, error) {
+	ctx, span := server.startTrace(ctx, tracing.FuncName())
+	defer span.End()
+
 	//nolint:gocritic // Provisionerd has specific authz rules.
 	ctx = dbauthz.AsProvisionerd(ctx)
 	jobID, err := uuid.Parse(completed.JobId)
@@ -1192,6 +1218,12 @@ func (server *Server) CompleteJob(ctx context.Context, completed *proto.Complete
 	return &proto.Empty{}, nil
 }
 
+func (server *Server) startTrace(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	return server.Tracer.Start(ctx, name, append(opts, trace.WithAttributes(
+		semconv.ServiceNameKey.String("coderd.provisionerd"),
+	))...)
+}
+
 func InsertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource, snapshot *telemetry.Snapshot) error {
 	resource, err := db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{
 		ID:         uuid.New(),

diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
 
 	"cdr.dev/slog/sloggers/slogtest"
@@ -59,6 +60,7 @@ func TestAcquireJob(t *testing.T) {
 			AcquireJobDebounce:    time.Hour,
 			Auditor:               mockAuditor(),
 			TemplateScheduleStore: testTemplateScheduleStore(),
+			Tracer:                trace.NewNoopTracerProvider().Tracer("noop"),
 		}
 		job, err := srv.AcquireJob(context.Background(), nil)
 		require.NoError(t, err)
@@ -1202,6 +1204,7 @@ func setup(t *testing.T, ignoreLogErrors bool) *provisionerdserver.Server {
 		Telemetry:             telemetry.NewNoop(),
 		Auditor:               mockAuditor(),
 		TemplateScheduleStore: testTemplateScheduleStore(),
+		Tracer:                trace.NewNoopTracerProvider().Tracer("noop"),
 	}
 }
 

diff --git a/coderd/templateversions.go b/coderd/templateversions.go
@@ -13,6 +13,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
 	"github.com/moby/moby/pkg/namesgenerator"
+	"github.com/tabbed/pqtype"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
@@ -25,6 +26,7 @@ import (
 	"github.com/coder/coder/coderd/parameter"
 	"github.com/coder/coder/coderd/provisionerdserver"
 	"github.com/coder/coder/coderd/rbac"
+	"github.com/coder/coder/coderd/tracing"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/examples"
 	sdkproto "github.com/coder/coder/provisionersdk/proto"
@@ -574,6 +576,15 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	metadataRaw, err := json.Marshal(tracing.MetadataFromContext(ctx))
+	if err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Internal error unmarshalling metadata.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
 	// Create a dry-run job
 	jobID := uuid.New()
 	provisionerJob, err := api.Database.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
@@ -589,6 +600,10 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques
 		Input:          input,
 		// Copy tags from the previous run.
 		Tags: job.Tags,
+		TraceMetadata: pqtype.NullRawMessage{
+			Valid:      true,
+			RawMessage: metadataRaw,
+		},
 	})
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -1408,6 +1423,10 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 		if err != nil {
 			return xerrors.Errorf("marshal job input: %w", err)
 		}
+		traceMetadataRaw, err := json.Marshal(tracing.MetadataFromContext(ctx))
+		if err != nil {
+			return xerrors.Errorf("marshal job metadata: %w", err)
+		}
 
 		provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
 			ID:             jobID,
@@ -1421,6 +1440,10 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 			Type:           database.ProvisionerJobTypeTemplateVersionImport,
 			Input:          jobInput,
 			Tags:           tags,
+			TraceMetadata: pqtype.NullRawMessage{
+				Valid:      true,
+				RawMessage: traceMetadataRaw,
+			},
 		})
 		if err != nil {
 			return xerrors.Errorf("insert provisioner job: %w", err)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE provisioner_jobs DROP COLUMN trace_metadata;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE provisioner_jobs ADD COLUMN trace_metadata jsonb;