Skip to content

feat: Move from datadog to generic otel #1567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
May 19, 2022
Merged

feat: Move from datadog to generic otel #1567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9249229
feat: Move from datadog to generic otel
f0ssel May 18, 2022
e0e433c
add telemetry pkg
f0ssel May 18, 2022
bdd60f5
add telemetry package
f0ssel May 18, 2022
33cfd74
cleanup data we send
f0ssel May 18, 2022
4d62113
add doc comment
f0ssel May 18, 2022
1974a1e
rename to tracing
f0ssel May 18, 2022
776d8fc
rename to tracing for real
f0ssel May 18, 2022
3d22060
remove datadog logger
f0ssel May 18, 2022
c0099b5
move to coderd
f0ssel May 18, 2022
195a1ba
pr comments
f0ssel May 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 20 additions & 16 deletions cli/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ import (
"golang.org/x/xerrors"
"google.golang.org/api/idtoken"
"google.golang.org/api/option"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"

sdktrace "go.opentelemetry.io/otel/sdk/trace"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
Expand All @@ -44,6 +45,7 @@ import (
"github.com/coder/coder/coderd/database/databasefake"
"github.com/coder/coder/coderd/devtunnel"
"github.com/coder/coder/coderd/gitsshkey"
"github.com/coder/coder/coderd/tracing"
"github.com/coder/coder/coderd/turnconn"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/cryptorand"
Expand Down Expand Up @@ -82,7 +84,7 @@ func server() *cobra.Command {
turnRelayAddress string
tunnel bool
stunServers []string
traceDatadog bool
trace bool
secureAuthCookie bool
sshKeygenAlgorithmRaw string
spooky bool
Expand All @@ -98,11 +100,20 @@ func server() *cobra.Command {
logger = logger.Leveled(slog.LevelDebug)
}

if traceDatadog {
tracer.Start(tracer.WithLogStartup(false), tracer.WithLogger(&datadogLogger{
logger: logger.Named("datadog"),
}))
defer tracer.Stop()
var tracerProvider *sdktrace.TracerProvider
var err error
if trace {
tracerProvider, err = tracing.TracerProvider(cmd.Context(), "coderd")
if err != nil {
logger.Warn(cmd.Context(), "failed to start telemetry exporter", slog.Error(err))
} else {
defer func() {
// allow time for traces to flush even if command context is cancelled
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = tracerProvider.Shutdown(ctx)
}()
}
}

printLogo(cmd, spooky)
Expand Down Expand Up @@ -216,6 +227,7 @@ func server() *cobra.Command {
SecureAuthCookie: secureAuthCookie,
SSHKeygenAlgorithm: sshKeygenAlgorithm,
TURNServer: turnServer,
TracerProvider: tracerProvider,
}

if oauth2GithubClientSecret != "" {
Expand Down Expand Up @@ -480,7 +492,7 @@ func server() *cobra.Command {
cliflag.StringArrayVarP(root.Flags(), &stunServers, "stun-server", "", "CODER_STUN_SERVERS", []string{
"stun:stun.l.google.com:19302",
}, "Specify URLs for STUN servers to enable P2P connections.")
cliflag.BoolVarP(root.Flags(), &traceDatadog, "trace-datadog", "", "CODER_TRACE_DATADOG", false, "Send tracing data to a datadog agent")
cliflag.BoolVarP(root.Flags(), &trace, "trace", "", "CODER_TRACE", false, "Specifies if application tracing data is collected")
cliflag.StringVarP(root.Flags(), &turnRelayAddress, "turn-relay-address", "", "CODER_TURN_RELAY_ADDRESS", "127.0.0.1",
"Specifies the address to bind TURN connections.")
cliflag.BoolVarP(root.Flags(), &secureAuthCookie, "secure-auth-cookie", "", "CODER_SECURE_AUTH_COOKIE", false, "Specifies if the 'Secure' property is set on browser session cookies")
Expand Down Expand Up @@ -719,11 +731,3 @@ func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler,

return func() { _ = srv.Close() }
}

type datadogLogger struct {
logger slog.Logger
}

func (d *datadogLogger) Log(msg string) {
d.logger.Debug(context.Background(), msg)
}
4 changes: 2 additions & 2 deletions cli/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,11 +280,11 @@ func TestServer(t *testing.T) {
require.NoError(t, err)
<-done
})
t.Run("DatadogTracerNoLeak", func(t *testing.T) {
t.Run("TracerNoLeak", func(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--trace-datadog=true")
root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--trace=true")
done := make(chan struct{})
go func() {
defer close(done)
Expand Down
6 changes: 4 additions & 2 deletions coderd/coderd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"golang.org/x/xerrors"
"google.golang.org/api/idtoken"

chitrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/go-chi/chi.v5"
sdktrace "go.opentelemetry.io/otel/sdk/trace"

"cdr.dev/slog"
"github.com/coder/coder/buildinfo"
Expand All @@ -25,6 +25,7 @@ import (
"github.com/coder/coder/coderd/httpapi"
"github.com/coder/coder/coderd/httpmw"
"github.com/coder/coder/coderd/rbac"
"github.com/coder/coder/coderd/tracing"
"github.com/coder/coder/coderd/turnconn"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/site"
Expand All @@ -51,6 +52,7 @@ type Options struct {
SSHKeygenAlgorithm gitsshkey.Algorithm
TURNServer *turnconn.Server
Authorizer rbac.Authorizer
TracerProvider *sdktrace.TracerProvider
}

// New constructs the Coder API into an HTTP handler.
Expand Down Expand Up @@ -92,7 +94,7 @@ func New(options *Options) (http.Handler, func()) {
})
},
httpmw.Prometheus,
chitrace.Middleware(),
tracing.HTTPMW(api.TracerProvider, "coderd.http"),
)

r.Route("/api/v2", func(r chi.Router) {
Expand Down
41 changes: 41 additions & 0 deletions coderd/tracing/exporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package tracing

import (
"context"

"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
"golang.org/x/xerrors"
)

// TracerProvider creates a grpc otlp exporter and configures a trace provider.
// Caller is responsible for calling TracerProvider.Shutdown to ensure all data is flushed.
func TracerProvider(ctx context.Context, service string) (*sdktrace.TracerProvider, error) {
res, err := resource.New(ctx,
resource.WithAttributes(
// the service name used to display traces in backends
semconv.ServiceNameKey.String(service),
),
)
if err != nil {
return nil, xerrors.Errorf("creating otlp resource: %w", err)
}

// By default we send span data to a local otel collector.
// The endpoint we push to can be configured with env vars.
// See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
exporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(otlptracegrpc.WithInsecure()))
if err != nil {
return nil, xerrors.Errorf("creating otlp exporter: %w", err)
}

tracerProvider := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
)

return tracerProvider, nil
}
68 changes: 68 additions & 0 deletions coderd/tracing/httpmw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package tracing

import (
"fmt"
"net/http"

"github.com/go-chi/chi/middleware"
"github.com/go-chi/chi/v5"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

// HTTPMW adds tracing to http routes.
func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
// do not trace if exporter has not be initialized
if tracerProvider == nil {
next.ServeHTTP(rw, r)
return
}

// start span with default span name. Span name will be updated to "method route" format once request finishes.
_, span := tracerProvider.Tracer(name).Start(r.Context(), fmt.Sprintf("%s %s", r.Method, r.RequestURI))
defer span.End()

wrw := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)

// pass the span through the request context and serve the request to the next middleware
next.ServeHTTP(rw, r)

// set the resource name as we get it only once the handler is executed
route := chi.RouteContext(r.Context()).RoutePattern()
if route != "" {
span.SetName(fmt.Sprintf("%s %s", r.Method, route))
}
span.SetAttributes(attribute.KeyValue{
Key: "http.method",
Value: attribute.StringValue(r.Method),
})
span.SetAttributes(attribute.KeyValue{
Key: "http.route",
Value: attribute.StringValue(route),
})
span.SetAttributes(attribute.KeyValue{
Key: "http.path",
Value: attribute.StringValue(r.URL.EscapedPath()),
})

// set the status code
status := wrw.Status()
// 0 status means one has not yet been sent in which case net/http library will write StatusOK
if status == 0 {
status = http.StatusOK
}
span.SetAttributes(attribute.KeyValue{
Key: "http.status_code",
Value: attribute.IntValue(status),
})

// if 5XX we set the span to "error" status
if status >= 500 {
span.SetStatus(codes.Error, fmt.Sprintf("%d: %s", status, http.StatusText(status)))
}
})
}
}
26 changes: 13 additions & 13 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ require (
github.com/tabbed/pqtype v0.1.1
github.com/unrolled/secure v1.10.0
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0
go.uber.org/atomic v1.9.0
go.uber.org/goleak v1.1.12
golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
Expand All @@ -118,7 +119,6 @@ require (
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f
google.golang.org/api v0.79.0
google.golang.org/protobuf v1.28.0
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
nhooyr.io/websocket v1.8.7
Expand All @@ -128,10 +128,6 @@ require (
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583 // indirect
github.com/DataDog/datadog-go v4.8.2+incompatible // indirect
github.com/DataDog/datadog-go/v5 v5.0.2 // indirect
github.com/DataDog/sketches-go v1.0.0 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/OneOfOne/xxhash v1.2.8 // indirect
Expand All @@ -141,7 +137,7 @@ require (
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/charmbracelet/bubbles v0.10.3 // indirect
github.com/charmbracelet/bubbletea v0.20.0 // indirect
Expand All @@ -150,46 +146,46 @@ require (
github.com/containerd/continuity v0.2.2 // indirect
github.com/coreos/go-oidc v2.2.1+incompatible // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/docker/cli v20.10.13+incompatible // indirect
github.com/docker/docker v20.10.13+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb // indirect
github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gin-gonic/gin v1.7.0 // indirect
github.com/go-chi/chi v1.5.4
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gobwas/ws v1.1.0 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
github.com/klauspost/reedsolomon v1.9.15 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
Expand All @@ -206,7 +202,6 @@ require (
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.7.0.20220408132554-2377ac4bc04c // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/pion/dtls/v2 v2.1.4 // indirect
github.com/pion/ice/v2 v2.2.6 // indirect
github.com/pion/interceptor v0.1.11 // indirect
Expand Down Expand Up @@ -234,7 +229,6 @@ require (
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect
github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b // indirect
github.com/tinylib/msgp v1.1.2 // indirect
github.com/tjfoc/gmsm v1.4.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
Expand All @@ -243,6 +237,12 @@ require (
github.com/zclconf/go-cty v1.10.0 // indirect
github.com/zeebo/errs v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v1.7.0
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0
go.opentelemetry.io/otel/sdk v1.7.0
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect
Expand Down
Loading