Skip to content

feat: tracing improvements #4988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 28, 2022
Merged

feat: tracing improvements #4988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f1affbe
feat: move tracing middleware higher up the middleware stack
deansheather Nov 9, 2022
032f1cf
fixup! feat: move tracing middleware higher up the middleware stack
deansheather Nov 9, 2022
d892c21
fixup! feat: move tracing middleware higher up the middleware stack
deansheather Nov 17, 2022
7383e8c
fixup! feat: move tracing middleware higher up the middleware stack
deansheather Nov 17, 2022
343e27d
fixup! feat: move tracing middleware higher up the middleware stack
deansheather Nov 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions coderd/coderd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,10 @@ func New(options *Options) *API {
})

r.Use(
httpmw.AttachRequestID,
httpmw.Recover(api.Logger),
tracing.StatusWriterMiddleware,
tracing.Middleware(api.TracerProvider),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will trace every static HTTP file... is that ok?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a filter within the middleware itself (and a test) to ensure that this is skipped on non-API routes.

httpmw.AttachRequestID,
httpmw.ExtractRealIP(api.RealIPConfig),
httpmw.Logger(api.Logger),
httpmw.Prometheus(options.PrometheusRegistry),
Expand Down Expand Up @@ -237,7 +239,6 @@ func New(options *Options) *API {

apps := func(r chi.Router) {
r.Use(
tracing.Middleware(api.TracerProvider),
httpmw.RateLimit(options.APIRateLimit, time.Minute),
httpmw.ExtractAPIKey(httpmw.ExtractAPIKeyConfig{
DB: options.Database,
Expand Down Expand Up @@ -285,7 +286,6 @@ func New(options *Options) *API {

r.NotFound(func(rw http.ResponseWriter, r *http.Request) { httpapi.RouteNotFound(rw) })
r.Use(
tracing.Middleware(api.TracerProvider),
// Specific routes can specify smaller limits.
httpmw.RateLimit(options.APIRateLimit, time.Minute),
)
Expand Down Expand Up @@ -509,14 +509,6 @@ func New(options *Options) *API {
r.Get("/listening-ports", api.workspaceAgentListeningPorts)
r.Get("/connection", api.workspaceAgentConnection)
r.Get("/coordinate", api.workspaceAgentClientCoordinate)
// TODO: This can be removed in October. It allows for a friendly
// error message when transitioning from WebRTC to Tailscale. See:
// https://github.com/coder/coder/issues/4126
r.Get("/dial", func(w http.ResponseWriter, r *http.Request) {
httpapi.Write(r.Context(), w, http.StatusGone, codersdk.Response{
Message: "Your Coder CLI is out of date, and requires v0.8.15+ to connect!",
})
})
})
})
r.Route("/workspaces", func(r chi.Router) {
Expand Down
2 changes: 0 additions & 2 deletions coderd/coderdtest/authorize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) {
"POST:/api/v2/csp/reports": {NoAuthorize: true},
"POST:/api/v2/authcheck": {NoAuthorize: true},
"GET:/api/v2/applications/host": {NoAuthorize: true},
// This is a dummy endpoint for compatibility with older CLI versions.
"GET:/api/v2/workspaceagents/{workspaceagent}/dial": {NoAuthorize: true},

// Has it's own auth
"GET:/api/v2/users/oauth2/github/callback": {NoAuthorize: true},
Expand Down
16 changes: 13 additions & 3 deletions coderd/httpmw/logger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package httpmw

import (
"context"
"fmt"
"net/http"
"time"

Expand All @@ -11,9 +13,13 @@ import (

func Logger(log slog.Logger) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
start := time.Now()
sw := &tracing.StatusWriter{ResponseWriter: w}

sw, ok := rw.(*tracing.StatusWriter)
if !ok {
panic(fmt.Sprintf("ResponseWriter not a *tracing.StatusWriter; got %T", rw))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's runtime, right? Isn't it risky to panic here? I understand that you assume that it will be recovered.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The response writer will always be a *tracing.StatusWriter, so I think it's fine for us to panic here as this is a developer error. We panic in a other places due to developer errors as well, like the middleware context value stuff also in the httpmw package.

We also already do this exact panic in the tracing middleware too

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I admit that I'm not a fan of it, but I understand why it's safe here 👍

}

httplog := log.With(
slog.F("host", httpapi.RequestHost(r)),
Expand Down Expand Up @@ -51,7 +57,11 @@ func Logger(log slog.Logger) func(next http.Handler) http.Handler {
logLevelFn = httplog.Warn
}

logLevelFn(r.Context(), r.Method)
// We already capture most of this information in the span (minus
// the response body which we don't want to capture anyways).
tracing.RunWithoutSpan(r.Context(), func(ctx context.Context) {
logLevelFn(ctx, r.Method)
})
})
}
}
50 changes: 50 additions & 0 deletions coderd/httpmw/patternmatcher/routepatterns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package patternmatcher

import (
"fmt"
"regexp"
"strings"

"golang.org/x/xerrors"
)

// RoutePatterns provides a method to generate a regex which will match a URL
// path against a collection of patterns. If any of the patterns match the path,
// the regex will return a successful match.
//
// Multiple patterns can be provided and they are matched in order. Example:
// - /api/* matches /api/1 but not /api or /api/1/2
// - /api/*/2 matches /api/1/2 but not /api/2 /api/1
// - /api/** matches /api/1, /api/1/2, /api/1/2/3 but not /api
// - /api/**/3 matches /api/1/2, /api/1/2/3 but not /api, /api/1 or /api/1/2
//
// All patterns support an optional trailing slash.
type RoutePatterns []string

func (rp RoutePatterns) MustCompile() *regexp.Regexp {
re, err := rp.Compile()
if err != nil {
panic(err)
}
return re
}

func (rp RoutePatterns) Compile() (*regexp.Regexp, error) {
patterns := make([]string, len(rp))
for i, p := range rp {
p = strings.ReplaceAll(p, "**", ".+")
p = strings.ReplaceAll(p, "*", "[^/]+")
if !strings.HasSuffix(p, "/") {
p += "/?"
}
patterns[i] = p
}

pattern := fmt.Sprintf("^(%s)$", strings.Join(patterns, "|"))
re, err := regexp.Compile(pattern)
if err != nil {
return nil, xerrors.Errorf("compile regex %q: %w", pattern, err)
}

return re, nil
}
135 changes: 135 additions & 0 deletions coderd/httpmw/patternmatcher/routepatterns_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package patternmatcher_test

import (
"testing"

"github.com/stretchr/testify/require"

"github.com/coder/coder/coderd/httpmw/patternmatcher"
)

func Test_RoutePatterns(t *testing.T) {
t.Parallel()

cases := []struct {
name string
patterns []string
errContains string
output string
}{
{
name: "Empty",
patterns: []string{},
output: "^()$",
},
{
name: "Single",
patterns: []string{
"/api",
},
output: "^(/api/?)$",
},
{
name: "TrailingSlash",
patterns: []string{
"/api/",
},
output: "^(/api/)$",
},
{
name: "Multiple",
patterns: []string{
"/api",
"/api2",
},
output: "^(/api/?|/api2/?)$",
},
{
name: "Star",
patterns: []string{
"/api/*",
},
output: "^(/api/[^/]+/?)$",
},
{
name: "StarStar",
patterns: []string{
"/api/**",
},
output: "^(/api/.+/?)$",
},
{
name: "TelemetryPatterns",
patterns: []string{
"/api",
"/api/**",
"/@*/*/apps/**",
"/%40*/*/apps/**",
"/gitauth/*/callback",
},
output: "^(/api/?|/api/.+/?|/@[^/]+/[^/]+/apps/.+/?|/%40[^/]+/[^/]+/apps/.+/?|/gitauth/[^/]+/callback/?)$",
},
{
name: "Slash",
patterns: []string{
"/",
},
output: "^(/)$",
},
{
name: "SlashStar",
patterns: []string{
"/*",
},
output: "^(/[^/]+/?)$",
},
{
name: "SlashStarStar",
patterns: []string{
"/**",
},
output: "^(/.+/?)$",
},
{
name: "SlashSlash",
patterns: []string{
"//",
"/api//v1",
},
output: "^(//|/api//v1/?)$",
},
{
name: "Invalid",
patterns: []string{
"/api(",
},
errContains: "compile regex",
},
}

for _, c := range cases {
c := c
t.Run(c.name, func(t *testing.T) {
t.Parallel()

rp := patternmatcher.RoutePatterns(c.patterns)
re, err := rp.Compile()
if c.errContains != "" {
require.Error(t, err)
require.ErrorContains(t, err, c.errContains)

require.Panics(t, func() {
_ = rp.MustCompile()
})
} else {
require.NoError(t, err)
require.Equal(t, c.output, re.String())

require.NotPanics(t, func() {
re := rp.MustCompile()
require.Equal(t, c.output, re.String())
})
}
})
}
}
16 changes: 15 additions & 1 deletion coderd/tracing/httpmw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,31 @@ import (
"go.opentelemetry.io/otel/propagation"
semconv "go.opentelemetry.io/otel/semconv/v1.11.0"
"go.opentelemetry.io/otel/trace"

"github.com/coder/coder/coderd/httpmw/patternmatcher"
)

// Middleware adds tracing to http routes.
func Middleware(tracerProvider trace.TracerProvider) func(http.Handler) http.Handler {
// We only want to create spans on the following route patterns, however
// we want the middleware to be very high in the middleware stack so it can
// capture the entire request.
re := patternmatcher.RoutePatterns{
"/api",
"/api/**",
"/@*/*/apps/**",
"/%40*/*/apps/**",
"/gitauth/*/callback",
}.MustCompile()

var tracer trace.Tracer
if tracerProvider != nil {
tracer = tracerProvider.Tracer(TracerName)
}

return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
if tracer == nil {
if tracer == nil || !re.MatchString(r.URL.Path) {
next.ServeHTTP(rw, r)
return
}
Expand Down
Loading