chore: instrument external oauth2 requests

coder · Emyrk · Jan 10, 2024 · Jan 9, 2024 · Jan 9, 2024 · Jan 9, 2024
commit b7f13fada0bbcf908eb0fd40df0d322bafc749f7
diff --git a/cli/server.go b/cli/server.go
@@ -80,6 +80,7 @@ import (
 	"github.com/coder/coder/v2/coderd/oauthpki"
 	"github.com/coder/coder/v2/coderd/prometheusmetrics"
 	"github.com/coder/coder/v2/coderd/prometheusmetrics/insights"
+	"github.com/coder/coder/v2/coderd/promoauth"
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/telemetry"
 	"github.com/coder/coder/v2/coderd/tracing"
@@ -102,7 +103,7 @@ import (
 	"github.com/coder/wgtunnel/tunnelsdk"
 )
 
-func createOIDCConfig(ctx context.Context, vals *codersdk.DeploymentValues) (*coderd.OIDCConfig, error) {
+func createOIDCConfig(ctx context.Context, instrument *promoauth.Factory, vals *codersdk.DeploymentValues) (*coderd.OIDCConfig, error) {
 	if vals.OIDC.ClientID == "" {
 		return nil, xerrors.Errorf("OIDC client ID must be set!")
 	}
@@ -133,7 +134,7 @@ func createOIDCConfig(ctx context.Context, vals *codersdk.DeploymentValues) (*co
 		Scopes:       vals.OIDC.Scopes,
 	}
 
-	var useCfg httpmw.OAuth2Config = oauthCfg
+	var useCfg promoauth.OAuth2Config = oauthCfg
 	if vals.OIDC.ClientKeyFile != "" {
 		// PKI authentication is done in the params. If a
 		// counter example is found, we can add a config option to
@@ -159,7 +160,7 @@ func createOIDCConfig(ctx context.Context, vals *codersdk.DeploymentValues) (*co
 	}
 
 	return &coderd.OIDCConfig{
-		OAuth2Config: useCfg,
+		OAuth2Config: instrument.New("oidc-login", useCfg),
 		Provider:     oidcProvider,
 		Verifier: oidcProvider.Verifier(&oidc.Config{
 			ClientID: vals.OIDC.ClientID.String(),
@@ -523,8 +524,11 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				return xerrors.Errorf("read external auth providers from env: %w", err)
 			}
 
+			promRegistry := prometheus.NewRegistry()
+			oauthIntrument := promoauth.NewFactory(promRegistry)
 			vals.ExternalAuthConfigs.Value = append(vals.ExternalAuthConfigs.Value, extAuthEnv...)
 			externalAuthConfigs, err := externalauth.ConvertConfig(
+				oauthIntrument,
 				vals.ExternalAuthConfigs.Value,
 				vals.AccessURL.Value(),
 			)
@@ -571,7 +575,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				// the DeploymentValues instead, this just serves to indicate the source of each
 				// option. This is just defensive to prevent accidentally leaking.
 				DeploymentOptions:           codersdk.DeploymentOptionsWithoutSecrets(opts),
-				PrometheusRegistry:          prometheus.NewRegistry(),
+				PrometheusRegistry:          promRegistry,
 				APIRateLimit:                int(vals.RateLimit.API.Value()),
 				LoginRateLimit:              loginRateLimit,
 				FilesRateLimit:              filesRateLimit,
@@ -617,7 +621,9 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			}
 
 			if vals.OAuth2.Github.ClientSecret != "" {
-				options.GithubOAuth2Config, err = configureGithubOAuth2(vals.AccessURL.Value(),
+				options.GithubOAuth2Config, err = configureGithubOAuth2(
+					oauthIntrument,
+					vals.AccessURL.Value(),
 					vals.OAuth2.Github.ClientID.String(),
 					vals.OAuth2.Github.ClientSecret.String(),
 					vals.OAuth2.Github.AllowSignups.Value(),
@@ -636,7 +642,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 					logger.Warn(ctx, "coder will not check email_verified for OIDC logins")
 				}
 
-				oc, err := createOIDCConfig(ctx, vals)
+				oc, err := createOIDCConfig(ctx, oauthIntrument, vals)
 				if err != nil {
 					return xerrors.Errorf("create oidc config: %w", err)
 				}
@@ -1737,7 +1743,7 @@ func configureCAPool(tlsClientCAFile string, tlsConfig *tls.Config) error {
 }
 
 //nolint:revive // Ignore flag-parameter: parameter 'allowEveryone' seems to be a control flag, avoid control coupling (revive)
-func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, allowSignups, allowEveryone bool, allowOrgs []string, rawTeams []string, enterpriseBaseURL string) (*coderd.GithubOAuth2Config, error) {
+func configureGithubOAuth2(instrument *promoauth.Factory, accessURL *url.URL, clientID, clientSecret string, allowSignups, allowEveryone bool, allowOrgs []string, rawTeams []string, enterpriseBaseURL string) (*coderd.GithubOAuth2Config, error) {
 	redirectURL, err := accessURL.Parse("/api/v2/users/oauth2/github/callback")
 	if err != nil {
 		return nil, xerrors.Errorf("parse github oauth callback url: %w", err)
@@ -1790,7 +1796,7 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al
 	}
 
 	return &coderd.GithubOAuth2Config{
-		OAuth2Config: &oauth2.Config{
+		OAuth2Config: instrument.New("github-login", &oauth2.Config{
 			ClientID:     clientID,
 			ClientSecret: clientSecret,
 			Endpoint:     endpoint,
@@ -1800,7 +1806,7 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al
 				"read:org",
 				"user:email",
 			},
-		},
+		}),
 		AllowSignups:       allowSignups,
 		AllowEveryone:      allowEveryone,
 		AllowOrganizations: allowOrgs,

diff --git a/coderd/externalauth/externalauth.go b/coderd/externalauth/externalauth.go
@@ -22,19 +22,14 @@ import (
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/promoauth"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/retry"
 )
 
-type OAuth2Config interface {
-	AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string
-	Exchange(ctx context.Context, code string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error)
-	TokenSource(context.Context, *oauth2.Token) oauth2.TokenSource
-}
-
 // Config is used for authentication for Git operations.
 type Config struct {
-	OAuth2Config
+	promoauth.OAuth2Config
 	// ID is a unique identifier for the authenticator.
 	ID string
 	// Type is the type of provider.
@@ -401,7 +396,7 @@ func (c *DeviceAuth) formatDeviceCodeURL() (string, error) {
 
 // ConvertConfig converts the SDK configuration entry format
 // to the parsed and ready-to-consume in coderd provider type.
-func ConvertConfig(entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([]*Config, error) {
+func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([]*Config, error) {
 	ids := map[string]struct{}{}
 	configs := []*Config{}
 	for _, entry := range entries {
@@ -453,7 +448,7 @@ func ConvertConfig(entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([
 			Scopes:      entry.Scopes,
 		}
 
-		var oauthConfig OAuth2Config = oc
+		var oauthConfig promoauth.OAuth2Config = oc
 		// Azure DevOps uses JWT token authentication!
 		if entry.Type == string(codersdk.EnhancedExternalAuthProviderAzureDevops) {
 			oauthConfig = &jwtConfig{oc}
@@ -463,7 +458,7 @@ func ConvertConfig(entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([
 		}
 
 		cfg := &Config{
-			OAuth2Config:        oauthConfig,
+			OAuth2Config:        instrument.New(entry.ID, oauthConfig),
 			ID:                  entry.ID,
 			Regex:               regex,
 			Type:                entry.Type,

diff --git a/coderd/httpmw/apikey.go b/coderd/httpmw/apikey.go
@@ -22,6 +22,7 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/promoauth"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -74,8 +75,8 @@ func UserAuthorization(r *http.Request) Authorization {
 // OAuth2Configs is a collection of configurations for OAuth-based authentication.
 // This should be extended to support other authentication types in the future.
 type OAuth2Configs struct {
-	Github OAuth2Config
-	OIDC   OAuth2Config
+	Github promoauth.OAuth2Config
+	OIDC   promoauth.OAuth2Config
 }
 
 func (c *OAuth2Configs) IsZero() bool {
@@ -270,7 +271,7 @@ func ExtractAPIKey(rw http.ResponseWriter, r *http.Request, cfg ExtractAPIKeyCon
 				})
 			}
 
-			var oauthConfig OAuth2Config
+			var oauthConfig promoauth.OAuth2Config
 			switch key.LoginType {
 			case database.LoginTypeGithub:
 				oauthConfig = cfg.OAuth2Configs.Github

diff --git a/coderd/httpmw/oauth2.go b/coderd/httpmw/oauth2.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/promoauth"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/cryptorand"
 )
@@ -22,14 +23,6 @@ type OAuth2State struct {
 	StateString string
 }
 
-// OAuth2Config exposes a subset of *oauth2.Config functions for easier testing.
-// *oauth2.Config should be used instead of implementing this in production.
-type OAuth2Config interface {
-	AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string
-	Exchange(ctx context.Context, code string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error)
-	TokenSource(context.Context, *oauth2.Token) oauth2.TokenSource
-}
-
 // OAuth2 returns the state from an oauth request.
 func OAuth2(r *http.Request) OAuth2State {
 	oauth, ok := r.Context().Value(oauth2StateKey{}).(OAuth2State)
@@ -44,7 +37,7 @@ func OAuth2(r *http.Request) OAuth2State {
 // a "code" URL parameter will be redirected.
 // AuthURLOpts are passed to the AuthCodeURL function. If this is nil,
 // the default option oauth2.AccessTypeOffline will be used.
-func ExtractOAuth2(config OAuth2Config, client *http.Client, authURLOpts map[string]string) func(http.Handler) http.Handler {
+func ExtractOAuth2(config promoauth.OAuth2Config, client *http.Client, authURLOpts map[string]string) func(http.Handler) http.Handler {
 	opts := make([]oauth2.AuthCodeOption, 0, len(authURLOpts)+1)
 	opts = append(opts, oauth2.AccessTypeOffline)
 	for k, v := range authURLOpts {

diff --git a/coderd/oauthpki/oidcpki.go b/coderd/oauthpki/oidcpki.go
@@ -20,7 +20,7 @@ import (
 	"golang.org/x/oauth2/jws"
 	"golang.org/x/xerrors"
 
-	"github.com/coder/coder/v2/coderd/httpmw"
+	"github.com/coder/coder/v2/coderd/promoauth"
 )
 
 // Config uses jwt assertions over client_secret for oauth2 authentication of
@@ -33,7 +33,7 @@ import (
 //
 //	https://datatracker.ietf.org/doc/html/rfc7523
 type Config struct {
-	cfg httpmw.OAuth2Config
+	cfg promoauth.OAuth2Config
 
 	// These values should match those provided in the oauth2.Config.
 	// Because the inner config is an interface, we need to duplicate these
@@ -57,7 +57,7 @@ type ConfigParams struct {
 	PemEncodedKey  []byte
 	PemEncodedCert []byte
 
-	Config httpmw.OAuth2Config
+	Config promoauth.OAuth2Config
 }
 
 // NewOauth2PKIConfig creates the oauth2 config for PKI based auth. It requires the certificate and it's private key.

diff --git a/coderd/promoauth/doc.go b/coderd/promoauth/doc.go
@@ -0,0 +1,4 @@
+// Package promoauth is for instrumenting oauth2 flows with prometheus metrics.
+// Specifically, it is intended to count the number of external requests made
+// by the underlying oauth2 exchanges.
+package promoauth
diff --git a/coderd/promoauth/oauth2.go b/coderd/promoauth/oauth2.go
@@ -0,0 +1,128 @@
+package promoauth
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"golang.org/x/oauth2"
+)
+
+// OAuth2Config exposes a subset of *oauth2.Config functions for easier testing.
+// *oauth2.Config should be used instead of implementing this in production.
+type OAuth2Config interface {
+	AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string
+	Exchange(ctx context.Context, code string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error)
+	TokenSource(context.Context, *oauth2.Token) oauth2.TokenSource
+}
+
+var _ OAuth2Config = (*Config)(nil)
+
+type Factory struct {
+	metrics *metrics
+}
+
+// metrics is the reusable metrics for all oauth2 providers.
+type metrics struct {
+	externalRequestCount *prometheus.CounterVec
+}
+
+func NewFactory(registry prometheus.Registerer) *Factory {
+	factory := promauto.With(registry)
+
+	return &Factory{
+		metrics: &metrics{
+			externalRequestCount: factory.NewCounterVec(prometheus.CounterOpts{
+				Namespace: "coderd",
+				Subsystem: "oauth2",
+				Name:      "external_requests_total",
+				Help:      "The total number of api calls made to external oauth2 providers. 'status_code' will be 0 if the request failed with no response.",
+			}, []string{
+				"name",
+				"status_code",
+				"domain",
+			}),
+		},
+	}
+}
+
+func (f *Factory) New(name string, under OAuth2Config) *Config {
+	return &Config{
+		name:       name,
+		underlying: under,
+		metrics:    f.metrics,
+	}
+}
+
+type Config struct {
+	// Name is a human friendly name to identify the oauth2 provider. This should be
+	// deterministic from restart to restart, as it is going to be used as a label in
+	// prometheus metrics.
+	name       string
+	underlying OAuth2Config
+	metrics    *metrics
+}
+
+func (c *Config) AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string {
+	// No external requests are made when constructing the auth code url.
+	return c.underlying.AuthCodeURL(state, opts...)
+}
+
+func (c *Config) Exchange(ctx context.Context, code string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error) {
+	return c.underlying.Exchange(c.wrapClient(ctx), code, opts...)
+}
+
+func (c *Config) TokenSource(ctx context.Context, token *oauth2.Token) oauth2.TokenSource {
+	return c.underlying.TokenSource(c.wrapClient(ctx), token)
+}
+
+// wrapClient is the only way we can accurately instrument the oauth2 client.
+// This is because method calls to the 'OAuth2Config' interface are not 1:1 with
+// network requests.
+//
+// For example, the 'TokenSource' method will return a token
+// source that will make a network request when the 'Token' method is called on
+// it if the token is expired.
+func (c *Config) wrapClient(ctx context.Context) context.Context {
+	cli := http.DefaultClient
+
+	// Check if the context has an http client already.
+	if hc, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok {
+		cli = hc
+	}
+
+	// The new tripper will instrument every request made by the oauth2 client.
+	cli.Transport = newInstrumentedTripper(c, cli.Transport)
+	return context.WithValue(ctx, oauth2.HTTPClient, cli)
+}
+
+type instrumentedTripper struct {
+	c          *Config
+	underlying http.RoundTripper
+}
+
+func newInstrumentedTripper(c *Config, under http.RoundTripper) *instrumentedTripper {
+	if under == nil {
+		under = http.DefaultTransport
+	}
+	return &instrumentedTripper{
+		c:          c,
+		underlying: under,
+	}
+}
+
+func (i *instrumentedTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	resp, err := i.underlying.RoundTrip(r)
+	var statusCode int
+	if resp != nil {
+		statusCode = resp.StatusCode
+	}
+	i.c.metrics.externalRequestCount.With(prometheus.Labels{
+		"name":        i.c.name,
+		"status_code": fmt.Sprintf("%d", statusCode),
+		"domain":      r.URL.Host,
+	}).Inc()
+	return resp, err
+}