Skip to content

Commit 3f9da67

Browse files
EmyrkEmyrk
authored
chore: instrument github oauth2 limits (#11532)
* chore: instrument github oauth2 limits Rate limit information for github oauth2 providers instrumented in prometheus
1 parent 50b78e3 commit 3f9da67

File tree

6 files changed

+421
-10
lines changed

6 files changed

+421
-10
lines changed

‎cli/server.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1802,7 +1802,7 @@ func configureGithubOAuth2(instrument *promoauth.Factory, accessURL *url.URL, cl
18021802
}
18031803

18041804
return &coderd.GithubOAuth2Config{
1805-
OAuth2Config: instrument.New("github-login", &oauth2.Config{
1805+
OAuth2Config: instrument.NewGithub("github-login", &oauth2.Config{
18061806
ClientID: clientID,
18071807
ClientSecret: clientSecret,
18081808
Endpoint: endpoint,

‎coderd/coderdtest/oidctest/idp.go

+9
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ type FakeIDP struct {
8585
// to test something like PKI auth vs a client_secret.
8686
hookAuthenticateClient func(t testing.TB, req *http.Request) (url.Values, error)
8787
serve bool
88+
// optional middlewares
89+
middlewares chi.Middlewares
8890
}
8991

9092
func StatusError(code int, err error) error {
@@ -115,6 +117,12 @@ func WithAuthorizedRedirectURL(hook func(redirectURL string) error) func(*FakeID
115117
}
116118
}
117119

120+
func WithMiddlewares(mws ...func(http.Handler) http.Handler) func(*FakeIDP) {
121+
return func(f *FakeIDP) {
122+
f.middlewares = append(f.middlewares, mws...)
123+
}
124+
}
125+
118126
// WithRefresh is called when a refresh token is used. The email is
119127
// the email of the user that is being refreshed assuming the claims are correct.
120128
func WithRefresh(hook func(email string) error) func(*FakeIDP) {
@@ -570,6 +578,7 @@ func (f *FakeIDP) httpHandler(t testing.TB) http.Handler {
570578
t.Helper()
571579

572580
mux := chi.NewMux()
581+
mux.Use(f.middlewares...)
573582
// This endpoint is required to initialize the OIDC provider.
574583
// It is used to get the OIDC configuration.
575584
mux.Get("/.well-known/openid-configuration", func(rw http.ResponseWriter, r *http.Request) {

‎coderd/externalauth/externalauth.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,13 @@ func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAut
464464
oauthConfig = &exchangeWithClientSecret{oc}
465465
}
466466

467+
instrumented := instrument.New(entry.ID, oauthConfig)
468+
if strings.EqualFold(entry.Type, string(codersdk.EnhancedExternalAuthProviderGitHub)) {
469+
instrumented = instrument.NewGithub(entry.ID, oauthConfig)
470+
}
471+
467472
cfg := &Config{
468-
InstrumentedOAuth2Config: instrument.New(entry.ID, oauthConfig),
473+
InstrumentedOAuth2Config: instrumented,
469474
ID: entry.ID,
470475
Regex: regex,
471476
Type: entry.Type,

‎coderd/promoauth/github.go

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package promoauth
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"strconv"
7+
"time"
8+
)
9+
10+
type rateLimits struct {
11+
Limit int
12+
Remaining int
13+
Used int
14+
Reset time.Time
15+
Resource string
16+
}
17+
18+
// githubRateLimits checks the returned response headers and
19+
func githubRateLimits(resp *http.Response, err error) (rateLimits, bool) {
20+
if err != nil || resp == nil {
21+
return rateLimits{}, false
22+
}
23+
24+
p := headerParser{header: resp.Header}
25+
// See
26+
// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
27+
limits := rateLimits{
28+
Limit: p.int("x-ratelimit-limit"),
29+
Remaining: p.int("x-ratelimit-remaining"),
30+
Used: p.int("x-ratelimit-used"),
31+
Resource: p.string("x-ratelimit-resource"),
32+
}
33+
34+
if limits.Limit == 0 &&
35+
limits.Remaining == 0 &&
36+
limits.Used == 0 {
37+
// For some requests, github has no rate limit. In which case,
38+
// it returns all 0s. We can just omit these.
39+
return limits, false
40+
}
41+
42+
// Reset is when the rate limit "used" will be reset to 0.
43+
// If it's unix 0, then we do not know when it will reset.
44+
// Change it to a zero time as that is easier to handle in golang.
45+
unix := p.int("x-ratelimit-reset")
46+
resetAt := time.Unix(int64(unix), 0)
47+
if unix == 0 {
48+
resetAt = time.Time{}
49+
}
50+
limits.Reset = resetAt
51+
52+
// Unauthorized requests have their own rate limit, so we should
53+
// track them separately.
54+
if resp.StatusCode == http.StatusUnauthorized {
55+
limits.Resource += "-unauthorized"
56+
}
57+
58+
// A 401 or 429 means too many requests. This might mess up the
59+
// "resource" string because we could hit the unauthorized limit,
60+
// and we do not want that to override the authorized one.
61+
// However, in testing, it seems a 401 is always a 401, even if
62+
// the limit is hit.
63+
64+
if len(p.errors) > 0 {
65+
// If we are missing any headers, then do not try and guess
66+
// what the rate limits are.
67+
return limits, false
68+
}
69+
return limits, true
70+
}
71+
72+
type headerParser struct {
73+
errors map[string]error
74+
header http.Header
75+
}
76+
77+
func (p *headerParser) string(key string) string {
78+
if p.errors == nil {
79+
p.errors = make(map[string]error)
80+
}
81+
82+
v := p.header.Get(key)
83+
if v == "" {
84+
p.errors[key] = fmt.Errorf("missing header %q", key)
85+
}
86+
return v
87+
}
88+
89+
func (p *headerParser) int(key string) int {
90+
v := p.string(key)
91+
if v == "" {
92+
return -1
93+
}
94+
95+
i, err := strconv.Atoi(v)
96+
if err != nil {
97+
p.errors[key] = err
98+
}
99+
return i
100+
}

‎coderd/promoauth/oauth2.go

+107
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7+
"time"
78

89
"github.com/prometheus/client_golang/prometheus"
910
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -46,11 +47,25 @@ var _ OAuth2Config = (*Config)(nil)
4647
// Primarily to avoid any prometheus errors registering duplicate metrics.
4748
type Factory struct {
4849
metrics *metrics
50+
// optional replace now func
51+
Now func() time.Time
4952
}
5053

5154
// metrics is the reusable metrics for all oauth2 providers.
5255
type metrics struct {
5356
externalRequestCount *prometheus.CounterVec
57+
58+
// if the oauth supports it, rate limit metrics.
59+
// rateLimit is the defined limit per interval
60+
rateLimit *prometheus.GaugeVec
61+
rateLimitRemaining *prometheus.GaugeVec
62+
rateLimitUsed *prometheus.GaugeVec
63+
// rateLimitReset is unix time of the next interval (when the rate limit resets).
64+
rateLimitReset *prometheus.GaugeVec
65+
// rateLimitResetIn is the time in seconds until the rate limit resets.
66+
// This is included because it is sometimes more helpful to know the limit
67+
// will reset in 600seconds, rather than at 1704000000 unix time.
68+
rateLimitResetIn *prometheus.GaugeVec
5469
}
5570

5671
func NewFactory(registry prometheus.Registerer) *Factory {
@@ -68,6 +83,53 @@ func NewFactory(registry prometheus.Registerer) *Factory {
6883
"source",
6984
"status_code",
7085
}),
86+
rateLimit: factory.NewGaugeVec(prometheus.GaugeOpts{
87+
Namespace: "coderd",
88+
Subsystem: "oauth2",
89+
Name: "external_requests_rate_limit_total",
90+
Help: "The total number of allowed requests per interval.",
91+
}, []string{
92+
"name",
93+
// Resource allows different rate limits for the same oauth2 provider.
94+
// Some IDPs have different buckets for different rate limits.
95+
"resource",
96+
}),
97+
rateLimitRemaining: factory.NewGaugeVec(prometheus.GaugeOpts{
98+
Namespace: "coderd",
99+
Subsystem: "oauth2",
100+
Name: "external_requests_rate_limit_remaining",
101+
Help: "The remaining number of allowed requests in this interval.",
102+
}, []string{
103+
"name",
104+
"resource",
105+
}),
106+
rateLimitUsed: factory.NewGaugeVec(prometheus.GaugeOpts{
107+
Namespace: "coderd",
108+
Subsystem: "oauth2",
109+
Name: "external_requests_rate_limit_used",
110+
Help: "The number of requests made in this interval.",
111+
}, []string{
112+
"name",
113+
"resource",
114+
}),
115+
rateLimitReset: factory.NewGaugeVec(prometheus.GaugeOpts{
116+
Namespace: "coderd",
117+
Subsystem: "oauth2",
118+
Name: "external_requests_rate_limit_next_reset_unix",
119+
Help: "Unix timestamp for when the next interval starts",
120+
}, []string{
121+
"name",
122+
"resource",
123+
}),
124+
rateLimitResetIn: factory.NewGaugeVec(prometheus.GaugeOpts{
125+
Namespace: "coderd",
126+
Subsystem: "oauth2",
127+
Name: "external_requests_rate_limit_reset_in_seconds",
128+
Help: "Seconds until the next interval",
129+
}, []string{
130+
"name",
131+
"resource",
132+
}),
71133
},
72134
}
73135
}
@@ -80,13 +142,53 @@ func (f *Factory) New(name string, under OAuth2Config) *Config {
80142
}
81143
}
82144

145+
// NewGithub returns a new instrumented oauth2 config for github. It tracks
146+
// rate limits as well as just the external request counts.
147+
//
148+
//nolint:bodyclose
149+
func (f *Factory) NewGithub(name string, under OAuth2Config) *Config {
150+
cfg := f.New(name, under)
151+
cfg.interceptors = append(cfg.interceptors, func(resp *http.Response, err error) {
152+
limits, ok := githubRateLimits(resp, err)
153+
if !ok {
154+
return
155+
}
156+
labels := prometheus.Labels{
157+
"name": cfg.name,
158+
"resource": limits.Resource,
159+
}
160+
// Default to -1 for "do not know"
161+
resetIn := float64(-1)
162+
if !limits.Reset.IsZero() {
163+
now := time.Now()
164+
if f.Now != nil {
165+
now = f.Now()
166+
}
167+
resetIn = limits.Reset.Sub(now).Seconds()
168+
if resetIn < 0 {
169+
// If it just reset, just make it 0.
170+
resetIn = 0
171+
}
172+
}
173+
174+
f.metrics.rateLimit.With(labels).Set(float64(limits.Limit))
175+
f.metrics.rateLimitRemaining.With(labels).Set(float64(limits.Remaining))
176+
f.metrics.rateLimitUsed.With(labels).Set(float64(limits.Used))
177+
f.metrics.rateLimitReset.With(labels).Set(float64(limits.Reset.Unix()))
178+
f.metrics.rateLimitResetIn.With(labels).Set(resetIn)
179+
})
180+
return cfg
181+
}
182+
83183
type Config struct {
84184
// Name is a human friendly name to identify the oauth2 provider. This should be
85185
// deterministic from restart to restart, as it is going to be used as a label in
86186
// prometheus metrics.
87187
name string
88188
underlying OAuth2Config
89189
metrics *metrics
190+
// interceptors are called after every request made by the oauth2 client.
191+
interceptors []func(resp *http.Response, err error)
90192
}
91193

92194
func (c *Config) Do(ctx context.Context, source Oauth2Source, req *http.Request) (*http.Response, error) {
@@ -169,5 +271,10 @@ func (i *instrumentedTripper) RoundTrip(r *http.Request) (*http.Response, error)
169271
"source": string(i.source),
170272
"status_code": fmt.Sprintf("%d", statusCode),
171273
}).Inc()
274+
275+
// Handle any extra interceptors.
276+
for _, interceptor := range i.c.interceptors {
277+
interceptor(resp, err)
278+
}
172279
return resp, err
173280
}

0 commit comments

Comments
 (0)