Skip to content

Commit 184fcf6

Browse files
EmyrkEmyrk
committed
chore: instrument github oauth2 limits
1 parent 50b78e3 commit 184fcf6

File tree

4 files changed

+196
-3
lines changed

4 files changed

+196
-3
lines changed

‎cli/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1802,7 +1802,7 @@ func configureGithubOAuth2(instrument *promoauth.Factory, accessURL *url.URL, cl
18021802
}
18031803

18041804
return &coderd.GithubOAuth2Config{
1805-
OAuth2Config: instrument.New("github-login", &oauth2.Config{
1805+
OAuth2Config: instrument.NewGithub("github-login", &oauth2.Config{
18061806
ClientID: clientID,
18071807
ClientSecret: clientSecret,
18081808
Endpoint: endpoint,

‎coderd/externalauth/externalauth.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,13 @@ func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAut
464464
oauthConfig = &exchangeWithClientSecret{oc}
465465
}
466466

467+
instrumented := instrument.New(entry.ID, oauthConfig)
468+
if strings.EqualFold(entry.Type, string(codersdk.EnhancedExternalAuthProviderGitHub)) {
469+
instrumented = instrument.NewGithub(entry.ID, oauthConfig)
470+
}
471+
467472
cfg := &Config{
468-
InstrumentedOAuth2Config: instrument.New(entry.ID, oauthConfig),
473+
InstrumentedOAuth2Config: instrumented,
469474
ID: entry.ID,
470475
Regex: regex,
471476
Type: entry.Type,

‎coderd/promoauth/github.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package promoauth
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"strconv"
7+
"time"
8+
)
9+
10+
type rateLimits struct {
11+
Limit int
12+
Remaining int
13+
Used int
14+
Reset time.Time
15+
Resource string
16+
}
17+
18+
// githubRateLimits checks the returned response headers and
19+
func githubRateLimits(resp *http.Response, err error) (rateLimits, bool) {
20+
if err != nil || resp == nil {
21+
return rateLimits{}, false
22+
}
23+
24+
p := headerParser{header: resp.Header}
25+
// See
26+
// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
27+
limits := rateLimits{
28+
Limit: p.int("x-ratelimit-limit"),
29+
Remaining: p.int("x-ratelimit-remaining"),
30+
Used: p.int("x-ratelimit-used"),
31+
Resource: p.header.Get("x-ratelimit-resource"),
32+
}
33+
34+
if limits.Limit == 0 &&
35+
limits.Remaining == 0 &&
36+
limits.Used == 0 {
37+
// For some requests, github has no rate limit. In which case,
38+
// it returns all 0s. We can just omit these.
39+
return limits, false
40+
}
41+
42+
// Reset is when the rate limit "used" will be reset to 0.
43+
// If it's unix 0, then we do not know when it will reset.
44+
// Change it to a zero time as that is easier to handle in golang.
45+
unix := p.int("x-ratelimit-reset")
46+
resetAt := time.Unix(int64(unix), 0)
47+
if unix == 0 {
48+
resetAt = time.Time{}
49+
}
50+
limits.Reset = resetAt
51+
52+
// Unauthorized requests have their own rate limit, so we should
53+
// track them separately.
54+
if resp.StatusCode == http.StatusUnauthorized {
55+
limits.Resource += "-unauthorized"
56+
}
57+
58+
// A 401 or 429 means too many requests. This might mess up the
59+
// "resource" string because we could hit the unauthorized limit,
60+
// and we do not want that to override the authorized one.
61+
// However, in testing, it seems a 401 is always a 401, even if
62+
// the limit is hit.
63+
64+
if len(p.errors) > 0 {
65+
return limits, false
66+
}
67+
return limits, true
68+
}
69+
70+
type headerParser struct {
71+
errors map[string]error
72+
header http.Header
73+
}
74+
75+
func (p *headerParser) int(key string) int {
76+
if p.errors == nil {
77+
p.errors = make(map[string]error)
78+
}
79+
80+
v := p.header.Get(key)
81+
if v == "" {
82+
p.errors[key] = fmt.Errorf("missing header %q", key)
83+
}
84+
85+
i, err := strconv.Atoi(v)
86+
if err != nil {
87+
p.errors[key] = err
88+
}
89+
return i
90+
}

‎coderd/promoauth/oauth2.go

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7+
"time"
78

89
"github.com/prometheus/client_golang/prometheus"
910
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -51,6 +52,16 @@ type Factory struct {
5152
// metrics is the reusable metrics for all oauth2 providers.
5253
type metrics struct {
5354
externalRequestCount *prometheus.CounterVec
55+
56+
// if the oauth supports it, rate limit metrics.
57+
// rateLimit is the defined limit per interval
58+
rateLimit *prometheus.GaugeVec
59+
rateLimitRemaining *prometheus.GaugeVec
60+
rateLimitUsed *prometheus.GaugeVec
61+
// rateLimitReset is the time in seconds the rate limit resets.
62+
rateLimitReset *prometheus.GaugeVec
63+
// rateLimitResetIn is the time in seconds until the rate limit resets.
64+
rateLimitResetIn *prometheus.GaugeVec
5465
}
5566

5667
func NewFactory(registry prometheus.Registerer) *Factory {
@@ -68,25 +79,107 @@ func NewFactory(registry prometheus.Registerer) *Factory {
6879
"source",
6980
"status_code",
7081
}),
82+
rateLimit: factory.NewGaugeVec(prometheus.GaugeOpts{
83+
Namespace: "coderd",
84+
Subsystem: "oauth2",
85+
Name: "external_requests_rate_limit_total",
86+
Help: "The total number of allowed requests per interval.",
87+
}, []string{
88+
"name",
89+
// Resource allows different rate limits for the same oauth2 provider.
90+
// Some IDPs have different buckets for different rate limits.
91+
"resource",
92+
}),
93+
rateLimitRemaining: factory.NewGaugeVec(prometheus.GaugeOpts{
94+
Namespace: "coderd",
95+
Subsystem: "oauth2",
96+
Name: "external_requests_rate_limit_remaining",
97+
Help: "The remaining number of allowed requests in this interval.",
98+
}, []string{
99+
"name",
100+
"resource",
101+
}),
102+
rateLimitUsed: factory.NewGaugeVec(prometheus.GaugeOpts{
103+
Namespace: "coderd",
104+
Subsystem: "oauth2",
105+
Name: "external_requests_rate_limit_used",
106+
Help: "The number of requests made in this interval.",
107+
}, []string{
108+
"name",
109+
"resource",
110+
}),
111+
rateLimitReset: factory.NewGaugeVec(prometheus.GaugeOpts{
112+
Namespace: "coderd",
113+
Subsystem: "oauth2",
114+
Name: "external_requests_rate_limit_next_reset_unix",
115+
Help: "Unix timestamp of the next interval",
116+
}, []string{
117+
"name",
118+
"resource",
119+
}),
120+
rateLimitResetIn: factory.NewGaugeVec(prometheus.GaugeOpts{
121+
Namespace: "coderd",
122+
Subsystem: "oauth2",
123+
Name: "external_requests_rate_limit_reset_in_seconds",
124+
Help: "Seconds until the next interval",
125+
}, []string{
126+
"name",
127+
"resource",
128+
}),
71129
},
72130
}
73131
}
74132

75-
func (f *Factory) New(name string, under OAuth2Config) *Config {
133+
func (f *Factory) New(name string, under OAuth2Config, opts ...func(cfg *Config)) *Config {
76134
return &Config{
77135
name: name,
78136
underlying: under,
79137
metrics: f.metrics,
80138
}
81139
}
82140

141+
// NewGithub returns a new instrumented oauth2 config for github. It tracks
142+
// rate limits as well as just the external request counts.
143+
func (f *Factory) NewGithub(name string, under OAuth2Config) *Config {
144+
cfg := f.New(name, under)
145+
cfg.interceptors = append(cfg.interceptors, func(resp *http.Response, err error) {
146+
limits, ok := githubRateLimits(resp, err)
147+
if !ok {
148+
return
149+
}
150+
labels := prometheus.Labels{
151+
"name": cfg.name,
152+
"resource": limits.Resource,
153+
}
154+
// Default to -1 for "do not know"
155+
resetIn := float64(-1)
156+
if !limits.Reset.IsZero() {
157+
now := time.Now()
158+
resetIn = float64(limits.Reset.Sub(now).Seconds())
159+
if resetIn < 0 {
160+
// If it just reset, just make it 0.
161+
resetIn = 0
162+
}
163+
}
164+
165+
f.metrics.rateLimit.With(labels).Set(float64(limits.Limit))
166+
f.metrics.rateLimitRemaining.With(labels).Set(float64(limits.Remaining))
167+
f.metrics.rateLimitUsed.With(labels).Set(float64(limits.Used))
168+
f.metrics.rateLimitReset.With(labels).Set(float64(limits.Reset.Unix()))
169+
f.metrics.rateLimitResetIn.With(labels).Set(resetIn)
170+
})
171+
return cfg
172+
}
173+
83174
type Config struct {
84175
// Name is a human friendly name to identify the oauth2 provider. This should be
85176
// deterministic from restart to restart, as it is going to be used as a label in
86177
// prometheus metrics.
87178
name string
88179
underlying OAuth2Config
89180
metrics *metrics
181+
// interceptors are called after every request made by the oauth2 client.
182+
interceptors []func(resp *http.Response, err error)
90183
}
91184

92185
func (c *Config) Do(ctx context.Context, source Oauth2Source, req *http.Request) (*http.Response, error) {
@@ -169,5 +262,10 @@ func (i *instrumentedTripper) RoundTrip(r *http.Request) (*http.Response, error)
169262
"source": string(i.source),
170263
"status_code": fmt.Sprintf("%d", statusCode),
171264
}).Inc()
265+
266+
// Handle any extra interceptors.
267+
for _, interceptor := range i.c.interceptors {
268+
interceptor(resp, err)
269+
}
172270
return resp, err
173271
}

0 commit comments

Comments
 (0)