Skip to content

Commit ec31915

Browse files
EmyrkEmyrk
committed
chore: instrument github oauth2 limits
1 parent 5222f66 commit ec31915

File tree

4 files changed

+196
-3
lines changed

4 files changed

+196
-3
lines changed

‎cli/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1802,7 +1802,7 @@ func configureGithubOAuth2(instrument *promoauth.Factory, accessURL *url.URL, cl
18021802
}
18031803

18041804
return &coderd.GithubOAuth2Config{
1805-
OAuth2Config: instrument.New("github-login", &oauth2.Config{
1805+
OAuth2Config: instrument.NewGithub("github-login", &oauth2.Config{
18061806
ClientID: clientID,
18071807
ClientSecret: clientSecret,
18081808
Endpoint: endpoint,

‎coderd/externalauth/externalauth.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,13 @@ func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAut
464464
oauthConfig = &exchangeWithClientSecret{oc}
465465
}
466466

467+
instrumented := instrument.New(entry.ID, oauthConfig)
468+
if strings.EqualFold(entry.Type, string(codersdk.EnhancedExternalAuthProviderGitHub)) {
469+
instrumented = instrument.NewGithub(entry.ID, oauthConfig)
470+
}
471+
467472
cfg := &Config{
468-
InstrumentedOAuth2Config: instrument.New(entry.ID, oauthConfig),
473+
InstrumentedOAuth2Config: instrumented,
469474
ID: entry.ID,
470475
Regex: regex,
471476
Type: entry.Type,

‎coderd/promoauth/github.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package promoauth
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"strconv"
7+
"time"
8+
)
9+
10+
type rateLimits struct {
11+
Limit int
12+
Remaining int
13+
Used int
14+
Reset time.Time
15+
Resource string
16+
}
17+
18+
// githubRateLimits checks the returned response headers and
19+
func githubRateLimits(resp *http.Response, err error) (rateLimits, bool) {
20+
if err != nil || resp == nil {
21+
return rateLimits{}, false
22+
}
23+
24+
p := headerParser{header: resp.Header}
25+
// See
26+
// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
27+
limits := rateLimits{
28+
Limit: p.int("x-ratelimit-limit"),
29+
Remaining: p.int("x-ratelimit-remaining"),
30+
Used: p.int("x-ratelimit-used"),
31+
Resource: p.header.Get("x-ratelimit-resource"),
32+
}
33+
34+
if limits.Limit == 0 &&
35+
limits.Remaining == 0 &&
36+
limits.Used == 0 {
37+
// For some requests, github has no rate limit. In which case,
38+
// it returns all 0s. We can just omit these.
39+
return limits, false
40+
}
41+
42+
// Reset is when the rate limit "used" will be reset to 0.
43+
// If it's unix 0, then we do not know when it will reset.
44+
// Change it to a zero time as that is easier to handle in golang.
45+
unix := p.int("x-ratelimit-reset")
46+
resetAt := time.Unix(int64(unix), 0)
47+
if unix == 0 {
48+
resetAt = time.Time{}
49+
}
50+
limits.Reset = resetAt
51+
52+
// Unauthorized requests have their own rate limit, so we should
53+
// track them separately.
54+
if resp.StatusCode == http.StatusUnauthorized {
55+
limits.Resource += "-unauthorized"
56+
}
57+
58+
// A 401 or 429 means too many requests. This might mess up the
59+
// "resource" string because we could hit the unauthorized limit,
60+
// and we do not want that to override the authorized one.
61+
// However, in testing, it seems a 401 is always a 401, even if
62+
// the limit is hit.
63+
64+
if len(p.errors) > 0 {
65+
return limits, false
66+
}
67+
return limits, true
68+
}
69+
70+
type headerParser struct {
71+
errors map[string]error
72+
header http.Header
73+
}
74+
75+
func (p *headerParser) int(key string) int {
76+
if p.errors == nil {
77+
p.errors = make(map[string]error)
78+
}
79+
80+
v := p.header.Get(key)
81+
if v == "" {
82+
p.errors[key] = fmt.Errorf("missing header %q", key)
83+
}
84+
85+
i, err := strconv.Atoi(v)
86+
if err != nil {
87+
p.errors[key] = err
88+
}
89+
return i
90+
}

‎coderd/promoauth/oauth2.go

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7+
"time"
78

89
"github.com/prometheus/client_golang/prometheus"
910
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -41,6 +42,16 @@ type Factory struct {
4142
// metrics is the reusable metrics for all oauth2 providers.
4243
type metrics struct {
4344
externalRequestCount *prometheus.CounterVec
45+
46+
// if the oauth supports it, rate limit metrics.
47+
// rateLimit is the defined limit per interval
48+
rateLimit *prometheus.GaugeVec
49+
rateLimitRemaining *prometheus.GaugeVec
50+
rateLimitUsed *prometheus.GaugeVec
51+
// rateLimitReset is the time in seconds the rate limit resets.
52+
rateLimitReset *prometheus.GaugeVec
53+
// rateLimitResetIn is the time in seconds until the rate limit resets.
54+
rateLimitResetIn *prometheus.GaugeVec
4455
}
4556

4657
func NewFactory(registry prometheus.Registerer) *Factory {
@@ -58,25 +69,107 @@ func NewFactory(registry prometheus.Registerer) *Factory {
5869
"source",
5970
"status_code",
6071
}),
72+
rateLimit: factory.NewGaugeVec(prometheus.GaugeOpts{
73+
Namespace: "coderd",
74+
Subsystem: "oauth2",
75+
Name: "external_requests_rate_limit_total",
76+
Help: "The total number of allowed requests per interval.",
77+
}, []string{
78+
"name",
79+
// Resource allows different rate limits for the same oauth2 provider.
80+
// Some IDPs have different buckets for different rate limits.
81+
"resource",
82+
}),
83+
rateLimitRemaining: factory.NewGaugeVec(prometheus.GaugeOpts{
84+
Namespace: "coderd",
85+
Subsystem: "oauth2",
86+
Name: "external_requests_rate_limit_remaining",
87+
Help: "The remaining number of allowed requests in this interval.",
88+
}, []string{
89+
"name",
90+
"resource",
91+
}),
92+
rateLimitUsed: factory.NewGaugeVec(prometheus.GaugeOpts{
93+
Namespace: "coderd",
94+
Subsystem: "oauth2",
95+
Name: "external_requests_rate_limit_used",
96+
Help: "The number of requests made in this interval.",
97+
}, []string{
98+
"name",
99+
"resource",
100+
}),
101+
rateLimitReset: factory.NewGaugeVec(prometheus.GaugeOpts{
102+
Namespace: "coderd",
103+
Subsystem: "oauth2",
104+
Name: "external_requests_rate_limit_next_reset_unix",
105+
Help: "Unix timestamp of the next interval",
106+
}, []string{
107+
"name",
108+
"resource",
109+
}),
110+
rateLimitResetIn: factory.NewGaugeVec(prometheus.GaugeOpts{
111+
Namespace: "coderd",
112+
Subsystem: "oauth2",
113+
Name: "external_requests_rate_limit_reset_in_seconds",
114+
Help: "Seconds until the next interval",
115+
}, []string{
116+
"name",
117+
"resource",
118+
}),
61119
},
62120
}
63121
}
64122

65-
func (f *Factory) New(name string, under OAuth2Config) *Config {
123+
func (f *Factory) New(name string, under OAuth2Config, opts ...func(cfg *Config)) *Config {
66124
return &Config{
67125
name: name,
68126
underlying: under,
69127
metrics: f.metrics,
70128
}
71129
}
72130

131+
// NewGithub returns a new instrumented oauth2 config for github. It tracks
132+
// rate limits as well as just the external request counts.
133+
func (f *Factory) NewGithub(name string, under OAuth2Config) *Config {
134+
cfg := f.New(name, under)
135+
cfg.interceptors = append(cfg.interceptors, func(resp *http.Response, err error) {
136+
limits, ok := githubRateLimits(resp, err)
137+
if !ok {
138+
return
139+
}
140+
labels := prometheus.Labels{
141+
"name": cfg.name,
142+
"resource": limits.Resource,
143+
}
144+
// Default to -1 for "do not know"
145+
resetIn := float64(-1)
146+
if !limits.Reset.IsZero() {
147+
now := time.Now()
148+
resetIn = float64(limits.Reset.Sub(now).Seconds())
149+
if resetIn < 0 {
150+
// If it just reset, just make it 0.
151+
resetIn = 0
152+
}
153+
}
154+
155+
f.metrics.rateLimit.With(labels).Set(float64(limits.Limit))
156+
f.metrics.rateLimitRemaining.With(labels).Set(float64(limits.Remaining))
157+
f.metrics.rateLimitUsed.With(labels).Set(float64(limits.Used))
158+
f.metrics.rateLimitReset.With(labels).Set(float64(limits.Reset.Unix()))
159+
f.metrics.rateLimitResetIn.With(labels).Set(resetIn)
160+
})
161+
return cfg
162+
}
163+
73164
type Config struct {
74165
// Name is a human friendly name to identify the oauth2 provider. This should be
75166
// deterministic from restart to restart, as it is going to be used as a label in
76167
// prometheus metrics.
77168
name string
78169
underlying OAuth2Config
79170
metrics *metrics
171+
// interceptors are called after every request made by the oauth2 client.
172+
interceptors []func(resp *http.Response, err error)
80173
}
81174

82175
func (c *Config) Do(ctx context.Context, source string, req *http.Request) (*http.Response, error) {
@@ -159,5 +252,10 @@ func (i *instrumentedTripper) RoundTrip(r *http.Request) (*http.Response, error)
159252
"source": i.source,
160253
"status_code": fmt.Sprintf("%d", statusCode),
161254
}).Inc()
255+
256+
// Handle any extra interceptors.
257+
for _, interceptor := range i.c.interceptors {
258+
interceptor(resp, err)
259+
}
162260
return resp, err
163261
}

0 commit comments

Comments
 (0)