Skip to content

Commit 2c7adcc

Browse files
bergquisttorkelo
bergquist
authored and
torkelo
committed
Use cache for http.client in tsdb package. (grafana#6833)
* datasource: move caching closer to datasource struct * tsdb: use cached version of datasource http transport closes grafana#6825
1 parent 24172fc commit 2c7adcc

File tree

20 files changed

+385
-363
lines changed

20 files changed

+385
-363
lines changed

‎pkg/api/dataproxy.go

Lines changed: 1 addition & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
package api
22

33
import (
4-
"crypto/tls"
5-
"crypto/x509"
6-
"net"
74
"net/http"
85
"net/http/httputil"
96
"net/url"
10-
"sync"
117
"time"
128

139
"github.com/grafana/grafana/pkg/api/cloudwatch"
@@ -19,75 +15,6 @@ import (
1915
"github.com/grafana/grafana/pkg/util"
2016
)
2117

22-
type proxyTransportCache struct {
23-
cache map[int64]cachedTransport
24-
sync.Mutex
25-
}
26-
27-
type cachedTransport struct {
28-
updated time.Time
29-
30-
*http.Transport
31-
}
32-
33-
var ptc = proxyTransportCache{
34-
cache: make(map[int64]cachedTransport),
35-
}
36-
37-
func DataProxyTransport(ds *m.DataSource) (*http.Transport, error) {
38-
ptc.Lock()
39-
defer ptc.Unlock()
40-
41-
if t, present := ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) {
42-
return t.Transport, nil
43-
}
44-
45-
transport := &http.Transport{
46-
TLSClientConfig: &tls.Config{
47-
InsecureSkipVerify: true,
48-
},
49-
Proxy: http.ProxyFromEnvironment,
50-
Dial: (&net.Dialer{
51-
Timeout: 30 * time.Second,
52-
KeepAlive: 30 * time.Second,
53-
}).Dial,
54-
TLSHandshakeTimeout: 10 * time.Second,
55-
}
56-
57-
var tlsAuth, tlsAuthWithCACert bool
58-
if ds.JsonData != nil {
59-
tlsAuth = ds.JsonData.Get("tlsAuth").MustBool(false)
60-
tlsAuthWithCACert = ds.JsonData.Get("tlsAuthWithCACert").MustBool(false)
61-
}
62-
63-
if tlsAuth {
64-
transport.TLSClientConfig.InsecureSkipVerify = false
65-
66-
decrypted := ds.SecureJsonData.Decrypt()
67-
68-
if tlsAuthWithCACert && len(decrypted["tlsCACert"]) > 0 {
69-
caPool := x509.NewCertPool()
70-
ok := caPool.AppendCertsFromPEM([]byte(decrypted["tlsCACert"]))
71-
if ok {
72-
transport.TLSClientConfig.RootCAs = caPool
73-
}
74-
}
75-
76-
cert, err := tls.X509KeyPair([]byte(decrypted["tlsClientCert"]), []byte(decrypted["tlsClientKey"]))
77-
if err != nil {
78-
return nil, err
79-
}
80-
transport.TLSClientConfig.Certificates = []tls.Certificate{cert}
81-
}
82-
83-
ptc.cache[ds.Id] = cachedTransport{
84-
Transport: transport,
85-
updated: ds.Updated,
86-
}
87-
88-
return transport, nil
89-
}
90-
9118
func NewReverseProxy(ds *m.DataSource, proxyPath string, targetUrl *url.URL) *httputil.ReverseProxy {
9219
director := func(req *http.Request) {
9320
req.URL.Scheme = targetUrl.Scheme
@@ -189,7 +116,7 @@ func ProxyDataSourceRequest(c *middleware.Context) {
189116
}
190117

191118
proxy := NewReverseProxy(ds, proxyPath, targetUrl)
192-
proxy.Transport, err = DataProxyTransport(ds)
119+
proxy.Transport, err = ds.GetHttpTransport()
193120
if err != nil {
194121
c.JsonApiErr(400, "Unable to load TLS certificate", err)
195122
return

‎pkg/api/dataproxy_test.go

Lines changed: 1 addition & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,18 @@ import (
44
"net/http"
55
"net/url"
66
"testing"
7-
"time"
87

98
. "github.com/smartystreets/goconvey/convey"
109

11-
"github.com/grafana/grafana/pkg/components/simplejson"
1210
m "github.com/grafana/grafana/pkg/models"
13-
"github.com/grafana/grafana/pkg/setting"
14-
"github.com/grafana/grafana/pkg/util"
1511
)
1612

1713
func TestDataSourceProxy(t *testing.T) {
18-
1914
Convey("When getting graphite datasource proxy", t, func() {
20-
clearCache()
2115
ds := m.DataSource{Url: "htttp://graphite:8080", Type: m.DS_GRAPHITE}
2216
targetUrl, err := url.Parse(ds.Url)
2317
proxy := NewReverseProxy(&ds, "/render", targetUrl)
24-
proxy.Transport, err = DataProxyTransport(&ds)
18+
proxy.Transport, err = ds.GetHttpTransport()
2519
So(err, ShouldBeNil)
2620

2721
transport, ok := proxy.Transport.(*http.Transport)
@@ -40,7 +34,6 @@ func TestDataSourceProxy(t *testing.T) {
4034
})
4135

4236
Convey("When getting influxdb datasource proxy", t, func() {
43-
clearCache()
4437
ds := m.DataSource{
4538
Type: m.DS_INFLUXDB_08,
4639
Url: "http://influxdb:8083",
@@ -67,148 +60,4 @@ func TestDataSourceProxy(t *testing.T) {
6760
So(queryVals["p"][0], ShouldEqual, "password")
6861
})
6962
})
70-
71-
Convey("When caching a datasource proxy", t, func() {
72-
clearCache()
73-
ds := m.DataSource{
74-
Id: 1,
75-
Url: "http://k8s:8001",
76-
Type: "Kubernetes",
77-
}
78-
79-
t1, err := DataProxyTransport(&ds)
80-
So(err, ShouldBeNil)
81-
82-
t2, err := DataProxyTransport(&ds)
83-
So(err, ShouldBeNil)
84-
85-
Convey("Should be using the cached proxy", func() {
86-
So(t2, ShouldEqual, t1)
87-
})
88-
})
89-
90-
Convey("When getting kubernetes datasource proxy", t, func() {
91-
clearCache()
92-
setting.SecretKey = "password"
93-
94-
json := simplejson.New()
95-
json.Set("tlsAuth", true)
96-
json.Set("tlsAuthWithCACert", true)
97-
98-
t := time.Now()
99-
ds := m.DataSource{
100-
Url: "http://k8s:8001",
101-
Type: "Kubernetes",
102-
Updated: t.Add(-2 * time.Minute),
103-
}
104-
105-
transport, err := DataProxyTransport(&ds)
106-
So(err, ShouldBeNil)
107-
108-
Convey("Should have no cert", func() {
109-
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, true)
110-
})
111-
112-
ds.JsonData = json
113-
ds.SecureJsonData = map[string][]byte{
114-
"tlsCACert": util.Encrypt([]byte(caCert), "password"),
115-
"tlsClientCert": util.Encrypt([]byte(clientCert), "password"),
116-
"tlsClientKey": util.Encrypt([]byte(clientKey), "password"),
117-
}
118-
ds.Updated = t.Add(-1 * time.Minute)
119-
120-
transport, err = DataProxyTransport(&ds)
121-
So(err, ShouldBeNil)
122-
123-
Convey("Should add cert", func() {
124-
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, false)
125-
So(len(transport.TLSClientConfig.Certificates), ShouldEqual, 1)
126-
})
127-
128-
ds.JsonData = nil
129-
ds.SecureJsonData = map[string][]byte{}
130-
ds.Updated = t
131-
132-
transport, err = DataProxyTransport(&ds)
133-
So(err, ShouldBeNil)
134-
135-
Convey("Should remove cert", func() {
136-
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, true)
137-
So(len(transport.TLSClientConfig.Certificates), ShouldEqual, 0)
138-
})
139-
})
140-
14163
}
142-
143-
func clearCache() {
144-
ptc.Lock()
145-
defer ptc.Unlock()
146-
147-
ptc.cache = make(map[int64]cachedTransport)
148-
}
149-
150-
const caCert string = `-----BEGIN CERTIFICATE-----
151-
MIIDATCCAemgAwIBAgIJAMQ5hC3CPDTeMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV
152-
BAMMDGNhLWs4cy1zdGhsbTAeFw0xNjEwMjcwODQyMjdaFw00NDAzMTQwODQyMjda
153-
MBcxFTATBgNVBAMMDGNhLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQADggEP
154-
ADCCAQoCggEBAMLe2AmJ6IleeUt69vgNchOjjmxIIxz5sp1vFu94m1vUip7CqnOg
155-
QkpUsHeBPrGYv8UGloARCL1xEWS+9FVZeXWQoDmbC0SxXhFwRIESNCET7Q8KMi/4
156-
4YPvnMLGZi3Fjwxa8BdUBCN1cx4WEooMVTWXm7RFMtZgDfuOAn3TNXla732sfT/d
157-
1HNFrh48b0wA+HhmA3nXoBnBEblA665hCeo7lIAdRr0zJxJpnFnWXkyTClsAUTMN
158-
iL905LdBiiIRenojipfKXvMz88XSaWTI7JjZYU3BvhyXndkT6f12cef3I96NY3WJ
159-
0uIK4k04WrbzdYXMU3rN6NqlvbHqnI+E7aMCAwEAAaNQME4wHQYDVR0OBBYEFHHx
160-
2+vSPw9bECHj3O51KNo5VdWOMB8GA1UdIwQYMBaAFHHx2+vSPw9bECHj3O51KNo5
161-
VdWOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH2eV5NcV3LBJHs9
162-
I+adbiTPg2vyumrGWwy73T0X8Dtchgt8wU7Q9b9Ucg2fOTmSSyS0iMqEu1Yb2ORB
163-
CknM9mixHC9PwEBbkGCom3VVkqdLwSP6gdILZgyLoH4i8sTUz+S1yGPepi+Vzhs7
164-
adOXtryjcGnwft6HdfKPNklMOHFnjw6uqpho54oj/z55jUpicY/8glDHdrr1bh3k
165-
MHuiWLGewHXPvxfG6UoUx1te65IhifVcJGFZDQwfEmhBflfCmtAJlZEsgTLlBBCh
166-
FHoXIyGOdq1chmRVocdGBCF8fUoGIbuF14r53rpvcbEKtKnnP8+96luKAZLq0a4n
167-
3lb92xM=
168-
-----END CERTIFICATE-----`
169-
170-
const clientCert string = `-----BEGIN CERTIFICATE-----
171-
MIICsjCCAZoCCQCcd8sOfstQLzANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQDDAxj
172-
YS1rOHMtc3RobG0wHhcNMTYxMTAyMDkyNTE1WhcNMTcxMTAyMDkyNTE1WjAfMR0w
173-
GwYDVQQDDBRhZG0tZGFuaWVsLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQAD
174-
ggEPADCCAQoCggEBAOMliaWyNEUJKM37vWCl5bGub3lMicyRAqGQyY/qxD9yKKM2
175-
FbucVcmWmg5vvTqQVl5rlQ+c7GI8OD6ptmFl8a26coEki7bFr8bkpSyBSEc5p27b
176-
Z0ORFSqBHWHQbr9PkxPLYW6T3gZYUtRYv3OQgGxLXlvUh85n/mQfuR3N1FgmShHo
177-
GtAFi/ht6leXa0Ms+jNSDLCmXpJm1GIEqgyKX7K3+g3vzo9coYqXq4XTa8Efs2v8
178-
SCwqWfBC3rHfgs/5DLB8WT4Kul8QzxkytzcaBQfRfzhSV6bkgm7oTzt2/1eRRsf4
179-
YnXzLE9YkCC9sAn+Owzqf+TYC1KRluWDfqqBTJUCAwEAATANBgkqhkiG9w0BAQsF
180-
AAOCAQEAdMsZg6edWGC+xngizn0uamrUg1ViaDqUsz0vpzY5NWLA4MsBc4EtxWRP
181-
ueQvjUimZ3U3+AX0YWNLIrH1FCVos2jdij/xkTUmHcwzr8rQy+B17cFi+a8jtpgw
182-
AU6WWoaAIEhhbWQfth/Diz3mivl1ARB+YqiWca2mjRPLTPcKJEURDVddQ423el0Q
183-
4JNxS5icu7T2zYTYHAo/cT9zVdLZl0xuLxYm3asK1IONJ/evxyVZima3il6MPvhe
184-
58Hwz+m+HdqHxi24b/1J/VKYbISG4huOQCdLzeNXgvwFlGPUmHSnnKo1/KbQDAR5
185-
llG/Sw5+FquFuChaA6l5KWy7F3bQyA==
186-
-----END CERTIFICATE-----`
187-
188-
const clientKey string = `-----BEGIN RSA PRIVATE KEY-----
189-
MIIEpQIBAAKCAQEA4yWJpbI0RQkozfu9YKXlsa5veUyJzJECoZDJj+rEP3IoozYV
190-
u5xVyZaaDm+9OpBWXmuVD5zsYjw4Pqm2YWXxrbpygSSLtsWvxuSlLIFIRzmnbttn
191-
Q5EVKoEdYdBuv0+TE8thbpPeBlhS1Fi/c5CAbEteW9SHzmf+ZB+5Hc3UWCZKEega
192-
0AWL+G3qV5drQyz6M1IMsKZekmbUYgSqDIpfsrf6De/Oj1yhiperhdNrwR+za/xI
193-
LCpZ8ELesd+Cz/kMsHxZPgq6XxDPGTK3NxoFB9F/OFJXpuSCbuhPO3b/V5FGx/hi
194-
dfMsT1iQIL2wCf47DOp/5NgLUpGW5YN+qoFMlQIDAQABAoIBAQCzy4u312XeW1Cs
195-
Mx6EuOwmh59/ESFmBkZh4rxZKYgrfE5EWlQ7i5SwG4BX+wR6rbNfy6JSmHDXlTkk
196-
CKvvToVNcW6fYHEivDnVojhIERFIJ4+rhQmpBtcNLOQ3/4cZ8X/GxE6b+3lb5l+x
197-
64mnjPLKRaIr5/+TVuebEy0xNTJmjnJ7yiB2HRz7uXEQaVSk/P7KAkkyl/9J3/LM
198-
8N9AX1w6qDaNQZ4/P0++1H4SQenosM/b/GqGTomarEk/GE0NcB9rzmR9VCXa7FRh
199-
WV5jyt9vUrwIEiK/6nUnOkGO8Ei3kB7Y+e+2m6WdaNoU5RAfqXmXa0Q/a0lLRruf
200-
vTMo2WrBAoGBAPRaK4cx76Q+3SJ/wfznaPsMM06OSR8A3ctKdV+ip/lyKtb1W8Pz
201-
k8MYQDH7GwPtSu5QD8doL00pPjugZL/ba7X9nAsI+pinyEErfnB9y7ORNEjIYYzs
202-
DiqDKup7ANgw1gZvznWvb9Ge0WUSXvWS0pFkgootQAf+RmnnbWGH6l6RAoGBAO35
203-
aGUrLro5u9RD24uSXNU3NmojINIQFK5dHAT3yl0BBYstL43AEsye9lX95uMPTvOQ
204-
Cqcn42Hjp/bSe3n0ObyOZeXVrWcDFAfE0wwB1BkvL1lpgnFO9+VQORlH4w3Ppnpo
205-
jcPkR2TFeDaAYtvckhxe/Bk3OnuFmnsQ3VzM75fFAoGBAI6PvS2XeNU+yA3EtA01
206-
hg5SQ+zlHswz2TMuMeSmJZJnhY78f5mHlwIQOAPxGQXlf/4iP9J7en1uPpzTK3S0
207-
M9duK4hUqMA/w5oiIhbHjf0qDnMYVbG+V1V+SZ+cPBXmCDihKreGr5qBKnHpkfV8
208-
v9WL6o1rcRw4wiQvnaV1gsvBAoGBALtzVTczr6gDKCAIn5wuWy+cQSGTsBunjRLX
209-
xuVm5iEiV+KMYkPvAx/pKzMLP96lRVR3ptyKgAKwl7LFk3u50+zh4gQLr35QH2wL
210-
Lw7rNc3srAhrItPsFzqrWX6/cGuFoKYVS239l/sZzRppQPXcpb7xVvTp2whHcir0
211-
Wtnpl+TdAoGAGqKqo2KU3JoY3IuTDUk1dsNAm8jd9EWDh+s1x4aG4N79mwcss5GD
212-
FF8MbFPneK7xQd8L6HisKUDAUi2NOyynM81LAftPkvN6ZuUVeFDfCL4vCA0HUXLD
213-
+VrOhtUZkNNJlLMiVRJuQKUOGlg8PpObqYbstQAf/0/yFJMRHG82Tcg=
214-
-----END RSA PRIVATE KEY-----`

‎pkg/api/datasources.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,9 @@ import (
55

66
"github.com/grafana/grafana/pkg/api/dtos"
77
"github.com/grafana/grafana/pkg/bus"
8-
"github.com/grafana/grafana/pkg/plugins"
9-
//"github.com/grafana/grafana/pkg/log"
108
"github.com/grafana/grafana/pkg/middleware"
119
m "github.com/grafana/grafana/pkg/models"
10+
"github.com/grafana/grafana/pkg/plugins"
1211
"github.com/grafana/grafana/pkg/util"
1312
)
1413

‎pkg/api/metrics.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/grafana/grafana/pkg/api/dtos"
99
"github.com/grafana/grafana/pkg/metrics"
1010
"github.com/grafana/grafana/pkg/middleware"
11+
"github.com/grafana/grafana/pkg/models"
1112
"github.com/grafana/grafana/pkg/tsdb"
1213
"github.com/grafana/grafana/pkg/tsdb/testdata"
1314
"github.com/grafana/grafana/pkg/util"
@@ -25,9 +26,9 @@ func QueryMetrics(c *middleware.Context, reqDto dtos.MetricRequest) Response {
2526
MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
2627
IntervalMs: query.Get("intervalMs").MustInt64(1000),
2728
Model: query,
28-
DataSource: &tsdb.DataSourceInfo{
29-
Name: "Grafana TestDataDB",
30-
PluginId: "grafana-testdata-datasource",
29+
DataSource: &models.DataSource{
30+
Name: "Grafana TestDataDB",
31+
Type: "grafana-testdata-datasource",
3132
},
3233
})
3334
}

0 commit comments

Comments
 (0)