Skip to content

Commit 362162d

Browse files
bergquistbergquist
committed
feat(mqe): add token cache
1 parent 69d6316 commit 362162d

File tree

14 files changed

+1536
-52
lines changed

14 files changed

+1536
-52
lines changed

‎pkg/tsdb/mqe/model_parser.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ import (
66
"github.com/grafana/grafana/pkg/tsdb"
77
)
88

9-
func NewQueryParser() *MQEQueryParser {
10-
return &MQEQueryParser{}
9+
func NewQueryParser() *QueryParser {
10+
return &QueryParser{}
1111
}
1212

13-
type MQEQueryParser struct{}
13+
type QueryParser struct{}
1414

15-
func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*MQEQuery, error) {
16-
query := &MQEQuery{TimeRange: queryContext.TimeRange}
15+
func (qp *QueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*Query, error) {
16+
query := &Query{TimeRange: queryContext.TimeRange}
1717
query.AddAppToAlias = model.Get("addAppToAlias").MustBool(false)
1818
query.AddHostToAlias = model.Get("addHostToAlias").MustBool(false)
1919
query.UseRawQuery = model.Get("rawQuery").MustBool(false)
@@ -22,11 +22,11 @@ func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSourc
2222
query.Apps = model.Get("apps").MustStringArray([]string{})
2323
query.Hosts = model.Get("hosts").MustStringArray([]string{})
2424

25-
var metrics []MQEMetric
25+
var metrics []Metric
2626
var err error
2727
for _, metricsObj := range model.Get("metrics").MustArray() {
2828
metricJson := simplejson.NewFromAny(metricsObj)
29-
var m MQEMetric
29+
var m Metric
3030

3131
m.Alias = metricJson.Get("alias").MustString("")
3232
m.Metric, err = metricJson.Get("metric").String()

‎pkg/tsdb/mqe/model_parser_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111

1212
func TestMQEQueryParser(t *testing.T) {
1313
Convey("MQE query parser", t, func() {
14-
parser := &MQEQueryParser{}
14+
parser := &QueryParser{}
1515

1616
dsInfo := &models.DataSource{JsonData: simplejson.New()}
1717
queryContext := &tsdb.QueryContext{}

‎pkg/tsdb/mqe/mqe.go

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,20 @@ import (
1212
"github.com/grafana/grafana/pkg/components/simplejson"
1313
"github.com/grafana/grafana/pkg/log"
1414
"github.com/grafana/grafana/pkg/models"
15+
"github.com/grafana/grafana/pkg/setting"
1516
"github.com/grafana/grafana/pkg/tsdb"
1617
)
1718

1819
/*
1920
TODO:
20-
* response serie names with wildcards
21-
* real caching
21+
* performance. outgoing requests in pararell.
22+
* frontend plugin. targetContainsTemplates
2223
*/
2324

2425
type MQEExecutor struct {
2526
*models.DataSource
26-
queryParser *MQEQueryParser
27-
responseParser *MQEResponseParser
27+
queryParser *QueryParser
28+
responseParser *ResponseParser
2829
httpClient *http.Client
2930
log log.Logger
3031
tokenClient *TokenClient
@@ -52,7 +53,7 @@ func init() {
5253

5354
type QueryToSend struct {
5455
RawQuery string
55-
QueryRef *MQEQuery
56+
QueryRef *Query
5657
}
5758

5859
func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult {
@@ -63,7 +64,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
6364
return result.WithError(err)
6465
}
6566

66-
var mqeQueries []*MQEQuery
67+
var mqeQueries []*Query
6768
for _, v := range queries {
6869
q, err := e.queryParser.Parse(v.Model, e.DataSource, queryContext)
6970
if err != nil {
@@ -82,9 +83,13 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
8283
rawQueries = append(rawQueries, queries...)
8384
}
8485

86+
e.log.Debug("Sending request", "url", e.DataSource.Url)
87+
8588
queryResult := &tsdb.QueryResult{}
8689
for _, v := range rawQueries {
87-
e.log.Info("Mqe executor", "query", v)
90+
if setting.Env == setting.DEV {
91+
e.log.Debug("Executing", "query", v)
92+
}
8893

8994
req, err := e.createRequest(v.RawQuery)
9095

@@ -108,7 +113,11 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
108113
}
109114

110115
func (e *MQEExecutor) createRequest(query string) (*http.Request, error) {
111-
u, _ := url.Parse(e.Url)
116+
u, err := url.Parse(e.Url)
117+
if err != nil {
118+
return nil, err
119+
}
120+
112121
u.Path = path.Join(u.Path, "query")
113122

114123
payload := simplejson.New()
@@ -131,6 +140,5 @@ func (e *MQEExecutor) createRequest(query string) (*http.Request, error) {
131140
req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword)
132141
}
133142

134-
e.log.Debug("Mqe request", "url", req.URL.String())
135143
return req, nil
136144
}

‎pkg/tsdb/mqe/response_parser.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313
"github.com/grafana/grafana/pkg/tsdb"
1414
)
1515

16-
func NewResponseParser() *MQEResponseParser {
17-
return &MQEResponseParser{
16+
func NewResponseParser() *ResponseParser {
17+
return &ResponseParser{
1818
log: log.New("tsdb.mqe"),
1919
}
2020
}
@@ -44,11 +44,11 @@ type MQESerie struct {
4444
Tagset map[string]string `json:"tagset"`
4545
}
4646

47-
type MQEResponseParser struct {
47+
type ResponseParser struct {
4848
log log.Logger
4949
}
5050

51-
func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (*tsdb.QueryResult, error) {
51+
func (parser *ResponseParser) Parse(res *http.Response, queryRef *Query) (*tsdb.QueryResult, error) {
5252
body, err := ioutil.ReadAll(res.Body)
5353
defer res.Body.Close()
5454
if err != nil {
@@ -63,12 +63,12 @@ func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (
6363
var data *MQEResponse = &MQEResponse{}
6464
err = json.Unmarshal(body, data)
6565
if err != nil {
66-
parser.log.Info("Failed to unmarshal mqe response", "error", err, "status", res.Status, "body", string(body))
66+
parser.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body))
6767
return nil, err
6868
}
6969

7070
if !data.Success {
71-
return nil, fmt.Errorf("MQE request failed.")
71+
return nil, fmt.Errorf("Request failed.")
7272
}
7373

7474
var series tsdb.TimeSeriesSlice

‎pkg/tsdb/mqe/response_parser_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func TestMQEResponseParser(t *testing.T) {
2020
parser := NewResponseParser()
2121

2222
Convey("Can parse response", func() {
23-
queryRef := &MQEQuery{
23+
queryRef := &Query{
2424
AddAppToAlias: true,
2525
AddHostToAlias: true,
2626
}

‎pkg/tsdb/mqe/token_client.go

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,25 @@ import (
88
"net/http"
99
"net/url"
1010
"path"
11+
"time"
1112

1213
"golang.org/x/net/context/ctxhttp"
1314

15+
"strconv"
16+
1417
"github.com/grafana/grafana/pkg/log"
1518
"github.com/grafana/grafana/pkg/models"
19+
"github.com/patrickmn/go-cache"
1620
)
1721

22+
var tokenCache *cache.Cache
23+
24+
func init() {
25+
tokenCache = cache.New(5*time.Minute, 30*time.Second)
26+
}
27+
1828
type TokenClient struct {
19-
tlog log.Logger
29+
log log.Logger
2030
Datasource *models.DataSource
2131
HttpClient *http.Client
2232
}
@@ -25,27 +35,30 @@ func NewTokenClient(datasource *models.DataSource) *TokenClient {
2535
httpClient, _ := datasource.GetHttpClient()
2636

2737
return &TokenClient{
28-
tlog: log.New("tsdb.mqe.tokenclient"),
38+
log: log.New("tsdb.mqe.tokenclient"),
2939
Datasource: datasource,
3040
HttpClient: httpClient,
3141
}
3242
}
3343

34-
var cache map[int64]*TokenBody = map[int64]*TokenBody{}
35-
36-
//Replace this stupid cache with internal cache from grafana master before merging
3744
func (client *TokenClient) GetTokenData(ctx context.Context) (*TokenBody, error) {
38-
_, excist := cache[client.Datasource.Id]
39-
if !excist {
40-
b, err := client.RequestTokenData(ctx)
41-
if err != nil {
42-
return nil, err
45+
key := strconv.FormatInt(client.Datasource.Id, 10)
46+
47+
item, found := tokenCache.Get(key)
48+
if found {
49+
if result, ok := item.(*TokenBody); ok {
50+
return result, nil
4351
}
52+
}
4453

45-
cache[client.Datasource.Id] = b
54+
b, err := client.RequestTokenData(ctx)
55+
if err != nil {
56+
return nil, err
4657
}
4758

48-
return cache[client.Datasource.Id], nil
59+
tokenCache.Set(key, b, cache.DefaultExpiration)
60+
61+
return b, nil
4962
}
5063

5164
func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, error) {
@@ -54,7 +67,7 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er
5467

5568
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
5669
if err != nil {
57-
client.tlog.Info("Failed to create request", "error", err)
70+
client.log.Info("Failed to create request", "error", err)
5871
}
5972

6073
res, err := ctxhttp.Do(ctx, client.HttpClient, req)
@@ -69,14 +82,14 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er
6982
}
7083

7184
if res.StatusCode/100 != 2 {
72-
client.tlog.Info("Request failed", "status", res.Status, "body", string(body))
85+
client.log.Info("Request failed", "status", res.Status, "body", string(body))
7386
return nil, fmt.Errorf("Request failed status: %v", res.Status)
7487
}
7588

7689
var result *TokenResponse
7790
err = json.Unmarshal(body, &result)
7891
if err != nil {
79-
client.tlog.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body))
92+
client.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body))
8093
return nil, err
8194
}
8295

‎pkg/tsdb/mqe/types.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ import (
1111
"github.com/grafana/grafana/pkg/tsdb"
1212
)
1313

14-
type MQEMetric struct {
14+
type Metric struct {
1515
Metric string
1616
Alias string
1717
}
1818

19-
type MQEQuery struct {
20-
Metrics []MQEMetric
19+
type Query struct {
20+
Metrics []Metric
2121
Hosts []string
2222
Apps []string
2323
AddAppToAlias bool
@@ -32,7 +32,7 @@ var (
3232
containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`)
3333
)
3434

35-
func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) {
35+
func (q *Query) Build(availableSeries []string) ([]QueryToSend, error) {
3636
var queriesToSend []QueryToSend
3737
where := q.buildWhereClause()
3838

@@ -90,7 +90,7 @@ func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) {
9090
return queriesToSend, nil
9191
}
9292

93-
func (q *MQEQuery) buildWhereClause() string {
93+
func (q *Query) buildWhereClause() string {
9494
hasApps := len(q.Apps) > 0
9595
hasHosts := len(q.Hosts) > 0
9696

‎pkg/tsdb/mqe/types_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ func TestWildcardExpansion(t *testing.T) {
2525

2626
Convey("Can expanding query", t, func() {
2727
Convey("Without wildcard series", func() {
28-
query := &MQEQuery{
29-
Metrics: []MQEMetric{
30-
MQEMetric{Metric: "os.cpu.3.idle", Alias: ""},
31-
MQEMetric{Metric: "os.cpu.2.idle", Alias: ""},
32-
MQEMetric{Metric: "os.cpu.1.idle", Alias: "cpu"},
28+
query := &Query{
29+
Metrics: []Metric{
30+
Metric{Metric: "os.cpu.3.idle", Alias: ""},
31+
Metric{Metric: "os.cpu.2.idle", Alias: ""},
32+
Metric{Metric: "os.cpu.1.idle", Alias: "cpu"},
3333
},
3434
Hosts: []string{"staples-lab-1", "staples-lab-2"},
3535
Apps: []string{"demoapp-1", "demoapp-2"},
@@ -47,9 +47,9 @@ func TestWildcardExpansion(t *testing.T) {
4747
})
4848

4949
Convey("Containg wildcard series", func() {
50-
query := &MQEQuery{
51-
Metrics: []MQEMetric{
52-
MQEMetric{Metric: "os.cpu*", Alias: ""},
50+
query := &Query{
51+
Metrics: []Metric{
52+
Metric{Metric: "os.cpu*", Alias: ""},
5353
},
5454
Hosts: []string{"staples-lab-1"},
5555
AddAppToAlias: false,

‎vendor/github.com/patrickmn/go-cache/CONTRIBUTORS

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/patrickmn/go-cache/LICENSE

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)