From 31f60c3c0c2b09ec5738e0e00c7ef4bbc05e87e9 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Sat, 29 Dec 2018 13:05:35 +0800 Subject: [PATCH 1/4] Update observer api to support queries pagination --- cmd/cql-observer/api.go | 136 ++++++++++++++++++++++++++++++++++++++-- types/request_type.go | 4 +- 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/cmd/cql-observer/api.go b/cmd/cql-observer/api.go index 8e4c0c483..279cc4931 100644 --- a/cmd/cql-observer/api.go +++ b/cmd/cql-observer/api.go @@ -57,6 +57,40 @@ type explorerAPI struct { service *Service } +type paginationOps struct { + page int + size int + queryType types.QueryType +} + +func newPaginationFromReq(r *http.Request) (op *paginationOps) { + op = &paginationOps{} + op.page, _ = strconv.Atoi(r.URL.Query().Get("page")) + op.size, _ = strconv.Atoi(r.URL.Query().Get("size")) + if r.URL.Query().Get("type") == types.ReadQuery.String() { + op.queryType = types.ReadQuery + } else if r.URL.Query().Get("type") == types.WriteQuery.String() { + op.queryType = types.WriteQuery + } else { + op.queryType = types.NumberOfQueryType + } + return +} + +func (op *paginationOps) normalize() { + if op == nil { + return + } + if op.page <= 0 { + op.page = 1 + } + if op.size <= 0 { + op.size = 10 + } + + return +} + func (a *explorerAPI) GetAck(rw http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -175,7 +209,9 @@ func (a *explorerAPI) GetBlockV3(rw http.ResponseWriter, r *http.Request) { return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetBlockByCount(rw http.ResponseWriter, r *http.Request) { @@ -239,7 +275,9 @@ func (a *explorerAPI) GetBlockByCountV3(rw http.ResponseWriter, r *http.Request) return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetBlockByHeight(rw http.ResponseWriter, r *http.Request) { @@ -303,7 +341,9 @@ func (a *explorerAPI) GetBlockByHeightV3(rw http.ResponseWriter, r *http.Request return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetHighestBlock(rw http.ResponseWriter, r *http.Request) { @@ -396,7 +436,9 @@ func (a *explorerAPI) GetHighestBlockV3(rw http.ResponseWriter, r *http.Request) return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) formatBlock(height int32, b *types.Block) (res map[string]interface{}) { @@ -425,7 +467,8 @@ func (a *explorerAPI) formatBlockV2(count, height int32, b *types.Block) (res ma return } -func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block) (res map[string]interface{}) { +func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, + pagination *paginationOps) (res map[string]interface{}) { res = a.formatBlockV2(count, height, b) blockRes := res["block"].(map[string]interface{}) blockRes["acks"] = func() (acks []interface{}) { @@ -437,18 +480,101 @@ func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block) (res ma return }() + + if pagination != nil { + pagination.normalize() + } + blockRes["queries"] = func() (tracks []interface{}) { tracks = make([]interface{}, 0, len(b.QueryTxs)) + var ( + offset = (pagination.page - 1) * pagination.size + end = pagination.page * pagination.size + pos = 0 + ) + for _, tx := range b.QueryTxs { + if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) && + tx.Request.Header.QueryType != pagination.queryType { + // count all + continue + } + + if pos < offset { + // skip + continue + } + + if pos >= end { + return + } + t := a.formatRequest(tx.Request) t["response"] = a.formatResponseHeader(tx.Response)["response"] + t["failed"] = false + tracks = append(tracks, t) + + pos++ + } + + for _, req := range b.FailedReqs { + if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) && + req.Header.QueryType != pagination.queryType { + // count all + continue + } + + if pos < offset { + // skip + continue + } + + if pos >= end { + break + } + + t := a.formatRequest(req) + t["failed"] = true tracks = append(tracks, t) + + pos++ } return }() + if pagination != nil { + blockRes["pagination"] = func() (res map[string]interface{}) { + // pagination features + res = map[string]interface{}{} + res["page"] = pagination.page + res["size"] = pagination.size + + if pagination.queryType != types.ReadQuery && pagination.queryType != types.WriteQuery { + res["total"] = len(b.QueryTxs) + len(b.FailedReqs) + } else { + var total int + + for _, tx := range b.QueryTxs { + if tx.Request.Header.QueryType == pagination.queryType { + total++ + } + } + + for _, req := range b.FailedReqs { + if req.Header.QueryType == pagination.queryType { + total++ + } + } + + res["total"] = total + } + + return + }() + } + return } diff --git a/types/request_type.go b/types/request_type.go index bf280a2f0..241d7c982 100644 --- a/types/request_type.go +++ b/types/request_type.go @@ -36,8 +36,8 @@ const ( ReadQuery QueryType = iota // WriteQuery defines a write query type. WriteQuery - // NumberOfPerm defines the number of query type. - NumberOfPerm + // NumberOfQueryType defines the number of query type. + NumberOfQueryType ) // NamedArg defines the named argument structure for database. From d09e8ba3193338571644a4e2417c56973a64ba3e Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Sat, 29 Dec 2018 16:12:41 +0800 Subject: [PATCH 2/4] Update observer pagination logic --- cmd/cql-observer/api.go | 46 +++++------------------ cmd/cql-observer/observation_test.go | 56 ++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/cmd/cql-observer/api.go b/cmd/cql-observer/api.go index 279cc4931..6738d97b6 100644 --- a/cmd/cql-observer/api.go +++ b/cmd/cql-observer/api.go @@ -74,20 +74,12 @@ func newPaginationFromReq(r *http.Request) (op *paginationOps) { } else { op.queryType = types.NumberOfQueryType } - return -} - -func (op *paginationOps) normalize() { - if op == nil { - return - } if op.page <= 0 { op.page = 1 } if op.size <= 0 { op.size = 10 } - return } @@ -471,22 +463,8 @@ func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, pagination *paginationOps) (res map[string]interface{}) { res = a.formatBlockV2(count, height, b) blockRes := res["block"].(map[string]interface{}) - blockRes["acks"] = func() (acks []interface{}) { - acks = make([]interface{}, 0, len(b.Acks)) - - for _, ack := range b.Acks { - acks = append(acks, a.formatAck(ack)["ack"]) - } - - return - }() - - if pagination != nil { - pagination.normalize() - } - blockRes["queries"] = func() (tracks []interface{}) { - tracks = make([]interface{}, 0, len(b.QueryTxs)) + tracks = make([]interface{}, 0, len(b.QueryTxs)+len(b.FailedReqs)) var ( offset = (pagination.page - 1) * pagination.size @@ -501,11 +479,6 @@ func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, continue } - if pos < offset { - // skip - continue - } - if pos >= end { return } @@ -513,7 +486,10 @@ func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, t := a.formatRequest(tx.Request) t["response"] = a.formatResponseHeader(tx.Response)["response"] t["failed"] = false - tracks = append(tracks, t) + + if pos >= offset { + tracks = append(tracks, t) + } pos++ } @@ -525,18 +501,16 @@ func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, continue } - if pos < offset { - // skip - continue - } - if pos >= end { - break + return } t := a.formatRequest(req) t["failed"] = true - tracks = append(tracks, t) + + if pos >= offset { + tracks = append(tracks, t) + } pos++ } diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index 99d739d4f..d601b759e 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -422,20 +422,54 @@ func TestFullProcess(t *testing.T) { genesisHash := ensureSuccess(res.String("block", "hash")).(string) // test get first containable block - res, err = getJSON("v3/height/%v/1", dbID) - So(err, ShouldBeNil) - So(ensureSuccess(res.Interface("block")), ShouldNotBeNil) - So(ensureSuccess(res.Int("block", "height")), ShouldEqual, 1) - So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty) - So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash) - So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty) - blockHash := ensureSuccess(res.String("block", "hash")).(string) - byHeightBlockResult := ensureSuccess(res.Interface()) + var ( + blockHash string + byHeightBlockResult interface{} + ) + + // access 5 blocks + for i := 1; i <= 5; i++ { + res, err = getJSON("v3/height/%v/%d", dbID, i) + So(err, ShouldBeNil) + So(ensureSuccess(res.Interface("block")), ShouldNotBeNil) + So(ensureSuccess(res.Int("block", "height")), ShouldEqual, i) + So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty) + So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash) + if len(ensureSuccess(res.ArrayOfObjects("block", "queries")).([]map[string]interface{})) == 0 { + // got empty block + log.WithField("block", res).Debugf("got empty block, try next index") + continue + } + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty) + blockHash = ensureSuccess(res.String("block", "hash")).(string) + byHeightBlockResult = ensureSuccess(res.Interface()) + } // test get block by hash - res, err = getJSON("v3/block/%v/%v", dbID, blockHash) + res, err = getJSON("v3/block/%v/%v?size=1000", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldResemble, + ensureSuccess(jsonq.NewQuery(byHeightBlockResult).ArrayOfObjects("block", "queries"))) + + // test get block by hash v3 with pagination + res, err = getJSON("v3/block/%v/%v?page=10000&size=10", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldBeEmpty) + + // test get block with page size = 1 + res, err = getJSON("v3/block/%v/%v?page=1&size=1", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1) + + // test get block with page size = 2 + res, err = getJSON("v3/block/%v/%v?page=1&size=2", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 2) + + // test get block with page size = 1, page = 2 + res, err = getJSON("v3/block/%v/%v?page=2&size=1", dbID, blockHash) So(err, ShouldBeNil) - So(ensureSuccess(res.Interface()), ShouldResemble, byHeightBlockResult) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1) // test get block by hash using v1 version, returns ack hashes as queries res, err = getJSON("v1/block/%v/%v", dbID, blockHash) From 2759228329df40818641d642f9711b41bf5476fc Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Sat, 29 Dec 2018 16:26:35 +0800 Subject: [PATCH 3/4] Break get block info on first non empty block --- cmd/cql-observer/observation_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index d601b759e..54c6e1e2d 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -443,6 +443,7 @@ func TestFullProcess(t *testing.T) { So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty) blockHash = ensureSuccess(res.String("block", "hash")).(string) byHeightBlockResult = ensureSuccess(res.Interface()) + break } // test get block by hash From 0c55ba6d595458acf2a067ac9ef4997928425952 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Sat, 29 Dec 2018 16:29:34 +0800 Subject: [PATCH 4/4] Revert back codecov comment config --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index f663a9238..b223a6cc2 100644 --- a/codecov.yml +++ b/codecov.yml @@ -24,7 +24,7 @@ parsers: comment: layout: "header, diff" - behavior: new + behavior: default require_changes: no ignore: