diff --git a/cmd/cql-observer/api.go b/cmd/cql-observer/api.go index 8e4c0c483..6738d97b6 100644 --- a/cmd/cql-observer/api.go +++ b/cmd/cql-observer/api.go @@ -57,6 +57,32 @@ type explorerAPI struct { service *Service } +type paginationOps struct { + page int + size int + queryType types.QueryType +} + +func newPaginationFromReq(r *http.Request) (op *paginationOps) { + op = &paginationOps{} + op.page, _ = strconv.Atoi(r.URL.Query().Get("page")) + op.size, _ = strconv.Atoi(r.URL.Query().Get("size")) + if r.URL.Query().Get("type") == types.ReadQuery.String() { + op.queryType = types.ReadQuery + } else if r.URL.Query().Get("type") == types.WriteQuery.String() { + op.queryType = types.WriteQuery + } else { + op.queryType = types.NumberOfQueryType + } + if op.page <= 0 { + op.page = 1 + } + if op.size <= 0 { + op.size = 10 + } + return +} + func (a *explorerAPI) GetAck(rw http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -175,7 +201,9 @@ func (a *explorerAPI) GetBlockV3(rw http.ResponseWriter, r *http.Request) { return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetBlockByCount(rw http.ResponseWriter, r *http.Request) { @@ -239,7 +267,9 @@ func (a *explorerAPI) GetBlockByCountV3(rw http.ResponseWriter, r *http.Request) return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetBlockByHeight(rw http.ResponseWriter, r *http.Request) { @@ -303,7 +333,9 @@ func (a *explorerAPI) GetBlockByHeightV3(rw http.ResponseWriter, r *http.Request return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) GetHighestBlock(rw http.ResponseWriter, r *http.Request) { @@ -396,7 +428,9 @@ func (a *explorerAPI) GetHighestBlockV3(rw http.ResponseWriter, r *http.Request) return } - sendResponse(200, true, "", a.formatBlockV3(count, height, block), rw) + op := newPaginationFromReq(r) + + sendResponse(200, true, "", a.formatBlockV3(count, height, block, op), rw) } func (a *explorerAPI) formatBlock(height int32, b *types.Block) (res map[string]interface{}) { @@ -425,30 +459,96 @@ func (a *explorerAPI) formatBlockV2(count, height int32, b *types.Block) (res ma return } -func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block) (res map[string]interface{}) { +func (a *explorerAPI) formatBlockV3(count, height int32, b *types.Block, + pagination *paginationOps) (res map[string]interface{}) { res = a.formatBlockV2(count, height, b) blockRes := res["block"].(map[string]interface{}) - blockRes["acks"] = func() (acks []interface{}) { - acks = make([]interface{}, 0, len(b.Acks)) - - for _, ack := range b.Acks { - acks = append(acks, a.formatAck(ack)["ack"]) - } - - return - }() blockRes["queries"] = func() (tracks []interface{}) { - tracks = make([]interface{}, 0, len(b.QueryTxs)) + tracks = make([]interface{}, 0, len(b.QueryTxs)+len(b.FailedReqs)) + + var ( + offset = (pagination.page - 1) * pagination.size + end = pagination.page * pagination.size + pos = 0 + ) for _, tx := range b.QueryTxs { + if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) && + tx.Request.Header.QueryType != pagination.queryType { + // count all + continue + } + + if pos >= end { + return + } + t := a.formatRequest(tx.Request) t["response"] = a.formatResponseHeader(tx.Response)["response"] - tracks = append(tracks, t) + t["failed"] = false + + if pos >= offset { + tracks = append(tracks, t) + } + + pos++ + } + + for _, req := range b.FailedReqs { + if (pagination.queryType == types.ReadQuery || pagination.queryType == types.WriteQuery) && + req.Header.QueryType != pagination.queryType { + // count all + continue + } + + if pos >= end { + return + } + + t := a.formatRequest(req) + t["failed"] = true + + if pos >= offset { + tracks = append(tracks, t) + } + + pos++ } return }() + if pagination != nil { + blockRes["pagination"] = func() (res map[string]interface{}) { + // pagination features + res = map[string]interface{}{} + res["page"] = pagination.page + res["size"] = pagination.size + + if pagination.queryType != types.ReadQuery && pagination.queryType != types.WriteQuery { + res["total"] = len(b.QueryTxs) + len(b.FailedReqs) + } else { + var total int + + for _, tx := range b.QueryTxs { + if tx.Request.Header.QueryType == pagination.queryType { + total++ + } + } + + for _, req := range b.FailedReqs { + if req.Header.QueryType == pagination.queryType { + total++ + } + } + + res["total"] = total + } + + return + }() + } + return } diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index 99d739d4f..54c6e1e2d 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -422,20 +422,55 @@ func TestFullProcess(t *testing.T) { genesisHash := ensureSuccess(res.String("block", "hash")).(string) // test get first containable block - res, err = getJSON("v3/height/%v/1", dbID) - So(err, ShouldBeNil) - So(ensureSuccess(res.Interface("block")), ShouldNotBeNil) - So(ensureSuccess(res.Int("block", "height")), ShouldEqual, 1) - So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty) - So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash) - So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty) - blockHash := ensureSuccess(res.String("block", "hash")).(string) - byHeightBlockResult := ensureSuccess(res.Interface()) + var ( + blockHash string + byHeightBlockResult interface{} + ) + + // access 5 blocks + for i := 1; i <= 5; i++ { + res, err = getJSON("v3/height/%v/%d", dbID, i) + So(err, ShouldBeNil) + So(ensureSuccess(res.Interface("block")), ShouldNotBeNil) + So(ensureSuccess(res.Int("block", "height")), ShouldEqual, i) + So(ensureSuccess(res.String("block", "hash")), ShouldNotBeEmpty) + So(ensureSuccess(res.String("block", "genesis_hash")), ShouldEqual, genesisHash) + if len(ensureSuccess(res.ArrayOfObjects("block", "queries")).([]map[string]interface{})) == 0 { + // got empty block + log.WithField("block", res).Debugf("got empty block, try next index") + continue + } + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldNotBeEmpty) + blockHash = ensureSuccess(res.String("block", "hash")).(string) + byHeightBlockResult = ensureSuccess(res.Interface()) + break + } // test get block by hash - res, err = getJSON("v3/block/%v/%v", dbID, blockHash) + res, err = getJSON("v3/block/%v/%v?size=1000", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldResemble, + ensureSuccess(jsonq.NewQuery(byHeightBlockResult).ArrayOfObjects("block", "queries"))) + + // test get block by hash v3 with pagination + res, err = getJSON("v3/block/%v/%v?page=10000&size=10", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldBeEmpty) + + // test get block with page size = 1 + res, err = getJSON("v3/block/%v/%v?page=1&size=1", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1) + + // test get block with page size = 2 + res, err = getJSON("v3/block/%v/%v?page=1&size=2", dbID, blockHash) + So(err, ShouldBeNil) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 2) + + // test get block with page size = 1, page = 2 + res, err = getJSON("v3/block/%v/%v?page=2&size=1", dbID, blockHash) So(err, ShouldBeNil) - So(ensureSuccess(res.Interface()), ShouldResemble, byHeightBlockResult) + So(ensureSuccess(res.ArrayOfObjects("block", "queries")), ShouldHaveLength, 1) // test get block by hash using v1 version, returns ack hashes as queries res, err = getJSON("v1/block/%v/%v", dbID, blockHash) diff --git a/codecov.yml b/codecov.yml index f663a9238..b223a6cc2 100644 --- a/codecov.yml +++ b/codecov.yml @@ -24,7 +24,7 @@ parsers: comment: layout: "header, diff" - behavior: new + behavior: default require_changes: no ignore: diff --git a/types/request_type.go b/types/request_type.go index bf280a2f0..241d7c982 100644 --- a/types/request_type.go +++ b/types/request_type.go @@ -36,8 +36,8 @@ const ( ReadQuery QueryType = iota // WriteQuery defines a write query type. WriteQuery - // NumberOfPerm defines the number of query type. - NumberOfPerm + // NumberOfQueryType defines the number of query type. + NumberOfQueryType ) // NamedArg defines the named argument structure for database.