Skip to content

Commit d9cc4ca

Browse files
micahrigganmicahriggan
authored
Merge pull request bitpay#1639 from micahriggan/feature/node-paging
Feature/node paging
2 parents 017689c + 4aa41b7 commit d9cc4ca

File tree

11 files changed

+120
-46
lines changed

11 files changed

+120
-46
lines changed

‎package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
},
77
"scripts": {
88
"build": "docker build -t bitcore-node .",
9-
"postinstall": "./node_modules/.bin/lerna bootstrap"
9+
"postinstall": "./node_modules/.bin/lerna bootstrap",
10+
"insight": "cd packages/insight && npm start",
11+
"node": "cd packages/bitcore-node && npm start"
1012
},
1113
"devDependencies": {
1214
"eslint": "^4.19.1",

‎packages/bitcore-node/src/models/base.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ export abstract class BaseModel<T> {
66
client?: MongoClient;
77
db?: Db;
88

9+
// each model must implement an array of keys that are indexed, for paging
10+
abstract allowedPaging: Array<{
11+
type: 'string' | 'number' | 'date';
12+
key: keyof T;
13+
}>;
14+
915
constructor(private collectionName: string) {
1016
this.handleConnection();
1117
}

‎packages/bitcore-node/src/models/block.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ export class Block extends BaseModel<IBlock> {
3232
super('blocks');
3333
}
3434

35+
allowedPaging = [
36+
{
37+
key: 'height' as 'height',
38+
type: 'number' as 'number'
39+
}
40+
];
41+
3542
async onConnect() {
3643
this.collection.createIndex({ hash: 1 });
3744
this.collection.createIndex({ chain: 1, network: 1, processed: 1, height: -1 });

‎packages/bitcore-node/src/models/coin.ts

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ class Coin extends BaseModel<ICoin> {
2525
super('coins');
2626
}
2727

28+
allowedPaging = [
29+
{ key: 'mintHeight' as 'mintHeight', type: 'number' as 'number' },
30+
{ key: 'spentHeight' as 'spentHeight', type: 'number' as 'number' }
31+
];
32+
2833
onConnect() {
2934
this.collection.createIndex({ mintTxid: 1 });
3035
this.collection.createIndex(
@@ -56,7 +61,7 @@ class Coin extends BaseModel<ICoin> {
5661
}
5762

5863
_apiTransform(coin: ICoin, options: { object: boolean }) {
59-
let sbuf = coin.script ? coin.script.buffer: Buffer.from('');
64+
let sbuf = coin.script ? coin.script.buffer : Buffer.from('');
6065
const script = Chain[coin.chain].lib.Script.fromBuffer(sbuf);
6166
let transform = {
6267
txid: coin.mintTxid,
@@ -66,7 +71,7 @@ class Coin extends BaseModel<ICoin> {
6671
address: coin.address,
6772
script: {
6873
type: script.classify(),
69-
asm: script.toASM(),
74+
asm: script.toASM()
7075
},
7176
value: coin.value
7277
};

‎packages/bitcore-node/src/models/transaction.ts

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ export class Transaction extends BaseModel<ITransaction> {
3232
super('transactions');
3333
}
3434

35+
allowedPaging = [{ key: 'blockHeight' as 'blockHeight', type: 'number' as 'number' }];
36+
3537
onConnect() {
3638
this.collection.createIndex({ txid: 1 });
3739
this.collection.createIndex({ chain: 1, network: 1, blockHeight: 1 });
@@ -155,12 +157,14 @@ export class Transaction extends BaseModel<ITransaction> {
155157
let mintOps = new Array<any>();
156158
let parentChainCoins = new Array<ICoin>();
157159
if (parentChain && forkHeight && height < forkHeight) {
158-
parentChainCoins = await CoinModel.collection.find({
159-
chain: parentChain,
160-
network,
161-
mintHeight: height,
162-
spentHeight: { $gt: -2, $lt: forkHeight }
163-
}).toArray();
160+
parentChainCoins = await CoinModel.collection
161+
.find({
162+
chain: parentChain,
163+
network,
164+
mintHeight: height,
165+
spentHeight: { $gt: -2, $lt: forkHeight }
166+
})
167+
.toArray();
164168
}
165169
for (let tx of txs) {
166170
tx._hash = tx.hash;

‎packages/bitcore-node/src/models/wallet.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export class Wallet extends BaseModel<IWallet> {
1717
constructor() {
1818
super('wallets');
1919
}
20+
allowedPaging = [];
2021

2122
onConnect() {
2223
this.collection.createIndex({ pubKey: 1 });

‎packages/bitcore-node/src/models/walletAddress.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ export class WalletAddress extends BaseModel<IWalletAddress> {
1818
super('walletaddresses');
1919
}
2020

21+
allowedPaging = [];
22+
2123
onConnect() {
2224
this.collection.createIndex({ address: 1, wallet: 1 });
2325
}

‎packages/bitcore-node/src/providers/chain-state/internal/internal.ts

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,23 +57,22 @@ export class InternalStateProvider implements CSP.IChainStateService {
5757
}
5858

5959
streamBlocks(params: CSP.StreamBlocksParams) {
60-
const { stream, args } = params;
61-
const { limit = 10 } = args;
62-
const query = this.getBlocksQuery(params);
63-
Storage.apiStreamingFind(BlockModel, query, { limit, sort: { height: -1 } }, stream);
60+
const { stream } = params;
61+
const { query, options } = this.getBlocksQuery(params);
62+
Storage.apiStreamingFind(BlockModel, query, options, stream);
6463
}
6564

6665
async getBlocks(params: CSP.StreamBlocksParams) {
67-
const { args = {} } = params;
68-
const { limit = 10 } = args;
69-
const query = this.getBlocksQuery(params);
70-
let blocks = await BlockModel.collection.find(query, { limit, sort: { height: -1 } }).toArray();
66+
const { query, options } = this.getBlocksQuery(params);
67+
let blocks = await BlockModel.collection.find(query, options).toArray();
7168
return blocks.map(block => BlockModel._apiTransform(block, { object: true }));
7269
}
7370

7471
private getBlocksQuery(params: CSP.StreamBlocksParams) {
75-
const { network, sinceBlock, blockId, args = {}} = params;
76-
let { startDate, endDate, date } = args;
72+
const { network, sinceBlock, blockId, args = {} } = params;
73+
let { startDate, endDate, date, since, direction, paging } = args;
74+
let { limit = 10, sort = { height: -1 } } = args;
75+
let options = { limit, sort, since, direction, paging };
7776
if (!this.chain || !network) {
7877
throw 'Missing required param';
7978
}
@@ -112,7 +111,7 @@ export class InternalStateProvider implements CSP.IChainStateService {
112111
nextDate.setDate(nextDate.getDate() + 1);
113112
query.time = { $gt: firstDate, $lt: nextDate };
114113
}
115-
return query;
114+
return { query, options };
116115
}
117116

118117
async getBlock(params: CSP.StreamBlocksParams) {

‎packages/bitcore-node/src/routes/api/block.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ const router = require('express').Router({ mergeParams: true });
44

55
router.get('/', async function(req: Request, res: Response) {
66
let { chain, network } = req.params;
7-
const { sinceBlock, date, limit } = req.query;
7+
const { sinceBlock, date, limit, since, direction, paging } = req.query;
88
try {
99
let payload = {
1010
chain,
1111
network,
1212
sinceBlock,
13-
args: { date, limit },
13+
args: { date, limit, since, direction, paging },
1414
stream: res
1515
};
16-
return ChainStateProvider.streamBlocks(payload);;
16+
return ChainStateProvider.streamBlocks(payload);
1717
} catch (err) {
1818
return res.status(500).send(err);
1919
}
@@ -22,7 +22,7 @@ router.get('/', async function(req: Request, res: Response) {
2222
router.get('/tip', async function(req: Request, res: Response) {
2323
let { chain, network } = req.params;
2424
try {
25-
let tip = await ChainStateProvider.getBlock({chain, network});
25+
let tip = await ChainStateProvider.getBlock({ chain, network });
2626
return res.json(tip);
2727
} catch (err) {
2828
return res.status(500).send(err);

‎packages/bitcore-node/src/services/storage.ts

Lines changed: 66 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
import { EventEmitter } from 'events';
2-
import { Response } from "express";
3-
import { TransformableModel } from "../types/TransformableModel";
2+
import { Response } from 'express';
3+
import { TransformableModel } from '../types/TransformableModel';
44
import logger from '../logger';
55
import config from '../config';
6-
import { LoggifyClass } from "../decorators/Loggify";
7-
import { MongoClient, Db, FindOneOptions } from "mongodb";
8-
import "../models"
6+
import { LoggifyClass } from '../decorators/Loggify';
7+
import { MongoClient, Db, FindOneOptions } from 'mongodb';
8+
import '../models';
9+
10+
export type StreamingFindOptions<T> = Partial<{
11+
paging: keyof T;
12+
since: T[keyof T];
13+
direction: 1 | -1;
14+
limit: number;
15+
}> &
16+
FindOneOptions;
917

1018
@LoggifyClass
1119
export class StorageService {
@@ -20,13 +28,16 @@ export class StorageService {
2028
let { dbName, dbHost } = options;
2129
const connectUrl = `mongodb://${dbHost}/${dbName}?socketTimeoutMS=3600000&noDelay=true`;
2230
let attemptConnect = async () => {
23-
return MongoClient.connect(connectUrl, {
24-
keepAlive: 1,
25-
poolSize: config.maxPoolSize,
26-
/*
27-
*nativeParser: true
28-
*/
29-
});
31+
return MongoClient.connect(
32+
connectUrl,
33+
{
34+
keepAlive: 1,
35+
poolSize: config.maxPoolSize
36+
/*
37+
*nativeParser: true
38+
*/
39+
}
40+
);
3041
};
3142
let attempted = 0;
3243
let attemptConnectId = setInterval(async () => {
@@ -42,7 +53,7 @@ export class StorageService {
4253
attempted++;
4354
if (attempted > 5) {
4455
clearInterval(attemptConnectId);
45-
reject(new Error("Failed to connect to database"));
56+
reject(new Error('Failed to connect to database'));
4657
}
4758
}
4859
}, 5000);
@@ -51,12 +62,48 @@ export class StorageService {
5162

5263
stop() {}
5364

54-
apiStreamingFind<T>(
55-
model: TransformableModel<T>,
56-
query: any,
57-
options: FindOneOptions,
58-
res: Response
59-
) {
65+
validPagingProperty<T>(model: TransformableModel<T>, property: keyof T) {
66+
return model.allowedPaging.some((prop) => prop.key === property);
67+
}
68+
69+
/**
70+
* castForDb
71+
*
72+
* For a given model, return the typecasted value based on a key and the type associated with that key
73+
*/
74+
typecastForDb<T>(model: TransformableModel<T>, modelKey: keyof T, modelValue: T[keyof T]) {
75+
let typecastedValue = modelValue;
76+
if (modelKey) {
77+
let oldValue = modelValue as any;
78+
let optionsType = model.allowedPaging.find(prop => prop.key === modelKey);
79+
if (optionsType) {
80+
switch (optionsType.type) {
81+
case 'number':
82+
typecastedValue = Number(oldValue) as any;
83+
break;
84+
case 'string':
85+
typecastedValue = (oldValue || '').toString() as any;
86+
break;
87+
case 'date':
88+
typecastedValue = new Date(oldValue) as any;
89+
break;
90+
}
91+
}
92+
}
93+
return typecastedValue;
94+
}
95+
96+
apiStreamingFind<T>(model: TransformableModel<T>, query: any, options: StreamingFindOptions<T>, res: Response) {
97+
if (options.since !== undefined && options.paging && this.validPagingProperty(model, options.paging)) {
98+
options.since = this.typecastForDb(model, options.paging, options.since);
99+
if (options.direction && Number(options.direction) === 1) {
100+
query[options.paging] = { $gt: options.since };
101+
options.sort = { [options.paging]: 1 };
102+
} else {
103+
query[options.paging] = { $lt: options.since };
104+
options.sort = { [options.paging]: -1 };
105+
}
106+
}
60107
options.limit = Math.min(options.limit || 100, 1000);
61108
let cursor = model.collection.find(query, options).stream({
62109
transform: model._apiTransform

‎packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { IBlock } from '../../models/block';
22
import { Response } from 'express';
33
import { IWallet } from '../../models/wallet';
44
import { ChainNetwork } from '../../types/ChainNetwork';
5+
import { StreamingFindOptions } from "../../services/storage";
56
export declare namespace CSP {
67
export type StreamWalletTransactionsArgs = {
78
startBlock: number;
@@ -30,12 +31,12 @@ export declare namespace CSP {
3031
export type StreamBlocksParams = ChainNetwork & {
3132
blockId?: string;
3233
sinceBlock: number | string;
33-
args: Partial<{ limit: number; startDate: Date; endDate: Date; date: Date }>;
34+
args: Partial<{ startDate: Date; endDate: Date; date: Date;} & StreamingFindOptions<IBlock>>;
3435
stream: Response;
3536
};
3637
export type GetEstimateSmartFeeParams = ChainNetwork & {
3738
target: number;
38-
}
39+
};
3940
export type BroadcastTransactionParams = ChainNetwork & {
4041
rawTx: string;
4142
};
@@ -102,7 +103,7 @@ export declare namespace CSP {
102103
streamWalletAddresses(params: StreamWalletAddressesParams): any;
103104
streamWalletTransactions(params: StreamWalletTransactionsParams): any;
104105
streamWalletUtxos(params: StreamWalletUtxosParams): any;
105-
getCoinsForTx(params: {chain: string, network: string, txid: string }): Promise<any>;
106+
getCoinsForTx(params: { chain: string; network: string; txid: string }): Promise<any>;
106107
}
107108

108109
type ChainStateServices = { [key: string]: IChainStateService };

0 commit comments

Comments
 (0)