From ff4d9e8dc9ddac44ad8e038bcfde1b9c7977b2ad Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Mon, 28 Jan 2019 19:29:39 +0800 Subject: [PATCH 01/35] Add set/clear support in blockNode struct --- blockproducer/blocknode.go | 25 +++++++++++++++++++------ blockproducer/branch.go | 16 +++++++++------- blockproducer/chain.go | 6 +++--- blockproducer/chain_io.go | 9 +++------ blockproducer/chain_test.go | 4 ++-- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/blockproducer/blocknode.go b/blockproducer/blocknode.go index 8a0aaacc8..1a5d2eafa 100644 --- a/blockproducer/blocknode.go +++ b/blockproducer/blocknode.go @@ -17,6 +17,8 @@ package blockproducer import ( + "sync/atomic" + "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/types" ) @@ -27,12 +29,13 @@ type blockNode struct { count uint32 height uint32 // Cached fields for quick reference - hash hash.Hash - block *types.BPBlock + hash hash.Hash + txCount int + block atomic.Value } -func newBlockNode(h uint32, b *types.BPBlock, p *blockNode) *blockNode { - return &blockNode{ +func newBlockNode(h uint32, b *types.BPBlock, p *blockNode) (node *blockNode) { + node = &blockNode{ parent: p, count: func() uint32 { @@ -43,9 +46,19 @@ func newBlockNode(h uint32, b *types.BPBlock, p *blockNode) *blockNode { }(), height: h, - hash: b.SignedHeader.BlockHash, - block: b, + hash: b.SignedHeader.BlockHash, + txCount: len(b.Transactions), } + node.block.Store(b) + return +} + +func (n *blockNode) load() *types.BPBlock { + return n.block.Load().(*types.BPBlock) +} + +func (n *blockNode) clear() { + n.block.Store(nil) } // fetchNodeList returns the block node list within range (from, n.count] from node head n. diff --git a/blockproducer/branch.go b/blockproducer/branch.go index fcba6e5dc..d1a5d98f3 100644 --- a/blockproducer/branch.go +++ b/blockproducer/branch.go @@ -57,11 +57,12 @@ func newBranch( } // Apply new blocks to view and pool for _, bn := range list { - if len(bn.block.Transactions) > conf.MaxTransactionsPerBlock { + var block = bn.load() + if len(block.Transactions) > conf.MaxTransactionsPerBlock { return nil, ErrTooManyTransactionsInBlock } - for _, v := range bn.block.Transactions { + for _, v := range block.Transactions { var k = v.Hash() // Check in tx pool if _, ok := inst.unpacked[k]; ok { @@ -126,17 +127,18 @@ func (b *branch) addTx(tx pi.Transaction) { } func (b *branch) applyBlock(n *blockNode) (br *branch, err error) { - if !b.head.hash.IsEqual(n.block.ParentHash()) { + var block = n.load() + if !b.head.hash.IsEqual(block.ParentHash()) { err = ErrParentNotMatch return } var cpy = b.makeArena() - if len(n.block.Transactions) > conf.MaxTransactionsPerBlock { + if len(block.Transactions) > conf.MaxTransactionsPerBlock { return nil, ErrTooManyTransactionsInBlock } - for _, v := range n.block.Transactions { + for _, v := range block.Transactions { var k = v.Hash() // Check in tx pool if _, ok := cpy.unpacked[k]; ok { @@ -258,13 +260,13 @@ func (b *branch) sprint(from uint32) (buff string) { if i == 0 { var p = v.parent buff += fmt.Sprintf("* #%d:%d %s {%d}", - p.height, p.count, p.hash.Short(4), len(p.block.Transactions)) + p.height, p.count, p.hash.Short(4), p.txCount) } if d := v.height - v.parent.height; d > 1 { buff += fmt.Sprintf(" <-- (skip %d blocks)", d-1) } buff += fmt.Sprintf(" <-- #%d:%d %s {%d}", - v.height, v.count, v.hash.Short(4), len(v.block.Transactions)) + v.height, v.count, v.hash.Short(4), v.txCount) } return } diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 8b8def50a..21a6b2c6d 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -649,7 +649,7 @@ func (c *Chain) replaceAndSwitchToBranch( resultTxPool[k] = v } for _, b := range newIrres { - for _, tx := range b.block.Transactions { + for _, tx := range b.load().Transactions { if err := c.immutable.apply(tx); err != nil { log.WithError(err).Fatal("failed to apply block to immutable database") } @@ -680,7 +680,7 @@ func (c *Chain) replaceAndSwitchToBranch( sps = append(sps, addBlock(height, newBlock)) sps = append(sps, buildBlockIndex(height, newBlock)) for _, n := range newIrres { - sps = append(sps, deleteTxs(n.block.Transactions)) + sps = append(sps, deleteTxs(n.load().Transactions)) } if len(expiredTxs) > 0 { sps = append(sps, deleteTxs(expiredTxs)) @@ -726,7 +726,7 @@ func (c *Chain) replaceAndSwitchToBranch( // Clear transactions in each branch for _, b := range newIrres { for _, br := range c.branches { - br.clearPackedTxs(b.block.Transactions) + br.clearPackedTxs(b.load().Transactions) } } for _, br := range c.branches { diff --git a/blockproducer/chain_io.go b/blockproducer/chain_io.go index 7e0755dc2..c93ef536f 100644 --- a/blockproducer/chain_io.go +++ b/blockproducer/chain_io.go @@ -49,8 +49,7 @@ func (c *Chain) fetchLastIrreversibleBlock() ( b *types.BPBlock, count uint32, height uint32, err error, ) { var node = c.lastIrreversibleBlock() - if node.block != nil { - b = node.block + if b = node.load(); b != nil { height = node.height count = node.count return @@ -71,8 +70,7 @@ func (c *Chain) fetchBlockByHeight(h uint32) (b *types.BPBlock, count uint32, er return } // OK, and block is cached - if node.block != nil { - b = node.block + if b = node.load(); b != nil { count = node.count return } @@ -91,8 +89,7 @@ func (c *Chain) fetchBlockByCount(count uint32) (b *types.BPBlock, height uint32 return } // OK, and block is cached - if node.block != nil { - b = node.block + if b = node.load(); b != nil { height = node.height return } diff --git a/blockproducer/chain_test.go b/blockproducer/chain_test.go index 58fd88bf0..d048268f0 100644 --- a/blockproducer/chain_test.go +++ b/blockproducer/chain_test.go @@ -344,7 +344,7 @@ func TestChain(t *testing.T) { // Try to use the no-cache version var node = chain.headBranch.head.ancestorByCount(5) - node.block = nil // Clear cached block + node.clear() bl, count, err = chain.fetchBlockByHeight(node.height) So(err, ShouldBeNil) So(count, ShouldEqual, node.count) @@ -355,7 +355,7 @@ func TestChain(t *testing.T) { So(bl.BlockHash(), ShouldResemble, &node.hash) var irreBlock = chain.lastIrre.block - chain.lastIrre.block = nil // Clear cached block + node.clear() bl, count, height, err = chain.fetchLastIrreversibleBlock() So(err, ShouldBeNil) So(bl, ShouldResemble, irreBlock) From a2b8db4ada24c274093e56e49f54f0d98c053d19 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Mon, 28 Jan 2019 20:59:01 +0800 Subject: [PATCH 02/35] Fix bug: bad critical section for multiple values --- rpc/rpcutil.go | 6 ++++-- rpc/rpcutil_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 147a32726..f7a5adb90 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -182,7 +182,9 @@ func recordRPCCost(startTime time.Time, method string, err error) { // Optimistically, val will not be nil except the first Call of method // expvar uses sync.Map // So, we try it first without lock - if val = expvar.Get(name); val == nil { + val = expvar.Get(name) + valC = expvar.Get(nameC) + if val == nil || valC == nil { callRPCExpvarLock.Lock() val = expvar.Get(name) if val == nil { @@ -191,9 +193,9 @@ func recordRPCCost(startTime time.Time, method string, err error) { } callRPCExpvarLock.Unlock() val = expvar.Get(name) + valC = expvar.Get(nameC) } val.(mw.Metric).Add(costTime.Seconds()) - valC = expvar.Get(nameC) valC.(mw.Metric).Add(1) return } diff --git a/rpc/rpcutil_test.go b/rpc/rpcutil_test.go index 90a432095..ebde85893 100644 --- a/rpc/rpcutil_test.go +++ b/rpc/rpcutil_test.go @@ -18,6 +18,7 @@ package rpc import ( "context" + "fmt" "os" "path/filepath" "runtime" @@ -443,3 +444,28 @@ func BenchmarkPersistentCaller_Call(b *testing.B) { server.Stop() } + +func TestRecordRPCCost(t *testing.T) { + Convey("Bug: bad critical section for multiple values", t, func(c C) { + var ( + start = time.Now() + rounds = 1000 + concurrent = 10 + wg = &sync.WaitGroup{} + body = func(i int) { + defer func() { + c.So(recover(), ShouldBeNil) + wg.Done() + }() + recordRPCCost(start, fmt.Sprintf("M%d", i), nil) + } + ) + defer wg.Wait() + for i := 0; i < rounds; i++ { + for j := 0; j < concurrent; j++ { + wg.Add(1) + go body(i) + } + } + }) +} From 89bb26eae886a547a51a493f3108e03683601add Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 10:42:06 +0800 Subject: [PATCH 03/35] Remove the `New` method of config struct Since all needed fields are exported, just use its literal notation. --- blockproducer/config.go | 17 ----------------- cmd/cqld/bootstrap.go | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/blockproducer/config.go b/blockproducer/config.go index d6f5befb7..5722a16cb 100644 --- a/blockproducer/config.go +++ b/blockproducer/config.go @@ -52,20 +52,3 @@ type Config struct { Period time.Duration Tick time.Duration } - -// NewConfig creates new config. -func NewConfig(genesis *types.BPBlock, dataFile string, - server *rpc.Server, peers *proto.Peers, - nodeID proto.NodeID, period time.Duration, tick time.Duration) *Config { - config := Config{ - Mode: BPMode, - Genesis: genesis, - DataFile: dataFile, - Server: server, - Peers: peers, - NodeID: nodeID, - Period: period, - Tick: tick, - } - return &config -} diff --git a/cmd/cqld/bootstrap.go b/cmd/cqld/bootstrap.go index b93eb8617..370231b36 100644 --- a/cmd/cqld/bootstrap.go +++ b/cmd/cqld/bootstrap.go @@ -153,16 +153,16 @@ func runNode(nodeID proto.NodeID, listenAddr string) (err error) { // init main chain service log.Info("register main chain service rpc") - chainConfig := bp.NewConfig( - genesis, - conf.GConf.BP.ChainFileName, - server, - peers, - nodeID, - conf.GConf.BPPeriod, - conf.GConf.BPTick, - ) - chainConfig.Mode = mode + chainConfig := &bp.Config{ + Mode: mode, + Genesis: genesis, + DataFile: conf.GConf.BP.ChainFileName, + Server: server, + Peers: peers, + NodeID: nodeID, + Period: conf.GConf.BPPeriod, + Tick: conf.GConf.BPTick, + } chain, err := bp.NewChain(chainConfig) if err != nil { log.WithError(err).Error("init chain failed") From 27033445e8fa99e455e60ddcd26d45c8ba2e9f53 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 10:50:11 +0800 Subject: [PATCH 04/35] Format license --- blockproducer/branch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/blockproducer/branch.go b/blockproducer/branch.go index d1a5d98f3..346b6bea0 100644 --- a/blockproducer/branch.go +++ b/blockproducer/branch.go @@ -10,7 +10,8 @@ * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and * limitations under the License. + * See the License for the specific language governing permissions and + * limitations under the License. */ package blockproducer From 0857e5160fd957fb5d59c9cc5247f895c829dac4 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 10:51:19 +0800 Subject: [PATCH 05/35] Format imports --- blockproducer/chain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 21a6b2c6d..7b61d0c63 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/pkg/errors" mw "github.com/zserge/metric" pi "github.com/CovenantSQL/CovenantSQL/blockproducer/interfaces" @@ -41,7 +42,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/types" "github.com/CovenantSQL/CovenantSQL/utils/log" xi "github.com/CovenantSQL/CovenantSQL/xenomint/interfaces" - "github.com/pkg/errors" ) // Chain defines the main chain. From e70d0eb4fa7f8ed5f769381bec7c1f8eb5056dd2 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 10:51:46 +0800 Subject: [PATCH 06/35] Add cached block setting --- conf/limits.go | 5 +++++ conf/parameters.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/limits.go b/conf/limits.go index 8c12c08cb..3b72ff0ec 100644 --- a/conf/limits.go +++ b/conf/limits.go @@ -25,3 +25,8 @@ const ( // MaxTransactionsPerBlock defines the limit of transactions per block. MaxTransactionsPerBlock = 10000 ) + +// These limits will not cause inconsistency within certain range. +const ( + MaxCachedBlock = 1000 +) diff --git a/conf/parameters.go b/conf/parameters.go index 4fa2296ef..0025fda57 100644 --- a/conf/parameters.go +++ b/conf/parameters.go @@ -21,7 +21,7 @@ const ( DefaultConfirmThreshold = float64(2) / 3.0 ) -// This parameters will not cause inconsistency within certain range. +// These parameters will not cause inconsistency within certain range. const ( BPStartupRequiredReachableCount = 2 // NOTE: this includes myself ) From 4f51f5792ab79a23cca1ec2be0af552c2c4dc7d0 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 11:53:10 +0800 Subject: [PATCH 07/35] Fix test case and other minor fixes --- blockproducer/blocknode.go | 2 +- blockproducer/chain.go | 16 ++++++++-------- blockproducer/chain_test.go | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/blockproducer/blocknode.go b/blockproducer/blocknode.go index 1a5d2eafa..5d2577536 100644 --- a/blockproducer/blocknode.go +++ b/blockproducer/blocknode.go @@ -58,7 +58,7 @@ func (n *blockNode) load() *types.BPBlock { } func (n *blockNode) clear() { - n.block.Store(nil) + n.block.Store((*types.BPBlock)(nil)) } // fetchNodeList returns the block node list within range (from, n.count] from node head n. diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 7b61d0c63..db3ad1793 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -44,6 +44,10 @@ import ( xi "github.com/CovenantSQL/CovenantSQL/xenomint/interfaces" ) +func init() { + expvar.Publish("height", mw.NewCounter("5m1s")) +} + // Chain defines the main chain. type Chain struct { // Routine controlling components @@ -61,6 +65,7 @@ type Chain struct { pendingAddTxReqs chan *types.AddTxReq // The following fields are read-only in runtime address proto.AccountAddress + mode RunMode genesisTime time.Time period time.Duration tick time.Duration @@ -78,16 +83,10 @@ type Chain struct { headBranch *branch branches []*branch txPool map[hash.Hash]pi.Transaction - mode RunMode } // NewChain creates a new blockchain. func NewChain(cfg *Config) (c *Chain, err error) { - // Normally, NewChain() should only be called once in app. - // So, we just check expvar without a lock - if expvar.Get("height") == nil { - expvar.Publish("height", mw.NewGauge("5m1s")) - } return NewChainWithContext(context.Background(), cfg) } @@ -218,6 +217,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) pendingAddTxReqs: make(chan *types.AddTxReq), address: addr, + mode: cfg.Mode, genesisTime: cfg.Genesis.SignedHeader.Timestamp, period: cfg.Period, tick: cfg.Tick, @@ -235,8 +235,8 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) headBranch: head, branches: branches, txPool: txPool, - mode: cfg.Mode, } + expvar.Get("height").(mw.Metric).Add(float64(c.nextHeight)) log.WithFields(log.Fields{ "local": c.getLocalBPInfo(), "period": c.period, @@ -369,7 +369,6 @@ func (c *Chain) advanceNextHeight(now time.Time, d time.Duration) { }).Warn("too much time elapsed in the new period, skip this block") return } - expvar.Get("height").(mw.Metric).Add(float64(c.getNextHeight())) log.WithField("height", c.getNextHeight()).Info("producing a new block") if err := c.produceBlock(now); err != nil { log.WithField("now", now.Format(time.RFC3339Nano)).WithError(err).Errorln( @@ -895,6 +894,7 @@ func (c *Chain) isMyTurn() bool { // increaseNextHeight prepares the chain state for the next turn. func (c *Chain) increaseNextHeight() { + expvar.Get("height").(mw.Metric).Add(1) c.Lock() defer c.Unlock() c.nextHeight++ diff --git a/blockproducer/chain_test.go b/blockproducer/chain_test.go index d048268f0..954a22702 100644 --- a/blockproducer/chain_test.go +++ b/blockproducer/chain_test.go @@ -340,7 +340,7 @@ func TestChain(t *testing.T) { So(err, ShouldBeNil) So(count, ShouldEqual, chain.lastIrre.count) So(height, ShouldEqual, chain.lastIrre.height) - So(bl, ShouldResemble, chain.lastIrre.block) + So(bl, ShouldResemble, chain.lastIrre.load()) // Try to use the no-cache version var node = chain.headBranch.head.ancestorByCount(5) @@ -354,7 +354,7 @@ func TestChain(t *testing.T) { So(height, ShouldEqual, node.height) So(bl.BlockHash(), ShouldResemble, &node.hash) - var irreBlock = chain.lastIrre.block + var irreBlock = chain.lastIrre.load() node.clear() bl, count, height, err = chain.fetchLastIrreversibleBlock() So(err, ShouldBeNil) From e5bddc07d8ef3d1c10febf457294e2088f1912e3 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 11:57:36 +0800 Subject: [PATCH 08/35] Move limits to correct section --- conf/limits.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/limits.go b/conf/limits.go index 3b72ff0ec..5d835080b 100644 --- a/conf/limits.go +++ b/conf/limits.go @@ -17,9 +17,6 @@ package conf const ( - // MaxTxBroadcastTTL defines the TTL limit of a AddTx request broadcasting within the - // block producers. - MaxTxBroadcastTTL = 1 // MaxPendingTxsPerAccount defines the limit of pending transactions of one account. MaxPendingTxsPerAccount = 1000 // MaxTransactionsPerBlock defines the limit of transactions per block. @@ -28,5 +25,8 @@ const ( // These limits will not cause inconsistency within certain range. const ( - MaxCachedBlock = 1000 + // MaxTxBroadcastTTL defines the TTL limit of a AddTx request broadcasting within the + // block producers. + MaxTxBroadcastTTL = 1 + MaxCachedBlock = 1000 ) From 5f09562f318c7edb21b27836fd7ca2df805dc127 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 16:13:38 +0800 Subject: [PATCH 09/35] Add metric values --- blockproducer/chain.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index db3ad1793..f756da3f7 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -44,8 +44,16 @@ import ( xi "github.com/CovenantSQL/CovenantSQL/xenomint/interfaces" ) +// Metric keys +const ( + mwKeyHeight = "service:bp:height" + mwKeyTxPooled = "service:bp:pooled" + mwKeyTxConfirmed = "service:bp:confirmed" +) + func init() { - expvar.Publish("height", mw.NewCounter("5m1s")) + expvar.Publish(mwKeyTxPooled, mw.NewCounter("5m1m")) + expvar.Publish(mwKeyTxConfirmed, mw.NewCounter("5m1m")) } // Chain defines the main chain. @@ -236,7 +244,14 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) branches: branches, txPool: txPool, } - expvar.Get("height").(mw.Metric).Add(float64(c.nextHeight)) + + // NOTE(leventeliu): this implies that BP chain is a singleton, otherwise we will need + // independent metric key for each chain instance. + if expvar.Get(mwKeyHeight) == nil { + expvar.Publish(mwKeyHeight, mw.NewGauge(fmt.Sprintf("5m%.0fs", cfg.Period.Seconds()))) + } + expvar.Get(mwKeyHeight).(mw.Metric).Add(float64(c.head().height)) + log.WithFields(log.Fields{ "local": c.getLocalBPInfo(), "period": c.period, @@ -354,7 +369,11 @@ func (c *Chain) advanceNextHeight(now time.Time, d time.Duration) { "elapsed_seconds": elapsed.Seconds(), }).Info("enclosing current height and advancing to next height") - defer c.increaseNextHeight() + defer func() { + c.increaseNextHeight() + expvar.Get(mwKeyHeight).(mw.Metric).Add(float64(c.head().height)) + }() + // Skip if it's not my turn if c.mode == APINodeMode || !c.isMyTurn() { return @@ -500,7 +519,9 @@ func (c *Chain) processAddTxReq(addTxReq *types.AddTxReq) { // Add to tx pool if err = c.storeTx(tx); err != nil { le.WithError(err).Error("failed to add transaction") + return } + expvar.Get(mwKeyTxPooled).(mw.Metric).Add(1) } func (c *Chain) processTxs(ctx context.Context) { @@ -629,6 +650,7 @@ func (c *Chain) replaceAndSwitchToBranch( newIrres []*blockNode sps []storageProcedure up storageCallback + txCount int height = c.heightOfTime(newBlock.Timestamp()) resultTxPool = make(map[hash.Hash]pi.Transaction) @@ -648,6 +670,7 @@ func (c *Chain) replaceAndSwitchToBranch( resultTxPool[k] = v } for _, b := range newIrres { + txCount += b.txCount for _, tx := range b.load().Transactions { if err := c.immutable.apply(tx); err != nil { log.WithError(err).Fatal("failed to apply block to immutable database") @@ -738,7 +761,9 @@ func (c *Chain) replaceAndSwitchToBranch( // Write to immutable database and update cache if err = store(c.storage, sps, up); err != nil { c.immutable.clean() + return } + expvar.Get(mwKeyTxConfirmed).(mw.Metric).Add(float64(txCount)) // TODO(leventeliu): trigger ChainBus.Publish. // ... return @@ -894,7 +919,6 @@ func (c *Chain) isMyTurn() bool { // increaseNextHeight prepares the chain state for the next turn. func (c *Chain) increaseNextHeight() { - expvar.Get("height").(mw.Metric).Add(1) c.Lock() defer c.Unlock() c.nextHeight++ From 36e828458ef070125c852a9be568c61718c00429 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 29 Jan 2019 16:24:45 +0800 Subject: [PATCH 10/35] Format query strings --- blockproducer/storage.go | 94 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/blockproducer/storage.go b/blockproducer/storage.go index 414bed504..0fca97223 100644 --- a/blockproducer/storage.go +++ b/blockproducer/storage.go @@ -36,71 +36,71 @@ var ( ddls = [...]string{ // Chain state tables `CREATE TABLE IF NOT EXISTS "blocks" ( - "height" INT, - "hash" TEXT, - "parent" TEXT, - "encoded" BLOB, - UNIQUE ("hash") - );`, + "height" INT, + "hash" TEXT, + "parent" TEXT, + "encoded" BLOB, + UNIQUE ("hash") +);`, `CREATE TABLE IF NOT EXISTS "txPool" ( - "type" INT, - "hash" TEXT, - "encoded" BLOB, - UNIQUE ("hash") - );`, + "type" INT, + "hash" TEXT, + "encoded" BLOB, + UNIQUE ("hash") +);`, `CREATE TABLE IF NOT EXISTS "irreversible" ( - "id" INT, - "hash" TEXT, - UNIQUE ("id") - );`, + "id" INT, + "hash" TEXT, + UNIQUE ("id") +);`, // Meta state tables `CREATE TABLE IF NOT EXISTS "accounts" ( - "address" TEXT, - "encoded" BLOB, - UNIQUE ("address") - );`, + "address" TEXT, + "encoded" BLOB, + UNIQUE ("address") +);`, `CREATE TABLE IF NOT EXISTS "shardChain" ( - "address" TEXT, - "id" TEXT, - "encoded" BLOB, - UNIQUE ("address", "id") - );`, + "address" TEXT, + "id" TEXT, + "encoded" BLOB, + UNIQUE ("address", "id") +);`, `CREATE TABLE IF NOT EXISTS "provider" ( - "address" TEXT, - "encoded" BLOB, - UNIQUE ("address") - );`, + "address" TEXT, + "encoded" BLOB, + UNIQUE ("address") +);`, `CREATE TABLE IF NOT EXISTS "indexed_blocks" ( - "height" INTEGER PRIMARY KEY, - "hash" TEXT, - "timestamp" INTEGER, - "version" INTEGER, - "producer" TEXT, - "merkle_root" TEXT, - "parent" TEXT, - "tx_count" INTEGER - );`, + "height" INTEGER PRIMARY KEY, + "hash" TEXT, + "timestamp" INTEGER, + "version" INTEGER, + "producer" TEXT, + "merkle_root" TEXT, + "parent" TEXT, + "tx_count" INTEGER +);`, `CREATE INDEX IF NOT EXISTS "idx__indexed_blocks__hash" ON "indexed_blocks" ("hash");`, `CREATE INDEX IF NOT EXISTS "idx__indexed_blocks__timestamp" ON "indexed_blocks" ("timestamp" DESC);`, `CREATE TABLE IF NOT EXISTS "indexed_transactions" ( - "block_height" INTEGER, - "tx_index" INTEGER, - "hash" TEXT, - "block_hash" TEXT, - "timestamp" INTEGER, - "tx_type" INTEGER, - "address" TEXT, - "raw" TEXT, - PRIMARY KEY ("block_height", "tx_index") - );`, + "block_height" INTEGER, + "tx_index" INTEGER, + "hash" TEXT, + "block_hash" TEXT, + "timestamp" INTEGER, + "tx_type" INTEGER, + "address" TEXT, + "raw" TEXT, + PRIMARY KEY ("block_height", "tx_index") +);`, `CREATE INDEX IF NOT EXISTS "idx__indexed_transactions__hash" ON "indexed_transactions" ("hash");`, `CREATE INDEX IF NOT EXISTS "idx__indexed_transactions__block_hash" ON "indexed_transactions" ("block_hash");`, From be6b10d2fe0c29d05252365667cdf98f0d33ee06 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 14:27:58 +0800 Subject: [PATCH 11/35] Add block cache LRU list --- blockproducer/chain.go | 22 ++++++++++++++++++++++ blockproducer/config.go | 2 ++ 2 files changed, 24 insertions(+) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 2eb1a8163..0ebdcd86a 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -25,6 +25,7 @@ import ( "sync" "time" + lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" mw "github.com/zserge/metric" @@ -67,6 +68,8 @@ type Chain struct { // Other components storage xi.Storage chainBus chainbus.Bus + // This LRU object is only used for block cache control, do NOT use it for query. + blockCache *lru.Cache // Channels for incoming blocks and transactions pendingBlocks chan *types.BPBlock pendingAddTxReqs chan *types.AddTxReq @@ -110,6 +113,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) m uint32 st xi.Storage + cache *lru.Cache irre *blockNode heads []*blockNode immutable *metaState @@ -151,6 +155,18 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) } }() + // Create block cache + if cfg.blockCacheSize > conf.MaxCachedBlock { + cfg.blockCacheSize = conf.MaxCachedBlock + } + if cache, err = lru.NewWithEvict(cfg.blockCacheSize, func(key interface{}, value interface{}) { + if node, ok := value.(*blockNode); ok && node != nil { + node.clear() + } + }); err != nil { + return + } + // Create initial state from genesis block and store if !existed { var init = newMetaState() @@ -239,6 +255,8 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) storage: st, chainBus: bus, + blockCache: cache, + pendingBlocks: make(chan *types.BPBlock), pendingAddTxReqs: make(chan *types.AddTxReq), @@ -755,6 +773,10 @@ func (c *Chain) replaceAndSwitchToBranch( } // Update txPool to result txPool (packed and expired transactions cleared!) c.txPool = resultTxPool + // Register new irreversible blocks to LRU cache list + for _, b := range newIrres { + c.blockCache.Add(b.count, b) + } } // Write to immutable database and update cache diff --git a/blockproducer/config.go b/blockproducer/config.go index 5722a16cb..3fb1b2ee1 100644 --- a/blockproducer/config.go +++ b/blockproducer/config.go @@ -51,4 +51,6 @@ type Config struct { Period time.Duration Tick time.Duration + + blockCacheSize int } From adf6bbdefcd9dfe66a89812842cfd8b5723b73ec Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 14:52:08 +0800 Subject: [PATCH 12/35] Use including notation for fetchNodeList method Which allows the genesis block to be included by a normal function call. --- blockproducer/blocknode.go | 6 +++--- blockproducer/blocknode_test.go | 6 +++--- blockproducer/branch.go | 2 +- blockproducer/chain.go | 18 ++++++++++++++---- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/blockproducer/blocknode.go b/blockproducer/blocknode.go index 2124c71f0..bc3e56c48 100644 --- a/blockproducer/blocknode.go +++ b/blockproducer/blocknode.go @@ -61,12 +61,12 @@ func (n *blockNode) clear() { n.block.Store((*types.BPBlock)(nil)) } -// fetchNodeList returns the block node list within range (from, n.count] from node head n. +// fetchNodeList returns the block node list within range [from, n.count] from node head n. func (n *blockNode) fetchNodeList(from uint32) (bl []*blockNode) { - if n.count <= from { + if n.count < from { return } - bl = make([]*blockNode, n.count-from) + bl = make([]*blockNode, n.count-from+1) var iter = n for i := len(bl) - 1; i >= 0; i-- { bl[i] = iter diff --git a/blockproducer/blocknode_test.go b/blockproducer/blocknode_test.go index 09299abdc..8c72f5111 100644 --- a/blockproducer/blocknode_test.go +++ b/blockproducer/blocknode_test.go @@ -115,11 +115,11 @@ func TestBlockNode(t *testing.T) { So(n0.count, ShouldEqual, 0) So(n1.count, ShouldEqual, n0.count+1) - So(n0.fetchNodeList(0), ShouldBeEmpty) So(n0.fetchNodeList(1), ShouldBeEmpty) So(n0.fetchNodeList(2), ShouldBeEmpty) - So(n3.fetchNodeList(0), ShouldResemble, []*blockNode{n1, n2, n3}) - So(n4p.fetchNodeList(2), ShouldResemble, []*blockNode{n3p, n4p}) + So(n0.fetchNodeList(3), ShouldBeEmpty) + So(n3.fetchNodeList(1), ShouldResemble, []*blockNode{n1, n2, n3}) + So(n4p.fetchNodeList(3), ShouldResemble, []*blockNode{n3p, n4p}) So(n0.ancestor(1), ShouldBeNil) So(n3.ancestor(3), ShouldEqual, n3) diff --git a/blockproducer/branch.go b/blockproducer/branch.go index 346b6bea0..83db975b1 100644 --- a/blockproducer/branch.go +++ b/blockproducer/branch.go @@ -44,7 +44,7 @@ func newBranch( br *branch, err error, ) { var ( - list = headNode.fetchNodeList(baseNode.count) + list = headNode.fetchNodeList(baseNode.count + 1) inst = &branch{ head: headNode, preview: baseState.makeCopy(), diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 0ebdcd86a..2f59895ab 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -185,16 +185,26 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) } } - // Load from database and rebuild branches + // Load from database if irre, heads, immutable, txPool, ierr = loadDatabase(st); ierr != nil { err = errors.Wrap(ierr, "failed to load data from storage") return } - if persistedGenesis := irre.ancestorByCount(0); persistedGenesis == nil || + + // Check genesis block + var irreBlocks = irre.fetchNodeList(0) + if persistedGenesis := irreBlocks[0]; persistedGenesis == nil || !persistedGenesis.hash.IsEqual(cfg.Genesis.BlockHash()) { err = ErrGenesisHashNotMatch return } + + // Add blocks to LRU list + for _, v := range irreBlocks { + cache.Add(v.count, v) + } + + // Rebuild branches for _, v := range heads { log.WithFields(log.Fields{ "irre_hash": irre.hash.Short(4), @@ -680,7 +690,7 @@ func (c *Chain) replaceAndSwitchToBranch( // May have multiple new irreversible blocks here if peer list shrinks. May also have // no new irreversible block at all if peer list expands. lastIrre = newBranch.head.lastIrreversible(c.confirms) - newIrres = lastIrre.fetchNodeList(c.lastIrre.count) + newIrres = lastIrre.fetchNodeList(c.lastIrre.count + 1) // Apply irreversible blocks to create dirty map on immutable cache for k, v := range c.txPool { @@ -800,7 +810,7 @@ func (c *Chain) stat() { } else { buff += fmt.Sprintf("[%04d] ", i) } - buff += v.sprint(c.lastIrre.count) + buff += v.sprint(c.lastIrre.count + 1) log.WithFields(log.Fields{ "branch": buff, }).Info("runtime state") From cea0aee04d399072bf36451c50c2ee763cf1326c Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 15:08:44 +0800 Subject: [PATCH 13/35] Rearrange some variable declarations --- blockproducer/chain.go | 45 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 2f59895ab..dbbb3e563 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -103,14 +103,7 @@ func NewChain(cfg *Config) (c *Chain, err error) { // NewChainWithContext creates a new blockchain with context. func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) { var ( - existed bool - ierr error - - cld context.Context - ccl context.CancelFunc - l = uint32(len(cfg.Peers.Servers)) - t float64 - m uint32 + ierr error st xi.Storage cache *lru.Cache @@ -119,11 +112,10 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) immutable *metaState txPool map[hash.Hash]pi.Transaction - branches []*branch - br, head *branch - headIndex int + branches []*branch + headBranch *branch + headIndex int - pubKey *asymmetric.PublicKey addr proto.AccountAddress bpInfos []*blockProducerInfo localBPInfo *blockProducerInfo @@ -142,6 +134,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) } // Open storage + var existed bool if fi, err := os.Stat(cfg.DataFile); err == nil && fi.Mode().IsRegular() { existed = true } @@ -213,6 +206,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) "head_count": v.count, }).Debug("checking head") if v.hasAncestor(irre) { + var br *branch if br, ierr = newBranch(irre, v, immutable, txPool); ierr != nil { err = errors.Wrapf(ierr, "failed to rebuild branch with head %s", v.hash.Short(4)) return @@ -227,13 +221,14 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) // Set head branch for i, v := range branches { - if head == nil || v.head.count > head.head.count { + if headBranch == nil || v.head.count > headBranch.head.count { headIndex = i - head = v + headBranch = v } } // Get accountAddress + var pubKey *asymmetric.PublicKey if pubKey, err = kms.GetLocalPublicKey(); err != nil { return } @@ -242,18 +237,24 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) } // Setup peer list + var ( + l = uint32(len(cfg.Peers.Servers)) + + threshold float64 + needConfirms uint32 + ) if localBPInfo, bpInfos, err = buildBlockProducerInfos(cfg.NodeID, cfg.Peers, cfg.Mode == APINodeMode); err != nil { return } - if t = cfg.ConfirmThreshold; t <= 0.0 { - t = conf.DefaultConfirmThreshold + if threshold = cfg.ConfirmThreshold; threshold <= 0.0 { + threshold = conf.DefaultConfirmThreshold } - if m = uint32(math.Ceil(float64(l)*t + 1)); m > l { - m = l + if needConfirms = uint32(math.Ceil(float64(l)*threshold + 1)); needConfirms > l { + needConfirms = l } // create chain - cld, ccl = context.WithCancel(ctx) + var cld, ccl = context.WithCancel(ctx) c = &Chain{ ctx: cld, cancel: ccl, @@ -279,14 +280,14 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) bpInfos: bpInfos, localBPInfo: localBPInfo, localNodeID: cfg.NodeID, - confirms: m, - nextHeight: head.head.height + 1, + confirms: needConfirms, + nextHeight: headBranch.head.height + 1, offset: time.Duration(0), // TODO(leventeliu): initialize offset lastIrre: irre, immutable: immutable, headIndex: headIndex, - headBranch: head, + headBranch: headBranch, branches: branches, txPool: txPool, } From 1474ad36322eaf505520db98fe199b626283e82b Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 15:18:25 +0800 Subject: [PATCH 14/35] Remove unused ChainBus field --- blockproducer/chain.go | 50 ++++++++++++++++++++-------------------- blockproducer/rpc.go | 7 ------ blockproducer/storage.go | 4 ++-- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index dbbb3e563..1b6d20711 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -30,7 +30,6 @@ import ( mw "github.com/zserge/metric" pi "github.com/CovenantSQL/CovenantSQL/blockproducer/interfaces" - "github.com/CovenantSQL/CovenantSQL/chainbus" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto" "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" @@ -62,17 +61,21 @@ type Chain struct { ctx context.Context cancel context.CancelFunc wg *sync.WaitGroup + // RPC components server *rpc.Server caller *rpc.Caller + // Other components - storage xi.Storage - chainBus chainbus.Bus - // This LRU object is only used for block cache control, do NOT use it for query. + storage xi.Storage + // NOTE(leventeliu): this LRU object is only used for block cache control, + // do NOT read it in any case. blockCache *lru.Cache + // Channels for incoming blocks and transactions pendingBlocks chan *types.BPBlock pendingAddTxReqs chan *types.AddTxReq + // The following fields are read-only in runtime address proto.AccountAddress mode RunMode @@ -107,7 +110,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) st xi.Storage cache *lru.Cache - irre *blockNode + lastIrre *blockNode heads []*blockNode immutable *metaState txPool map[hash.Hash]pi.Transaction @@ -119,8 +122,6 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) addr proto.AccountAddress bpInfos []*blockProducerInfo localBPInfo *blockProducerInfo - - bus = chainbus.New() ) // Verify genesis block in config @@ -179,13 +180,13 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) } // Load from database - if irre, heads, immutable, txPool, ierr = loadDatabase(st); ierr != nil { + if lastIrre, heads, immutable, txPool, ierr = loadDatabase(st); ierr != nil { err = errors.Wrap(ierr, "failed to load data from storage") return } // Check genesis block - var irreBlocks = irre.fetchNodeList(0) + var irreBlocks = lastIrre.fetchNodeList(0) if persistedGenesis := irreBlocks[0]; persistedGenesis == nil || !persistedGenesis.hash.IsEqual(cfg.Genesis.BlockHash()) { err = ErrGenesisHashNotMatch @@ -200,14 +201,14 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) // Rebuild branches for _, v := range heads { log.WithFields(log.Fields{ - "irre_hash": irre.hash.Short(4), - "irre_count": irre.count, + "irre_hash": lastIrre.hash.Short(4), + "irre_count": lastIrre.count, "head_hash": v.hash.Short(4), "head_count": v.count, }).Debug("checking head") - if v.hasAncestor(irre) { + if v.hasAncestor(lastIrre) { var br *branch - if br, ierr = newBranch(irre, v, immutable, txPool); ierr != nil { + if br, ierr = newBranch(lastIrre, v, immutable, txPool); ierr != nil { err = errors.Wrapf(ierr, "failed to rebuild branch with head %s", v.hash.Short(4)) return } @@ -219,7 +220,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) return } - // Set head branch + // Select head branch for i, v := range branches { if headBranch == nil || v.head.count > headBranch.head.count { headIndex = i @@ -243,7 +244,9 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) threshold float64 needConfirms uint32 ) - if localBPInfo, bpInfos, err = buildBlockProducerInfos(cfg.NodeID, cfg.Peers, cfg.Mode == APINodeMode); err != nil { + if localBPInfo, bpInfos, err = buildBlockProducerInfos( + cfg.NodeID, cfg.Peers, cfg.Mode == APINodeMode, + ); err != nil { return } if threshold = cfg.ConfirmThreshold; threshold <= 0.0 { @@ -263,9 +266,7 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) server: cfg.Server, caller: rpc.NewCaller(), - storage: st, - chainBus: bus, - + storage: st, blockCache: cache, pendingBlocks: make(chan *types.BPBlock), @@ -283,13 +284,12 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) confirms: needConfirms, nextHeight: headBranch.head.height + 1, offset: time.Duration(0), // TODO(leventeliu): initialize offset - - lastIrre: irre, - immutable: immutable, - headIndex: headIndex, - headBranch: headBranch, - branches: branches, - txPool: txPool, + lastIrre: lastIrre, + immutable: immutable, + headIndex: headIndex, + headBranch: headBranch, + branches: branches, + txPool: txPool, } // NOTE(leventeliu): this implies that BP chain is a singleton, otherwise we will need diff --git a/blockproducer/rpc.go b/blockproducer/rpc.go index 1b021614b..61e46b723 100644 --- a/blockproducer/rpc.go +++ b/blockproducer/rpc.go @@ -138,13 +138,6 @@ func (s *ChainRPCService) QueryTxState( return } -// Sub is the RPC method to subscribe some event. -func (s *ChainRPCService) Sub(req *types.SubReq, resp *types.SubResp) (err error) { - return s.chain.chainBus.Subscribe(req.Topic, func(request interface{}, response interface{}) { - s.chain.caller.CallNode(req.NodeID.ToNodeID(), req.Callback, request, response) - }) -} - // WaitDatabaseCreation waits for database creation complete. func WaitDatabaseCreation( ctx context.Context, diff --git a/blockproducer/storage.go b/blockproducer/storage.go index 0fca97223..23c1adaad 100644 --- a/blockproducer/storage.go +++ b/blockproducer/storage.go @@ -416,7 +416,7 @@ func loadTxPool(st xi.Storage) (txPool map[hash.Hash]pi.Transaction, err error) } func loadBlocks( - st xi.Storage, irreHash hash.Hash) (irre *blockNode, heads []*blockNode, err error, + st xi.Storage, irreHash hash.Hash) (lastIrre *blockNode, heads []*blockNode, err error, ) { var ( rows *sql.Rows @@ -494,7 +494,7 @@ func loadBlocks( headsIndex[bh] = bn } - if irre, ok = index[irreHash]; !ok { + if lastIrre, ok = index[irreHash]; !ok { err = errors.Wrapf(ErrParentNotFound, "irreversible block %s not found", ph.Short(4)) return } From 5d225a206a964e009451ce5345c51439bf8fdc10 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 15:23:43 +0800 Subject: [PATCH 15/35] Fix issue: must provide a positive size for LRU cache --- blockproducer/chain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index 1b6d20711..d6e60b375 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -153,6 +153,9 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) if cfg.blockCacheSize > conf.MaxCachedBlock { cfg.blockCacheSize = conf.MaxCachedBlock } + if cfg.blockCacheSize <= 0 { + cfg.blockCacheSize = 1 // Must provide a positive size + } if cache, err = lru.NewWithEvict(cfg.blockCacheSize, func(key interface{}, value interface{}) { if node, ok := value.(*blockNode); ok && node != nil { node.clear() From 34858f00e0a6f87e55156a267d36551040da9236 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 17:03:41 +0800 Subject: [PATCH 16/35] Export block cache size field in config --- blockproducer/chain.go | 10 +++++----- blockproducer/config.go | 2 +- cmd/cqld/bootstrap.go | 17 +++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/blockproducer/chain.go b/blockproducer/chain.go index d6e60b375..86789e2fc 100644 --- a/blockproducer/chain.go +++ b/blockproducer/chain.go @@ -150,13 +150,13 @@ func NewChainWithContext(ctx context.Context, cfg *Config) (c *Chain, err error) }() // Create block cache - if cfg.blockCacheSize > conf.MaxCachedBlock { - cfg.blockCacheSize = conf.MaxCachedBlock + if cfg.BlockCacheSize > conf.MaxCachedBlock { + cfg.BlockCacheSize = conf.MaxCachedBlock } - if cfg.blockCacheSize <= 0 { - cfg.blockCacheSize = 1 // Must provide a positive size + if cfg.BlockCacheSize <= 0 { + cfg.BlockCacheSize = 1 // Must provide a positive size } - if cache, err = lru.NewWithEvict(cfg.blockCacheSize, func(key interface{}, value interface{}) { + if cache, err = lru.NewWithEvict(cfg.BlockCacheSize, func(key interface{}, value interface{}) { if node, ok := value.(*blockNode); ok && node != nil { node.clear() } diff --git a/blockproducer/config.go b/blockproducer/config.go index 3fb1b2ee1..9c167a50f 100644 --- a/blockproducer/config.go +++ b/blockproducer/config.go @@ -52,5 +52,5 @@ type Config struct { Period time.Duration Tick time.Duration - blockCacheSize int + BlockCacheSize int } diff --git a/cmd/cqld/bootstrap.go b/cmd/cqld/bootstrap.go index 2b90d262b..80bf2d423 100644 --- a/cmd/cqld/bootstrap.go +++ b/cmd/cqld/bootstrap.go @@ -159,14 +159,15 @@ func runNode(nodeID proto.NodeID, listenAddr string) (err error) { // init main chain service log.Info("register main chain service rpc") chainConfig := &bp.Config{ - Mode: mode, - Genesis: genesis, - DataFile: conf.GConf.BP.ChainFileName, - Server: server, - Peers: peers, - NodeID: nodeID, - Period: conf.GConf.BPPeriod, - Tick: conf.GConf.BPTick, + Mode: mode, + Genesis: genesis, + DataFile: conf.GConf.BP.ChainFileName, + Server: server, + Peers: peers, + NodeID: nodeID, + Period: conf.GConf.BPPeriod, + Tick: conf.GConf.BPTick, + BlockCacheSize: 1000, } chain, err := bp.NewChain(chainConfig) if err != nil { From ec33085dca1264966e50e4ad72dca86b404747e0 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 12 Feb 2019 17:10:54 +0800 Subject: [PATCH 17/35] Make use of txCount field in blockNode --- blockproducer/branch.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blockproducer/branch.go b/blockproducer/branch.go index 83db975b1..6182d3f7d 100644 --- a/blockproducer/branch.go +++ b/blockproducer/branch.go @@ -58,11 +58,11 @@ func newBranch( } // Apply new blocks to view and pool for _, bn := range list { - var block = bn.load() - if len(block.Transactions) > conf.MaxTransactionsPerBlock { + if bn.txCount > conf.MaxTransactionsPerBlock { return nil, ErrTooManyTransactionsInBlock } + var block = bn.load() for _, v := range block.Transactions { var k = v.Hash() // Check in tx pool @@ -135,7 +135,7 @@ func (b *branch) applyBlock(n *blockNode) (br *branch, err error) { } var cpy = b.makeArena() - if len(block.Transactions) > conf.MaxTransactionsPerBlock { + if n.txCount > conf.MaxTransactionsPerBlock { return nil, ErrTooManyTransactionsInBlock } From 44adc797c1df20fc11b8ba2e53f2fe2e78020d3e Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Wed, 13 Feb 2019 18:22:53 +0800 Subject: [PATCH 18/35] Stablize genesis block generation --- blockproducer/interfaces/mixins.go | 2 +- blockproducer/metastate.go | 31 ++++------------ client/helper_test.go | 31 +++++----------- cmd/cql-minerd/dbms.go | 31 +++++----------- proto/nodeinfo.go | 2 +- sqlchain/chain_test.go | 12 ------- sqlchain/xxx_test.go | 18 +++++----- types/block.go | 57 ++++++++++++++++-------------- types/block_test.go | 55 +++------------------------- types/errors.go | 2 ++ types/xxx_test.go | 31 +++++----------- worker/helper_test.go | 30 +++++----------- 12 files changed, 88 insertions(+), 214 deletions(-) diff --git a/blockproducer/interfaces/mixins.go b/blockproducer/interfaces/mixins.go index 88ef8d353..35c547e5a 100644 --- a/blockproducer/interfaces/mixins.go +++ b/blockproducer/interfaces/mixins.go @@ -30,7 +30,7 @@ type TransactionTypeMixin struct { func NewTransactionTypeMixin(txType TransactionType) *TransactionTypeMixin { return &TransactionTypeMixin{ TxType: txType, - Timestamp: time.Now(), + Timestamp: time.Now().UTC(), } } diff --git a/blockproducer/metastate.go b/blockproducer/metastate.go index ec3152630..3fda7ee4c 100644 --- a/blockproducer/metastate.go +++ b/blockproducer/metastate.go @@ -19,14 +19,10 @@ package blockproducer import ( "bytes" "sort" - "time" pi "github.com/CovenantSQL/CovenantSQL/blockproducer/interfaces" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto" - "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" - "github.com/CovenantSQL/CovenantSQL/crypto/hash" - "github.com/CovenantSQL/CovenantSQL/crypto/kms" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/types" "github.com/CovenantSQL/CovenantSQL/utils" @@ -651,7 +647,7 @@ func (s *metaState) matchProvidersWithUser(tx *types.CreateDatabase) (err error) AdvancePayment: tx.AdvancePayment, } // generate genesis block - gb, err := s.generateGenesisBlock(dbID, tx.ResourceMeta) + gb, err := s.generateGenesisBlock(dbID, tx) if err != nil { log.WithFields(log.Fields{ "dbID": dbID, @@ -1161,32 +1157,19 @@ func (s *metaState) applyTransaction(tx pi.Transaction) (err error) { return } -func (s *metaState) generateGenesisBlock(dbID proto.DatabaseID, resourceMeta types.ResourceMeta) (genesisBlock *types.Block, err error) { - // TODO(xq262144): following is stub code, real logic should be implemented in the future - emptyHash := hash.Hash{} - - var privKey *asymmetric.PrivateKey - if privKey, err = kms.GetLocalPrivateKey(); err != nil { - return - } - var nodeID proto.NodeID - if nodeID, err = kms.GetLocalNodeID(); err != nil { - return - } - +func (s *metaState) generateGenesisBlock(dbID proto.DatabaseID, tx *types.CreateDatabase) (genesisBlock *types.Block, err error) { + emptyNode := &proto.RawNodeID{} genesisBlock = &types.Block{ SignedHeader: types.SignedHeader{ Header: types.Header{ - Version: 0x01000000, - Producer: nodeID, - GenesisHash: emptyHash, - ParentHash: emptyHash, - Timestamp: time.Now().UTC(), + Version: 0x01000000, + Producer: emptyNode.ToNodeID(), + Timestamp: tx.Timestamp, }, }, } - err = genesisBlock.PackAndSignBlock(privKey) + err = genesisBlock.PackAsGenesis() return } diff --git a/client/helper_test.go b/client/helper_test.go index ae7dda08d..bfae11f71 100644 --- a/client/helper_test.go +++ b/client/helper_test.go @@ -34,7 +34,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/pow/cpuminer" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/rpc" @@ -271,7 +270,7 @@ func initNode() (cleanupFunc func(), tempDir string, server *rpc.Server, err err // copied from sqlchain.xxx_test. func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { // Generate key pair - priv, pub, err := asymmetric.GenSecp256k1KeyPair() + priv, _, err := asymmetric.GenSecp256k1KeyPair() if err != nil { return @@ -293,26 +292,14 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er } if isGenesis { - // Compute nonce with public key - nonceCh := make(chan cpuminer.NonceInfo) - quitCh := make(chan struct{}) - miner := cpuminer.NewCPUMiner(quitCh) - go miner.ComputeBlockNonce(cpuminer.MiningBlock{ - Data: pub.Serialize(), - NonceChan: nonceCh, - Stop: nil, - }, cpuminer.Uint256{A: 0, B: 0, C: 0, D: 0}, 4) - nonce := <-nonceCh - close(quitCh) - close(nonceCh) - // Add public key to KMS - id := cpuminer.HashBlock(pub.Serialize(), nonce.Nonce) - b.SignedHeader.Header.Producer = proto.NodeID(id.String()) - err = kms.SetPublicKey(proto.NodeID(id.String()), nonce.Nonce, pub) - - if err != nil { - return nil, err - } + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} + b.SignedHeader.GenesisHash = hash.Hash{} + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} + + err = b.PackAsGenesis() + return } err = b.PackAndSignBlock(priv) diff --git a/cmd/cql-minerd/dbms.go b/cmd/cql-minerd/dbms.go index bcd234c8d..d9b76fc3e 100644 --- a/cmd/cql-minerd/dbms.go +++ b/cmd/cql-minerd/dbms.go @@ -27,7 +27,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/pow/cpuminer" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/rpc" "github.com/CovenantSQL/CovenantSQL/types" @@ -150,7 +149,7 @@ func loadGenesisBlock(fixture *conf.MinerDatabaseFixture) (block *types.Block, e // copied from sqlchain.xxx_test. func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { // Generate key pair - priv, pub, err := asymmetric.GenSecp256k1KeyPair() + priv, _, err := asymmetric.GenSecp256k1KeyPair() if err != nil { return @@ -172,26 +171,14 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er } if isGenesis { - // Compute nonce with public key - nonceCh := make(chan cpuminer.NonceInfo) - quitCh := make(chan struct{}) - miner := cpuminer.NewCPUMiner(quitCh) - go miner.ComputeBlockNonce(cpuminer.MiningBlock{ - Data: pub.Serialize(), - NonceChan: nonceCh, - Stop: nil, - }, cpuminer.Uint256{}, 4) - nonce := <-nonceCh - close(quitCh) - close(nonceCh) - // Add public key to KMS - id := cpuminer.HashBlock(pub.Serialize(), nonce.Nonce) - b.SignedHeader.Header.Producer = proto.NodeID(id.String()) - err = kms.SetPublicKey(proto.NodeID(id.String()), nonce.Nonce, pub) - - if err != nil { - return nil, err - } + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} + b.SignedHeader.GenesisHash = hash.Hash{} + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} + + err = b.PackAsGenesis() + return } err = b.PackAndSignBlock(priv) diff --git a/proto/nodeinfo.go b/proto/nodeinfo.go index c6ef33b49..b855a9d73 100644 --- a/proto/nodeinfo.go +++ b/proto/nodeinfo.go @@ -149,7 +149,7 @@ func (id *NodeID) ToRawNodeID() *RawNodeID { // IsEmpty test if a nodeID is empty. func (id *NodeID) IsEmpty() bool { - return id == nil || "" == string(*id) + return id == nil || "" == string(*id) || id.ToRawNodeID().IsEqual(&hash.Hash{}) } // IsEqual returns if two node id is equal. diff --git a/sqlchain/chain_test.go b/sqlchain/chain_test.go index fd6085e4d..4fbba3cf9 100644 --- a/sqlchain/chain_test.go +++ b/sqlchain/chain_test.go @@ -96,12 +96,6 @@ func TestMultiChain(t *testing.T) { t.Fatalf("error occurred: %v", err) } - gnonce, err := kms.GetNodeInfo(genesis.Producer()) - - if err != nil { - t.Fatalf("error occurred: %v", err) - } - // Create peer list: `testPeersNumber` miners + 1 block producer nis, peers, err := createTestPeers(testPeersNumber + 1) @@ -268,12 +262,6 @@ func TestMultiChain(t *testing.T) { // Test chain data reloading before exit for _, v := range chains { defer func(p *chainParams) { - if _, err := kms.GetPublicKey(genesis.Producer()); err != nil { - if err = kms.SetPublicKey(genesis.Producer(), gnonce.Nonce, genesis.Signee()); err != nil { - t.Errorf("error occurred: %v", err) - } - } - if chain, err := NewChain(p.config); err != nil { t.Errorf("error occurred: %v", err) } else { diff --git a/sqlchain/xxx_test.go b/sqlchain/xxx_test.go index ba2c184af..29d1e649a 100644 --- a/sqlchain/xxx_test.go +++ b/sqlchain/xxx_test.go @@ -237,7 +237,7 @@ func registerNodesWithPublicKey(pub *asymmetric.PublicKey, diff int, num int) ( func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { // Generate key pair - priv, pub, err := asymmetric.GenSecp256k1KeyPair() + priv, _, err := asymmetric.GenSecp256k1KeyPair() if err != nil { return @@ -271,16 +271,14 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er } if isGenesis { - // Register node for genesis verification - var nis []cpuminer.NonceInfo - nis, err = registerNodesWithPublicKey(pub, testDifficulty, 1) - - if err != nil { - return - } - + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Header.Producer = proto.NodeID(nis[0].Hash.String()) + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} + b.Acks = nil + + err = b.PackAsGenesis() } err = b.PackAndSignBlock(priv) diff --git a/types/block.go b/types/block.go index 27ec40b3b..c421fd952 100644 --- a/types/block.go +++ b/types/block.go @@ -21,11 +21,10 @@ import ( ca "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" - "github.com/CovenantSQL/CovenantSQL/crypto/kms" "github.com/CovenantSQL/CovenantSQL/crypto/verifier" "github.com/CovenantSQL/CovenantSQL/merkle" "github.com/CovenantSQL/CovenantSQL/proto" - "github.com/CovenantSQL/CovenantSQL/utils/log" + "github.com/pkg/errors" ) //go:generate hsp @@ -56,24 +55,14 @@ func (s *SignedHeader) Verify() error { return s.HSV.Verify(&s.Header) } -// VerifyAsGenesis verifies the signed header as a genesis block header. -func (s *SignedHeader) VerifyAsGenesis() (err error) { - var pk *ca.PublicKey - log.WithFields(log.Fields{ - "producer": s.Producer, - "root": s.GenesisHash.String(), - "parent": s.ParentHash.String(), - "merkle": s.MerkleRoot.String(), - "block": s.HSV.Hash().String(), - }).Debug("verifying genesis block header") - if pk, err = kms.GetPublicKey(s.Producer); err != nil { - return - } - if !pk.IsEqual(s.HSV.Signee) { - err = ErrNodePublicKeyNotMatch - return - } - return s.Verify() +// VerifyHash verifies the hash of the signed header. +func (s *SignedHeader) VerifyHash() error { + return s.HSV.VerifyHash(&s.Header) +} + +// ComputeHash computes the hash of the signed header. +func (s *SignedHeader) ComputeHash() error { + return s.HSV.SetHash(&s.Header) } // QueryAsTx defines a tx struct which is combined with request and signed response header @@ -115,6 +104,11 @@ func (b *Block) PackAndSignBlock(signer *ca.PrivateKey) (err error) { return b.SignedHeader.Sign(signer) } +// PackAsGenesis generates the hash of the genesis block. +func (b *Block) PackAsGenesis() (err error) { + return b.SignedHeader.ComputeHash() +} + // Verify verifies the merkle root and header signature of the block. func (b *Block) Verify() (err error) { // Verify merkle root @@ -126,15 +120,24 @@ func (b *Block) Verify() (err error) { // VerifyAsGenesis verifies the block as a genesis block. func (b *Block) VerifyAsGenesis() (err error) { - var pk *ca.PublicKey - if pk, err = kms.GetPublicKey(b.SignedHeader.Producer); err != nil { - return + if !b.SignedHeader.Producer.IsEmpty() { + // not empty + return errors.Wrap(ErrInvalidGenesis, "invalid producer") + } + if !b.SignedHeader.GenesisHash.IsEqual(&hash.Hash{}) { + // not empty + return errors.Wrap(ErrInvalidGenesis, "invalid genesis hash") } - if !pk.IsEqual(b.SignedHeader.HSV.Signee) { - err = ErrNodePublicKeyNotMatch - return + if !b.SignedHeader.ParentHash.IsEqual(&hash.Hash{}) { + // not empty + return errors.Wrap(ErrInvalidGenesis, "invalid parent hash") } - return b.Verify() + if !b.SignedHeader.MerkleRoot.IsEqual(&hash.Hash{}) { + // not empty + return errors.Wrap(ErrInvalidGenesis, "invalid merkle root") + } + + return b.SignedHeader.VerifyHash() } // Timestamp returns the timestamp field of the block header. diff --git a/types/block_test.go b/types/block_test.go index 70e42c4ed..a33001bee 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -18,11 +18,10 @@ package types import ( "bytes" - "math/big" + "math/rand" "reflect" "testing" - "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/verifier" "github.com/CovenantSQL/CovenantSQL/utils" @@ -31,7 +30,7 @@ import ( ) func TestSignAndVerify(t *testing.T) { - block, err := createRandomBlock(genesisHash, true) + block, err := createRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -97,7 +96,7 @@ func TestHeaderMarshalUnmarshaler(t *testing.T) { } func TestSignedHeaderMarshaleUnmarshaler(t *testing.T) { - block, err := createRandomBlock(genesisHash, true) + block, err := createRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -219,10 +218,6 @@ func TestGenesis(t *testing.T) { t.Fatalf("error occurred: %v", err) } - if err = genesis.SignedHeader.VerifyAsGenesis(); err != nil { - t.Fatalf("error occurred: %v", err) - } - // Test non-genesis block genesis, err = createRandomBlock(genesisHash, false) @@ -236,57 +231,17 @@ func TestGenesis(t *testing.T) { t.Fatal("unexpected result: returned nil while expecting an error") } - if err = genesis.SignedHeader.VerifyAsGenesis(); err != nil { - t.Logf("Error occurred as expected: %v", err) - } else { - t.Fatal("unexpected result: returned nil while expecting an error") - } - - // Test altered public key block - genesis, err = createRandomBlock(genesisHash, true) - - if err != nil { - t.Fatalf("error occurred: %v", err) - } - - _, pub, err := asymmetric.GenSecp256k1KeyPair() - - if err != nil { - t.Fatalf("error occurred: %v", err) - } - - genesis.SignedHeader.HSV.Signee = pub - - if err = genesis.VerifyAsGenesis(); err != nil { - t.Logf("Error occurred as expected: %v", err) - } else { - t.Fatal("unexpected result: returned nil while expecting an error") - } - - if err = genesis.SignedHeader.VerifyAsGenesis(); err != nil { - t.Logf("Error occurred as expected: %v", err) - } else { - t.Fatal("unexpected result: returned nil while expecting an error") - } - - // Test altered signature + // Test altered block genesis, err = createRandomBlock(genesisHash, true) if err != nil { t.Fatalf("error occurred: %v", err) } - genesis.SignedHeader.HSV.Signature.R.Add(genesis.SignedHeader.HSV.Signature.R, big.NewInt(int64(1))) - genesis.SignedHeader.HSV.Signature.S.Add(genesis.SignedHeader.HSV.Signature.S, big.NewInt(int64(1))) + rand.Read(genesis.SignedHeader.ParentHash[:]) if err = genesis.VerifyAsGenesis(); err != nil { t.Logf("Error occurred as expected: %v", err) - } else { - t.Fatalf("unexpected error: %v", err) - } - - if err = genesis.SignedHeader.VerifyAsGenesis(); err != nil { - t.Logf("Error occurred as expected: %v", err) } else { t.Fatal("unexpected result: returned nil while expecting an error") } diff --git a/types/errors.go b/types/errors.go index 86ea4154d..5ba1176f4 100644 --- a/types/errors.go +++ b/types/errors.go @@ -32,4 +32,6 @@ var ( ErrBillingNotMatch = errors.New("billing request doesn't match") // ErrHashVerification indicates a failed hash verification. ErrHashVerification = errors.New("hash verification failed") + // ErrInvalidGenesis indicates a failed genesis block verification. + ErrInvalidGenesis = errors.New("invalid genesis block") ) diff --git a/types/xxx_test.go b/types/xxx_test.go index 7da8dffa6..8e472a798 100644 --- a/types/xxx_test.go +++ b/types/xxx_test.go @@ -27,7 +27,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/pow/cpuminer" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/utils/log" ) @@ -302,7 +301,7 @@ func setup() { func createRandomBlock(parent hash.Hash, isGenesis bool) (b *Block, err error) { // Generate key pair - priv, pub, err := asymmetric.GenSecp256k1KeyPair() + priv, _, err := asymmetric.GenSecp256k1KeyPair() if err != nil { return @@ -324,28 +323,14 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *Block, err error) { } if isGenesis { - // Compute nonce with public key - nonceCh := make(chan cpuminer.NonceInfo) - quitCh := make(chan struct{}) - miner := cpuminer.NewCPUMiner(quitCh) - go miner.ComputeBlockNonce(cpuminer.MiningBlock{ - Data: pub.Serialize(), - NonceChan: nonceCh, - Stop: nil, - }, cpuminer.Uint256{A: 0, B: 0, C: 0, D: 0}, 4) - nonce := <-nonceCh - close(quitCh) - close(nonceCh) - // Add public key to KMS - id := cpuminer.HashBlock(pub.Serialize(), nonce.Nonce) - b.SignedHeader.Header.Producer = proto.NodeID(id.String()) - - if err = kms.SetPublicKey(proto.NodeID(id.String()), nonce.Nonce, pub); err != nil { - return nil, err - } - - // Set genesis hash as zero value + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} b.SignedHeader.GenesisHash = hash.Hash{} + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} + + err = b.PackAsGenesis() + return } err = b.PackAndSignBlock(priv) diff --git a/worker/helper_test.go b/worker/helper_test.go index 7157de32b..22c02183d 100644 --- a/worker/helper_test.go +++ b/worker/helper_test.go @@ -31,8 +31,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/consistent" "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" - "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/pow/cpuminer" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/rpc" @@ -249,7 +247,7 @@ func initNode() (cleanupFunc func(), server *rpc.Server, err error) { // copied from sqlchain.xxx_test. func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { // Generate key pair - priv, pub, err := asymmetric.GenSecp256k1KeyPair() + priv, _, err := asymmetric.GenSecp256k1KeyPair() if err != nil { return @@ -271,26 +269,14 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er } if isGenesis { - // Compute nonce with public key - nonceCh := make(chan cpuminer.NonceInfo) - quitCh := make(chan struct{}) - miner := cpuminer.NewCPUMiner(quitCh) - go miner.ComputeBlockNonce(cpuminer.MiningBlock{ - Data: pub.Serialize(), - NonceChan: nonceCh, - Stop: nil, - }, cpuminer.Uint256{A: 0, B: 0, C: 0, D: 0}, 4) - nonce := <-nonceCh - close(quitCh) - close(nonceCh) - // Add public key to KMS - id := cpuminer.HashBlock(pub.Serialize(), nonce.Nonce) - b.SignedHeader.Header.Producer = proto.NodeID(id.String()) - err = kms.SetPublicKey(proto.NodeID(id.String()), nonce.Nonce, pub) + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} + b.SignedHeader.GenesisHash = hash.Hash{} + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} - if err != nil { - return nil, err - } + err = b.PackAsGenesis() + return } err = b.PackAndSignBlock(priv) From fb9d2f7dc12b9d0b96e154662ec94397b961cad8 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Thu, 14 Feb 2019 10:28:48 +0800 Subject: [PATCH 19/35] Remove conflict check --- xenomint/state.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xenomint/state.go b/xenomint/state.go index bdf84735e..334283cdd 100644 --- a/xenomint/state.go +++ b/xenomint/state.go @@ -522,10 +522,7 @@ func (s *State) ReplayBlockWithContext(ctx context.Context, block *types.Block) } // Match and skip already pooled query if q.Response.ResponseHeader.LogOffset < lastsp { - if !s.pool.match(q.Response.ResponseHeader.LogOffset, q.Request) { - err = ErrQueryConflict - return - } + // TODO(), recover logic after sqlchain forks by multiple write point continue } // Replay query From 64c179e8823e0a6b4a4e6814d848485fc96ce9f4 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Mon, 18 Feb 2019 15:38:43 +0800 Subject: [PATCH 20/35] Fix replay block test case --- xenomint/state_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xenomint/state_test.go b/xenomint/state_test.go index 48965d2e4..f2d397403 100644 --- a/xenomint/state_test.go +++ b/xenomint/state_test.go @@ -25,7 +25,6 @@ import ( "sync" "testing" - "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/verifier" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/types" @@ -458,9 +457,10 @@ INSERT INTO t1 (k, v) VALUES (?, ?)`, concat(values[2:4])...), }, }, } - blockx.QueryTxs[0].Request.Header.DataHash = hash.Hash{0x0, 0x0, 0x0, 0x1} + // modify response offset + blockx.QueryTxs[0].Response.ResponseHeader.LogOffset = 10000 err = st2.ReplayBlock(blockx) - So(err, ShouldEqual, ErrQueryConflict) + So(errors.Cause(err), ShouldEqual, ErrMissingParent) }, ) Convey( From d9ba4fbb520680a6bd231626b969fe6fba27e2cd Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Mon, 18 Feb 2019 15:40:03 +0800 Subject: [PATCH 21/35] Fix sqlchain createRandomBlock util bug --- sqlchain/xxx_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlchain/xxx_test.go b/sqlchain/xxx_test.go index 29d1e649a..25b5e8350 100644 --- a/sqlchain/xxx_test.go +++ b/sqlchain/xxx_test.go @@ -279,6 +279,7 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er b.Acks = nil err = b.PackAsGenesis() + return } err = b.PackAndSignBlock(priv) From d7ee083b500388d671b1b891126b3244c5dc5697 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Thu, 31 Jan 2019 19:15:49 +0800 Subject: [PATCH 22/35] Refactor observer to pull mode --- cmd/cql-observer/config_test.go | 2 + cmd/cql-observer/main.go | 10 +- cmd/cql-observer/node.go | 24 +-- cmd/cql-observer/observation_test.go | 7 +- cmd/cql-observer/observer.go | 16 +- cmd/cql-observer/service.go | 133 +++----------- cmd/cql-observer/worker.go | 162 +++++++++++++++++ route/acl.go | 18 +- route/acl_test.go | 2 +- sqlchain/blockindex.go | 15 ++ sqlchain/chain.go | 148 +++++---------- sqlchain/observer.go | 257 --------------------------- worker/dbms.go | 69 ------- worker/dbms_rpc.go | 40 +---- worker/dbms_test.go | 10 +- worker/observer.go | 98 ++++++++++ 16 files changed, 370 insertions(+), 641 deletions(-) create mode 100644 cmd/cql-observer/worker.go delete mode 100644 sqlchain/observer.go create mode 100644 worker/observer.go diff --git a/cmd/cql-observer/config_test.go b/cmd/cql-observer/config_test.go index e542af247..5c3583bfd 100644 --- a/cmd/cql-observer/config_test.go +++ b/cmd/cql-observer/config_test.go @@ -1,3 +1,5 @@ +// +build !testbinary + /* * Copyright 2018 The CovenantSQL Authors. * diff --git a/cmd/cql-observer/main.go b/cmd/cql-observer/main.go index 34162cbc3..d1d6c9dcf 100644 --- a/cmd/cql-observer/main.go +++ b/cmd/cql-observer/main.go @@ -30,7 +30,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/kms" "github.com/CovenantSQL/CovenantSQL/proto" - "github.com/CovenantSQL/CovenantSQL/rpc" "github.com/CovenantSQL/CovenantSQL/utils" "github.com/CovenantSQL/CovenantSQL/utils/log" ) @@ -85,15 +84,14 @@ func main() { kms.InitBP() - // start rpc - var server *rpc.Server - if server, err = initNode(); err != nil { + // init node + if err = initNode(); err != nil { log.WithError(err).Fatal("init node failed") } // start service var service *Service - if service, err = startService(server); err != nil { + if service, err = startService(); err != nil { log.WithError(err).Fatal("start observation failed") } @@ -144,7 +142,7 @@ func main() { } // stop subscriptions - if err = stopService(service, server); err != nil { + if err = stopService(service); err != nil { log.WithError(err).Fatal("stop service failed") } diff --git a/cmd/cql-observer/node.go b/cmd/cql-observer/node.go index 49bb83513..29b9c5351 100644 --- a/cmd/cql-observer/node.go +++ b/cmd/cql-observer/node.go @@ -18,18 +18,16 @@ package main import ( "fmt" - "os" "syscall" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto/kms" "github.com/CovenantSQL/CovenantSQL/route" - "github.com/CovenantSQL/CovenantSQL/rpc" "github.com/CovenantSQL/CovenantSQL/utils/log" "golang.org/x/crypto/ssh/terminal" ) -func initNode() (server *rpc.Server, err error) { +func initNode() (err error) { var masterKey []byte if !conf.GConf.IsTestMode { fmt.Print("Type in Master key to continue:") @@ -50,25 +48,5 @@ func initNode() (server *rpc.Server, err error) { // init kms routing route.InitKMS(conf.GConf.PubKeyStoreFile) - // init server - if server, err = createServer( - conf.GConf.PrivateKeyFile, conf.GConf.PubKeyStoreFile, masterKey, conf.GConf.ListenAddr); err != nil { - log.WithError(err).Error("create server failed") - return - } - - return -} - -func createServer(privateKeyPath, pubKeyStorePath string, masterKey []byte, listenAddr string) (server *rpc.Server, err error) { - os.Remove(pubKeyStorePath) - - server = rpc.NewServer() - if err != nil { - return - } - - err = server.InitRPCServer(listenAddr, privateKeyPath, masterKey) - return } diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index bd3b9e620..532672371 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -500,7 +500,7 @@ func TestFullProcess(t *testing.T) { observerCmd.Cmd.Wait() }() - // wait for the observer to collect blocks, two periods is enough + // wait for the observer to collect blocks time.Sleep(conf.GConf.SQLChainPeriod * 5) // test get genesis block by height @@ -686,11 +686,14 @@ func TestFullProcess(t *testing.T) { }) So(err, ShouldBeNil) + // wait for the observer to be enabled query by miner, and collect blocks + time.Sleep(conf.GConf.SQLChainPeriod * 5) + // test get genesis block by height res, err = getJSON("v3/head/%v", dbID2) So(err, ShouldBeNil) So(ensureSuccess(res.Interface("block")), ShouldNotBeNil) - So(ensureSuccess(res.Int("block", "height")), ShouldEqual, 0) + So(ensureSuccess(res.Int("block", "height")), ShouldBeGreaterThanOrEqualTo, 0) log.Info(err, res) err = client.Drop(dsn) diff --git a/cmd/cql-observer/observer.go b/cmd/cql-observer/observer.go index ff7ed1b22..2190f170e 100644 --- a/cmd/cql-observer/observer.go +++ b/cmd/cql-observer/observer.go @@ -20,7 +20,6 @@ import ( "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto/kms" "github.com/CovenantSQL/CovenantSQL/proto" - "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/rpc" ) @@ -41,33 +40,22 @@ func registerNode() (err error) { return } -func startService(server *rpc.Server) (service *Service, err error) { +func startService() (service *Service, err error) { // register observer service to rpc server service, err = NewService() if err != nil { return } - if err = server.RegisterService(route.ObserverRPCName, service); err != nil { - return - } - - // start service rpc, observer acts as client role but listen to - go server.Serve() - // start observer service service.start() return } -func stopService(service *Service, server *rpc.Server) (err error) { +func stopService(service *Service) (err error) { // stop subscription service.stop() - // stop rpc service - server.Listener.Close() - server.Stop() - return } diff --git a/cmd/cql-observer/service.go b/cmd/cql-observer/service.go index a2465a368..c49138860 100644 --- a/cmd/cql-observer/service.go +++ b/cmd/cql-observer/service.go @@ -30,11 +30,9 @@ import ( "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/rpc" - "github.com/CovenantSQL/CovenantSQL/sqlchain" "github.com/CovenantSQL/CovenantSQL/types" "github.com/CovenantSQL/CovenantSQL/utils" "github.com/CovenantSQL/CovenantSQL/utils/log" - "github.com/CovenantSQL/CovenantSQL/worker" bolt "github.com/coreos/bbolt" ) @@ -95,9 +93,8 @@ var ( // Service defines the observer service structure. type Service struct { - lock sync.Mutex - subscription map[proto.DatabaseID]int32 - upstreamServers sync.Map + subscription sync.Map // map[proto.DatabaseID]*subscribeWorker + upstreamServers sync.Map // map[proto.DatabaseID]*types.ServiceInstance db *bolt.DB caller *rpc.Caller @@ -147,17 +144,16 @@ func NewService() (service *Service, err error) { // init service service = &Service{ - subscription: make(map[proto.DatabaseID]int32), - db: db, - caller: rpc.NewCaller(), + db: db, + caller: rpc.NewCaller(), } // load previous subscriptions if err = db.View(func(tx *bolt.Tx) error { - return tx.Bucket(subscriptionBucket).ForEach(func(rawDBID, rawHeight []byte) (err error) { + return tx.Bucket(subscriptionBucket).ForEach(func(rawDBID, rawCount []byte) (err error) { dbID := proto.DatabaseID(string(rawDBID)) - h := bytesToInt32(rawHeight) - service.subscription[dbID] = h + count := bytesToInt32(rawCount) + service.subscription.Store(dbID, newSubscribeWorker(dbID, count, service)) return }) }); err != nil { @@ -182,10 +178,6 @@ func (s *Service) subscribe(dbID proto.DatabaseID, resetSubscribePosition string return ErrStopped } - s.lock.Lock() - - shouldStartSubscribe := false - if resetSubscribePosition != "" { var fromPos int32 @@ -198,46 +190,25 @@ func (s *Service) subscribe(dbID proto.DatabaseID, resetSubscribePosition string fromPos = types.ReplicateFromNewest } - s.subscription[dbID] = fromPos - - // send start subscription request - // TODO(leventeliu): should also clean up obsolete data in db file! - shouldStartSubscribe = true + unpackWorker(s.subscription.LoadOrStore(dbID, + newSubscribeWorker(dbID, fromPos, s))).reset(fromPos) } else { // not resetting - if _, exists := s.subscription[dbID]; !exists { - s.subscription[dbID] = types.ReplicateFromNewest - shouldStartSubscribe = true - } - } - - s.lock.Unlock() - - if shouldStartSubscribe { - return s.startSubscribe(dbID) + unpackWorker(s.subscription.LoadOrStore(dbID, + newSubscribeWorker(dbID, types.ReplicateFromNewest, s))).start() } return } -// AdviseNewBlock handles block replication request from the remote database chain service. -func (s *Service) AdviseNewBlock(req *sqlchain.MuxAdviseNewBlockReq, resp *sqlchain.MuxAdviseNewBlockResp) (err error) { - if atomic.LoadInt32(&s.stopped) == 1 { - // stopped - return ErrStopped - } - - if req.Block == nil { - log.WithField("node", req.GetNodeID().String()).Warning("received empty block") +func unpackWorker(actual interface{}, _ ...interface{}) (worker *subscribeWorker) { + if actual == nil { return } - log.WithFields(log.Fields{ - "node": req.GetNodeID().String(), - "block": req.Block.BlockHash(), - }).Debug("received block") + worker, _ = actual.(*subscribeWorker) - return s.addBlock(req.DatabaseID, req.Count, req.Block) + return } func (s *Service) start() (err error) { @@ -246,54 +217,14 @@ func (s *Service) start() (err error) { return ErrStopped } - s.lock.Lock() - dbs := make([]proto.DatabaseID, len(s.subscription)) - for dbID := range s.subscription { - dbs = append(dbs, dbID) - } - s.lock.Unlock() - - for _, dbID := range dbs { - if err = s.startSubscribe(dbID); err != nil { - log.WithField("db", dbID).WithError(err).Warning("start subscription failed") - } - } + s.subscription.Range(func(_, rawWorker interface{}) bool { + unpackWorker(rawWorker).start() + return true + }) return nil } -func (s *Service) startSubscribe(dbID proto.DatabaseID) (err error) { - if atomic.LoadInt32(&s.stopped) == 1 { - // stopped - return ErrStopped - } - - s.lock.Lock() - defer s.lock.Unlock() - - // start subscribe on first node of each sqlchain server peers - log.WithField("db", dbID).Info("start subscribing transactions") - - instance, err := s.getUpstream(dbID) - if err != nil { - return - } - - // store the genesis block - if err = s.addBlock(dbID, 0, instance.GenesisBlock); err != nil { - return - } - - req := &worker.SubscribeTransactionsReq{} - resp := &worker.SubscribeTransactionsResp{} - req.Height = s.subscription[dbID] - req.DatabaseID = dbID - - err = s.minerRequest(dbID, route.DBSSubscribeTransactions.String(), req, resp) - - return -} - func (s *Service) addAck(dbID proto.DatabaseID, height int32, offset int32, ack *types.SignedAckHeader) (err error) { log.WithFields(log.Fields{ "height": height, @@ -306,9 +237,6 @@ func (s *Service) addAck(dbID proto.DatabaseID, height int32, offset int32, ack return ErrStopped } - s.lock.Lock() - defer s.lock.Unlock() - if err = ack.Verify(); err != nil { return } @@ -335,9 +263,6 @@ func (s *Service) addQueryTracker(dbID proto.DatabaseID, height int32, offset in return ErrStopped } - s.lock.Lock() - defer s.lock.Unlock() - if err = qt.Request.Verify(); err != nil { return } @@ -433,23 +358,13 @@ func (s *Service) stop() (err error) { return ErrStopped } - s.lock.Lock() - defer s.lock.Unlock() - // send cancel subscription to all databases log.Info("stop subscribing all databases") - for dbID := range s.subscription { - // send cancel subscription rpc - req := &worker.CancelSubscriptionReq{} - resp := &worker.CancelSubscriptionResp{} - req.DatabaseID = dbID - - if err = s.minerRequest(dbID, route.DBSCancelSubscription.String(), req, resp); err != nil { - // cancel subscription failed - log.WithField("db", dbID).WithError(err).Warning("cancel subscription") - } - } + s.subscription.Range(func(_, rawWorker interface{}) bool { + unpackWorker(rawWorker).stop() + return true + }) // close the subscription database s.db.Close() @@ -467,7 +382,7 @@ func (s *Service) minerRequest(dbID proto.DatabaseID, method string, request int } func (s *Service) getUpstream(dbID proto.DatabaseID) (instance *types.ServiceInstance, err error) { - log.WithField("db", dbID).Info("get peers info for database") + log.WithField("db", dbID).Debug("get peers info for database") if iInstance, exists := s.upstreamServers.Load(dbID); exists { instance = iInstance.(*types.ServiceInstance) diff --git a/cmd/cql-observer/worker.go b/cmd/cql-observer/worker.go new file mode 100644 index 000000000..23f12e24d --- /dev/null +++ b/cmd/cql-observer/worker.go @@ -0,0 +1,162 @@ +/* + * Copyright 2019 The CovenantSQL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/CovenantSQL/CovenantSQL/conf" + "github.com/CovenantSQL/CovenantSQL/proto" + "github.com/CovenantSQL/CovenantSQL/route" + "github.com/CovenantSQL/CovenantSQL/utils/log" + "github.com/CovenantSQL/CovenantSQL/worker" +) + +type subscribeWorker struct { + l sync.Mutex + s *Service + dbID proto.DatabaseID + head int32 + wg *sync.WaitGroup + stopCh chan struct{} +} + +func newSubscribeWorker(dbID proto.DatabaseID, head int32, s *Service) *subscribeWorker { + return &subscribeWorker{ + dbID: dbID, + head: head, + s: s, + } +} + +func (w *subscribeWorker) run() { + defer w.wg.Done() + + // calc next tick + var nextTick time.Duration + + for { + + select { + case <-w.stopCh: + return + case <-time.After(nextTick): + if err := w.pull(atomic.LoadInt32(&w.head)); err != nil { + // calc next tick + nextTick = conf.GConf.SQLChainPeriod + } else { + nextTick /= 10 + } + } + } +} + +func (w *subscribeWorker) pull(count int32) (err error) { + var ( + req = new(worker.ObserverFetchBlockReq) + resp = new(worker.ObserverFetchBlockResp) + next int32 + ) + + defer func() { + lf := log.WithFields(log.Fields{ + "req_count": count, + "count": resp.Count, + }) + + if err != nil { + lf.WithError(err).Debug("sync block failed") + } else { + if resp.Block != nil { + lf = lf.WithField("block", resp.Block.BlockHash()) + } else { + lf = lf.WithField("block", nil) + } + lf.WithField("next", next).Debug("sync block success") + } + }() + + req.DatabaseID = w.dbID + req.Count = count + + if err = w.s.minerRequest(w.dbID, route.DBSObserverFetchBlock.String(), req, resp); err != nil { + return + } + + if resp.Block == nil { + err = errors.New("nil block, try later") + return + } + + if err = w.s.addBlock(w.dbID, count, resp.Block); err != nil { + return + } + + if count < 0 { + next = resp.Count + 1 + } else { + next = count + 1 + } + + atomic.CompareAndSwapInt32(&w.head, count, next) + + return +} + +func (w *subscribeWorker) reset(head int32) { + atomic.StoreInt32(&w.head, head) + w.start() +} + +func (w *subscribeWorker) start() { + w.l.Lock() + defer w.l.Unlock() + + if w.isStopped() { + w.stopCh = make(chan struct{}) + w.wg = new(sync.WaitGroup) + w.wg.Add(1) + go w.run() + } +} + +func (w *subscribeWorker) stop() { + w.l.Lock() + defer w.l.Unlock() + + if !w.isStopped() { + // stop + close(w.stopCh) + w.wg.Wait() + } +} + +func (w *subscribeWorker) isStopped() bool { + if w.stopCh == nil { + return true + } + + select { + case <-w.stopCh: + return true + default: + return false + } +} diff --git a/route/acl.go b/route/acl.go index d6a42b3f5..032aff54e 100644 --- a/route/acl.go +++ b/route/acl.go @@ -77,10 +77,8 @@ const ( DBSAck // DBSDeploy is used by BP to create/drop/update database DBSDeploy - // DBSSubscribeTransactions is used by dbms to handle observer subscription request - DBSSubscribeTransactions - // DBSCancelSubscription is used by dbms to handle observer subscription cancellation request - DBSCancelSubscription + // DBSObserverFetchBlock is used by observer to fetch block. + DBSObserverFetchBlock // DBCCall is used by Miner for data consistency DBCCall // SQLCAdviseNewBlock is used by sqlchain to advise new block between adjacent node @@ -95,8 +93,6 @@ const ( SQLCSignBilling // SQLCLaunchBilling is used by blockproducer to trigger the billing process in sqlchain SQLCLaunchBilling - // OBSAdviseNewBlock is used by sqlchain to push new block to observers - OBSAdviseNewBlock // MCCAdviseNewBlock is used by block producer to push block to adjacent nodes MCCAdviseNewBlock // MCCAdviseTxBilling is used by block producer to push billing transaction to adjacent nodes @@ -131,8 +127,6 @@ const ( SQLChainRPCName = "SQLC" // DBRPCName defines the sql chain db service rpc name DBRPCName = "DBS" - // ObserverRPCName defines the observer node service rpc name - ObserverRPCName = "OBS" ) // String returns the RemoteFunc string. @@ -154,10 +148,8 @@ func (s RemoteFunc) String() string { return "DBS.Ack" case DBSDeploy: return "DBS.Deploy" - case DBSSubscribeTransactions: - return "DBS.SubscribeTransactions" - case DBSCancelSubscription: - return "DBS.CancelSubscription" + case DBSObserverFetchBlock: + return "DBS.ObserverFetchBlock" case DBCCall: return "DBC.Call" case SQLCAdviseNewBlock: @@ -172,8 +164,6 @@ func (s RemoteFunc) String() string { return "SQLC.SignBilling" case SQLCLaunchBilling: return "SQLC.LaunchBilling" - case OBSAdviseNewBlock: - return "OBS.AdviseNewBlock" case MCCAdviseNewBlock: return "MCC.AdviseNewBlock" case MCCAdviseTxBilling: diff --git a/route/acl_test.go b/route/acl_test.go index c59e85811..60732f031 100644 --- a/route/acl_test.go +++ b/route/acl_test.go @@ -59,7 +59,7 @@ func TestIsPermitted(t *testing.T) { }) Convey("string RemoteFunc", t, func() { - for i := DHTPing; i <= OBSAdviseNewBlock; i++ { + for i := DHTPing; i <= MCCQueryTxState; i++ { So(fmt.Sprintf("%s", RemoteFunc(i)), ShouldContainSubstring, ".") } So(fmt.Sprintf("%s", RemoteFunc(9999)), ShouldContainSubstring, "Unknown") diff --git a/sqlchain/blockindex.go b/sqlchain/blockindex.go index 04bf7d97f..4256b675f 100644 --- a/sqlchain/blockindex.go +++ b/sqlchain/blockindex.go @@ -77,6 +77,21 @@ func (n *blockNode) ancestor(height int32) (ancestor *blockNode) { return } +func (n *blockNode) ancestorByCount(count int32) (ancestor *blockNode) { + if count < 0 || count > n.count { + return nil + } + + for ancestor = n; ancestor != nil && ancestor.count > count; ancestor = ancestor.parent { + } + + if ancestor != nil && ancestor.count < count { + ancestor = nil + } + + return +} + func (n *blockNode) indexKey() (key []byte) { key = make([]byte, hash.HashSize+4) binary.BigEndian.PutUint32(key[0:4], uint32(n.height)) diff --git a/sqlchain/chain.go b/sqlchain/chain.go index 2ee56f765..dae801222 100644 --- a/sqlchain/chain.go +++ b/sqlchain/chain.go @@ -119,15 +119,6 @@ type Chain struct { gasPrice uint64 updatePeriod uint64 - // observerLock defines the lock of observer update operations. - observerLock sync.Mutex - // observers defines the observer nodes of current chain. - observers map[proto.NodeID]int32 - // observerReplicators defines the observer states of current chain. - observerReplicators map[proto.NodeID]*observerReplicator - // replCh defines the replication trigger channel for replication check. - replCh chan struct{} - // Cached fileds, may need to renew some of this fields later. // // pk is the private key of the local miner. @@ -215,11 +206,6 @@ func NewChainWithContext(ctx context.Context, c *Config) (chain *Chain, err erro updatePeriod: c.UpdatePeriod, databaseID: c.DatabaseID, - // Observer related - observers: make(map[proto.NodeID]int32), - observerReplicators: make(map[proto.NodeID]*observerReplicator), - replCh: make(chan struct{}), - pk: pk, addr: &addr, } @@ -295,11 +281,6 @@ func LoadChainWithContext(ctx context.Context, c *Config) (chain *Chain, err err updatePeriod: c.UpdatePeriod, databaseID: c.DatabaseID, - // Observer related - observers: make(map[proto.NodeID]int32), - observerReplicators: make(map[proto.NodeID]*observerReplicator), - replCh: make(chan struct{}), - pk: pk, addr: &addr, } @@ -657,8 +638,6 @@ func (c *Chain) produceBlock(now time.Time) (err error) { } wg.Wait() - // fire replication to observers - c.startStopReplication(c.rt.ctx) return } @@ -939,8 +918,6 @@ func (c *Chain) processBlocks(ctx context.Context) { } } } - // fire replication to observers - c.startStopReplication(c.rt.ctx) case <-ctx.Done(): return } @@ -955,7 +932,6 @@ func (c *Chain) Start() (err error) { c.rt.goFunc(c.processBlocks) c.rt.goFunc(c.mainCycle) - c.rt.goFunc(c.replicationCycle) c.rt.startService(c) return } @@ -1007,21 +983,53 @@ func (c *Chain) Stop() (err error) { // FetchBlock fetches the block at specified height from local cache. func (c *Chain) FetchBlock(height int32) (b *types.Block, err error) { if n := c.rt.getHead().node.ancestor(height); n != nil { - k := utils.ConcatAll(metaBlockIndex[:], n.indexKey()) - var v []byte - v, err = c.bdb.Get(k, nil) + b, err = c.fetchBlockByIndexKey(n.indexKey()) if err != nil { - err = errors.Wrapf(err, "fetch block %s", string(k)) return } + } + + return +} - b = &types.Block{} - statBlock(b) - err = utils.DecodeMsgPack(v, b) +// FetchBlockByCount fetches the block at specified count from local cache. +func (c *Chain) FetchBlockByCount(count int32) (b *types.Block, realCount int32, height int32, err error) { + var n *blockNode + + if count < 0 { + n = c.rt.getHead().node + } else { + n = c.rt.getHead().node.ancestorByCount(count) + } + + if n != nil { + b, err = c.fetchBlockByIndexKey(n.indexKey()) if err != nil { - err = errors.Wrapf(err, "fetch block %s", string(k)) return } + + height = n.height + realCount = n.count + } + + return +} + +func (c *Chain) fetchBlockByIndexKey(indexKey []byte) (b *types.Block, err error) { + k := utils.ConcatAll(metaBlockIndex[:], indexKey) + var v []byte + v, err = c.bdb.Get(k, nil) + if err != nil { + err = errors.Wrapf(err, "fetch block %s", string(k)) + return + } + + b = &types.Block{} + statBlock(b) + err = utils.DecodeMsgPack(v, b) + if err != nil { + err = errors.Wrapf(err, "fetch block %s", string(k)) + return } return @@ -1121,82 +1129,6 @@ func (c *Chain) UpdatePeers(peers *proto.Peers) error { return c.rt.updatePeers(peers) } -// AddSubscription is used by dbms to add an observer. -func (c *Chain) AddSubscription(nodeID proto.NodeID, startHeight int32) (err error) { - // send previous height and transactions using AdviseAckedQuery/AdviseNewBlock RPC method - // add node to subscriber list - c.observerLock.Lock() - defer c.observerLock.Unlock() - c.observers[nodeID] = startHeight - c.startStopReplication(c.rt.ctx) - return -} - -// CancelSubscription is used by dbms to cancel an observer. -func (c *Chain) CancelSubscription(nodeID proto.NodeID) (err error) { - // remove node from subscription list - c.observerLock.Lock() - defer c.observerLock.Unlock() - delete(c.observers, nodeID) - c.startStopReplication(c.rt.ctx) - return -} - -func (c *Chain) startStopReplication(ctx context.Context) { - if c.replCh != nil { - select { - case c.replCh <- struct{}{}: - case <-ctx.Done(): - default: - } - } -} - -func (c *Chain) populateObservers() { - c.observerLock.Lock() - defer c.observerLock.Unlock() - - // handle replication threads - for nodeID, startHeight := range c.observers { - if replicator, exists := c.observerReplicators[nodeID]; exists { - // already started - if startHeight >= 0 { - replicator.setNewHeight(startHeight) - c.observers[nodeID] = int32(-1) - } - } else { - // start new replication routine - replicator := newObserverReplicator(nodeID, startHeight, c) - c.observerReplicators[nodeID] = replicator - c.rt.goFunc(replicator.run) - } - } - - // stop replicators - for nodeID, replicator := range c.observerReplicators { - if _, exists := c.observers[nodeID]; !exists { - replicator.stop() - delete(c.observerReplicators, nodeID) - } - } -} - -func (c *Chain) replicationCycle(ctx context.Context) { - for { - select { - case <-c.replCh: - // populateObservers - c.populateObservers() - // send triggers to replicators - for _, replicator := range c.observerReplicators { - replicator.tick() - } - case <-ctx.Done(): - return - } - } -} - // Query queries req from local chain state and returns the query results in resp. func (c *Chain) Query( req *types.Request, isLeader bool) (tracker *x.QueryTracker, resp *types.Response, err error, diff --git a/sqlchain/observer.go b/sqlchain/observer.go deleted file mode 100644 index 169b4892d..000000000 --- a/sqlchain/observer.go +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright 2018 The CovenantSQL Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package sqlchain - -import ( - "context" - "sync" - - "github.com/CovenantSQL/CovenantSQL/proto" - "github.com/CovenantSQL/CovenantSQL/route" - "github.com/CovenantSQL/CovenantSQL/types" - "github.com/CovenantSQL/CovenantSQL/utils/log" -) - -/* -Observer implements method AdviseNewBlock to receive blocks from sqlchain node. -Request/Response entity from sqlchain api is re-used for simplicity. - -type Observer interface { - AdviseNewBlock(*MuxAdviseNewBlockReq, *MuxAdviseNewBlockResp) error -} -*/ - -// observerReplicator defines observer replication state. -type observerReplicator struct { - nodeID proto.NodeID - height int32 - triggerCh chan struct{} - stopOnce sync.Once - stopCh chan struct{} - replLock sync.Mutex - c *Chain -} - -// newObserverReplicator creates new observer. -func newObserverReplicator(nodeID proto.NodeID, startHeight int32, c *Chain) *observerReplicator { - return &observerReplicator{ - nodeID: nodeID, - height: startHeight, - triggerCh: make(chan struct{}, 1), - stopCh: make(chan struct{}, 1), - c: c, - } -} - -func (r *observerReplicator) setNewHeight(newHeight int32) { - r.replLock.Lock() - defer r.replLock.Unlock() - r.height = newHeight -} - -func (r *observerReplicator) stop() { - r.stopOnce.Do(func() { - select { - case <-r.stopCh: - default: - close(r.stopCh) - } - }) -} - -func (r *observerReplicator) replicate() { - r.replLock.Lock() - defer r.replLock.Unlock() - - var err error - - defer func() { - if err != nil { - // TODO(xq262144), add backoff logic to prevent sqlchain node from flooding the observer - } - }() - - curHeight := r.c.rt.getHead().Height - - if r.height == types.ReplicateFromNewest { - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": curHeight, - }).Warning("observer being set to read from the newest block") - r.height = curHeight - } else if r.height > curHeight+1 { - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": r.height, - }).Warning("observer subscribes to height not yet produced") - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": curHeight + 1, - }).Warning("reset observer to height") - r.height = curHeight + 1 - } else if r.height == curHeight+1 { - // wait for next block - log.WithField("node", r.nodeID).Info("no more blocks for observer to read") - return - } - - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": r.height, - }).Debug("try replicating block for observer") - - // replicate one record - var block *types.Block - if block, err = r.c.FetchBlock(r.height); err != nil { - // fetch block failed - log.WithField("height", r.height).WithError(err).Warning("fetch block with height failed") - return - } else if block == nil { - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": r.height, - }).Debug("no block of height for observer") - - // black hole in chain? - // find last available block - log.Debug("start block height hole detection") - - var lastBlock, nextBlock *types.Block - var lastHeight, nextHeight int32 - - for h := r.height - 1; h >= 0; h-- { - if lastBlock, err = r.c.FetchBlock(h); err == nil && lastBlock != nil { - lastHeight = h - log.WithFields(log.Fields{ - "block": lastBlock.BlockHash().String(), - "height": lastHeight, - }).Debug("found last available block of height") - break - } - } - - if lastBlock == nil { - // could not find last available block, this should be a fatal issue - log.Warning("could not found last available block during hole detection") - return - } - - // find next available block - for h := r.height + 1; h <= curHeight; h++ { - if nextBlock, err = r.c.FetchBlock(h); err == nil && nextBlock != nil { - if !nextBlock.ParentHash().IsEqual(lastBlock.BlockHash()) { - // inconsistency - log.WithFields(log.Fields{ - "lastHeight": lastHeight, - "lastHash": lastBlock.BlockHash().String(), - "nextHeight": h, - "nextHash": nextBlock.BlockHash().String(), - "actualParentHash": nextBlock.ParentHash().String(), - }).Warning("inconsistency detected during hole detection") - - return - } - - nextHeight = h - log.WithFields(log.Fields{ - "block": nextBlock.BlockHash().String(), - "height": nextHeight, - }).Debug("found next available block of height") - break - } - } - - if nextBlock == nil { - // could not find next available block, try next time - log.Debug("could not found next available block during hole detection") - return - } - - // successfully found a hole in chain - log.WithFields(log.Fields{ - "fromBlock": lastBlock.BlockHash().String(), - "fromHeight": lastHeight, - "toBlock": nextBlock.BlockHash().String(), - "toHeight": nextHeight, - "skipped": nextHeight - lastHeight - 1, - }).Debug("found a hole in chain, skipping") - - r.height = nextHeight - block = nextBlock - - log.WithFields(log.Fields{ - "block": block.BlockHash().String(), - "height": r.height, - }).Debug("finish block height hole detection, skipping") - } - - // send block - req := &MuxAdviseNewBlockReq{ - Envelope: proto.Envelope{}, - DatabaseID: r.c.databaseID, - AdviseNewBlockReq: AdviseNewBlockReq{ - Block: block, - Count: func() int32 { - if nd := r.c.bi.lookupNode(block.BlockHash()); nd != nil { - return nd.count - } - if pn := r.c.bi.lookupNode(block.ParentHash()); pn != nil { - return pn.count + 1 - } - return -1 - }(), - }, - } - resp := &MuxAdviseNewBlockResp{} - err = r.c.cl.CallNode(r.nodeID, route.OBSAdviseNewBlock.String(), req, resp) - if err != nil { - log.WithFields(log.Fields{ - "node": r.nodeID, - "height": r.height, - }).WithError(err).Warning("send block advise to observer failed") - return - } - - // advance to next height - r.height++ - - if r.height <= r.c.rt.getHead().Height { - // send ticks to myself - r.tick() - } -} - -func (r *observerReplicator) tick() { - select { - case r.triggerCh <- struct{}{}: - default: - } -} -func (r *observerReplicator) run(ctx context.Context) { - for { - select { - case <-r.triggerCh: - // replication - r.replicate() - case <-ctx.Done(): - r.stop() - return - case <-r.stopCh: - return - } - } -} diff --git a/worker/dbms.go b/worker/dbms.go index 6aa7834f9..d608281af 100644 --- a/worker/dbms.go +++ b/worker/dbms.go @@ -572,75 +572,6 @@ func (dbms *DBMS) checkPermission(addr proto.AccountAddress, return } -func (dbms *DBMS) addTxSubscription(dbID proto.DatabaseID, nodeID proto.NodeID, startHeight int32) (err error) { - // check permission - pubkey, err := kms.GetPublicKey(nodeID) - if err != nil { - log.WithFields(log.Fields{ - "databaseID": dbID, - "nodeID": nodeID, - }).WithError(err).Warning("get public key failed in addTxSubscription") - return - } - addr, err := crypto.PubKeyHash(pubkey) - if err != nil { - log.WithFields(log.Fields{ - "databaseID": dbID, - "nodeID": nodeID, - }).WithError(err).Warning("generate addr failed in addTxSubscription") - return - } - - log.WithFields(log.Fields{ - "dbID": dbID, - "nodeID": nodeID, - "addr": addr.String(), - "startHeight": startHeight, - }).Debugf("addTxSubscription") - - err = dbms.checkPermission(addr, dbID, types.ReadQuery, nil) - if err != nil { - log.WithFields(log.Fields{"databaseID": dbID, "addr": addr}).WithError(err).Warning("permission deny") - return - } - - rawDB, ok := dbms.dbMap.Load(dbID) - if !ok { - err = ErrNotExists - log.WithFields(log.Fields{ - "databaseID": dbID, - "nodeID": nodeID, - "startHeight": startHeight, - }).WithError(err).Warning("unexpected error in addTxSubscription") - return - } - db := rawDB.(*Database) - err = db.chain.AddSubscription(nodeID, startHeight) - return -} - -func (dbms *DBMS) cancelTxSubscription(dbID proto.DatabaseID, nodeID proto.NodeID) (err error) { - rawDB, ok := dbms.dbMap.Load(dbID) - if !ok { - err = ErrNotExists - log.WithFields(log.Fields{ - "databaseID": dbID, - "nodeID": nodeID, - }).WithError(err).Warning("unexpected error in cancelTxSubscription") - return - } - db := rawDB.(*Database) - err = db.chain.CancelSubscription(nodeID) - if err != nil { - log.WithFields(log.Fields{ - "databaseID": dbID, - "nodeID": nodeID, - }).WithError(err).Warning("unexpected error in cancelTxSubscription") - return - } - return -} - // Shutdown defines dbms shutdown logic. func (dbms *DBMS) Shutdown() (err error) { dbms.dbMap.Range(func(_, rawDB interface{}) bool { diff --git a/worker/dbms_rpc.go b/worker/dbms_rpc.go index e3d5dda3b..b2289c7f0 100644 --- a/worker/dbms_rpc.go +++ b/worker/dbms_rpc.go @@ -30,27 +30,17 @@ var ( dbQueryFailCounter metrics.Meter ) -// SubscribeTransactionsReq defines a request of SubscribeTransaction RPC method. -type SubscribeTransactionsReq struct { +// ObserverFetchBlockReq defines the request for observer to fetch block. +type ObserverFetchBlockReq struct { proto.Envelope - DatabaseID proto.DatabaseID - Height int32 + proto.DatabaseID + Count int32 } -// SubscribeTransactionsResp defines a response of SubscribeTransaction RPC method. -type SubscribeTransactionsResp struct { - proto.Envelope -} - -// CancelSubscriptionReq defines a request of CancelSubscription RPC method. -type CancelSubscriptionReq struct { - proto.Envelope - DatabaseID proto.DatabaseID -} - -// CancelSubscriptionResp defines a response of CancelSubscription RPC method. -type CancelSubscriptionResp struct { - proto.Envelope +// ObserverFetchBlockResp defines the response for observer to fetch block. +type ObserverFetchBlockResp struct { + Count int32 + Block *types.Block } // DBMSRPCService is the rpc endpoint of database management. @@ -143,17 +133,3 @@ func (rpc *DBMSRPCService) Deploy(req *types.UpdateService, _ *types.UpdateServi return } - -// SubscribeTransactions is the RPC method to fetch subscribe new packed and confirmed transactions from the target server. -func (rpc *DBMSRPCService) SubscribeTransactions(req *SubscribeTransactionsReq, resp *SubscribeTransactionsResp) (err error) { - subscribeID := req.GetNodeID().ToNodeID() - err = rpc.dbms.addTxSubscription(req.DatabaseID, subscribeID, req.Height) - return -} - -// CancelSubscription is the RPC method to cancel subscription in the target server. -func (rpc *DBMSRPCService) CancelSubscription(req *CancelSubscriptionReq, _ *CancelSubscriptionResp) (err error) { - nodeID := req.GetNodeID().ToNodeID() - err = rpc.dbms.cancelTxSubscription(req.DatabaseID, nodeID) - return -} diff --git a/worker/dbms_test.go b/worker/dbms_test.go index 5984bb6c9..db29cead8 100644 --- a/worker/dbms_test.go +++ b/worker/dbms_test.go @@ -192,11 +192,9 @@ func TestDBMS(t *testing.T) { err = testRequest(route.DBSAck, ack, &ackRes) So(err, ShouldBeNil) - err = dbms.addTxSubscription(dbID2, nodeID, 1) + _, _, err = dbms.observerFetchBlock(dbID2, nodeID, 1) So(err.Error(), ShouldContainSubstring, ErrPermissionDeny.Error()) - err = dbms.addTxSubscription(dbID, nodeID, 1) - So(err, ShouldBeNil) - err = dbms.cancelTxSubscription(dbID, nodeID) + _, _, err = dbms.observerFetchBlock(dbID, nodeID, 1) So(err, ShouldBeNil) // revoke write permission @@ -235,7 +233,7 @@ func TestDBMS(t *testing.T) { err = testRequest(route.DBSQuery, readQuery, &queryRes) So(err, ShouldBeNil) - err = dbms.addTxSubscription(dbID, nodeID, 1) + _, _, err = dbms.observerFetchBlock(dbID, nodeID, 1) So(err, ShouldBeNil) }) @@ -315,7 +313,7 @@ func TestDBMS(t *testing.T) { err = testRequest(route.DBSQuery, readQuery, &queryRes) So(err.Error(), ShouldContainSubstring, ErrPermissionDeny.Error()) - err = dbms.addTxSubscription(dbID, nodeID, 1) + _, _, err = dbms.observerFetchBlock(dbID, nodeID, 1) So(err.Error(), ShouldContainSubstring, ErrPermissionDeny.Error()) }) diff --git a/worker/observer.go b/worker/observer.go new file mode 100644 index 000000000..37e580c38 --- /dev/null +++ b/worker/observer.go @@ -0,0 +1,98 @@ +/* + * Copyright 2019 The CovenantSQL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package worker + +import ( + "github.com/CovenantSQL/CovenantSQL/crypto" + "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" + "github.com/CovenantSQL/CovenantSQL/crypto/kms" + "github.com/CovenantSQL/CovenantSQL/proto" + "github.com/CovenantSQL/CovenantSQL/types" + "github.com/CovenantSQL/CovenantSQL/utils/log" +) + +// ObserverFetchBlock handles observer fetch block logic. +func (rpc *DBMSRPCService) ObserverFetchBlock(req *ObserverFetchBlockReq, resp *ObserverFetchBlockResp) (err error) { + subscriberID := req.GetNodeID().ToNodeID() + resp.Block, resp.Count, err = rpc.dbms.observerFetchBlock(req.DatabaseID, subscriberID, req.Count) + return +} + +func (dbms *DBMS) observerFetchBlock(dbID proto.DatabaseID, nodeID proto.NodeID, count int32) ( + block *types.Block, realCount int32, err error) { + var ( + pubKey *asymmetric.PublicKey + addr proto.AccountAddress + height int32 + ) + + // node parameters + pubKey, err = kms.GetPublicKey(nodeID) + if err != nil { + log.WithFields(log.Fields{ + "databaseID": dbID, + "nodeID": nodeID, + }).WithError(err).Warning("get public key failed in observerFetchBlock") + return + } + + addr, err = crypto.PubKeyHash(pubKey) + if err != nil { + log.WithFields(log.Fields{ + "databaseID": dbID, + "nodeID": nodeID, + }).WithError(err).Warning("generate addr failed in observerFetchBlock") + return + } + + defer func() { + lf := log.WithFields(log.Fields{ + "dbID": dbID, + "nodeID": nodeID, + "addr": addr.String(), + "count": count, + }) + + if err != nil { + lf.WithError(err).Debug("observer fetch block") + } else { + if block != nil { + lf = lf.WithField("block", block.BlockHash()) + } + lf.WithField("height", height).Debug("observer fetch block") + } + }() + + // check permission + err = dbms.checkPermission(addr, dbID, types.ReadQuery, nil) + if err != nil { + log.WithFields(log.Fields{ + "databaseID": dbID, + "addr": addr, + }).WithError(err).Warning("permission deny") + return + } + + rawDB, ok := dbms.dbMap.Load(dbID) + if !ok { + err = ErrNotExists + return + } + db := rawDB.(*Database) + block, realCount, height, err = db.chain.FetchBlockByCount(count) + return +} From baf34342b24f99d94a9f13a3bbfaf5e2c7702b71 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Mon, 18 Feb 2019 17:16:49 +0800 Subject: [PATCH 23/35] Fix bug of observer restart recover --- cmd/cql-observer/api.go | 11 +++++++++++ cmd/cql-observer/observation_test.go | 26 ++++++++++++++++++++++++++ cmd/cql-observer/service.go | 24 ++++++++++++++++++++++++ cmd/cql-observer/worker.go | 14 ++++++++++++-- 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/cmd/cql-observer/api.go b/cmd/cql-observer/api.go index 4026c3a89..b22a0eac4 100644 --- a/cmd/cql-observer/api.go +++ b/cmd/cql-observer/api.go @@ -83,6 +83,16 @@ func newPaginationFromReq(r *http.Request) (op *paginationOps) { return } +func (a *explorerAPI) GetAllSubscriptions(rw http.ResponseWriter, r *http.Request) { + subscriptions, err := a.service.getAllSubscriptions() + if err != nil { + sendResponse(500, false, err, nil, rw) + return + } + + sendResponse(200, true, "", subscriptions, rw) +} + func (a *explorerAPI) GetAck(rw http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -677,6 +687,7 @@ func startAPI(service *Service, listenAddr string) (server *http.Server, err err v3Router.HandleFunc("/count/{db}/{count:[0-9]+}", api.GetBlockByCountV3).Methods("GET") v3Router.HandleFunc("/height/{db}/{height:[0-9]+}", api.GetBlockByHeightV3).Methods("GET") v3Router.HandleFunc("/head/{db}", api.GetHighestBlockV3).Methods("GET") + v3Router.HandleFunc("/subscriptions", api.GetAllSubscriptions).Methods("GET") server = &http.Server{ Addr: listenAddr, diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index 532672371..a0d0d7d87 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -701,6 +701,32 @@ func TestFullProcess(t *testing.T) { err = client.Drop(dsn2) So(err, ShouldBeNil) + + observerCmd.Cmd.Process.Signal(os.Interrupt) + observerCmd.Cmd.Wait() + + // start observer again + observerCmd, err = utils.RunCommandNB( + FJ(baseDir, "./bin/cql-observer.test"), + []string{"-config", FJ(testWorkingDir, "./observation/node_observer/config.yaml"), + "-database", string(dbID), "-reset", "oldest", + "-test.coverprofile", FJ(baseDir, "./cmd/cql-observer/observer.cover.out"), + }, + "observer", testWorkingDir, logDir, false, + ) + So(err, ShouldBeNil) + + // call observer subscription status + // wait for observer to start + time.Sleep(time.Second * 3) + + res, err = getJSON("v3/subscriptions") + So(err, ShouldBeNil) + subscriptions, err := res.Object() + So(subscriptions, ShouldContainKey, string(dbID)) + So(subscriptions, ShouldContainKey, string(dbID2)) + So(subscriptions[string(dbID)], ShouldBeGreaterThanOrEqualTo, 1) + So(subscriptions[string(dbID2)], ShouldBeGreaterThanOrEqualTo, 0) }) } diff --git a/cmd/cql-observer/service.go b/cmd/cql-observer/service.go index c49138860..fc6379606 100644 --- a/cmd/cql-observer/service.go +++ b/cmd/cql-observer/service.go @@ -225,6 +225,19 @@ func (s *Service) start() (err error) { return nil } +func (s *Service) saveSubscriptionStatus(dbID proto.DatabaseID, count int32) (err error) { + log.WithFields(log.Fields{}).Debug("save subscription status") + + if atomic.LoadInt32(&s.stopped) == 1 { + // stopped + return ErrStopped + } + + return s.db.Update(func(tx *bolt.Tx) error { + return tx.Bucket(subscriptionBucket).Put([]byte(dbID), int32ToBytes(count)) + }) +} + func (s *Service) addAck(dbID proto.DatabaseID, height int32, offset int32, ack *types.SignedAckHeader) (err error) { log.WithFields(log.Fields{ "height": height, @@ -773,3 +786,14 @@ func (s *Service) getBlock(dbID proto.DatabaseID, h *hash.Hash) (count int32, he return } + +func (s *Service) getAllSubscriptions() (subscriptions map[proto.DatabaseID]int32, err error) { + subscriptions = map[proto.DatabaseID]int32{} + s.subscription.Range(func(_, rawWorker interface{}) bool { + worker := unpackWorker(rawWorker) + subscriptions[worker.dbID] = worker.getHead() + return true + }) + + return +} diff --git a/cmd/cql-observer/worker.go b/cmd/cql-observer/worker.go index 23f12e24d..fdc8e6920 100644 --- a/cmd/cql-observer/worker.go +++ b/cmd/cql-observer/worker.go @@ -58,7 +58,7 @@ func (w *subscribeWorker) run() { case <-w.stopCh: return case <-time.After(nextTick): - if err := w.pull(atomic.LoadInt32(&w.head)); err != nil { + if err := w.pull(w.getHead()); err != nil { // calc next tick nextTick = conf.GConf.SQLChainPeriod } else { @@ -115,7 +115,10 @@ func (w *subscribeWorker) pull(count int32) (err error) { next = count + 1 } - atomic.CompareAndSwapInt32(&w.head, count, next) + if atomic.CompareAndSwapInt32(&w.head, count, next) { + // update subscription status to database + _ = w.s.saveSubscriptionStatus(w.dbID, next) + } return } @@ -129,6 +132,9 @@ func (w *subscribeWorker) start() { w.l.Lock() defer w.l.Unlock() + // update subscription status to database + _ = w.s.saveSubscriptionStatus(w.dbID, w.getHead()) + if w.isStopped() { w.stopCh = make(chan struct{}) w.wg = new(sync.WaitGroup) @@ -137,6 +143,10 @@ func (w *subscribeWorker) start() { } } +func (w *subscribeWorker) getHead() int32 { + return atomic.LoadInt32(&w.head) +} + func (w *subscribeWorker) stop() { w.l.Lock() defer w.l.Unlock() From 64f76b57140cb899c6ac0188e61bf5a79edc57fa Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Mon, 18 Feb 2019 17:25:32 +0800 Subject: [PATCH 24/35] Add comments for count in ObserverFetchBlock rpc --- worker/dbms_rpc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/dbms_rpc.go b/worker/dbms_rpc.go index b2289c7f0..e0cff5b8c 100644 --- a/worker/dbms_rpc.go +++ b/worker/dbms_rpc.go @@ -34,12 +34,12 @@ var ( type ObserverFetchBlockReq struct { proto.Envelope proto.DatabaseID - Count int32 + Count int32 // sqlchain block serial number since genesis block (0) } // ObserverFetchBlockResp defines the response for observer to fetch block. type ObserverFetchBlockResp struct { - Count int32 + Count int32 // sqlchain block serial number since genesis block (0) Block *types.Block } From f3fce74a8dd32544751c3734460deb7e2d18e597 Mon Sep 17 00:00:00 2001 From: laodouya Date: Mon, 18 Feb 2019 18:22:55 +0800 Subject: [PATCH 25/35] Combine all createRandomBlock funcs to one(exclude sqlchain/xxx_test.go) --- client/helper_test.go | 43 +-------------------------------------- cmd/cql-minerd/dbms.go | 43 +-------------------------------------- types/block_test.go | 16 +++++++-------- types/util.go | 46 ++++++++++++++++++++++++++++++++++++++++++ types/xxx_test.go | 39 ----------------------------------- worker/db_test.go | 6 +++--- worker/dbms_test.go | 2 +- worker/helper_test.go | 42 -------------------------------------- 8 files changed, 60 insertions(+), 177 deletions(-) diff --git a/client/helper_test.go b/client/helper_test.go index bfae11f71..d6447b530 100644 --- a/client/helper_test.go +++ b/client/helper_test.go @@ -19,13 +19,11 @@ package client import ( "database/sql" "io/ioutil" - "math/rand" "os" "path/filepath" "runtime" "sync" "sync/atomic" - "time" pi "github.com/CovenantSQL/CovenantSQL/blockproducer/interfaces" "github.com/CovenantSQL/CovenantSQL/conf" @@ -143,7 +141,7 @@ func startTestService() (stopTestService func(), tempDir string, err error) { dbID := proto.DatabaseID("db") // create sqlchain block - block, err = createRandomBlock(rootHash, true) + block, err = types.CreateRandomBlock(rootHash, true) // get database peers if peers, err = genPeers(1); err != nil { @@ -267,45 +265,6 @@ func initNode() (cleanupFunc func(), tempDir string, server *rpc.Server, err err return } -// copied from sqlchain.xxx_test. -func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { - // Generate key pair - priv, _, err := asymmetric.GenSecp256k1KeyPair() - - if err != nil { - return - } - - h := hash.Hash{} - rand.Read(h[:]) - - b = &types.Block{ - SignedHeader: types.SignedHeader{ - Header: types.Header{ - Version: 0x01000000, - Producer: proto.NodeID(h.String()), - GenesisHash: rootHash, - ParentHash: parent, - Timestamp: time.Now().UTC(), - }, - }, - } - - if isGenesis { - emptyNode := &proto.RawNodeID{} - b.SignedHeader.ParentHash = hash.Hash{} - b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Producer = emptyNode.ToNodeID() - b.SignedHeader.MerkleRoot = hash.Hash{} - - err = b.PackAsGenesis() - return - } - - err = b.PackAndSignBlock(priv) - return -} - func testRequest(method route.RemoteFunc, req interface{}, response interface{}) (err error) { // get node id var nodeID proto.NodeID diff --git a/cmd/cql-minerd/dbms.go b/cmd/cql-minerd/dbms.go index d9b76fc3e..ab6696f3f 100644 --- a/cmd/cql-minerd/dbms.go +++ b/cmd/cql-minerd/dbms.go @@ -19,9 +19,7 @@ package main import ( "bytes" "io/ioutil" - "math/rand" "os" - "time" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" @@ -123,7 +121,7 @@ func loadGenesisBlock(fixture *conf.MinerDatabaseFixture) (block *types.Block, e if os.IsNotExist(err) && fixture.AutoGenerateGenesisBlock { // generate - if block, err = createRandomBlock(rootHash, true); err != nil { + if block, err = types.CreateRandomBlock(rootHash, true); err != nil { err = errors.Wrap(err, "create random block failed") return } @@ -145,42 +143,3 @@ func loadGenesisBlock(fixture *conf.MinerDatabaseFixture) (block *types.Block, e return } - -// copied from sqlchain.xxx_test. -func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { - // Generate key pair - priv, _, err := asymmetric.GenSecp256k1KeyPair() - - if err != nil { - return - } - - h := hash.Hash{} - rand.Read(h[:]) - - b = &types.Block{ - SignedHeader: types.SignedHeader{ - Header: types.Header{ - Version: 0x01000000, - Producer: proto.NodeID(h.String()), - GenesisHash: rootHash, - ParentHash: parent, - Timestamp: time.Now().UTC(), - }, - }, - } - - if isGenesis { - emptyNode := &proto.RawNodeID{} - b.SignedHeader.ParentHash = hash.Hash{} - b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Producer = emptyNode.ToNodeID() - b.SignedHeader.MerkleRoot = hash.Hash{} - - err = b.PackAsGenesis() - return - } - - err = b.PackAndSignBlock(priv) - return -} diff --git a/types/block_test.go b/types/block_test.go index a33001bee..5c5a26eea 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -30,7 +30,7 @@ import ( ) func TestSignAndVerify(t *testing.T) { - block, err := createRandomBlock(genesisHash, false) + block, err := CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -58,7 +58,7 @@ func TestSignAndVerify(t *testing.T) { } func TestHeaderMarshalUnmarshaler(t *testing.T) { - block, err := createRandomBlock(genesisHash, false) + block, err := CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -96,7 +96,7 @@ func TestHeaderMarshalUnmarshaler(t *testing.T) { } func TestSignedHeaderMarshaleUnmarshaler(t *testing.T) { - block, err := createRandomBlock(genesisHash, false) + block, err := CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -143,11 +143,11 @@ func TestSignedHeaderMarshaleUnmarshaler(t *testing.T) { } func TestBlockMarshalUnmarshaler(t *testing.T) { - origin, err := createRandomBlock(genesisHash, false) + origin, err := CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) } - origin2, err := createRandomBlock(genesisHash, false) + origin2, err := CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) } @@ -208,7 +208,7 @@ func TestBlockMarshalUnmarshaler(t *testing.T) { } func TestGenesis(t *testing.T) { - genesis, err := createRandomBlock(genesisHash, true) + genesis, err := CreateRandomBlock(genesisHash, true) if err != nil { t.Fatalf("error occurred: %v", err) @@ -219,7 +219,7 @@ func TestGenesis(t *testing.T) { } // Test non-genesis block - genesis, err = createRandomBlock(genesisHash, false) + genesis, err = CreateRandomBlock(genesisHash, false) if err != nil { t.Fatalf("error occurred: %v", err) @@ -232,7 +232,7 @@ func TestGenesis(t *testing.T) { } // Test altered block - genesis, err = createRandomBlock(genesisHash, true) + genesis, err = CreateRandomBlock(genesisHash, true) if err != nil { t.Fatalf("error occurred: %v", err) diff --git a/types/util.go b/types/util.go index ab09e4310..7773106f0 100644 --- a/types/util.go +++ b/types/util.go @@ -17,11 +17,18 @@ package types import ( + "crypto/rand" + "time" + + "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/crypto/verifier" + "github.com/CovenantSQL/CovenantSQL/proto" "github.com/pkg/errors" ) +var genesisHash = hash.Hash{} + type canMarshalHash interface { MarshalHash() ([]byte, error) } @@ -46,3 +53,42 @@ func buildHash(data canMarshalHash, h *hash.Hash) (err error) { copy(h[:], newHash[:]) return } + +// CreateRandomBlock create a new random block +func CreateRandomBlock(parent hash.Hash, isGenesis bool) (b *Block, err error) { + // Generate key pair + priv, _, err := asymmetric.GenSecp256k1KeyPair() + + if err != nil { + return + } + + h := hash.Hash{} + rand.Read(h[:]) + + b = &Block{ + SignedHeader: SignedHeader{ + Header: Header{ + Version: 0x01000000, + Producer: proto.NodeID(h.String()), + GenesisHash: genesisHash, + ParentHash: parent, + Timestamp: time.Now().UTC(), + }, + }, + } + + if isGenesis { + emptyNode := &proto.RawNodeID{} + b.SignedHeader.ParentHash = hash.Hash{} + b.SignedHeader.GenesisHash = hash.Hash{} + b.SignedHeader.Producer = emptyNode.ToNodeID() + b.SignedHeader.MerkleRoot = hash.Hash{} + + err = b.PackAsGenesis() + return + } + + err = b.PackAndSignBlock(priv) + return +} diff --git a/types/xxx_test.go b/types/xxx_test.go index 8e472a798..a15dd8666 100644 --- a/types/xxx_test.go +++ b/types/xxx_test.go @@ -33,7 +33,6 @@ import ( var ( uuidLen = 32 - genesisHash = hash.Hash{} testingPrivateKey *asymmetric.PrivateKey testingPublicKey *asymmetric.PublicKey ) @@ -299,44 +298,6 @@ func setup() { log.SetLevel(log.DebugLevel) } -func createRandomBlock(parent hash.Hash, isGenesis bool) (b *Block, err error) { - // Generate key pair - priv, _, err := asymmetric.GenSecp256k1KeyPair() - - if err != nil { - return - } - - h := hash.Hash{} - rand.Read(h[:]) - - b = &Block{ - SignedHeader: SignedHeader{ - Header: Header{ - Version: 0x01000000, - Producer: proto.NodeID(h.String()), - GenesisHash: genesisHash, - ParentHash: parent, - Timestamp: time.Now().UTC(), - }, - }, - } - - if isGenesis { - emptyNode := &proto.RawNodeID{} - b.SignedHeader.ParentHash = hash.Hash{} - b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Producer = emptyNode.ToNodeID() - b.SignedHeader.MerkleRoot = hash.Hash{} - - err = b.PackAsGenesis() - return - } - - err = b.PackAndSignBlock(priv) - return -} - func TestMain(m *testing.M) { setup() os.Exit(m.Run()) diff --git a/worker/db_test.go b/worker/db_test.go index 9a7e08421..ea9a00279 100644 --- a/worker/db_test.go +++ b/worker/db_test.go @@ -79,7 +79,7 @@ func TestSingleDatabase(t *testing.T) { // create genesis block var block *types.Block - block, err = createRandomBlock(rootHash, true) + block, err = types.CreateRandomBlock(rootHash, true) So(err, ShouldBeNil) // create database @@ -418,7 +418,7 @@ func TestInitFailed(t *testing.T) { // create genesis block var block *types.Block - block, err = createRandomBlock(rootHash, true) + block, err = types.CreateRandomBlock(rootHash, true) So(err, ShouldBeNil) // broken peers configuration @@ -472,7 +472,7 @@ func TestDatabaseRecycle(t *testing.T) { // create genesis block var block *types.Block - block, err = createRandomBlock(rootHash, true) + block, err = types.CreateRandomBlock(rootHash, true) So(err, ShouldBeNil) // create database diff --git a/worker/dbms_test.go b/worker/dbms_test.go index db29cead8..e30745f67 100644 --- a/worker/dbms_test.go +++ b/worker/dbms_test.go @@ -86,7 +86,7 @@ func TestDBMS(t *testing.T) { So(err, ShouldBeNil) // create sqlchain block - block, err = createRandomBlock(rootHash, true) + block, err = types.CreateRandomBlock(rootHash, true) So(err, ShouldBeNil) // get peers diff --git a/worker/helper_test.go b/worker/helper_test.go index 22c02183d..b9a09a186 100644 --- a/worker/helper_test.go +++ b/worker/helper_test.go @@ -17,19 +17,16 @@ package worker import ( - "crypto/rand" "io/ioutil" "os" "path/filepath" "runtime" "sync" "sync/atomic" - "time" "github.com/CovenantSQL/CovenantSQL/blockproducer/interfaces" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/consistent" - "github.com/CovenantSQL/CovenantSQL/crypto/asymmetric" "github.com/CovenantSQL/CovenantSQL/crypto/hash" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" @@ -243,42 +240,3 @@ func initNode() (cleanupFunc func(), server *rpc.Server, err error) { return } - -// copied from sqlchain.xxx_test. -func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { - // Generate key pair - priv, _, err := asymmetric.GenSecp256k1KeyPair() - - if err != nil { - return - } - - h := hash.Hash{} - rand.Read(h[:]) - - b = &types.Block{ - SignedHeader: types.SignedHeader{ - Header: types.Header{ - Version: 0x01000000, - Producer: proto.NodeID(h.String()), - GenesisHash: rootHash, - ParentHash: parent, - Timestamp: time.Now().UTC(), - }, - }, - } - - if isGenesis { - emptyNode := &proto.RawNodeID{} - b.SignedHeader.ParentHash = hash.Hash{} - b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Producer = emptyNode.ToNodeID() - b.SignedHeader.MerkleRoot = hash.Hash{} - - err = b.PackAsGenesis() - return - } - - err = b.PackAndSignBlock(priv) - return -} From 27fb7350e31ed62d858291d17130ac40f9cc67a7 Mon Sep 17 00:00:00 2001 From: laodouya Date: Mon, 18 Feb 2019 18:36:42 +0800 Subject: [PATCH 26/35] Change sqlchain/xxx_test.go createRandomBlock function to reuse types.CreateRandomBlock --- sqlchain/xxx_test.go | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/sqlchain/xxx_test.go b/sqlchain/xxx_test.go index 25b5e8350..5f2fcb45e 100644 --- a/sqlchain/xxx_test.go +++ b/sqlchain/xxx_test.go @@ -236,26 +236,19 @@ func registerNodesWithPublicKey(pub *asymmetric.PublicKey, diff int, num int) ( } func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err error) { - // Generate key pair - priv, _, err := asymmetric.GenSecp256k1KeyPair() - + b, err = types.CreateRandomBlock(parent, isGenesis) if err != nil { return } - h := hash.Hash{} - rand.Read(h[:]) + if isGenesis { + return + } - b = &types.Block{ - SignedHeader: types.SignedHeader{ - Header: types.Header{ - Version: 0x01000000, - Producer: proto.NodeID(h.String()), - GenesisHash: genesisHash, - ParentHash: parent, - Timestamp: time.Now().UTC(), - }, - }, + // Generate key pair + priv, _, err := asymmetric.GenSecp256k1KeyPair() + if err != nil { + return } for i, n := 0, rand.Intn(10)+10; i < n; i++ { @@ -270,18 +263,6 @@ func createRandomBlock(parent hash.Hash, isGenesis bool) (b *types.Block, err er } } - if isGenesis { - emptyNode := &proto.RawNodeID{} - b.SignedHeader.ParentHash = hash.Hash{} - b.SignedHeader.GenesisHash = hash.Hash{} - b.SignedHeader.Producer = emptyNode.ToNodeID() - b.SignedHeader.MerkleRoot = hash.Hash{} - b.Acks = nil - - err = b.PackAsGenesis() - return - } - err = b.PackAndSignBlock(priv) return } From ad0622887f2ea94d47410cb29e5f4f2e659d168f Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 19 Feb 2019 11:12:46 +0800 Subject: [PATCH 27/35] Add transaction hash as return value in create/drop --- client/clientbench_test.go | 4 ++-- client/driver.go | 7 ++++--- client/driver_test.go | 14 +++++++------- cmd/cql-adapter/storage/covenantsql.go | 4 ++-- cmd/cql-fuse/block_test.go | 2 +- cmd/cql-minerd/integration_test.go | 10 +++++----- cmd/cql-observer/observation_test.go | 4 ++-- cmd/cql/main.go | 18 ++++++++---------- 8 files changed, 31 insertions(+), 32 deletions(-) diff --git a/client/clientbench_test.go b/client/clientbench_test.go index 85573172a..782b8f3c2 100644 --- a/client/clientbench_test.go +++ b/client/clientbench_test.go @@ -66,7 +66,7 @@ func BenchmarkCovenantSQLDriver(b *testing.B) { // create meta := ResourceMeta{} meta.Node = 3 - dsn, err := Create(meta) + _, dsn, err := Create(meta) if err != nil { b.Fatal(err) } @@ -109,7 +109,7 @@ func BenchmarkCovenantSQLDriver(b *testing.B) { if err != nil { b.Fatal(err) } - err = Drop(dsn) + _, err = Drop(dsn) if err != nil { b.Fatal(err) } diff --git a/client/driver.go b/client/driver.go index c54afb2e9..055a03b75 100644 --- a/client/driver.go +++ b/client/driver.go @@ -144,8 +144,8 @@ func Init(configFile string, masterKey []byte) (err error) { return } -// Create send create database operation to block producer. -func Create(meta ResourceMeta) (dsn string, err error) { +// Create sends create database operation to block producer. +func Create(meta ResourceMeta) (txHash hash.Hash, dsn string, err error) { if atomic.LoadUint32(&driverInitialized) == 0 { err = ErrNotInitialized return @@ -202,6 +202,7 @@ func Create(meta ResourceMeta) (dsn string, err error) { return } + txHash = req.Tx.Hash() cfg := NewConfig() cfg.DatabaseID = string(proto.FromAccountAndNonce(clientAddr, uint32(nonceResp.Nonce))) dsn = cfg.FormatDSN() @@ -269,7 +270,7 @@ func WaitBPDatabaseCreation( } // Drop send drop database operation to block producer. -func Drop(dsn string) (err error) { +func Drop(dsn string) (txHash hash.Hash, err error) { if atomic.LoadUint32(&driverInitialized) == 0 { err = ErrNotInitialized return diff --git a/client/driver_test.go b/client/driver_test.go index 903e9dd42..200e5d592 100644 --- a/client/driver_test.go +++ b/client/driver_test.go @@ -118,12 +118,12 @@ func TestCreate(t *testing.T) { var err error var dsn string - dsn, err = Create(ResourceMeta{}) + _, dsn, err = Create(ResourceMeta{}) So(err, ShouldEqual, ErrNotInitialized) // fake driver initialized atomic.StoreUint32(&driverInitialized, 1) - dsn, err = Create(ResourceMeta{}) + _, dsn, err = Create(ResourceMeta{}) So(err, ShouldNotBeNil) // reset driver not initialized atomic.StoreUint32(&driverInitialized, 0) @@ -131,7 +131,7 @@ func TestCreate(t *testing.T) { stopTestService, _, err = startTestService() So(err, ShouldBeNil) defer stopTestService() - dsn, err = Create(ResourceMeta{}) + _, dsn, err = Create(ResourceMeta{}) So(err, ShouldBeNil) dsnCfg, err := ParseDSN(dsn) So(err, ShouldBeNil) @@ -171,16 +171,16 @@ func TestDrop(t *testing.T) { var stopTestService func() var err error - err = Drop("covenantsql://db") + _, err = Drop("covenantsql://db") So(err, ShouldEqual, ErrNotInitialized) stopTestService, _, err = startTestService() So(err, ShouldBeNil) defer stopTestService() - err = Drop("covenantsql://db") + _, err = Drop("covenantsql://db") So(err, ShouldBeNil) - err = Drop("invalid dsn") + _, err = Drop("invalid dsn") So(err, ShouldNotBeNil) }) } @@ -259,7 +259,7 @@ func TestWaitDBCreation(t *testing.T) { err = WaitDBCreation(ctx, "covenantsql://db") So(err, ShouldBeNil) - dsn, err = Create(ResourceMeta{}) + _, dsn, err = Create(ResourceMeta{}) So(err, ShouldBeNil) err = WaitDBCreation(ctx, dsn) So(err, ShouldNotBeNil) diff --git a/cmd/cql-adapter/storage/covenantsql.go b/cmd/cql-adapter/storage/covenantsql.go index f52d468eb..fb1772d9d 100644 --- a/cmd/cql-adapter/storage/covenantsql.go +++ b/cmd/cql-adapter/storage/covenantsql.go @@ -37,7 +37,7 @@ func (s *CovenantSQLStorage) Create(nodeCnt int) (dbID string, err error) { meta.Node = uint16(nodeCnt) var dsn string - if dsn, err = client.Create(meta); err != nil { + if _, dsn, err = client.Create(meta); err != nil { return } @@ -54,7 +54,7 @@ func (s *CovenantSQLStorage) Create(nodeCnt int) (dbID string, err error) { func (s *CovenantSQLStorage) Drop(dbID string) (err error) { cfg := client.NewConfig() cfg.DatabaseID = dbID - err = client.Drop(cfg.FormatDSN()) + _, err = client.Drop(cfg.FormatDSN()) return } diff --git a/cmd/cql-fuse/block_test.go b/cmd/cql-fuse/block_test.go index 817bbe91e..bf2a5103b 100644 --- a/cmd/cql-fuse/block_test.go +++ b/cmd/cql-fuse/block_test.go @@ -256,7 +256,7 @@ func initTestDB() (*sql.DB, func()) { // create meta := client.ResourceMeta{} meta.Node = 1 - dsn, err := client.Create(meta) + _, dsn, err := client.Create(meta) if err != nil { log.Errorf("create db failed: %v", err) return nil, stopNodes diff --git a/cmd/cql-minerd/integration_test.go b/cmd/cql-minerd/integration_test.go index d033c00b7..0f6df6f40 100644 --- a/cmd/cql-minerd/integration_test.go +++ b/cmd/cql-minerd/integration_test.go @@ -395,7 +395,7 @@ func TestFullProcess(t *testing.T) { t.Fatalf("wait for chain service failed: %v", err) } - dsn, err := client.Create(meta) + _, dsn, err := client.Create(meta) So(err, ShouldBeNil) dsnCfg, err := client.ParseDSN(dsn) So(err, ShouldBeNil) @@ -769,7 +769,7 @@ func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualCon // create meta := client.ResourceMeta{ ResourceMeta: types.ResourceMeta{ - Node: minerCount, + Node: minerCount, UseEventualConsistency: useEventualConsistency, }, } @@ -781,7 +781,7 @@ func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualCon b.Fatalf("wait for chain service failed: %v", err) } - dsn, err = client.Create(meta) + _, dsn, err = client.Create(meta) So(err, ShouldBeNil) log.Infof("the created database dsn is %v", dsn) err = ioutil.WriteFile(dsnFile, []byte(dsn), 0666) @@ -804,7 +804,7 @@ func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualCon benchDB(b, db, minerCount > 0) - err = client.Drop(dsn) + _, err = client.Drop(dsn) So(err, ShouldBeNil) time.Sleep(5 * time.Second) stopNodes() @@ -892,7 +892,7 @@ func benchOutsideMinerWithTargetMinerList( b.Fatalf("wait for chain service failed: %v", err) } - dsn, err = client.Create(meta) + _, dsn, err = client.Create(meta) So(err, ShouldBeNil) log.Infof("the created database dsn is %v", dsn) diff --git a/cmd/cql-observer/observation_test.go b/cmd/cql-observer/observation_test.go index a0d0d7d87..17414efc3 100644 --- a/cmd/cql-observer/observation_test.go +++ b/cmd/cql-observer/observation_test.go @@ -696,10 +696,10 @@ func TestFullProcess(t *testing.T) { So(ensureSuccess(res.Int("block", "height")), ShouldBeGreaterThanOrEqualTo, 0) log.Info(err, res) - err = client.Drop(dsn) + _, err = client.Drop(dsn) So(err, ShouldBeNil) - err = client.Drop(dsn2) + _, err = client.Drop(dsn2) So(err, ShouldBeNil) observerCmd.Cmd.Process.Signal(os.Interrupt) diff --git a/cmd/cql/main.go b/cmd/cql/main.go index 825e9f669..6ef7395e6 100644 --- a/cmd/cql/main.go +++ b/cmd/cql/main.go @@ -316,12 +316,17 @@ func main() { dropDB = cfg.FormatDSN() } - if err := client.Drop(dropDB); err != nil { + txHash, err := client.Drop(dropDB) + if err != nil { // drop database failed log.WithField("db", dropDB).WithError(err).Error("drop database failed") return } + if waitTxConfirmation { + wait(txHash) + } + // drop database success log.Infof("drop database %#v success", dropDB) return @@ -346,7 +351,7 @@ func main() { meta.Node = uint16(nodeCnt) } - dsn, err := client.Create(meta) + txHash, dsn, err := client.Create(meta) if err != nil { log.WithError(err).Error("create database failed") os.Exit(-1) @@ -354,14 +359,7 @@ func main() { } if waitTxConfirmation { - var ctx, cancel = context.WithTimeout(context.Background(), waitTxConfirmationMaxDuration) - defer cancel() - err = client.WaitDBCreation(ctx, dsn) - if err != nil { - log.WithError(err).Error("create database failed durating creation") - os.Exit(-1) - return - } + wait(txHash) } log.Infof("the newly created database is: %#v", dsn) From 7c9e5d75e27a195542dc208b8fa877148d1576be Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 19 Feb 2019 11:17:59 +0800 Subject: [PATCH 28/35] Minor fix --- client/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver.go b/client/driver.go index 055a03b75..114357f09 100644 --- a/client/driver.go +++ b/client/driver.go @@ -269,7 +269,7 @@ func WaitBPDatabaseCreation( } } -// Drop send drop database operation to block producer. +// Drop sends drop database operation to block producer. func Drop(dsn string) (txHash hash.Hash, err error) { if atomic.LoadUint32(&driverInitialized) == 0 { err = ErrNotInitialized From 970277b653032d4a95d2ec2f11deed0c21526cf7 Mon Sep 17 00:00:00 2001 From: Levente Liu Date: Tue, 19 Feb 2019 16:13:33 +0800 Subject: [PATCH 29/35] Also wait for creation after transaction is confirmed --- cmd/cql/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/cql/main.go b/cmd/cql/main.go index 6ef7395e6..91b1442a3 100644 --- a/cmd/cql/main.go +++ b/cmd/cql/main.go @@ -360,6 +360,13 @@ func main() { if waitTxConfirmation { wait(txHash) + var ctx, cancel = context.WithTimeout(context.Background(), waitTxConfirmationMaxDuration) + defer cancel() + err = client.WaitDBCreation(ctx, dsn) + if err != nil { + log.WithError(err).Error("wait database failed during creation") + os.Exit(-1) + } } log.Infof("the newly created database is: %#v", dsn) From f6f665303c4497626ea36c8422a87446b94cfbf2 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Tue, 19 Feb 2019 15:48:48 +0800 Subject: [PATCH 30/35] Update dht consensus using gossip method --- cmd/cqld/adapter.go | 413 +++++++++++------------------------------ cmd/cqld/bench_test.go | 2 +- cmd/cqld/bootstrap.go | 94 ++-------- cmd/cqld/gossip.go | 43 +++++ cmd/cqld/kayak.go | 55 ------ cmd/cqld/main.go | 2 +- route/acl.go | 14 +- route/acl_test.go | 4 +- rpc/rpcutil.go | 5 +- 9 files changed, 186 insertions(+), 446 deletions(-) create mode 100644 cmd/cqld/gossip.go delete mode 100644 cmd/cqld/kayak.go diff --git a/cmd/cqld/adapter.go b/cmd/cqld/adapter.go index 8b41a2769..96437dd25 100644 --- a/cmd/cqld/adapter.go +++ b/cmd/cqld/adapter.go @@ -17,46 +17,29 @@ package main import ( - "bytes" "context" "database/sql" "os" + "sync" + "time" - bp "github.com/CovenantSQL/CovenantSQL/blockproducer" "github.com/CovenantSQL/CovenantSQL/consistent" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/kayak" - kt "github.com/CovenantSQL/CovenantSQL/kayak/types" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" + "github.com/CovenantSQL/CovenantSQL/rpc" "github.com/CovenantSQL/CovenantSQL/storage" - "github.com/CovenantSQL/CovenantSQL/types" "github.com/CovenantSQL/CovenantSQL/utils" "github.com/CovenantSQL/CovenantSQL/utils/log" "github.com/pkg/errors" ) -const ( - // CmdSet is the command to set node - CmdSet = "set" - // CmdSetDatabase is the command to set database - CmdSetDatabase = "set_database" - // CmdDeleteDatabase is the command to del database - CmdDeleteDatabase = "delete_database" -) - // LocalStorage holds consistent and storage struct type LocalStorage struct { consistent *consistent.Consistent *storage.Storage } -type compiledLog struct { - cmdType string - queries []storage.Query - nodeToSet *proto.Node -} - func initStorage(dbFile string) (stor *LocalStorage, err error) { var st *storage.Storage if st, err = storage.New(dbFile); err != nil { @@ -67,9 +50,6 @@ func initStorage(dbFile string) (stor *LocalStorage, err error) { { Pattern: "CREATE TABLE IF NOT EXISTS `dht` (`id` TEXT NOT NULL PRIMARY KEY, `node` BLOB);", }, - { - Pattern: "CREATE TABLE IF NOT EXISTS `databases` (`id` TEXT NOT NULL PRIMARY KEY, `meta` BLOB);", - }, }) if err != nil { wd, _ := os.Getwd() @@ -84,339 +64,129 @@ func initStorage(dbFile string) (stor *LocalStorage, err error) { return } -// EncodePayload implements kayak.types.Handler.EncodePayload. -func (s *LocalStorage) EncodePayload(request interface{}) (data []byte, err error) { - var buf *bytes.Buffer - if buf, err = utils.EncodeMsgPack(request); err != nil { - err = errors.Wrap(err, "encode kayak payload failed") - return - } - - data = buf.Bytes() - return -} - -// DecodePayload implements kayak.types.Handler.DecodePayload. -func (s *LocalStorage) DecodePayload(data []byte) (request interface{}, err error) { - var kp *KayakPayload +// SetNode handles dht storage node update. +func (s *LocalStorage) SetNode(node *proto.Node) (err error) { + query := "INSERT OR REPLACE INTO `dht` (`id`, `node`) VALUES (?, ?);" + log.Debugf("sql: %#v", query) - if err = utils.DecodeMsgPack(data, &kp); err != nil { - err = errors.Wrap(err, "decode kayak payload failed") + nodeBuf, err := utils.EncodeMsgPack(node) + if err != nil { + err = errors.Wrap(err, "encode node failed") return } - request = kp - return -} - -// Check implements kayak.types.Handler.Check. -func (s *LocalStorage) Check(req interface{}) (err error) { - return nil -} - -// Commit implements kayak.types.Handler.Commit. -func (s *LocalStorage) Commit(req interface{}, isLeader bool) (_ interface{}, err error) { - var kp *KayakPayload - var cl *compiledLog - var ok bool - - if kp, ok = req.(*KayakPayload); !ok || kp == nil { - err = errors.Wrapf(kt.ErrInvalidLog, "invalid kayak payload %#v", req) - return + err = route.SetNodeAddrCache(node.ID.ToRawNodeID(), node.Addr) + if err != nil { + log.WithFields(log.Fields{ + "id": node.ID, + "addr": node.Addr, + }).WithError(err).Error("set node addr cache failed") } - - if cl, err = s.compileLog(kp); err != nil { - err = errors.Wrap(err, "compile log failed") - return + err = kms.SetNode(node) + if err != nil { + log.WithField("node", node).WithError(err).Error("kms set node failed") } - if cl.nodeToSet != nil { - err = route.SetNodeAddrCache(cl.nodeToSet.ID.ToRawNodeID(), cl.nodeToSet.Addr) - if err != nil { - log.WithFields(log.Fields{ - "id": cl.nodeToSet.ID, - "addr": cl.nodeToSet.Addr, - }).WithError(err).Error("set node addr cache failed") - } - err = kms.SetNode(cl.nodeToSet) - if err != nil { - log.WithField("node", cl.nodeToSet).WithError(err).Error("kms set node failed") - } - - // if s.consistent == nil, it is called during Init. and AddCache will be called by consistent.InitConsistent - if s.consistent != nil { - s.consistent.AddCache(*cl.nodeToSet) - } + // if s.consistent == nil, it is called during Init. and AddCache will be called by consistent.InitConsistent + if s.consistent != nil { + s.consistent.AddCache(*node) } // execute query - if _, err = s.Storage.Exec(context.Background(), cl.queries); err != nil { + if _, err = s.Storage.Exec(context.Background(), []storage.Query{ + { + Pattern: query, + Args: []sql.NamedArg{ + sql.Named("", node.ID), + sql.Named("", nodeBuf.Bytes()), + }, + }, + }); err != nil { err = errors.Wrap(err, "execute query in dht database failed") } return } -func (s *LocalStorage) compileLog(payload *KayakPayload) (result *compiledLog, err error) { - switch payload.Command { - case CmdSet: - var nodeToSet proto.Node - err = utils.DecodeMsgPack(payload.Data, &nodeToSet) - if err != nil { - log.WithError(err).Error("compileLog: unmarshal node from payload failed") - return - } - query := "INSERT OR REPLACE INTO `dht` (`id`, `node`) VALUES (?, ?);" - log.Debugf("sql: %#v", query) - result = &compiledLog{ - cmdType: payload.Command, - queries: []storage.Query{ - { - Pattern: query, - Args: []sql.NamedArg{ - sql.Named("", nodeToSet.ID), - sql.Named("", payload.Data), - }, - }, - }, - nodeToSet: &nodeToSet, - } - case CmdSetDatabase: - var instance types.ServiceInstance - if err = utils.DecodeMsgPack(payload.Data, &instance); err != nil { - log.WithError(err).Error("compileLog: unmarshal instance meta failed") - return - } - query := "INSERT OR REPLACE INTO `databases` (`id`, `meta`) VALUES (? ,?);" - result = &compiledLog{ - cmdType: payload.Command, - queries: []storage.Query{ - { - Pattern: query, - Args: []sql.NamedArg{ - sql.Named("", string(instance.DatabaseID)), - sql.Named("", payload.Data), - }, - }, - }, - } - case CmdDeleteDatabase: - var instance types.ServiceInstance - if err = utils.DecodeMsgPack(payload.Data, &instance); err != nil { - log.WithError(err).Error("compileLog: unmarshal instance id failed") - return - } - // TODO(xq262144), should add additional limit 1 after delete clause - // however, currently the go-sqlite3 - query := "DELETE FROM `databases` WHERE `id` = ?" - result = &compiledLog{ - cmdType: payload.Command, - queries: []storage.Query{ - { - Pattern: query, - Args: []sql.NamedArg{ - sql.Named("", string(instance.DatabaseID)), - }, - }, - }, - } - default: - err = errors.Errorf("undefined command: %v", payload.Command) - log.WithError(err).Error("compile log failed") - } - return +// KVServer holds LocalStorage instance and implements consistent persistence interface. +type KVServer struct { + current proto.NodeID + peers *proto.Peers + storage *LocalStorage + ctx context.Context + cancelCtx context.CancelFunc + timeout time.Duration + wg sync.WaitGroup } -// KayakKVServer holds kayak.Runtime and LocalStorage -type KayakKVServer struct { - Runtime *kayak.Runtime - KVStorage *LocalStorage +// NewKVServer returns the kv server instance. +func NewKVServer(currentNode proto.NodeID, peers *proto.Peers, storage *LocalStorage, timeout time.Duration) (s *KVServer) { + ctx, cancelCtx := context.WithCancel(context.Background()) + + return &KVServer{ + current: currentNode, + peers: peers, + storage: storage, + ctx: ctx, + cancelCtx: cancelCtx, + timeout: timeout, + } } // Init implements consistent.Persistence -func (s *KayakKVServer) Init(storePath string, initNodes []proto.Node) (err error) { +func (s *KVServer) Init(storePath string, initNodes []proto.Node) (err error) { for _, n := range initNodes { - var nodeBuf *bytes.Buffer - nodeBuf, err = utils.EncodeMsgPack(n) - if err != nil { - log.WithError(err).Error("marshal node failed") - return - } - payload := &KayakPayload{ - Command: CmdSet, - Data: nodeBuf.Bytes(), - } - _, err = s.KVStorage.Commit(payload, true) + err = s.storage.SetNode(&n) if err != nil { - log.WithError(err).Error("init kayak KV commit node failed") + log.WithError(err).Error("init dht kv server failed") return } } return } -// KayakPayload is the payload used in kayak Leader and Follower -type KayakPayload struct { - Command string - Data []byte +// SetNode implements consistent.Persistence +func (s *KVServer) SetNode(node *proto.Node) (err error) { + return s.SetNodeEx(node, 1, s.current) } -// SetNode implements consistent.Persistence -func (s *KayakKVServer) SetNode(node *proto.Node) (err error) { - nodeBuf, err := utils.EncodeMsgPack(node) - if err != nil { - log.WithError(err).Error("marshal node failed") +// SetNodeEx is used by gossip service to broadcast to other nodes. +func (s *KVServer) SetNodeEx(node *proto.Node, ttl uint32, origin proto.NodeID) (err error) { + log.WithFields(log.Fields{ + "node": node.ID, + "ttl": ttl, + "origin": origin, + }).Debug("update node to kv storage") + + // set local + if err = s.storage.SetNode(node); err != nil { + err = errors.Wrap(err, "set node failed") return } - payload := &KayakPayload{ - Command: CmdSet, - Data: nodeBuf.Bytes(), - } - _, _, err = s.Runtime.Apply(context.Background(), payload) - if err != nil { - log.Errorf("apply set node failed: %#v\nPayload:\n %#v", err, payload) + if ttl > 0 { + s.nonBlockingSync(node, origin, ttl-1) } return } // DelNode implements consistent.Persistence -func (s *KayakKVServer) DelNode(nodeID proto.NodeID) (err error) { +func (s *KVServer) DelNode(nodeID proto.NodeID) (err error) { // no need to del node currently return } // Reset implements consistent.Persistence -func (s *KayakKVServer) Reset() (err error) { - // no need to reset for kayak - return -} - -// GetDatabase implements blockproducer.DBMetaPersistence. -func (s *KayakKVServer) GetDatabase(dbID proto.DatabaseID) (instance types.ServiceInstance, err error) { - var result [][]interface{} - query := "SELECT `meta` FROM `databases` WHERE `id` = ? LIMIT 1" - _, _, result, err = s.KVStorage.Query(context.Background(), []storage.Query{ - { - Pattern: query, - Args: []sql.NamedArg{ - sql.Named("", string(dbID)), - }, - }, - }) - if err != nil { - log.WithField("db", dbID).WithError(err).Error("query database instance meta failed") - return - } - - if len(result) <= 0 || len(result[0]) <= 0 { - err = bp.ErrNoSuchDatabase - return - } - - var rawInstanceMeta []byte - var ok bool - if rawInstanceMeta, ok = result[0][0].([]byte); !ok { - err = bp.ErrNoSuchDatabase - return - } - - err = utils.DecodeMsgPack(rawInstanceMeta, &instance) - return -} - -// SetDatabase implements blockproducer.DBMetaPersistence. -func (s *KayakKVServer) SetDatabase(meta types.ServiceInstance) (err error) { - var metaBuf *bytes.Buffer - if metaBuf, err = utils.EncodeMsgPack(meta); err != nil { - return - } - - payload := &KayakPayload{ - Command: CmdSetDatabase, - Data: metaBuf.Bytes(), - } - - _, _, err = s.Runtime.Apply(context.Background(), payload) - if err != nil { - log.Errorf("apply set database failed: %#v\nPayload:\n %#v", err, payload) - } - - return -} - -// DeleteDatabase implements blockproducer.DBMetaPersistence. -func (s *KayakKVServer) DeleteDatabase(dbID proto.DatabaseID) (err error) { - meta := types.ServiceInstance{ - DatabaseID: dbID, - } - - var metaBuf *bytes.Buffer - if metaBuf, err = utils.EncodeMsgPack(meta); err != nil { - return - } - payload := &KayakPayload{ - Command: CmdDeleteDatabase, - Data: metaBuf.Bytes(), - } - - _, _, err = s.Runtime.Apply(context.Background(), payload) - if err != nil { - log.Errorf("apply set database failed: %#v\nPayload:\n %#v", err, payload) - } - - return -} - -// GetAllDatabases implements blockproducer.DBMetaPersistence. -func (s *KayakKVServer) GetAllDatabases() (instances []types.ServiceInstance, err error) { - var result [][]interface{} - query := "SELECT `meta` FROM `databases`" - _, _, result, err = s.KVStorage.Query(context.Background(), []storage.Query{ - { - Pattern: query, - }, - }) - if err != nil { - log.WithError(err).Error("query all database instance meta failed") - return - } - - instances = make([]types.ServiceInstance, 0, len(result)) - - for _, row := range result { - if len(row) <= 0 { - continue - } - - var instance types.ServiceInstance - var rawInstanceMeta []byte - var ok bool - if rawInstanceMeta, ok = row[0].([]byte); !ok { - err = bp.ErrNoSuchDatabase - continue - } - - if err = utils.DecodeMsgPack(rawInstanceMeta, &instance); err != nil { - continue - } - - instances = append(instances, instance) - } - - if len(instances) > 0 { - err = nil - } - +func (s *KVServer) Reset() (err error) { return } // GetAllNodeInfo implements consistent.Persistence -func (s *KayakKVServer) GetAllNodeInfo() (nodes []proto.Node, err error) { +func (s *KVServer) GetAllNodeInfo() (nodes []proto.Node, err error) { var result [][]interface{} query := "SELECT `node` FROM `dht`;" - _, _, result, err = s.KVStorage.Query(context.Background(), []storage.Query{ + _, _, result, err = s.storage.Query(context.Background(), []storage.Query{ { Pattern: query, }, @@ -451,5 +221,38 @@ func (s *KayakKVServer) GetAllNodeInfo() (nodes []proto.Node, err error) { if len(nodes) > 0 { err = nil } + return } + +// Stop stops the dht node server and wait for inflight gossip requests. +func (s *KVServer) Stop() { + if s.cancelCtx != nil { + s.cancelCtx() + } + s.wg.Wait() +} + +func (s *KVServer) nonBlockingSync(node *proto.Node, origin proto.NodeID, ttl uint32) { + if s.peers == nil { + return + } + + c, cancel := context.WithTimeout(s.ctx, s.timeout) + defer cancel() + for _, n := range s.peers.Servers { + if n != s.current && n != origin { + // sync + req := &GossipRequest{ + Node: node, + TTL: ttl, + } + + s.wg.Add(1) + go func(node proto.NodeID) { + defer s.wg.Done() + _ = rpc.NewCaller().CallNodeWithContext(c, node, route.DHTGSetNode.String(), req, nil) + }(n) + } + } +} diff --git a/cmd/cqld/bench_test.go b/cmd/cqld/bench_test.go index 37771d128..3bbd21fd0 100644 --- a/cmd/cqld/bench_test.go +++ b/cmd/cqld/bench_test.go @@ -235,7 +235,7 @@ func TestStartBP_CallRPC(t *testing.T) { } } -func BenchmarkKayakKVServer_GetAllNodeInfo(b *testing.B) { +func BenchmarkKVServer_GetAllNodeInfo(b *testing.B) { log.SetLevel(log.DebugLevel) start3BPs() diff --git a/cmd/cqld/bootstrap.go b/cmd/cqld/bootstrap.go index 80bf2d423..72dca2816 100644 --- a/cmd/cqld/bootstrap.go +++ b/cmd/cqld/bootstrap.go @@ -20,18 +20,13 @@ import ( "fmt" "os" "os/signal" - "path/filepath" "syscall" "time" "github.com/CovenantSQL/CovenantSQL/api" - bp "github.com/CovenantSQL/CovenantSQL/blockproducer" "github.com/CovenantSQL/CovenantSQL/conf" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - "github.com/CovenantSQL/CovenantSQL/kayak" - kt "github.com/CovenantSQL/CovenantSQL/kayak/types" - kl "github.com/CovenantSQL/CovenantSQL/kayak/wal" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/rpc" @@ -42,18 +37,11 @@ import ( ) const ( - kayakServiceName = "Kayak" - kayakApplyMethodName = "Apply" - kayakFetchMethodName = "Fetch" - kayakWalFileName = "kayak.ldb" - kayakPrepareTimeout = 5 * time.Second - kayakCommitTimeout = time.Minute - kayakLogWaitTimeout = 10 * time.Second + dhtGossipServiceName = "DHTG" + dhtGossipTimeout = time.Second * 20 ) func runNode(nodeID proto.NodeID, listenAddr string) (err error) { - rootPath := conf.GConf.WorkingRoot - genesis, err := loadGenesis() if err != nil { return @@ -124,30 +112,29 @@ func runNode(nodeID proto.NodeID, listenAddr string) (err error) { return err } - // init kayak - log.Info("init kayak runtime") - var kayakRuntime *kayak.Runtime - if kayakRuntime, err = initKayakTwoPC(rootPath, thisNode, peers, st, server); err != nil { - log.WithError(err).Error("init kayak runtime failed") - return err - } - - // init kayak and consistent - log.Info("init kayak and consistent runtime") - kvServer := &KayakKVServer{ - Runtime: kayakRuntime, - KVStorage: st, - } + // init dht node server + log.Info("init consistent runtime") + kvServer := NewKVServer(thisNode.ID, peers, st, dhtGossipTimeout) dht, err := route.NewDHTService(conf.GConf.DHTFileName, kvServer, true) if err != nil { log.WithError(err).Error("init consistent hash failed") return err } + defer kvServer.Stop() - // set consistent handler to kayak storage - kvServer.KVStorage.consistent = dht.Consistent + // set consistent handler to local storage + kvServer.storage.consistent = dht.Consistent - // register service rpc + // register gossip service rpc + gossipService := NewGossipService(kvServer) + log.Info("register dht gossip service rpc") + err = server.RegisterService(route.DHTGossipRPCName, gossipService) + if err != nil { + log.WithError(err).Error("register dht gossip service failed") + return err + } + + // register dht service rpc log.Info("register dht service rpc") err = server.RegisterService(route.DHTRPCName, dht) if err != nil { @@ -211,49 +198,8 @@ func createServer(privateKeyPath, pubKeyStorePath string, masterKey []byte, list return } -func initKayakTwoPC(rootDir string, node *proto.Node, peers *proto.Peers, h kt.Handler, server *rpc.Server) (runtime *kayak.Runtime, err error) { - // create kayak config - log.Info("create kayak config") - - walPath := filepath.Join(rootDir, kayakWalFileName) - - var logWal kt.Wal - if logWal, err = kl.NewLevelDBWal(walPath); err != nil { - err = errors.Wrap(err, "init kayak log pool failed") - return - } - - config := &kt.RuntimeConfig{ - Handler: h, - PrepareThreshold: 1.0, - CommitThreshold: 1.0, - PrepareTimeout: kayakPrepareTimeout, - CommitTimeout: kayakCommitTimeout, - LogWaitTimeout: kayakLogWaitTimeout, - Peers: peers, - Wal: logWal, - NodeID: node.ID, - ServiceName: kayakServiceName, - ApplyMethodName: kayakApplyMethodName, - FetchMethodName: kayakFetchMethodName, - } - - // create kayak runtime - log.Info("init kayak runtime") - if runtime, err = kayak.NewRuntime(config); err != nil { - err = errors.Wrap(err, "init kayak runtime failed") - return - } - - // register rpc service - if _, err = NewKayakService(server, kayakServiceName, runtime); err != nil { - err = errors.Wrap(err, "init kayak rpc service failed") - return - } - - // init runtime - log.Info("start kayak runtime") - runtime.Start() +func initDHTGossip() (err error) { + log.Info("init gossip service") return } diff --git a/cmd/cqld/gossip.go b/cmd/cqld/gossip.go new file mode 100644 index 000000000..46125332c --- /dev/null +++ b/cmd/cqld/gossip.go @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The CovenantSQL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import "github.com/CovenantSQL/CovenantSQL/proto" + +// GossipRequest defines the gossip request payload. +type GossipRequest struct { + proto.Envelope + Node *proto.Node + TTL uint32 +} + +// GossipService defines the gossip service instance. +type GossipService struct { + s *KVServer +} + +// NewGossipService returns new gossip service. +func NewGossipService(s *KVServer) *GossipService { + return &GossipService{ + s: s, + } +} + +// SetNode update current node info and broadcast node update request. +func (s *GossipService) SetNode(req *GossipRequest, resp *interface{}) (err error) { + return s.s.SetNodeEx(req.Node, req.TTL, req.GetNodeID().ToNodeID()) +} diff --git a/cmd/cqld/kayak.go b/cmd/cqld/kayak.go deleted file mode 100644 index ee36f886a..000000000 --- a/cmd/cqld/kayak.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2018 The CovenantSQL Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "github.com/CovenantSQL/CovenantSQL/kayak" - kt "github.com/CovenantSQL/CovenantSQL/kayak/types" - "github.com/CovenantSQL/CovenantSQL/rpc" -) - -// KayakService defines the leader service kayak. -type KayakService struct { - serviceName string - rt *kayak.Runtime -} - -// NewKayakService returns new kayak service instance for block producer consensus. -func NewKayakService(server *rpc.Server, serviceName string, rt *kayak.Runtime) (s *KayakService, err error) { - s = &KayakService{ - serviceName: serviceName, - rt: rt, - } - err = server.RegisterService(serviceName, s) - return -} - -// Apply handles kayak apply call. -func (s *KayakService) Apply(req *kt.ApplyRequest, _ *interface{}) (err error) { - return s.rt.FollowerApply(req.Log) -} - -// Fetch handles kayak log fetch call. -func (s *KayakService) Fetch(req *kt.FetchRequest, resp *kt.FetchResponse) (err error) { - var l *kt.Log - if l, err = s.rt.Fetch(req.GetContext(), req.Index); err != nil { - return - } - - resp.Log = l - return -} diff --git a/cmd/cqld/main.go b/cmd/cqld/main.go index 723dee0f0..00e7fd1bb 100644 --- a/cmd/cqld/main.go +++ b/cmd/cqld/main.go @@ -143,7 +143,7 @@ func main() { defer utils.StopProfile() if err := runNode(conf.GConf.ThisNodeID, conf.GConf.ListenAddr); err != nil { - log.WithError(err).Fatal("run kayak failed") + log.WithError(err).Fatal("run block producer node failed") } log.Info("server stopped") diff --git a/route/acl.go b/route/acl.go index 032aff54e..ae7a465ee 100644 --- a/route/acl.go +++ b/route/acl.go @@ -67,8 +67,8 @@ const ( DHTFindNeighbor // DHTFindNode gets node info DHTFindNode - // KayakCall is used by BP for data consistency - KayakCall + // DHTGSetNode is used by BP for dht data gossip + DHTGSetNode // MetricUploadMetrics uploads node metrics MetricUploadMetrics // DBSQuery is used by client to read/write database @@ -121,6 +121,8 @@ const ( // DHTRPCName defines the block producer dh-rpc service name DHTRPCName = "DHT" + // DHTGossipRPCName defines the block producer dh-rpc gossip service name + DHTGossipRPCName = "DHTG" // BlockProducerRPCName defines main chain rpc name BlockProducerRPCName = "MCC" // SQLChainRPCName defines the sql chain rpc name @@ -138,10 +140,10 @@ func (s RemoteFunc) String() string { return "DHT.FindNeighbor" case DHTFindNode: return "DHT.FindNode" + case DHTGSetNode: + return "DHTG.SetNode" case MetricUploadMetrics: return "Metric.UploadMetrics" - case KayakCall: - return "Kayak.Call" case DBSQuery: return "DBS.Query" case DBSAck: @@ -213,8 +215,8 @@ func IsPermitted(callerEnvelope *proto.Envelope, funcName RemoteFunc) (ok bool) // DHT related case DHTPing, DHTFindNode, DHTFindNeighbor, MetricUploadMetrics: return true - // Kayak related - case KayakCall: + // DHTGSetNode is for block producer to update node info + case DHTGSetNode: return false // DBSDeploy case DBSDeploy: diff --git a/route/acl_test.go b/route/acl_test.go index 60732f031..630e649fb 100644 --- a/route/acl_test.go +++ b/route/acl_test.go @@ -51,8 +51,8 @@ func TestIsPermitted(t *testing.T) { nodeID := proto.NodeID("0000") testEnv := &proto.Envelope{NodeID: nodeID.ToRawNodeID()} testAnonymous := &proto.Envelope{NodeID: kms.AnonymousRawNodeID} - So(IsPermitted(&proto.Envelope{NodeID: &conf.GConf.BP.RawNodeID}, KayakCall), ShouldBeTrue) - So(IsPermitted(testEnv, KayakCall), ShouldBeFalse) + So(IsPermitted(&proto.Envelope{NodeID: &conf.GConf.BP.RawNodeID}, DHTGSetNode), ShouldBeTrue) + So(IsPermitted(testEnv, DHTGSetNode), ShouldBeFalse) So(IsPermitted(testEnv, DHTFindNode), ShouldBeTrue) So(IsPermitted(testEnv, RemoteFunc(9999)), ShouldBeFalse) So(IsPermitted(testAnonymous, DHTFindNode), ShouldBeFalse) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 3588ca5a5..27e011538 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -430,7 +430,9 @@ func RegisterNodeToBP(timeout time.Duration) (err error) { err := PingBP(localNodeInfo, id) if err == nil { log.Infof("ping BP succeed: %v", localNodeInfo) - ch <- id + select { + case ch <- id: + } return } if strings.Contains(err.Error(), kt.ErrNotLeader.Error()) { @@ -446,7 +448,6 @@ func RegisterNodeToBP(timeout time.Duration) (err error) { select { case bp := <-pingWaitCh: - close(pingWaitCh) log.WithField("BP", bp).Infof("ping BP succeed") case <-time.After(timeout): return errors.New("ping BP timeout") From c25c9b549e55fee4dc5ed8832720cec76b14af8c Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Tue, 19 Feb 2019 15:52:02 +0800 Subject: [PATCH 31/35] Remove useless block producer leader/follower role check --- rpc/rpcutil.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 27e011538..1ee8204a6 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -28,7 +28,6 @@ import ( "time" "github.com/CovenantSQL/CovenantSQL/crypto/kms" - kt "github.com/CovenantSQL/CovenantSQL/kayak/types" "github.com/CovenantSQL/CovenantSQL/proto" "github.com/CovenantSQL/CovenantSQL/route" "github.com/CovenantSQL/CovenantSQL/utils/log" @@ -435,10 +434,6 @@ func RegisterNodeToBP(timeout time.Duration) (err error) { } return } - if strings.Contains(err.Error(), kt.ErrNotLeader.Error()) { - log.Debug("stop ping non leader BP node") - return - } log.Warnf("ping BP failed: %v", err) time.Sleep(3 * time.Second) From 8956e11ccfdc3ac2a32a6b0d1d8afe8ccc00c6a5 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Tue, 19 Feb 2019 17:02:01 +0800 Subject: [PATCH 32/35] Fix bug in RegisterNodeToBP feature --- rpc/rpcutil.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 1ee8204a6..535acb891 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -431,6 +431,7 @@ func RegisterNodeToBP(timeout time.Duration) (err error) { log.Infof("ping BP succeed: %v", localNodeInfo) select { case ch <- id: + default: } return } From 2bf6642073ee3b2742a6a8eeb9e6b107bf86889e Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Wed, 20 Feb 2019 14:56:03 +0800 Subject: [PATCH 33/35] Try all block producer to find locally unknown nodes --- rpc/rpcutil.go | 74 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 535acb891..8e356984e 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -249,26 +249,13 @@ func GetNodeAddr(id *proto.RawNodeID) (addr string, err error) { if err != nil { //log.WithField("target", id.String()).WithError(err).Debug("get node addr from cache failed") if err == route.ErrUnknownNodeID { - BPs := route.GetBPs() - if len(BPs) == 0 { - err = errors.New("no available BP") - return - } - client := NewCaller() - reqFN := &proto.FindNodeReq{ - ID: proto.NodeID(id.String()), - } - respFN := new(proto.FindNodeResp) - - bp := BPs[rand.Intn(len(BPs))] - method := "DHT.FindNode" - err = client.CallNode(bp, method, reqFN, respFN) + var node *proto.Node + node, err = FindNodeInBP(id) if err != nil { - err = errors.Wrapf(err, "call dht rpc %s to %s failed", method, bp) return } - route.SetNodeAddrCache(id, respFN.Node.Addr) - addr = respFN.Node.Addr + route.SetNodeAddrCache(id, node.Addr) + addr = node.Addr } } return @@ -280,24 +267,7 @@ func GetNodeInfo(id *proto.RawNodeID) (nodeInfo *proto.Node, err error) { if err != nil { //log.WithField("target", id.String()).WithError(err).Info("get node info from KMS failed") if errors.Cause(err) == kms.ErrKeyNotFound { - BPs := route.GetBPs() - if len(BPs) == 0 { - err = errors.New("no available BP") - return - } - client := NewCaller() - reqFN := &proto.FindNodeReq{ - ID: proto.NodeID(id.String()), - } - respFN := new(proto.FindNodeResp) - bp := BPs[rand.Intn(len(BPs))] - method := "DHT.FindNode" - err = client.CallNode(bp, method, reqFN, respFN) - if err != nil { - err = errors.Wrapf(err, "call dht rpc %s to %s failed", method, bp) - return - } - nodeInfo = respFN.Node + nodeInfo, err = FindNodeInBP(id) errSet := route.SetNodeAddrCache(id, nodeInfo.Addr) if errSet != nil { log.WithError(errSet).Warning("set node addr cache failed") @@ -311,6 +281,40 @@ func GetNodeInfo(id *proto.RawNodeID) (nodeInfo *proto.Node, err error) { return } +// FindNodeInBP find node in block producer dht service. +func FindNodeInBP(id *proto.RawNodeID) (node *proto.Node, err error) { + bps := route.GetBPs() + if len(bps) == 0 { + err = errors.New("no available BP") + return + } + client := NewCaller() + req := &proto.FindNodeReq{ + ID: proto.NodeID(id.String()), + } + resp := new(proto.FindNodeResp) + bpCount := len(bps) + offset := rand.Intn(bpCount) + method := route.DHTFindNode.String() + + for i := 0; i != bpCount; i++ { + bp := bps[(offset+i)%bpCount] + err = client.CallNode(bp, method, req, resp) + if err == nil { + node = resp.Node + return + } + + log.WithFields(log.Fields{ + "method": method, + "bp": bp, + }).WithError(err).Error("call dht rpc failed") + } + + err = errors.Wrapf(err, "could not find node in all block producers") + return +} + // PingBP Send DHT.Ping Request with Anonymous ETLS session. func PingBP(node *proto.Node, BPNodeID proto.NodeID) (err error) { client := NewCaller() From 40a75cada3ac3f296674c88087c41e7c6341d9cd Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Wed, 20 Feb 2019 15:10:35 +0800 Subject: [PATCH 34/35] Fix bug in rpcutil not checking FindNodeInBP result --- rpc/rpcutil.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpc/rpcutil.go b/rpc/rpcutil.go index 8e356984e..c2bacb5f4 100644 --- a/rpc/rpcutil.go +++ b/rpc/rpcutil.go @@ -268,6 +268,9 @@ func GetNodeInfo(id *proto.RawNodeID) (nodeInfo *proto.Node, err error) { //log.WithField("target", id.String()).WithError(err).Info("get node info from KMS failed") if errors.Cause(err) == kms.ErrKeyNotFound { nodeInfo, err = FindNodeInBP(id) + if err != nil { + return + } errSet := route.SetNodeAddrCache(id, nodeInfo.Addr) if errSet != nil { log.WithError(errSet).Warning("set node addr cache failed") @@ -308,7 +311,7 @@ func FindNodeInBP(id *proto.RawNodeID) (node *proto.Node, err error) { log.WithFields(log.Fields{ "method": method, "bp": bp, - }).WithError(err).Error("call dht rpc failed") + }).WithError(err).Warning("call dht rpc failed") } err = errors.Wrapf(err, "could not find node in all block producers") From 2b66081852964d1bc09c74d2fe8311de0bc41f90 Mon Sep 17 00:00:00 2001 From: Qi Xiao Date: Mon, 25 Feb 2019 19:01:59 +0800 Subject: [PATCH 35/35] Remove imports blank lines --- api/service_test.go | 9 ++++----- cmd/cql-minerd/integration_test.go | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/api/service_test.go b/api/service_test.go index a66e12ad8..52bd09e52 100644 --- a/api/service_test.go +++ b/api/service_test.go @@ -8,15 +8,14 @@ import ( "testing" "time" - "github.com/pkg/errors" - - "github.com/CovenantSQL/CovenantSQL/api" - "github.com/CovenantSQL/CovenantSQL/api/models" - "github.com/gorilla/websocket" + "github.com/pkg/errors" . "github.com/smartystreets/goconvey/convey" "github.com/sourcegraph/jsonrpc2" wsstream "github.com/sourcegraph/jsonrpc2/websocket" + + "github.com/CovenantSQL/CovenantSQL/api" + "github.com/CovenantSQL/CovenantSQL/api/models" ) const ( diff --git a/cmd/cql-minerd/integration_test.go b/cmd/cql-minerd/integration_test.go index 9d8202538..ff818f234 100644 --- a/cmd/cql-minerd/integration_test.go +++ b/cmd/cql-minerd/integration_test.go @@ -770,7 +770,7 @@ func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualCon // create meta := client.ResourceMeta{ ResourceMeta: types.ResourceMeta{ - Node: minerCount, + Node: minerCount, UseEventualConsistency: useEventualConsistency, }, }