Skip to content

Commit 6e75ca9

Browse files
zeligzelig
committed
swarm/cmd: fix directory checksum-ing for correctness tests; add bootstrap-test.sh script
swarm/network: config tweaks and log improved swarm/storage: cloud delivery in own go routine, dpa worker settings
1 parent 2ef669b commit 6e75ca9

File tree

8 files changed

+66
-23
lines changed

8 files changed

+66
-23
lines changed

‎swarm/cmd/bootstrap-testnet.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#
2+
swarm remote-run nodes.lst 'swarm netstatkill; swarm stop all; rm -rf bin bzz .ethash /tmp/.eth* /tmp/swarm*; sudo reboot'
3+
sleep 60
4+
# update the control scripts on each host
5+
swarm remote-update-scripts nodes.lst
6+
# update the binary on each host node
7+
swarm remote-update-bin nodes.lst
8+
9+
# spawn a 2-instance local cluster on each of our 10 host nodes
10+
swarm remote-run nodes.lst 'swarm init 2; swarm netstatconf sworm; swarm netstatrun'
11+
12+
#collect enodes from all instances on all hosts
13+
swarm remote-run nodes.lst 'swarm enode all' | tr -d '"' |grep -v running > enodes.lst
14+
15+
# swarm remote-run nodes.lst swarm addpeers <(enode)'
16+
17+
# copy enodes file to each host node
18+
for node in `cat nodes.lst`; do scp enodes.lst $node:; done
19+
20+
# inject all enodes as peers to
21+
22+
swarm remote-run nodes.lst swarm addpeers all enodes.lst
23+
24+
# to bootstrap with i
25+
26+
# restart an instance on the first two host nodes to get the blockchain rolling
27+
swarm remote-run <(head -2 nodes.lst) swarm restart 00 --mine
28+

‎swarm/cmd/swarm/swarm

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ function stop {
287287
function shutdown {
288288
echo -n "stopping $1..."
289289
kill -2 $1
290+
((i=0))
290291
while true; do
291292
ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
292293
if ((i++>5)); then
@@ -434,7 +435,7 @@ function up {
434435
id=$1
435436
path=$2
436437
file=$3
437-
if [ $file = "" ]; then
438+
if [ "$file" = "" ]; then
438439
file=`basename $path`
439440
fi
440441
echo "Upload file '$path' to node $id... " 1>&2
@@ -590,20 +591,26 @@ function update {
590591
}
591592

592593
function checksum {
593-
tar -cf - $1 | md5sum|awk '{print $1}'
594+
if [ -f $1 ]; then
595+
cat $1 | md5sum | awk '{print $1}'
596+
else
597+
cd $1 && md5deep -r -l . | sort | md5sum| awk '{print $1}'
598+
fi
594599
}
595600

596601
function checkaccess {
597602
nodes=$1
598603
path=$2
599604
target=`basename $2`
600-
chsum=`md5sum $2|cut -f1 -d' '`
605+
chsum=`checksum $2`
601606
shift
602607
shift
603608
master=`head -1 $nodes`
604-
echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)"
605-
scp $path $master:$target
609+
echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $path|cut -f1`)"
610+
remote-run $master rm -rf $target
611+
scp -r $path $master:$target
606612
hash=`swarm remote-run $master "swarm up 00 $target $*"|tr -d '"'`
613+
echo $hash
607614
remote-run $nodes swarm checkdownload all $hash $chsum
608615
}
609616

@@ -618,25 +625,27 @@ function checkdownload {
618625
done
619626
else
620627
hash=$2
621-
target=$3
622628
datetag=`date "+%Y-%m-%d-%H:%M:%S"`
623629
file="swarm-$datetag"
624-
rm -rf $tmpdir/$file
625-
/usr/bin/time -o $tmpdir/$file.log -f "%e" swarm download $id $hash $tmpdir/$file > /dev/null
626-
echo
630+
dest=$tmpdir/$file
631+
target=$3
632+
rm -rf $dest
633+
mkdir -p $dest
634+
/usr/bin/time -o $dest.log -f "%e" swarm download $id $hash $dest > /dev/null
635+
echo "instance $id download $hash to $dest"
627636
if [ -f $target ]; then
628-
cmp --silent $target $tmpdir/$file/* && echo PASS || echo FAIL
637+
cmp --silent $target $dest/* && echo PASS || echo FAIL
629638
elif [ -r $target ]; then
630-
diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL
639+
diff -r $target $dest >/dev/null && echo PASS || echo FAIL
631640
else
632-
exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '`
641+
exp=`checksum $dest/*`
633642
if [ "$exp" = "$target" ]; then
634643
echo -n PASS
635644
else
636645
echo FAIL "$exp = $target"
637646
fi
638647
fi
639-
echo " latency: " `cat $tmpdir/$file.log`
648+
echo " latency: " `cat $dest.log`
640649
fi
641650
}
642651

@@ -676,6 +685,8 @@ case $cmd in
676685
info $*;;
677686
"enode" )
678687
enode $*;;
688+
"checksum" )
689+
checksum $*;;
679690
"status" )
680691
status $*;;
681692
"peers" )

‎swarm/network/hive.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
128128
node, need, proxLimit := self.kad.Suggest()
129129

130130
if node != nil && len(node.Url) > 0 {
131-
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
131+
glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
132132
// enode or any lower level connection address is unnecessary in future
133133
// discovery table is used to look it up.
134134
connectPeer(node.Url)
@@ -142,7 +142,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
142142
req := &retrieveRequestMsgData{
143143
Key: storage.Key(randAddr[:]),
144144
}
145-
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
145+
glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
146146
peers[0].(*peer).retrieve(req)
147147
} else {
148148
glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")

‎swarm/network/kademlia/kademlia.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ const (
2222
var (
2323
purgeInterval = 42 * time.Hour
2424
initialRetryInterval = 42 * time.Millisecond
25-
maxIdleInterval = 42 * 100 * time.Millisecond
25+
maxIdleInterval = 42 * 1000 * time.Millisecond
26+
// maxIdleInterval = 42 * 10 0 * time.Millisecond
2627
)
2728

2829
type KadParams struct {

‎swarm/network/protocol.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ func (self *bzz) handle() error {
272272
return self.protoError(ErrDecode, "<- %v: %v", msg, err)
273273
}
274274
glog.V(logger.Debug).Infof("[BZZ] <- sync request: %v", req)
275+
self.lastActive = time.Now()
275276
self.sync(req.SyncState)
276277

277278
case unsyncedKeysMsg:
@@ -282,6 +283,7 @@ func (self *bzz) handle() error {
282283
}
283284
glog.V(logger.Debug).Infof("[BZZ] <- unsynced keys : %s", req.String())
284285
err := self.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: self})
286+
self.lastActive = time.Now()
285287
if err != nil {
286288
return self.protoError(ErrDecode, "<- %v: %v", msg, err)
287289
}
@@ -295,6 +297,7 @@ func (self *bzz) handle() error {
295297
}
296298
glog.V(logger.Debug).Infof("[BZZ] <- delivery request: %s", req.String())
297299
err := self.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: self})
300+
self.lastActive = time.Now()
298301
if err != nil {
299302
return self.protoError(ErrDecode, "<- %v: %v", msg, err)
300303
}

‎swarm/network/syncer.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ LOOP:
426426
keys = self.keys[priority]
427427
break PRIORITIES
428428
}
429-
glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
429+
glog.V(logger.Detail).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
430430
// if the input queue is empty on this level, resort to history if there is any
431431
if uint(priority) == histPrior && history != nil {
432432
glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key)

‎swarm/storage/dpa.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ implementation for storage or retrieval.
2323
*/
2424

2525
const (
26-
storeChanCapacity = 1000
27-
retrieveChanCapacity = 1000
26+
storeChanCapacity = 100
27+
retrieveChanCapacity = 100
2828
singletonSwarmDbCapacity = 50000
2929
singletonSwarmCacheCapacity = 500
30-
maxStoreProcesses = 100
31-
maxRetrieveProcesses = 100
30+
maxStoreProcesses = 8
31+
maxRetrieveProcesses = 8
3232
)
3333

3434
var (

‎swarm/storage/netstore.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,12 @@ func (self *NetStore) Put(entry *Chunk) {
8787
// that the chunk is has been retrieved
8888
close(entry.Req.C)
8989
// deliver the chunk to requesters upstream
90-
self.cloud.Deliver(entry)
90+
go self.cloud.Deliver(entry)
9191
} else {
9292
glog.V(logger.Detail).Infof("[BZZ] NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())
9393
// handle propagating store requests
9494
// go self.cloud.Store(entry)
95-
self.cloud.Store(entry)
95+
go self.cloud.Store(entry)
9696
}
9797
}
9898

0 commit comments

Comments
 (0)