Skip to content

Commit cbd1ea0

Browse files
corecodecorecode
committed
sbft: refactor + document future directions
Squashed commit of the following: commit ad540eeb38effe086851c6d9c58a0192e7034b10 Author: Simon Schubert <[email protected]> Date: Tue Oct 4 15:09:38 2016 +0200 address golint concerns Change-Id: I59ae215d771a128cac740489a1d6308ebd1fb2a5 Signed-off-by: Simon Schubert <[email protected]> commit d60d5f54bb5ca04e5857be13cd5dbfd3d67e5ac6 Author: Simon Schubert <[email protected]> Date: Tue Oct 4 14:51:14 2016 +0200 add comments on backlog and state transfer strategy Change-Id: Iedc84f196c76e1cd12c88f2e74505cae64d80752 Signed-off-by: Simon Schubert <[email protected]> commit 6169fd180918940e622a580a55d3be4243ccd919 Author: Simon Schubert <[email protected]> Date: Mon Oct 3 15:35:00 2016 +0200 properly process queued checkpoint messages Change-Id: If6d8e58f8a02178a4a435542162cba52b233df92 Signed-off-by: Simon Schubert <[email protected]> commit 017c174700e0a47015a6dfc8967576cd8872de79 Author: Simon Schubert <[email protected]> Date: Mon Oct 3 15:33:57 2016 +0200 send hello message on connect Change-Id: Id76553a7403d730182ca5f8bd445053c28969f91 Signed-off-by: Simon Schubert <[email protected]> commit b11ed71a695a96f103626d1db8629a39d39db7c4 Author: Simon Schubert <[email protected]> Date: Mon Oct 3 14:28:52 2016 +0200 rename message Seq to SeqView Change-Id: I2cb6b4082b72642342ca09eeb4c0be7eb382356d Signed-off-by: Simon Schubert <[email protected]> commit 12b1d77eb8e7e0e4e8f1a68e04c5183ddf6a40c0 Author: Simon Schubert <[email protected]> Date: Mon Oct 3 14:24:22 2016 +0200 record signature origin in batch Change-Id: I9416e90ae01c35d548eeedb56ac6f345e9a0b63b Signed-off-by: Simon Schubert <[email protected]> commit f18a7896183b94edd0ed1285fe2d8d827435e5bc Author: Simon Schubert <[email protected]> Date: Mon Oct 3 12:50:47 2016 +0200 change info to warning Change-Id: If25917123731484784deec45af589e84a17d9c26 Signed-off-by: Simon Schubert <[email protected]> commit 59e79929ffb18f7fd834f9bf9bd667109f6c9e5d Author: Simon Schubert <[email protected]> Date: Mon Oct 3 12:50:28 2016 +0200 conform to protobuf style guide Change-Id: I851476b6449c17c88f31539c78221fc81338cadd Signed-off-by: Simon Schubert <[email protected]> Change-Id: Iaa289455d6ebc7c31a1e4130b9d07f8759d05c88 Signed-off-by: Simon Schubert <[email protected]>
1 parent 52c8407 commit cbd1ea0

15 files changed

+307
-212
lines changed

‎consensus/simplebft/backlog.go

+50-14
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,33 @@ func (s *SBFT) recordBacklogMsg(m *Msg, src uint64) {
4949
if src == s.id {
5050
panic("should never have to backlog my own message")
5151
}
52-
// TODO prevent DoS by limiting the number of messages per replica
52+
// TODO
53+
//
54+
// Prevent DoS by limiting the number of messages per replica.
55+
//
56+
// If the backlog limit is exceeded, discard all messages with
57+
// Seq before the replica's hello message (we can, because we
58+
// can play forward to this batch via state transfer). If
59+
// there is no hello message, we must be really slow or the
60+
// replica must be byzantine. In this case we probably should
61+
// re-establish the connection.
62+
//
63+
// After the connection has been re-established, we will
64+
// receive a hello, and the following messages will trigger
65+
// the pruning of old messages. If this pruning lead us not
66+
// to make progress, the backlog processing algorithm as lined
67+
// out below will take care of starting a state transfer,
68+
// using the hello message we received on reconnect.
5369
s.backLog[src] = append(s.backLog[src], m)
5470
}
5571

5672
func (s *SBFT) processBacklog() {
5773
processed := true
74+
notReady := uint64(0)
5875

5976
for processed {
6077
processed = false
61-
notReady := uint64(0)
62-
for src, _ := range s.backLog {
78+
for src := range s.backLog {
6379
for len(s.backLog[src]) > 0 {
6480
m, rest := s.backLog[src][0], s.backLog[src][1:]
6581
if s.testBacklog2(m, src) {
@@ -74,16 +90,36 @@ func (s *SBFT) processBacklog() {
7490
processed = true
7591
}
7692
}
77-
78-
// all minus us
79-
if notReady >= s.config.N-1 {
80-
// This is a problem - we consider all other replicas
81-
// too far ahead for us. We need to do a state transfer
82-
// to get out of this rut.
83-
for src := range s.backLog {
84-
delete(s.backLog, src)
85-
}
86-
// TODO trigger state transfer
87-
}
8893
}
94+
95+
// TODO
96+
//
97+
// Detect when we need to reconsider our options.
98+
//
99+
// We arrived here because either all is fine, we're with the
100+
// pack. Or we have messages in the backlog because we're
101+
// connected asymmetrically, and a close replica already
102+
// started talking about the next batch while we're still
103+
// waiting for rounds to arrive for our current batch. That's
104+
// still fine.
105+
//
106+
// We might also be here because we lost connectivity, and we
107+
// either missed some messages, or our connection is bad and
108+
// we should reconnect to get a working connection going
109+
// again.
110+
//
111+
// If a noFaultyQuorum (-1, because we're not faulty, just
112+
// were disconnected) is backlogged, we know that we need to
113+
// perform a state transfer. Of course, f of these might be
114+
// byzantine, and the remaining f that are not backlogged will
115+
// allow us to get unstuck. To check against that, we need to
116+
// only consider backlogged replicas of which we have a hello
117+
// message that talks about a future Seq.
118+
//
119+
// We need to pick the highest Seq of all the hello messages
120+
// we received, perform a state transfer to that Batch, and
121+
// discard all backlogged messages that refer to a lower Seq.
122+
//
123+
// Do we need to detect that a connection is stuck and we
124+
// should reconnect?
89125
}

‎consensus/simplebft/batch.go

+1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ func (s *SBFT) checkBatch(b *Batch) (*BatchHeader, error) {
5959

6060
////////////////////////////////////////
6161

62+
// Hash returns the hash of the Batch.
6263
func (b *Batch) Hash() []byte {
6364
return hash(b.Header)
6465
}

‎consensus/simplebft/checkpoint.go

+6-9
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,10 @@ func (s *SBFT) handleCheckpoint(c *Checkpoint, src uint64) {
7575

7676
// got a weak checkpoint
7777

78-
cpset := &CheckpointSet{make(map[uint64]*Checkpoint)}
79-
var sigs [][]byte
78+
cpset := make(map[uint64][]byte)
8079
for _, r := range replicas {
8180
cp := s.cur.checkpoint[r]
82-
cpset.CheckpointSet[r] = cp
83-
sigs = append(sigs, cp.Signature)
81+
cpset[r] = cp.Signature
8482
}
8583
s.cur.checkpointDone = true
8684

@@ -93,11 +91,10 @@ func (s *SBFT) handleCheckpoint(c *Checkpoint, src uint64) {
9391
}
9492

9593
// ignore null requests
96-
if s.cur.preprep.Batch != nil {
97-
batch := *s.cur.preprep.Batch
98-
batch.Signatures = sigs
99-
s.sys.Deliver(&batch)
100-
}
94+
batch := *s.cur.preprep.Batch
95+
batch.Signatures = cpset
96+
s.sys.Deliver(&batch)
97+
10198
s.cur.timeout.Cancel()
10299
log.Infof("request %s %s completed on %d", s.cur.subject.Seq, hash2str(s.cur.subject.Digest), s.id)
103100

‎consensus/simplebft/commit.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func (s *SBFT) handleCommit(c *Subject, src uint64) {
3939
}
4040

4141
if !reflect.DeepEqual(c, &s.cur.subject) {
42-
log.Infof("commit does not match expected subject %v, got %v", &s.cur.subject, c)
42+
log.Warningf("commit does not match expected subject %v, got %v", &s.cur.subject, c)
4343
return
4444
}
4545
if _, ok := s.cur.commit[src]; ok {

‎consensus/simplebft/connection.go

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package simplebft
18+
19+
// Connection is an event from system to notify a new connection with
20+
// replica.
21+
// On connection, we send our latest (weak) checkpoint, and we expect
22+
// to receive one from replica.
23+
func (s *SBFT) Connection(replica uint64) {
24+
batch := *s.sys.LastBatch()
25+
batch.Payloads = nil // don't send the big payload
26+
s.sys.Send(&Msg{&Msg_Hello{&batch}}, replica)
27+
28+
// TODO
29+
//
30+
// A reconnecting replica can play forward its blockchain to
31+
// the batch listed in the hello message. However, the
32+
// currently in-flight batch will not be reflected in the
33+
// Hello message, nor will all messages be present to actually
34+
// commit the in-flight batch at the reconnecting replica.
35+
//
36+
// Therefore we also send the most recent (pre)prepare,
37+
// commit, checkpoint so that the reconnecting replica can
38+
// catch up on the in-flight batch.
39+
}
40+
41+
func (s *SBFT) handleHello(h *Batch, src uint64) {
42+
}

‎consensus/simplebft/crypto.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,10 @@ func merkleHashDigests(digests [][]byte) []byte {
6161
digests = nextDigests
6262
}
6363

64-
if len(digests) > 0 {
65-
return digests[0]
66-
} else {
64+
if len(digests) == 0 {
6765
return nil
6866
}
67+
return digests[0]
6968
}
7069

7170
////////////////////////////////////////////////

‎consensus/simplebft/newview.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,12 @@ func (s *SBFT) maybeSendNewView() {
6868
}
6969

7070
func (s *SBFT) handleNewView(nv *NewView, src uint64) {
71-
if src != s.primaryIdView(nv.View) {
71+
if src != s.primaryIDView(nv.View) {
7272
log.Warningf("invalid new view from %d for %d", src, nv.View)
7373
return
7474
}
7575

76-
if onv, ok := s.newview[s.primaryIdView(nv.View)]; ok && onv.View >= nv.View {
76+
if onv, ok := s.newview[s.primaryIDView(nv.View)]; ok && onv.View >= nv.View {
7777
log.Debugf("discarding duplicate new view for %d", nv.View)
7878
return
7979
}
@@ -128,7 +128,7 @@ func (s *SBFT) handleNewView(nv *NewView, src uint64) {
128128
return
129129
}
130130

131-
s.newview[s.primaryIdView(nv.View)] = nv
131+
s.newview[s.primaryIDView(nv.View)] = nv
132132

133133
s.processNewView()
134134
}
@@ -138,7 +138,7 @@ func (s *SBFT) processNewView() {
138138
return
139139
}
140140

141-
nv, ok := s.newview[s.primaryIdView(s.seq.View)]
141+
nv, ok := s.newview[s.primaryIDView(s.seq.View)]
142142
if !ok || nv.View != s.seq.View {
143143
return
144144
}

‎consensus/simplebft/newview_test.go

+29-29
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,26 @@ import (
2222
)
2323

2424
func TestXsetNoByz(t *testing.T) {
25-
s := &SBFT{config: Config{N: 4, F: 1}, seq: Seq{3, 1}}
25+
s := &SBFT{config: Config{N: 4, F: 1}, seq: SeqView{3, 1}}
2626
vcs := []*ViewChange{
2727
&ViewChange{
2828
View: 3,
2929
Pset: nil,
30-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")},
31-
&Subject{&Seq{2, 2}, []byte("val2")}},
30+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")},
31+
&Subject{&SeqView{2, 2}, []byte("val2")}},
3232
Executed: 1,
3333
},
3434
&ViewChange{
3535
View: 3,
36-
Pset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
37-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
36+
Pset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
37+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
3838
Executed: 1,
3939
},
4040
&ViewChange{
4141
View: 3,
42-
Pset: []*Subject{&Subject{&Seq{2, 2}, []byte("val2")}},
43-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")},
44-
&Subject{&Seq{2, 2}, []byte("val2")}},
42+
Pset: []*Subject{&Subject{&SeqView{2, 2}, []byte("val2")}},
43+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")},
44+
&Subject{&SeqView{2, 2}, []byte("val2")}},
4545
Executed: 1,
4646
},
4747
}
@@ -51,13 +51,13 @@ func TestXsetNoByz(t *testing.T) {
5151
t.Fatal("no xset")
5252
}
5353

54-
if !reflect.DeepEqual(xset, &Subject{&Seq{3, 2}, []byte("val2")}) {
54+
if !reflect.DeepEqual(xset, &Subject{&SeqView{3, 2}, []byte("val2")}) {
5555
t.Error(xset)
5656
}
5757
}
5858

5959
func TestXsetByz0(t *testing.T) {
60-
s := &SBFT{config: Config{N: 4, F: 1}, seq: Seq{3, 1}}
60+
s := &SBFT{config: Config{N: 4, F: 1}, seq: SeqView{3, 1}}
6161
vcs := []*ViewChange{
6262
&ViewChange{
6363
View: 3,
@@ -67,15 +67,15 @@ func TestXsetByz0(t *testing.T) {
6767
},
6868
&ViewChange{
6969
View: 3,
70-
Pset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
71-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
70+
Pset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
71+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
7272
Executed: 1,
7373
},
7474
&ViewChange{
7575
View: 3,
76-
Pset: []*Subject{&Subject{&Seq{2, 2}, []byte("val2")}},
77-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")},
78-
&Subject{&Seq{2, 2}, []byte("val2")}},
76+
Pset: []*Subject{&Subject{&SeqView{2, 2}, []byte("val2")}},
77+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")},
78+
&Subject{&SeqView{2, 2}, []byte("val2")}},
7979
Executed: 1,
8080
},
8181
}
@@ -87,41 +87,41 @@ func TestXsetByz0(t *testing.T) {
8787

8888
vcs = append(vcs, &ViewChange{
8989
View: 3,
90-
Pset: []*Subject{&Subject{&Seq{2, 2}, []byte("val2")}},
91-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")},
92-
&Subject{&Seq{2, 2}, []byte("val2")}},
90+
Pset: []*Subject{&Subject{&SeqView{2, 2}, []byte("val2")}},
91+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")},
92+
&Subject{&SeqView{2, 2}, []byte("val2")}},
9393
Executed: 2,
9494
})
9595

9696
xset, ok = s.makeXset(vcs)
9797
if !ok {
9898
t.Error("no xset")
9999
}
100-
if !reflect.DeepEqual(xset, &Subject{&Seq{3, 2}, []byte("val2")}) {
100+
if !reflect.DeepEqual(xset, &Subject{&SeqView{3, 2}, []byte("val2")}) {
101101
t.Error(xset)
102102
}
103103
}
104104

105105
func TestXsetByz2(t *testing.T) {
106-
s := &SBFT{config: Config{N: 4, F: 1}, seq: Seq{3, 1}}
106+
s := &SBFT{config: Config{N: 4, F: 1}, seq: SeqView{3, 1}}
107107
vcs := []*ViewChange{
108108
&ViewChange{
109109
View: 3,
110110
Pset: nil,
111-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
111+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
112112
Executed: 1,
113113
},
114114
&ViewChange{
115115
View: 3,
116-
Pset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
117-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
116+
Pset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
117+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
118118
Executed: 1,
119119
},
120120
&ViewChange{
121121
View: 3,
122-
Pset: []*Subject{&Subject{&Seq{2, 2}, []byte("val2")}},
123-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")},
124-
&Subject{&Seq{2, 2}, []byte("val2")}},
122+
Pset: []*Subject{&Subject{&SeqView{2, 2}, []byte("val2")}},
123+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")},
124+
&Subject{&SeqView{2, 2}, []byte("val2")}},
125125
Executed: 1,
126126
},
127127
}
@@ -133,16 +133,16 @@ func TestXsetByz2(t *testing.T) {
133133

134134
vcs = append(vcs, &ViewChange{
135135
View: 3,
136-
Pset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
137-
Qset: []*Subject{&Subject{&Seq{1, 2}, []byte("val1")}},
136+
Pset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
137+
Qset: []*Subject{&Subject{&SeqView{1, 2}, []byte("val1")}},
138138
Executed: 2,
139139
})
140140

141141
xset, ok = s.makeXset(vcs)
142142
if !ok {
143143
t.Error("no xset")
144144
}
145-
if !reflect.DeepEqual(xset, &Subject{&Seq{3, 2}, []byte("val1")}) {
145+
if !reflect.DeepEqual(xset, &Subject{&SeqView{3, 2}, []byte("val1")}) {
146146
t.Error(xset)
147147
}
148148
}

‎consensus/simplebft/preprepare.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (s *SBFT) sendPreprepare(batch []*Request) {
4141
}
4242

4343
func (s *SBFT) handlePreprepare(pp *Preprepare, src uint64) {
44-
if src != s.primaryId() {
44+
if src != s.primaryID() {
4545
log.Infof("preprepare from non-primary %d", src)
4646
return
4747
}

‎consensus/simplebft/request.go

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package simplebft
1818

1919
import "time"
2020

21+
// Request proposes a new request to the BFT network.
2122
func (s *SBFT) Request(req []byte) {
2223
s.broadcast(&Msg{&Msg_Request{&Request{req}}})
2324
}

0 commit comments

Comments
 (0)