Skip to content

Commit 248d48c

Browse files
yacovmyacovm
committed
[FAB-2198] Adjust gossip membership layer
Continuation of: https://gerrit.hyperledger.org/r/#/c/5903/ Introduce envelopes to gossip message The gossip membership module has a storage of GossipMessages. It needs to be refactored to hold SignedMessages instead, but SignedMessages don't have any meaningful methods because they are only envelopes to GossipMessages. Therefore, this commit introduces an internal message type to the gossip membership layer, that has both the SignedGossipMessage and the GossipMessage as pointers. Also- previously the cachedMembership field was used but now can't be used because it's a raw protobuf structure. Therefore, I refactored it to a 'membershipStore' which is just a nice wrapper around a map from PKI-ID --> message. Signed-off-by: Yacov Manevich <[email protected]> Change-Id: I2251c912b6a4610f6565f51c67bbb89d5d26b94f
1 parent 5dbe29e commit 248d48c

File tree

3 files changed

+201
-69
lines changed

3 files changed

+201
-69
lines changed

‎gossip/discovery/discovery_impl.go

+46-69
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,6 @@ func SetReconnectInterval(interval time.Duration) {
5656
reconnectInterval = interval
5757
}
5858

59-
func samePKIidAliveMessage(a interface{}, b interface{}) bool {
60-
return equalPKIid(a.(*proto.GossipMessage).GetAliveMsg().Membership.PkiID, b.(*proto.GossipMessage).GetAliveMsg().Membership.PkiID)
61-
}
62-
6359
type timestamp struct {
6460
incTime time.Time
6561
seqNum uint64
@@ -71,13 +67,14 @@ func (ts *timestamp) String() string {
7167
}
7268

7369
type gossipDiscoveryImpl struct {
74-
incTime uint64
75-
seqNum uint64
76-
self NetworkMember
77-
deadLastTS map[string]*timestamp // H
78-
aliveLastTS map[string]*timestamp // V
79-
id2Member map[string]*NetworkMember // all known members
80-
cachedMembership *proto.MembershipResponse
70+
incTime uint64
71+
seqNum uint64
72+
self NetworkMember
73+
deadLastTS map[string]*timestamp // H
74+
aliveLastTS map[string]*timestamp // V
75+
id2Member map[string]*NetworkMember // all known members
76+
aliveMembership membershipStore
77+
deadMembership membershipStore
8178

8279
bootstrapPeers []string
8380

@@ -93,22 +90,20 @@ type gossipDiscoveryImpl struct {
9390
// NewDiscoveryService returns a new discovery service with the comm module passed and the crypto service passed
9491
func NewDiscoveryService(bootstrapPeers []string, self NetworkMember, comm CommService, crypt CryptoService) Discovery {
9592
d := &gossipDiscoveryImpl{
96-
self: self,
97-
incTime: uint64(time.Now().UnixNano()),
98-
seqNum: uint64(0),
99-
deadLastTS: make(map[string]*timestamp),
100-
aliveLastTS: make(map[string]*timestamp),
101-
id2Member: make(map[string]*NetworkMember),
102-
cachedMembership: &proto.MembershipResponse{
103-
Alive: make([]*proto.GossipMessage, 0),
104-
Dead: make([]*proto.GossipMessage, 0),
105-
},
106-
crypt: crypt,
107-
comm: comm,
108-
lock: &sync.RWMutex{},
109-
toDieChan: make(chan struct{}, 1),
110-
toDieFlag: int32(0),
111-
logger: util.GetLogger(util.LoggingDiscoveryModule, self.InternalEndpoint.Endpoint),
93+
self: self,
94+
incTime: uint64(time.Now().UnixNano()),
95+
seqNum: uint64(0),
96+
deadLastTS: make(map[string]*timestamp),
97+
aliveLastTS: make(map[string]*timestamp),
98+
id2Member: make(map[string]*NetworkMember),
99+
aliveMembership: make(membershipStore, 0),
100+
deadMembership: make(membershipStore, 0),
101+
crypt: crypt,
102+
comm: comm,
103+
lock: &sync.RWMutex{},
104+
toDieChan: make(chan struct{}, 1),
105+
toDieFlag: int32(0),
106+
logger: util.GetLogger(util.LoggingDiscoveryModule, self.InternalEndpoint.Endpoint),
112107
}
113108

114109
go d.periodicalSendAlive()
@@ -188,14 +183,15 @@ func (d *gossipDiscoveryImpl) InitiateSync(peerNum int) {
188183

189184
d.lock.RLock()
190185

191-
n := len(d.cachedMembership.Alive)
186+
n := len(d.aliveMembership)
192187
k := peerNum
193188
if k > n {
194189
k = n
195190
}
196191

192+
aliveMembersAsSlice := d.aliveMembership.ToSlice()
197193
for _, i := range util.GetRandomIndices(k, n-1) {
198-
pulledPeer := d.cachedMembership.Alive[i].GetAliveMsg().Membership
194+
pulledPeer := aliveMembersAsSlice[i].GetAliveMsg().Membership
199195
netMember := &NetworkMember{
200196
Endpoint: pulledPeer.Endpoint,
201197
Metadata: pulledPeer.Metadata,
@@ -335,7 +331,7 @@ func (d *gossipDiscoveryImpl) createMembershipResponse(known [][]byte) *proto.Me
335331

336332
deadPeers := []*proto.GossipMessage{}
337333

338-
for _, dm := range d.cachedMembership.Dead {
334+
for _, dm := range d.deadMembership.ToSlice() {
339335
isKnown := false
340336
for _, knownPeer := range known {
341337
if equalPKIid(knownPeer, dm.GetAliveMsg().Membership.PkiID) {
@@ -344,13 +340,19 @@ func (d *gossipDiscoveryImpl) createMembershipResponse(known [][]byte) *proto.Me
344340
}
345341
}
346342
if !isKnown {
347-
deadPeers = append(deadPeers, dm)
343+
deadPeers = append(deadPeers, dm.GossipMessage)
348344
break
349345
}
350346
}
351347

348+
aliveMembersAsSlice := d.aliveMembership.ToSlice()
349+
aliveSnapshot := make([]*proto.GossipMessage, len(aliveMembersAsSlice))
350+
for i, msg := range aliveMembersAsSlice {
351+
aliveSnapshot[i] = msg.GossipMessage
352+
}
353+
352354
return &proto.MembershipResponse{
353-
Alive: append(d.cachedMembership.Alive, aliveMsg),
355+
Alive: append(aliveSnapshot, aliveMsg),
354356
Dead: deadPeers,
355357
}
356358
}
@@ -441,24 +443,10 @@ func (d *gossipDiscoveryImpl) resurrectMember(am *proto.GossipMessage, t proto.P
441443
PKIid: member.PkiID,
442444
InternalEndpoint: member.InternalEndpoint,
443445
}
444-
delete(d.deadLastTS, string(pkiID))
445446

446-
aliveMsgWithID := &proto.GossipMessage{
447-
Content: &proto.GossipMessage_AliveMsg{
448-
AliveMsg: &proto.AliveMessage{
449-
Membership: &proto.Member{PkiID: pkiID},
450-
},
451-
},
452-
}
453-
454-
i := util.IndexInSlice(d.cachedMembership.Dead, aliveMsgWithID, samePKIidAliveMessage)
455-
if i != -1 {
456-
d.cachedMembership.Dead = append(d.cachedMembership.Dead[:i], d.cachedMembership.Dead[i+1:]...)
457-
}
458-
459-
if util.IndexInSlice(d.cachedMembership.Alive, am, samePKIidAliveMessage) == -1 {
460-
d.cachedMembership.Alive = append(d.cachedMembership.Alive, am)
461-
}
447+
delete(d.deadLastTS, string(pkiID))
448+
d.deadMembership.Remove(common.PKIidType(pkiID))
449+
d.aliveMembership.Put(common.PKIidType(pkiID), &message{GossipMessage: am})
462450
}
463451

464452
func (d *gossipDiscoveryImpl) periodicalReconnectToDead() {
@@ -559,19 +547,9 @@ func (d *gossipDiscoveryImpl) expireDeadMembers(dead []common.PKIidType) {
559547
delete(d.aliveLastTS, string(pkiID))
560548
}
561549

562-
aliveMsgWithPKIid := &proto.GossipMessage{
563-
Content: &proto.GossipMessage_AliveMsg{
564-
AliveMsg: &proto.AliveMessage{
565-
Membership: &proto.Member{PkiID: pkiID},
566-
},
567-
},
568-
}
569-
aliveMemberIndex := util.IndexInSlice(d.cachedMembership.Alive, aliveMsgWithPKIid, samePKIidAliveMessage)
570-
if aliveMemberIndex != -1 {
571-
// Move the alive member to the dead members
572-
d.cachedMembership.Dead = append(d.cachedMembership.Dead, d.cachedMembership.Alive[aliveMemberIndex])
573-
// Delete the alive member from the cached membership
574-
d.cachedMembership.Alive = append(d.cachedMembership.Alive[:aliveMemberIndex], d.cachedMembership.Alive[aliveMemberIndex+1:]...)
550+
if am := d.aliveMembership.msgByID(pkiID); am != nil {
551+
d.deadMembership.Put(pkiID, am)
552+
d.aliveMembership.Remove(pkiID)
575553
}
576554
}
577555

@@ -677,13 +655,12 @@ func (d *gossipDiscoveryImpl) learnExistingMembers(aliveArr []*proto.GossipMessa
677655
alive.lastSeen = time.Now()
678656
alive.seqNum = am.Timestamp.SeqNum
679657

680-
i := util.IndexInSlice(d.cachedMembership.Alive, m, samePKIidAliveMessage)
681-
if i == -1 {
658+
if am := d.aliveMembership.msgByID(m.GetAliveMsg().Membership.PkiID); am == nil {
682659
d.logger.Debug("Appended", am, "to d.cachedMembership.Alive")
683-
d.cachedMembership.Alive = append(d.cachedMembership.Alive, m)
660+
d.aliveMembership.Put(m.GetAliveMsg().Membership.PkiID, &message{GossipMessage: m})
684661
} else {
685662
d.logger.Debug("Replaced", am, "in d.cachedMembership.Alive")
686-
d.cachedMembership.Alive[i] = m
663+
am.GossipMessage = m
687664
}
688665
}
689666
}
@@ -706,7 +683,7 @@ func (d *gossipDiscoveryImpl) learnNewMembers(aliveMembers []*proto.GossipMessag
706683
seqNum: am.GetAliveMsg().Timestamp.SeqNum,
707684
}
708685

709-
d.cachedMembership.Alive = append(d.cachedMembership.Alive, am)
686+
d.aliveMembership.Put(am.GetAliveMsg().Membership.PkiID, &message{GossipMessage: am})
710687
d.logger.Infof("Learned about a new alive member: %v", am)
711688
}
712689

@@ -720,7 +697,7 @@ func (d *gossipDiscoveryImpl) learnNewMembers(aliveMembers []*proto.GossipMessag
720697
seqNum: dm.GetAliveMsg().Timestamp.SeqNum,
721698
}
722699

723-
d.cachedMembership.Dead = append(d.cachedMembership.Dead, dm)
700+
d.deadMembership.Put(dm.GetAliveMsg().Membership.PkiID, &message{GossipMessage: dm})
724701
d.logger.Infof("Learned about a new dead member: %v", dm)
725702
}
726703

@@ -750,7 +727,7 @@ func (d *gossipDiscoveryImpl) GetMembership() []NetworkMember {
750727
defer d.lock.RUnlock()
751728

752729
response := []NetworkMember{}
753-
for _, m := range d.cachedMembership.Alive {
730+
for _, m := range d.aliveMembership.ToSlice() {
754731
member := m.GetAliveMsg()
755732
response = append(response, NetworkMember{
756733
PKIid: member.Membership.PkiID,

‎gossip/discovery/msgs.go

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
Copyright IBM Corp. 2017 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package discovery
18+
19+
import (
20+
"github.com/hyperledger/fabric/gossip/common"
21+
proto "github.com/hyperledger/fabric/protos/gossip"
22+
)
23+
24+
type message struct {
25+
*proto.SignedGossipMessage
26+
*proto.GossipMessage
27+
}
28+
29+
type membershipStore map[string]*message
30+
31+
// msgByID returns a message stored by a certain ID, or nil
32+
// if such an ID isn't found
33+
func (m membershipStore) msgByID(pkiID common.PKIidType) *message {
34+
if msg, exists := m[string(pkiID)]; exists {
35+
return msg
36+
}
37+
return nil
38+
}
39+
40+
// Put associates msg with the given pkiID
41+
func (m membershipStore) Put(pkiID common.PKIidType, msg *message) {
42+
m[string(pkiID)] = msg
43+
}
44+
45+
// Remove removes a message with a given pkiID
46+
func (m membershipStore) Remove(pkiID common.PKIidType) {
47+
delete(m, string(pkiID))
48+
}
49+
50+
// ToSlice returns a slice backed by the elements
51+
// of the membershipStore
52+
func (m membershipStore) ToSlice() []*message {
53+
members := make([]*message, len(m))
54+
i := 0
55+
for _, member := range m {
56+
members[i] = member
57+
i++
58+
}
59+
return members
60+
}

‎gossip/discovery/msgs_test.go

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
Copyright IBM Corp. 2017 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package discovery
18+
19+
import (
20+
"testing"
21+
22+
"github.com/hyperledger/fabric/gossip/common"
23+
proto "github.com/hyperledger/fabric/protos/gossip"
24+
"github.com/stretchr/testify/assert"
25+
)
26+
27+
func TestMembershipStore(t *testing.T) {
28+
membershipStore := make(membershipStore, 0)
29+
30+
id1 := common.PKIidType("id1")
31+
id2 := common.PKIidType("id2")
32+
33+
msg1 := &message{}
34+
msg2 := &message{SignedGossipMessage: &proto.SignedGossipMessage{}}
35+
36+
// Test initially created store is empty
37+
assert.Nil(t, membershipStore.msgByID(id1))
38+
assert.Len(t, membershipStore, 0)
39+
// Test put works as expected
40+
membershipStore.Put(id1, msg1)
41+
assert.NotNil(t, membershipStore.msgByID(id1))
42+
// Test msgByID returns the right instance stored
43+
membershipStore.Put(id2, msg2)
44+
assert.Equal(t, msg1, membershipStore.msgByID(id1))
45+
assert.NotEqual(t, msg2, membershipStore.msgByID(id1))
46+
// Test capacity grows
47+
assert.Len(t, membershipStore, 2)
48+
// Test remove works
49+
membershipStore.Remove(id1)
50+
assert.Nil(t, membershipStore.msgByID(id1))
51+
assert.Len(t, membershipStore, 1)
52+
// Test returned instance is not a copy
53+
msg3 := &message{GossipMessage: &proto.GossipMessage{}}
54+
msg3Clone := &message{GossipMessage: &proto.GossipMessage{}}
55+
id3 := common.PKIidType("id3")
56+
membershipStore.Put(id3, msg3)
57+
assert.Equal(t, msg3Clone, msg3)
58+
membershipStore.msgByID(id3).Channel = []byte{0, 1, 2, 3}
59+
assert.NotEqual(t, msg3Clone, msg3)
60+
}
61+
62+
func TestToSlice(t *testing.T) {
63+
membershipStore := make(membershipStore, 0)
64+
id1 := common.PKIidType("id1")
65+
id2 := common.PKIidType("id2")
66+
id3 := common.PKIidType("id3")
67+
id4 := common.PKIidType("id4")
68+
69+
msg1 := &message{}
70+
msg2 := &message{SignedGossipMessage: &proto.SignedGossipMessage{}}
71+
msg3 := &message{GossipMessage: &proto.GossipMessage{}}
72+
msg4 := &message{GossipMessage: &proto.GossipMessage{}, SignedGossipMessage: &proto.SignedGossipMessage{}}
73+
74+
membershipStore.Put(id1, msg1)
75+
membershipStore.Put(id2, msg2)
76+
membershipStore.Put(id3, msg3)
77+
membershipStore.Put(id4, msg4)
78+
79+
assert.Len(t, membershipStore.ToSlice(), 4)
80+
81+
existsInSlice := func(slice []*message, msg *message) bool {
82+
for _, m := range slice {
83+
if assert.ObjectsAreEqual(m, msg) {
84+
return true
85+
}
86+
}
87+
return false
88+
}
89+
90+
expectedMsgs := []*message{msg1, msg2, msg3, msg4}
91+
for _, msg := range membershipStore.ToSlice() {
92+
assert.True(t, existsInSlice(expectedMsgs, msg))
93+
}
94+
95+
}

0 commit comments

Comments
 (0)