[FAB-1924]: Rework delivery client

Move ordering service delivery client management into
gossip service, this to be able to maintain single
connection to the ordering service. Moreover this to
be widely used after gossip based leader election
integrated.

Change-Id: Iea9a70a1d6ba82caa55716444c54f3ddbc19673b
Signed-off-by: Artem Barger <[email protected]>

Author: C0rWin

core/deliverservice/client.go core/deliverservice/blocksprovider/blocksprovider.go

@@ -1,5 +1,5 @@
 /*
-Copyright IBM Corp. 2016 All Rights Reserved.
+Copyright IBM Corp. 2017 All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,150 +14,172 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package deliverclient
+package blocksprovider
 
 import (
 	"math"
-	"time"
+	"sync/atomic"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/hyperledger/fabric/core/committer"
 	gossipcommon "github.com/hyperledger/fabric/gossip/common"
+	"github.com/hyperledger/fabric/gossip/discovery"
+
 	gossip_proto "github.com/hyperledger/fabric/gossip/proto"
-	"github.com/hyperledger/fabric/gossip/service"
 	"github.com/hyperledger/fabric/protos/common"
 	"github.com/hyperledger/fabric/protos/orderer"
 	"github.com/hyperledger/fabric/protos/utils"
 	"github.com/op/go-logging"
-	"github.com/spf13/viper"
-	"golang.org/x/net/context"
-	"google.golang.org/grpc"
 )
 
-var logger *logging.Logger // package-level logger
+// LedgerInfo an adapter to provide the interface to query
+// the ledger committer for current ledger height
+type LedgerInfo interface {
+	// LedgerHeight returns current local ledger height
+	LedgerHeight() (uint64, error)
+}
 
-func init() {
-	logger = logging.MustGetLogger("deliveryService")
+// GossipServiceAdapter serves to provide basic functionality
+// required from gossip service by delivery service
+type GossipServiceAdapter interface {
+	// PeersOfChannel returns slice with members of specified channel
+	PeersOfChannel(gossipcommon.ChainID) []discovery.NetworkMember
+
+	// AddPayload adds payload to the local state sync buffer
+	AddPayload(chainID string, payload *gossip_proto.Payload) error
+
+	// Gossip the message across the peers
+	Gossip(msg *gossip_proto.GossipMessage)
+}
+
+// BlocksProvider used to read blocks from the ordering service
+// for specified chain it subscribed to
+type BlocksProvider interface {
+	// RequestBlock acquire new blocks from ordering service based on
+	// information provided by ledger info instance
+	RequestBlocks(ledgerInfoProvider LedgerInfo) error
+
+	// DeliverBlocks starts delivering and disseminating blocks
+	DeliverBlocks()
+
+	// Stop shutdowns blocks provider and stops delivering new blocks
+	Stop()
 }
 
-// DeliverService used to communicate with orderers to obtain
-// new block and send the to the committer service
-type DeliverService struct {
-	client orderer.AtomicBroadcast_DeliverClient
+// BlocksDeliverer defines interface which actually helps
+// to abstract the AtomicBroadcast_DeliverClient with only
+// required method for blocks provider. This also help to
+// build up mocking facilities for testing purposes
+type BlocksDeliverer interface {
+	// Recv capable to bring new blocks from the ordering service
+	Recv() (*orderer.DeliverResponse, error)
 
+	// Send used to send request to the ordering service to obtain new blocks
+	Send(*common.Envelope) error
+}
+
+// blocksProviderImpl the actual implementation for BlocksProvider interface
+type blocksProviderImpl struct {
 	chainID string
-	conn    *grpc.ClientConn
+
+	client BlocksDeliverer
+
+	gossip GossipServiceAdapter
+
+	done int32
 }
 
-// StopDeliveryService sends stop to the delivery service reference
-func StopDeliveryService(service *DeliverService) {
-	if service != nil {
-		service.Stop()
+var logger *logging.Logger // package-level logger
+
+func init() {
+	logger = logging.MustGetLogger("blocksProvider")
+}
+
+// NewBlocksProvider constructor function to creare blocks deliverer instance
+func NewBlocksProvider(chainID string, client BlocksDeliverer, gossip GossipServiceAdapter) BlocksProvider {
+	return &blocksProviderImpl{
+		chainID: chainID,
+		client:  client,
+		gossip:  gossip,
 	}
 }
 
-// NewDeliverService construction function to create and initilize
-// delivery service instance
-func NewDeliverService(chainID string) *DeliverService {
-	if viper.GetBool("peer.committer.enabled") {
-		logger.Infof("Creating committer for single noops endorser")
-		deliverService := &DeliverService{
-			// Instance of RawLedger
-			chainID: chainID,
+// DeliverBlocks used to pull out blocks from the ordering service to
+// distributed them across peers
+func (b *blocksProviderImpl) DeliverBlocks() {
+	for !b.isDone() {
+		msg, err := b.client.Recv()
+		if err != nil {
+			logger.Warningf("Receive error: %s", err.Error())
+			return
 		}
+		switch t := msg.Type.(type) {
+		case *orderer.DeliverResponse_Status:
+			if t.Status == common.Status_SUCCESS {
+				logger.Warning("ERROR! Received success for a seek that should never complete")
+				return
+			}
+			logger.Warning("Got error ", t)
+		case *orderer.DeliverResponse_Block:
+			seqNum := t.Block.Header.Number
 
-		return deliverService
+			numberOfPeers := len(b.gossip.PeersOfChannel(gossipcommon.ChainID(b.chainID)))
+			// Create payload with a block received
+			payload := createPayload(seqNum, t.Block)
+			// Use payload to create gossip message
+			gossipMsg := createGossipMsg(b.chainID, payload)
+
+			logger.Debugf("Adding payload locally, buffer seqNum = [%d], peers number [%d]", seqNum, numberOfPeers)
+			// Add payload to local state payloads buffer
+			b.gossip.AddPayload(b.chainID, payload)
+
+			// Gossip messages with other nodes
+			logger.Debugf("Gossiping block [%d], peers number [%d]", seqNum, numberOfPeers)
+			b.gossip.Gossip(gossipMsg)
+		default:
+			logger.Warning("Received unknown: ", t)
+			return
+		}
 	}
-	logger.Infof("Committer disabled")
-	return nil
 }
 
-func (d *DeliverService) startDeliver(committer committer.Committer) error {
-	logger.Info("Starting deliver service client")
-	err := d.initDeliver()
+// Stops blocks delivery provider
+func (b *blocksProviderImpl) Stop() {
+	atomic.StoreInt32(&b.done, 1)
+}
 
-	if err != nil {
-		logger.Errorf("Can't initiate deliver protocol [%s]", err)
-		return err
-	}
+// Check whenever provider is stopped
+func (b *blocksProviderImpl) isDone() bool {
+	return atomic.LoadInt32(&b.done) == 1
+}
 
-	height, err := committer.LedgerHeight()
+func (b *blocksProviderImpl) RequestBlocks(ledgerInfoProvider LedgerInfo) error {
+	height, err := ledgerInfoProvider.LedgerHeight()
 	if err != nil {
 		logger.Errorf("Can't get legder height from committer [%s]", err)
 		return err
 	}
 
 	if height > 0 {
 		logger.Debugf("Starting deliver with block [%d]", height)
-		if err := d.seekLatestFromCommitter(height); err != nil {
+		if err := b.seekLatestFromCommitter(height); err != nil {
 			return err
 		}
-
 	} else {
 		logger.Debug("Starting deliver with olders block")
-		if err := d.seekOldest(); err != nil {
+		if err := b.seekOldest(); err != nil {
 			return err
 		}
-
 	}
 
-	d.readUntilClose()
-
-	return nil
-}
-
-func (d *DeliverService) initDeliver() error {
-	opts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithTimeout(3 * time.Second), grpc.WithBlock()}
-	endpoint := viper.GetString("peer.committer.ledger.orderer")
-	conn, err := grpc.Dial(endpoint, opts...)
-	if err != nil {
-		logger.Errorf("Cannot dial to %s, because of %s", endpoint, err)
-		return err
-	}
-	var abc orderer.AtomicBroadcast_DeliverClient
-	abc, err = orderer.NewAtomicBroadcastClient(conn).Deliver(context.TODO())
-	if err != nil {
-		logger.Errorf("Unable to initialize atomic broadcast, due to %s", err)
-		return err
-	}
-
-	// Atomic Broadcast Deliver Client
-	d.client = abc
-	d.conn = conn
 	return nil
-
 }
 
-func (d *DeliverService) stopDeliver() {
-	if d.conn != nil {
-		d.conn.Close()
-	}
-}
-
-// Stop all service and release resources
-func (d *DeliverService) Stop() {
-	d.stopDeliver()
-}
-
-// Start delivery service
-func (d *DeliverService) Start(committer committer.Committer) {
-	go d.checkLeaderAndRunDeliver(committer)
-}
-
-func (d *DeliverService) checkLeaderAndRunDeliver(committer committer.Committer) {
-	isLeader := viper.GetBool("peer.gossip.orgLeader")
-
-	if isLeader {
-		d.startDeliver(committer)
-	}
-}
-
-func (d *DeliverService) seekOldest() error {
-	return d.client.Send(&common.Envelope{
+func (b *blocksProviderImpl) seekOldest() error {
+	return b.client.Send(&common.Envelope{
 		Payload: utils.MarshalOrPanic(&common.Payload{
 			Header: &common.Header{
 				ChainHeader: &common.ChainHeader{
-					ChainID: d.chainID,
+					ChainID: b.chainID,
 				},
 				SignatureHeader: &common.SignatureHeader{},
 			},
@@ -170,12 +192,12 @@ func (d *DeliverService) seekOldest() error {
 	})
 }
 
-func (d *DeliverService) seekLatestFromCommitter(height uint64) error {
-	return d.client.Send(&common.Envelope{
+func (b *blocksProviderImpl) seekLatestFromCommitter(height uint64) error {
+	return b.client.Send(&common.Envelope{
 		Payload: utils.MarshalOrPanic(&common.Payload{
 			Header: &common.Header{
 				ChainHeader: &common.ChainHeader{
-					ChainID: d.chainID,
+					ChainID: b.chainID,
 				},
 				SignatureHeader: &common.SignatureHeader{},
 			},
@@ -188,43 +210,6 @@ func (d *DeliverService) seekLatestFromCommitter(height uint64) error {
 	})
 }
 
-func (d *DeliverService) readUntilClose() {
-	for {
-		msg, err := d.client.Recv()
-		if err != nil {
-			logger.Warningf("Receive error: %s", err.Error())
-			return
-		}
-		switch t := msg.Type.(type) {
-		case *orderer.DeliverResponse_Status:
-			if t.Status == common.Status_SUCCESS {
-				logger.Warning("ERROR! Received success for a seek that should never complete")
-				return
-			}
-			logger.Warning("Got error ", t)
-		case *orderer.DeliverResponse_Block:
-			seqNum := t.Block.Header.Number
-
-			numberOfPeers := len(service.GetGossipService().PeersOfChannel(gossipcommon.ChainID(d.chainID)))
-			// Create payload with a block received
-			payload := createPayload(seqNum, t.Block)
-			// Use payload to create gossip message
-			gossipMsg := createGossipMsg(d.chainID, payload)
-
-			logger.Debugf("Adding payload locally, buffer seqNum = [%d], peers number [%d]", seqNum, numberOfPeers)
-			// Add payload to local state payloads buffer
-			service.GetGossipService().AddPayload(d.chainID, payload)
-
-			// Gossip messages with other nodes
-			logger.Debugf("Gossiping block [%d], peers number [%d]", seqNum, numberOfPeers)
-			service.GetGossipService().Gossip(gossipMsg)
-		default:
-			logger.Warning("Received unknown: ", t)
-			return
-		}
-	}
-}
-
 func createGossipMsg(chainID string, payload *gossip_proto.Payload) *gossip_proto.GossipMessage {
 	gossipMsg := &gossip_proto.GossipMessage{
 		Nonce:   0,