[FAB-3715] Harden deliveryService TLS credentials

This change set makes the transport credentials to be obtained
per channel and not globally, in order to prevent a spoofing attempt
of a node that's not in the channel but has a TLS cert
(with the Subject Alternative Name of the host of the real orderer)
signed by a CA that its org is a member of some other channel
that the peer is also a member of.

Full details in the JIRA item.

Change-Id: I79b64ff7729c92d0a4b9cf2e2d9c7b523218fcbc
Signed-off-by: Yacov Manevich <[email protected]>

Author: yacovm

core/comm/connection.go

@@ -10,17 +10,17 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"os"
 	"sync"
 	"time"
 
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/grpclog"
-
 	"github.com/hyperledger/fabric/common/flogging"
 	"github.com/hyperledger/fabric/core/config"
 	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/grpclog"
 )
 
 const defaultTimeout = time.Second * 3
@@ -74,16 +74,25 @@ func (cas *CASupport) GetServerRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
 	return appRootCAs, ordererRootCAs
 }
 
-// GetDeliverServiceCredentials returns GRPC transport credentials for use by GRPC
+// GetDeliverServiceCredentials returns GRPC transport credentials for given channel to be used by GRPC
 // clients which communicate with ordering service endpoints.
-func (cas *CASupport) GetDeliverServiceCredentials() credentials.TransportCredentials {
+// If the channel isn't found, error is returned.
+func (cas *CASupport) GetDeliverServiceCredentials(channelID string) (credentials.TransportCredentials, error) {
+	cas.RLock()
+	defer cas.RUnlock()
+
 	var creds credentials.TransportCredentials
 	var tlsConfig = &tls.Config{}
 	var certPool = x509.NewCertPool()
-	// loop through the orderer CAs
-	_, roots := cas.GetServerRootCAs()
-	for _, root := range roots {
-		block, _ := pem.Decode(root)
+
+	rootCACerts, exists := cas.OrdererRootCAsByChain[channelID]
+	if !exists {
+		commLogger.Errorf("Attempted to obtain root CA certs of a non existent channel: %s", channelID)
+		return nil, fmt.Errorf("didn't find any root CA certs for channel %s", channelID)
+	}
+
+	for _, cert := range rootCACerts {
+		block, _ := pem.Decode(cert)
 		if block != nil {
 			cert, err := x509.ParseCertificate(block.Bytes)
 			if err == nil {
@@ -97,7 +106,7 @@ func (cas *CASupport) GetDeliverServiceCredentials() credentials.TransportCreden
 	}
 	tlsConfig.RootCAs = certPool
 	creds = credentials.NewTLS(tlsConfig)
-	return creds
+	return creds, nil
 }
 
 // GetPeerCredentials returns GRPC transport credentials for use by GRPC

core/comm/connection_test.go

@@ -17,17 +17,20 @@ limitations under the License.
 package comm
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"path/filepath"
+	"sync/atomic"
 	"testing"
+	"time"
 
-	"github.com/spf13/viper"
-
-	"crypto/tls"
-
+	testpb "github.com/hyperledger/fabric/core/comm/testdata/grpc"
 	"github.com/hyperledger/fabric/core/testutil"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
 
@@ -141,7 +144,7 @@ func TestCASupport(t *testing.T) {
 	casClone := GetCASupport()
 	assert.Exactly(t, casClone, cas, "Expected GetCASupport to be a singleton")
 
-	creds := cas.GetDeliverServiceCredentials()
+	creds, _ := cas.GetDeliverServiceCredentials("channel1")
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 	creds = cas.GetPeerCredentials(tls.Certificate{})
@@ -151,11 +154,115 @@ func TestCASupport(t *testing.T) {
 	// append some bad certs and make sure things still work
 	cas.ServerRootCAs = append(cas.ServerRootCAs, []byte("badcert"))
 	cas.ServerRootCAs = append(cas.ServerRootCAs, []byte(badPEM))
-	creds = cas.GetDeliverServiceCredentials()
+	creds, _ = cas.GetDeliverServiceCredentials("channel1")
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 	creds = cas.GetPeerCredentials(tls.Certificate{})
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 
 }
+
+type srv struct {
+	port int
+	GRPCServer
+	caCert   []byte
+	serviced uint32
+}
+
+func (s *srv) assertServiced(t *testing.T) {
+	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.serviced))
+	atomic.StoreUint32(&s.serviced, 0)
+}
+
+func (s *srv) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+	atomic.StoreUint32(&s.serviced, 1)
+	return &testpb.Empty{}, nil
+}
+
+func newServer(org string, port int) *srv {
+	certs := map[string][]byte{
+		"ca.crt":     nil,
+		"server.crt": nil,
+		"server.key": nil,
+	}
+	for suffix := range certs {
+		fName := filepath.Join("testdata", "impersonation", org, suffix)
+		cert, err := ioutil.ReadFile(fName)
+		if err != nil {
+			panic(fmt.Errorf("Failed reading %s: %v", fName, err))
+		}
+		certs[suffix] = cert
+	}
+	l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		panic(fmt.Errorf("Failed listening on port %d: %v", port, err))
+	}
+	gSrv, err := NewGRPCServerFromListener(l, SecureServerConfig{
+		ServerCertificate: certs["server.crt"],
+		ServerKey:         certs["server.key"],
+		UseTLS:            true,
+	})
+	if err != nil {
+		panic(fmt.Errorf("Failed starting gRPC server: %v", err))
+	}
+	s := &srv{
+		port:       port,
+		caCert:     certs["ca.crt"],
+		GRPCServer: gSrv,
+	}
+	testpb.RegisterTestServiceServer(gSrv.Server(), s)
+	go s.Start()
+	return s
+}
+
+func TestImpersonation(t *testing.T) {
+	// Scenario: We have 2 organizations: orgA, orgB
+	// and each of them are in their respected channels- A, B.
+	// The test would obtain credentials.TransportCredentials by calling GetDeliverServiceCredentials.
+	// Each organization would have its own gRPC server (srvA and srvB) with a TLS certificate
+	// signed by its root CA and with a SAN entry of 'localhost'.
+	// We test the following assertions:
+	// 1) Invocation with GetDeliverServiceCredentials("A") to srvA succeeds
+	// 2) Invocation with GetDeliverServiceCredentials("B") to srvB succeeds
+	// 3) Invocation with GetDeliverServiceCredentials("A") to srvB fails
+	// 4) Invocation with GetDeliverServiceCredentials("B") to srvA fails
+
+	osA := newServer("orgA", 7070)
+	defer osA.Stop()
+	osB := newServer("orgB", 7080)
+	defer osB.Stop()
+	time.Sleep(time.Second)
+
+	cas := GetCASupport()
+	_, err := GetCASupport().GetDeliverServiceCredentials("C")
+	assert.Error(t, err)
+
+	cas.OrdererRootCAsByChain["A"] = [][]byte{osA.caCert}
+	cas.OrdererRootCAsByChain["B"] = [][]byte{osB.caCert}
+
+	testInvoke(t, "A", osA, true)
+	testInvoke(t, "B", osB, true)
+	testInvoke(t, "A", osB, false)
+	testInvoke(t, "B", osA, false)
+
+}
+
+func testInvoke(t *testing.T, channelID string, s *srv, shouldSucceed bool) {
+	creds, err := GetCASupport().GetDeliverServiceCredentials(channelID)
+	assert.NoError(t, err)
+	endpoint := fmt.Sprintf("localhost:%d", s.port)
+	conn, err := grpc.Dial(endpoint, grpc.WithTimeout(time.Second*3), grpc.WithTransportCredentials(creds), grpc.WithBlock())
+	if shouldSucceed {
+		assert.NoError(t, err)
+		defer conn.Close()
+	} else {
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "certificate signed by unknown authority")
+		return
+	}
+	client := testpb.NewTestServiceClient(conn)
+	_, err = client.EmptyCall(context.Background(), &testpb.Empty{})
+	assert.NoError(t, err)
+	s.assertServiced(t)
+}

core/comm/testdata/impersonation/orgA/ca.crt

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICHDCCAcOgAwIBAgIQHaelrvVzGi4qoc91lIEEzjAKBggqhkjOPQQDAjBbMQsw
+CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy
+YW5jaXNjbzENMAsGA1UEChMEb3JnQTEQMA4GA1UEAxMHY2Eub3JnQTAeFw0xNzA2
+MDgxMjAzMzFaFw0yNzA2MDYxMjAzMzFaMFsxCzAJBgNVBAYTAlVTMRMwEQYDVQQI
+EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRv
+cmdBMRAwDgYDVQQDEwdjYS5vcmdBMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE
+vHfWYfXiQwG58vR+iIlAj0TJV09fEXUElfnOY0T1uqhBftrQCEonzT8j5kxjuksl
+cYCB+nJOZTOeNZyLWygslaNpMGcwDgYDVR0PAQH/BAQDAgGmMBkGA1UdJQQSMBAG
+BFUdJQAGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wKQYDVR0OBCIEIFz7UdB7
+Lxhy5u8mM20w0z1oimfCdf4ju3PSF1z27ePRMAoGCCqGSM49BAMCA0cAMEQCIEtY
+oJWa2n3oPJbofWEHZBy8weAyS8idNpdZ215U/Db/AiAIkxL9MUk7M476ZxUctXj+
+Ll7oLHhcRudsmIeUJu7xww==
+-----END CERTIFICATE-----

core/comm/testdata/impersonation/orgA/server.crt

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICNjCCAd2gAwIBAgIRAIVdca7gRBEyyp/DyCmPRV4wCgYIKoZIzj0EAwIwWzEL
+MAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNhbiBG
+cmFuY2lzY28xDTALBgNVBAoTBG9yZ0ExEDAOBgNVBAMTB2NhLm9yZ0EwHhcNMTcw
+NjA4MTIwMzMxWhcNMjcwNjA2MTIwMzMxWjBTMQswCQYDVQQGEwJVUzETMBEGA1UE
+CBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEXMBUGA1UEAxMO
+bG9jYWxob3N0Lm9yZ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQlAdoVacaS
+Eo8MrjemN+oN7WMPNCTDoobJw+/KiLR6ItoaDrMhzvoIaP4DJP8xvn9YIloUODaK
+pX/CZXjtxs8Xo4GJMIGGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEF
+BQcDATAMBgNVHRMBAf8EAjAAMCsGA1UdIwQkMCKAIFz7UdB7Lxhy5u8mM20w0z1o
+imfCdf4ju3PSF1z27ePRMCQGA1UdEQQdMBuCDmxvY2FsaG9zdC5vcmdBgglsb2Nh
+bGhvc3QwCgYIKoZIzj0EAwIDRwAwRAIgJFESyVamhihN9adp8ekl78/80DmfSd2N
+ZVAC2g4Gp+0CIFquN7/Z+75yMW4jYrXFLlR6iM1Iy4oRR1S9kQ1e0h18
+-----END CERTIFICATE-----

core/comm/testdata/impersonation/orgA/server.key

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgiiG7wI1OuYPHb9Ft
+6o/uVjSH6CM2Yq63yvXE+bg9JJyhRANCAAQlAdoVacaSEo8MrjemN+oN7WMPNCTD
+oobJw+/KiLR6ItoaDrMhzvoIaP4DJP8xvn9YIloUODaKpX/CZXjtxs8X
+-----END PRIVATE KEY-----

core/comm/testdata/impersonation/orgB/ca.crt

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICHTCCAcSgAwIBAgIRAKZxKXGx9VIoK2vAWHF2u1MwCgYIKoZIzj0EAwIwWzEL
+MAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNhbiBG
+cmFuY2lzY28xDTALBgNVBAoTBG9yZ0IxEDAOBgNVBAMTB2NhLm9yZ0IwHhcNMTcw
+NjA4MTIwMzMxWhcNMjcwNjA2MTIwMzMxWjBbMQswCQYDVQQGEwJVUzETMBEGA1UE
+CBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChME
+b3JnQjEQMA4GA1UEAxMHY2Eub3JnQjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA
+BKP7/XEYcHqW2915PucqcZtiT86bkPqU52W7t2Xe8hELEsad8XMjZcQ0Kwcs++O/
+BVrMv5R8UoS3H5hx+LLVu8+jaTBnMA4GA1UdDwEB/wQEAwIBpjAZBgNVHSUEEjAQ
+BgRVHSUABggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdDgQiBCAZB75b
+7YJN/QFjWiX7B4mcwI80mwkaR6vB5y0Ne6iszDAKBggqhkjOPQQDAgNHADBEAiBn
+lF1PNfg6X9IKj39y/TmPIHVBbmClchTEeNE2D6SdDQIgb9iqNW+/UekuSTn/XVtT
+jtpdqcbKE8pDNp3Gi9Fv1as=
+-----END CERTIFICATE-----

core/comm/testdata/impersonation/orgB/server.crt

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICNTCCAdygAwIBAgIQEytB52SKNRuBcfMledS0LzAKBggqhkjOPQQDAjBbMQsw
+CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy
+YW5jaXNjbzENMAsGA1UEChMEb3JnQjEQMA4GA1UEAxMHY2Eub3JnQjAeFw0xNzA2
+MDgxMjAzMzFaFw0yNzA2MDYxMjAzMzFaMFMxCzAJBgNVBAYTAlVTMRMwEQYDVQQI
+EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRcwFQYDVQQDEw5s
+b2NhbGhvc3Qub3JnQjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABNeIm7orPT/U
+NzM43TaEg3PfyY6pNCyD1bswq8+ykVayDuBz71TidBJL/HHn02RLhSHJ+oHBRV0w
+xel1QTQGjYujgYkwgYYwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUF
+BwMBMAwGA1UdEwEB/wQCMAAwKwYDVR0jBCQwIoAgGQe+W+2CTf0BY1ol+weJnMCP
+NJsJGkerwectDXuorMwwJAYDVR0RBB0wG4IObG9jYWxob3N0Lm9yZ0KCCWxvY2Fs
+aG9zdDAKBggqhkjOPQQDAgNHADBEAiBDQsH0omgTJlznoFdN9U7Cn11mdRKgNReA
+/iB8p63rmgIgf8e7ErCPgoz+zbUPSkOFtZPW+0hgdPRXhczhZwIgS8A=
+-----END CERTIFICATE-----

core/comm/testdata/impersonation/orgB/server.key

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgpEGx/HVbldUaoaiP
+9n3DFtxUr1RSvCdQFHFVIRya2uShRANCAATXiJu6Kz0/1DczON02hINz38mOqTQs
+g9W7MKvPspFWsg7gc+9U4nQSS/xx59NkS4UhyfqBwUVdMMXpdUE0Bo2L
+-----END PRIVATE KEY-----

core/deliverservice/deliveryclient.go

@@ -63,8 +63,8 @@ type deliverServiceImpl struct {
 // how it verifies messages received from it,
 // and how it disseminates the messages to other peers
 type Config struct {
-	// ConnFactory creates a connection to an endpoint
-	ConnFactory func(endpoint string) (*grpc.ClientConn, error)
+	// ConnFactory returns a function that creates a connection to an endpoint
+	ConnFactory func(channelID string) func(endpoint string) (*grpc.ClientConn, error)
 	// ABCFactory creates an AtomicBroadcastClient out of a connection
 	ABCFactory func(*grpc.ClientConn) orderer.AtomicBroadcastClient
 	// CryptoSvc performs cryptographic actions like message verification and signing
@@ -183,27 +183,33 @@ func (d *deliverServiceImpl) newClient(chainID string, ledgerInfoProvider blocks
 		}
 		return time.Duration(math.Pow(2, float64(attemptNum))) * time.Millisecond * 500, true
 	}
-	connProd := comm.NewConnectionProducer(d.conf.ConnFactory, d.conf.Endpoints)
+	connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID), d.conf.Endpoints)
 	bClient := NewBroadcastClient(connProd, d.conf.ABCFactory, broadcastSetup, backoffPolicy)
 	requester.client = bClient
 	return bClient
 }
 
-func DefaultConnectionFactory(endpoint string) (*grpc.ClientConn, error) {
-	dialOpts := []grpc.DialOption{grpc.WithTimeout(connTimeout), grpc.WithBlock()}
-	// set max send/recv msg sizes
-	dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(comm.MaxRecvMsgSize()),
-		grpc.MaxCallSendMsgSize(comm.MaxSendMsgSize())))
-	// set the keepalive options
-	dialOpts = append(dialOpts, comm.ClientKeepaliveOptions()...)
-
-	if comm.TLSEnabled() {
-		dialOpts = append(dialOpts, grpc.WithTransportCredentials(comm.GetCASupport().GetDeliverServiceCredentials()))
-	} else {
-		dialOpts = append(dialOpts, grpc.WithInsecure())
+func DefaultConnectionFactory(channelID string) func(endpoint string) (*grpc.ClientConn, error) {
+	return func(endpoint string) (*grpc.ClientConn, error) {
+		dialOpts := []grpc.DialOption{grpc.WithTimeout(connTimeout), grpc.WithBlock()}
+		// set max send/recv msg sizes
+		dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(comm.MaxRecvMsgSize()),
+			grpc.MaxCallSendMsgSize(comm.MaxSendMsgSize())))
+		// set the keepalive options
+		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions()...)
+
+		if comm.TLSEnabled() {
+			creds, err := comm.GetCASupport().GetDeliverServiceCredentials(channelID)
+			if err != nil {
+				return nil, fmt.Errorf("Failed obtaining credentials for channel %s: %v", channelID, err)
+			}
+			dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
+		} else {
+			dialOpts = append(dialOpts, grpc.WithInsecure())
+		}
+		grpc.EnableTracing = true
+		return grpc.Dial(endpoint, dialOpts...)
 	}
-	grpc.EnableTracing = true
-	return grpc.Dial(endpoint, dialOpts...)
 }
 
 func DefaultABCFactory(conn *grpc.ClientConn) orderer.AtomicBroadcastClient {

core/deliverservice/deliveryclient_test.go

@@ -95,10 +95,12 @@ func TestNewDeliverService(t *testing.T) {
 		return &mocks.MockAtomicBroadcastClient{blocksDeliverer}
 	}
 
-	connFactory := func(endpoint string) (*grpc.ClientConn, error) {
-		lock.Lock()
-		defer lock.Unlock()
-		return newConnection(), nil
+	connFactory := func(_ string) func(string) (*grpc.ClientConn, error) {
+		return func(endpoint string) (*grpc.ClientConn, error) {
+			lock.Lock()
+			defer lock.Unlock()
+			return newConnection(), nil
+		}
 	}
 	service, err := NewDeliverService(&Config{
 		Endpoints:   []string{"a"},