Skip to content

Commit f8a49c0

Browse files
mastersingh24mastersingh24
committed
[FAB-2745] Update e2e_cli to work with TLS
https://jira.hyperledger.org/browse/FAB-2745 NOTE: needed to touch multiple parts of the system to get TLS working e2e but this seemed like the best approach - changed default compose files to require TLS - renamed original compose file to *-no-tls.yaml - modified peer to pass peer.tls.rootcert instead of peer.tls.cert to client connections and chaincode container - modified go chaincode to use rootcert and fallback to cert - fixed some logic in the getTLSCert method - modified TLS startup logic for orderer - moved casupport to core/comm package - modified configtx.yaml to use orderer0 for orderer address to match its certificate - regenerated orderer.block and channel.tx using updated configtx.yaml - added singleton CASupport object to the core/comm package - added flags and supporting code for TLS to CLI commands which communicate with an orderer endpoint - modified deliverclient to support TLS credentials which are dynamically updated via config blocks Change-Id: Ifd69b08ada90f3d579697c2b7ef027789a799f04 Signed-off-by: Gari Singh <[email protected]>
1 parent b3d3254 commit f8a49c0

20 files changed

+440
-108
lines changed

‎core/chaincode/platforms/platforms.go

+14-9
Original file line numberDiff line numberDiff line change
@@ -75,17 +75,22 @@ func GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte, error) {
7575
}
7676

7777
func getPeerTLSCert() ([]byte, error) {
78-
path := viper.GetString("peer.tls.cert.file")
79-
if _, err := os.Stat(path); err != nil {
80-
81-
if os.IsNotExist(err) && viper.GetBool("peer.tls.enabled") == false {
82-
// It's not an error if the file doesn't exist but TLS is disabled anyway
83-
return nil, nil
84-
}
8578

79+
if viper.GetBool("peer.tls.enabled") == false {
80+
// no need for certificates if TLS is not enabled
81+
return nil, nil
82+
}
83+
var path string
84+
// first we check for the rootcert
85+
path = viper.GetString("peer.tls.rootcert.file")
86+
if path == "" {
87+
// check for tls cert
88+
path = viper.GetString("peer.tls.cert.file")
89+
}
90+
// this should not happen if the peer is running with TLS enabled
91+
if _, err := os.Stat(path); err != nil {
8692
return nil, err
8793
}
88-
8994
// FIXME: FAB-2037 - ensure we sanely resolve relative paths specified in the yaml
9095
return ioutil.ReadFile(path)
9196
}
@@ -119,7 +124,7 @@ func generateDockerfile(platform Platform, cds *pb.ChaincodeDeploymentSpec, tls
119124
if tls {
120125
const guestTLSPath = "/etc/hyperledger/fabric/peer.crt"
121126

122-
buf = append(buf, "ENV CORE_PEER_TLS_CERT_FILE="+guestTLSPath)
127+
buf = append(buf, "ENV CORE_PEER_TLS_ROOTCERT_FILE="+guestTLSPath)
123128
buf = append(buf, "COPY peer.crt "+guestTLSPath)
124129
}
125130

‎core/comm/connection.go

+105-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@ limitations under the License.
1717
package comm
1818

1919
import (
20+
"crypto/tls"
21+
"crypto/x509"
22+
"encoding/pem"
2023
"os"
24+
"sync"
2125
"time"
2226

2327
"google.golang.org/grpc"
@@ -31,6 +35,105 @@ import (
3135
const defaultTimeout = time.Second * 3
3236

3337
var commLogger = logging.MustGetLogger("comm")
38+
var caSupport *CASupport
39+
var once sync.Once
40+
41+
// CASupport type manages certificate authorities scoped by channel
42+
type CASupport struct {
43+
sync.RWMutex
44+
AppRootCAsByChain map[string][][]byte
45+
OrdererRootCAsByChain map[string][][]byte
46+
ClientRootCAs [][]byte
47+
ServerRootCAs [][]byte
48+
}
49+
50+
// GetCASupport returns the signleton CASupport instance
51+
func GetCASupport() *CASupport {
52+
53+
once.Do(func() {
54+
caSupport = &CASupport{
55+
AppRootCAsByChain: make(map[string][][]byte),
56+
OrdererRootCAsByChain: make(map[string][][]byte),
57+
}
58+
})
59+
return caSupport
60+
}
61+
62+
// GetServerRootCAs returns the PEM-encoded root certificates for all of the
63+
// application and orderer organizations defined for all chains. The root
64+
// certificates returned should be used to set the trusted server roots for
65+
// TLS clients.
66+
func (cas *CASupport) GetServerRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
67+
cas.RLock()
68+
defer cas.RUnlock()
69+
70+
appRootCAs = [][]byte{}
71+
ordererRootCAs = [][]byte{}
72+
73+
for _, appRootCA := range cas.AppRootCAsByChain {
74+
appRootCAs = append(appRootCAs, appRootCA...)
75+
}
76+
77+
for _, ordererRootCA := range cas.AppRootCAsByChain {
78+
ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
79+
}
80+
81+
// also need to append statically configured root certs
82+
appRootCAs = append(appRootCAs, cas.ServerRootCAs...)
83+
ordererRootCAs = append(ordererRootCAs, cas.ServerRootCAs...)
84+
return appRootCAs, ordererRootCAs
85+
}
86+
87+
// GetDeliverServiceCredentials returns GRPC transport credentials for use by GRPC
88+
// clients which communicate with ordering service endpoints.
89+
func (cas *CASupport) GetDeliverServiceCredentials() credentials.TransportCredentials {
90+
var creds credentials.TransportCredentials
91+
var tlsConfig = &tls.Config{}
92+
var certPool = x509.NewCertPool()
93+
// loop through the orderer CAs
94+
_, roots := cas.GetServerRootCAs()
95+
for _, root := range roots {
96+
block, _ := pem.Decode(root)
97+
if block != nil {
98+
cert, err := x509.ParseCertificate(block.Bytes)
99+
if err == nil {
100+
certPool.AddCert(cert)
101+
} else {
102+
commLogger.Warningf("Failed to add root cert to credentials (%s)", err)
103+
}
104+
} else {
105+
commLogger.Warning("Failed to add root cert to credentials")
106+
}
107+
}
108+
tlsConfig.RootCAs = certPool
109+
creds = credentials.NewTLS(tlsConfig)
110+
return creds
111+
}
112+
113+
// GetClientRootCAs returns the PEM-encoded root certificates for all of the
114+
// application and orderer organizations defined for all chains. The root
115+
// certificates returned should be used to set the trusted client roots for
116+
// TLS servers.
117+
func (cas *CASupport) GetClientRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
118+
cas.RLock()
119+
defer cas.RUnlock()
120+
121+
appRootCAs = [][]byte{}
122+
ordererRootCAs = [][]byte{}
123+
124+
for _, appRootCA := range cas.AppRootCAsByChain {
125+
appRootCAs = append(appRootCAs, appRootCA...)
126+
}
127+
128+
for _, ordererRootCA := range cas.AppRootCAsByChain {
129+
ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
130+
}
131+
132+
// also need to append statically configured root certs
133+
appRootCAs = append(appRootCAs, cas.ClientRootCAs...)
134+
ordererRootCAs = append(ordererRootCAs, cas.ClientRootCAs...)
135+
return appRootCAs, ordererRootCAs
136+
}
34137

35138
func getEnv(key, def string) string {
36139
val := os.Getenv(key)
@@ -71,9 +174,9 @@ func InitTLSForPeer() credentials.TransportCredentials {
71174
sn = viper.GetString("peer.tls.serverhostoverride")
72175
}
73176
var creds credentials.TransportCredentials
74-
if viper.GetString("peer.tls.cert.file") != "" {
177+
if viper.GetString("peer.tls.rootcert.file") != "" {
75178
var err error
76-
creds, err = credentials.NewClientTLSFromFile(viper.GetString("peer.tls.cert.file"), sn)
179+
creds, err = credentials.NewClientTLSFromFile(viper.GetString("peer.tls.rootcert.file"), sn)
77180
if err != nil {
78181
grpclog.Fatalf("Failed to create TLS credentials %v", err)
79182
}

‎core/deliverservice/deliveryclient.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,11 @@ func NewDeliverService(gossip blocksprovider.GossipServiceAdapter, endpoints []s
112112
dialOpts := []grpc.DialOption{grpc.WithTimeout(3 * time.Second), grpc.WithBlock()}
113113

114114
if comm.TLSEnabled() {
115-
dialOpts = append(dialOpts, grpc.WithTransportCredentials(comm.InitTLSForPeer()))
115+
dialOpts = append(dialOpts, grpc.WithTransportCredentials(comm.GetCASupport().GetDeliverServiceCredentials()))
116116
} else {
117117
dialOpts = append(dialOpts, grpc.WithInsecure())
118118
}
119-
119+
grpc.EnableTracing = true
120120
conn, err := grpc.Dial(endpoints[idx], dialOpts...)
121121
if err != nil {
122122
logger.Errorf("Cannot dial to %s, because of %s", endpoints[idx], err)

‎core/peer/peer.go

+6-42
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,8 @@ var peerLogger = logging.MustGetLogger("peer")
5050

5151
var peerServer comm.GRPCServer
5252

53-
var rootCASupport = struct {
54-
sync.RWMutex
55-
appRootCAsByChain map[string][][]byte
56-
ordererRootCAsByChain map[string][][]byte
57-
}{
58-
appRootCAsByChain: make(map[string][][]byte),
59-
ordererRootCAsByChain: make(map[string][][]byte),
60-
}
53+
// singleton instance to manage CAs for the peer across channel config changes
54+
var rootCASupport = comm.GetCASupport()
6155

6256
type chainSupport struct {
6357
configtxapi.Manager
@@ -317,7 +311,7 @@ func GetCurrConfigBlock(cid string) *common.Block {
317311
// updates the trusted roots for the peer based on updates to channels
318312
func updateTrustedRoots(cm configtxapi.Manager) {
319313
// this is triggered on per channel basis so first update the roots for the channel
320-
314+
peerLogger.Debugf("Updating trusted root authorities for channel %s", cm.ChainID())
321315
var secureConfig comm.SecureServerConfig
322316
var err error
323317
// only run is TLS is enabled
@@ -329,7 +323,7 @@ func updateTrustedRoots(cm configtxapi.Manager) {
329323
trustedRoots := [][]byte{}
330324
rootCASupport.RLock()
331325
defer rootCASupport.RUnlock()
332-
for _, roots := range rootCASupport.appRootCAsByChain {
326+
for _, roots := range rootCASupport.AppRootCAsByChain {
333327
trustedRoots = append(trustedRoots, roots...)
334328
}
335329
// also need to append statically configured root certs
@@ -395,41 +389,11 @@ func buildTrustedRootsForChain(cm configtxapi.Manager) {
395389
}
396390
// TODO: separate app and orderer CAs
397391
ordererRootCAs = appRootCAs
398-
rootCASupport.appRootCAsByChain[cid] = appRootCAs
399-
rootCASupport.ordererRootCAsByChain[cid] = ordererRootCAs
392+
rootCASupport.AppRootCAsByChain[cid] = appRootCAs
393+
rootCASupport.OrdererRootCAsByChain[cid] = ordererRootCAs
400394
}
401395
}
402396

403-
// GetRootCAs returns the PEM-encoded root certificates for all of the
404-
// application and orderer organizations defined for all chains
405-
func GetRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
406-
rootCASupport.RLock()
407-
defer rootCASupport.RUnlock()
408-
409-
appRootCAs = [][]byte{}
410-
ordererRootCAs = [][]byte{}
411-
412-
for _, appRootCA := range rootCASupport.appRootCAsByChain {
413-
appRootCAs = append(appRootCAs, appRootCA...)
414-
}
415-
// also need to append statically configured root certs
416-
secureConfig, err := GetSecureConfig()
417-
if err == nil {
418-
if len(secureConfig.ClientRootCAs) > 0 {
419-
appRootCAs = append(appRootCAs, secureConfig.ClientRootCAs...)
420-
}
421-
if len(secureConfig.ServerRootCAs) > 0 {
422-
appRootCAs = append(appRootCAs, secureConfig.ServerRootCAs...)
423-
}
424-
}
425-
426-
for _, ordererRootCA := range rootCASupport.appRootCAsByChain {
427-
ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
428-
}
429-
430-
return appRootCAs, ordererRootCAs
431-
}
432-
433397
// GetMSPIDs returns the ID of each application MSP defined on this chain
434398
func GetMSPIDs(cid string) []string {
435399
chains.RLock()

‎core/peer/peer_test.go

-6
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,6 @@ func (*mockDeliveryClientFactory) Service(g service.GossipService, endpoints []s
6565
return &mockDeliveryClient{}, nil
6666
}
6767

68-
func TestGetRootCAsNoChains(t *testing.T) {
69-
appRootCAs, ordererRootCAs := GetRootCAs()
70-
assert.Equal(t, len(appRootCAs), 0, "Expected zero appRootCAs")
71-
assert.Equal(t, len(ordererRootCAs), 0, "Expected zero ordererRootCAs")
72-
}
73-
7468
func TestInitialize(t *testing.T) {
7569
viper.Set("peer.fileSystemPath", "/var/hyperledger/test/")
7670

‎core/peer/pkg_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func TestCreatePeerServer(t *testing.T) {
152152
t.Fatalf("Failed to create config block (%s)", err)
153153
}
154154
t.Logf("Channel %s MSPIDs: (%s)", cid, peer.GetMSPIDs(cid))
155-
appCAs, orgCAs := peer.GetRootCAs()
155+
appCAs, orgCAs := comm.GetCASupport().GetClientRootCAs()
156156
t.Logf("appCAs after update for channel %s: %d", cid, len(appCAs))
157157
t.Logf("orgCAs after update for channel %s: %d", cid, len(orgCAs))
158158
}

‎examples/ccchecker/ccchecker.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func CCCheckerInit() {
9292
//CCCheckerRun main loops that will run the tests and cleanup
9393
func CCCheckerRun(orderingEndpoint string, report bool, verbose bool) error {
9494
//connect with Broadcast client
95-
bc, err := common.GetBroadcastClient(orderingEndpoint)
95+
bc, err := common.GetBroadcastClient(orderingEndpoint, false, "")
9696
if err != nil {
9797
return err
9898
}

‎examples/e2e_cli/configtx.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ Orderer: &OrdererDefaults
128128
OrdererType: solo
129129

130130
Addresses:
131-
- orderer:7050
131+
- orderer0:7050
132132

133133
# Batch Timeout: The amount of time to wait before creating a batch
134134
BatchTimeout: 10s

0 commit comments

Comments
 (0)