[FAB-3751] Add support to configure kafka in BDD

Added concept of composition to step ordererBootstrapAdmin
creating the genesis block.  This makes mor explicit
the requirement that the orderer boostrap requires network
configuration information.

Now the addition of the orderer addresses and kafka specific
genesis block configuration information depend on the composition
information.

Increased the topology graph view.

Change-Id: Icf1fc49265a24898c6c924c56c8778dc3b8e74d6
Signed-off-by: Jeff Garratt <[email protected]>

Author: jeffgarratt

bddtests/features/bootstrap.feature

@@ -68,9 +68,9 @@ Feature: Bootstrap
 
     # Order info includes orderer admin/orderer information and address (host:port) from previous steps
     # Only the peer organizations can vary.
-    And the ordererBootstrapAdmin using cert alias "bootstrapCertAlias" creates the genesis block "ordererGenesisBlock" for chain "OrdererSystemChainId" for network config policy "<PolicyType>" and consensus "<ConsensusType>" using consortiums:
-      |   Consortium    |
-      |  consortium1    |
+    And the ordererBootstrapAdmin using cert alias "bootstrapCertAlias" creates the genesis block "ordererGenesisBlock" for chain "OrdererSystemChainId" for composition "<ComposeFile>" and consensus "<ConsensusType>" using consortiums:
+      | Consortium  |
+      | consortium1 |
 
 
     And the orderer admins inspect and approve the genesis block for chain "OrdererSystemChainId"

bddtests/steps/bootstrap_impl.py

@@ -62,8 +62,8 @@ def step_impl(context):
     # Simply create the user
     bootstrap_util.getOrdererBootstrapAdmin(context, shouldCreate=True)
 
-@given(u'the ordererBootstrapAdmin using cert alias "{certAlias}" creates the genesis block "{ordererGenesisBlockName}" for chain "{ordererSystemChainIdName}" for network config policy "{networkConfigPolicy}" and consensus "{consensusType}" using consortiums')
-def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdName, networkConfigPolicy, consensusType):
+@given(u'the ordererBootstrapAdmin using cert alias "{certAlias}" creates the genesis block "{ordererGenesisBlockName}" for chain "{ordererSystemChainIdName}" for composition "{composeFile}" and consensus "{consensusType}" using consortiums')
+def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdName, composeFile, consensusType):
     directory = bootstrap_util.getDirectory(context=context)
     ordererBootstrapAdmin = bootstrap_util.getOrdererBootstrapAdmin(context)
     ordererSystemChainIdGUUID = ordererBootstrapAdmin.tags[ordererSystemChainIdName]
@@ -74,12 +74,17 @@ def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdN
         configGroups += ordererBootstrapAdmin.tags[configGroupName]
     # Concatenate signedConfigItems
 
+    service_names = compose.Composition(context, composeFilesYaml=composeFile, register_and_up=False).getServiceNames()
+
     # Construct block
     nodeAdminTuple = ordererBootstrapAdmin.tags[certAlias]
     bootstrapCert = directory.findCertForNodeAdminTuple(nodeAdminTuple=nodeAdminTuple)
-    (genesisBlock, envelope, genesis_block_channel_config) = bootstrap_util.createGenesisBlock(context, ordererSystemChainIdGUUID, consensusType,
-                                                                 nodeAdminTuple=nodeAdminTuple,
-                                                                 signedConfigItems=configGroups)
+    (genesisBlock, envelope, genesis_block_channel_config) = bootstrap_util.createGenesisBlock(context=context,
+                                                                                               service_names=service_names,
+                                                                                               chainId=ordererSystemChainIdGUUID,
+                                                                                               consensusType=consensusType,
+                                                                                               nodeAdminTuple=nodeAdminTuple,
+                                                                                               signedConfigItems=configGroups)
     ordererBootstrapAdmin.setTagValue(ordererGenesisBlockName + "_genesis_channel_config", genesis_block_channel_config)
     ordererBootstrapAdmin.setTagValue(ordererGenesisBlockName, genesisBlock)
     ordererBootstrapAdmin.setTagValue("ConsensusType", consensusType)

bddtests/steps/bootstrap_util.py

@@ -595,7 +595,7 @@ def setDefaultPoliciesForOrgs(channel, orgs, group_name, version=0, policy_versi
 
 
 
-def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType="solo", batchTimeout="1s", batchSizeMaxMessageCount=10, batchSizeAbsoluteMaxBytes=100000000, batchSizePreferredMaxBytes=512 * 1024, channel_max_count=0):
+def createChannelConfigGroup(directory, service_names, hashingAlgoName="SHA256", consensusType="solo", batchTimeout="1s", batchSizeMaxMessageCount=10, batchSizeAbsoluteMaxBytes=100000000, batchSizePreferredMaxBytes=512 * 1024, channel_max_count=0):
 
     channel = common_dot_configtx_pb2.ConfigGroup()
     # channel.groups[ApplicationGroup] = common_dot_configtx_pb2.ConfigGroup()
@@ -647,12 +647,12 @@ def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType=
         channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].value = toValue(
             getMSPConfig(org=ordererOrg, directory=directory))
         channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].mod_policy=BootstrapHelper.KEY_POLICY_ADMINS
-    # #Kafka specific
-    # matchingNATs = [nat for nat in directory.getNamedCtxTuples() if (("orderer" in nat.user) and ("Signer" in nat.user) and ((compose_service in nat.nodeName)))]
-    # for broker in [org for org in directory.getOrganizations().values() if Network.Orderer in org.networks]:
-    #     channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].value = toValue(
-    #         ordererOrg.getMSPConfig())
-    # channel.groups[OrdererGroup].values[BootstrapHelper.KEY_ORDERER_KAFKA_BROKERS].value = toValue(orderer_dot_configuration_pb2.KafkaBrokers(brokers=["kafka0:9092"]))
+
+    #Kafka specific
+    kafka_brokers = ["{0}:9092".format(service_name) for service_name in service_names if "kafka" in service_name]
+    if len(kafka_brokers) > 0:
+        channel.groups[OrdererGroup].values[BootstrapHelper.KEY_ORDERER_KAFKA_BROKERS].value = toValue(
+            orderer_dot_configuration_pb2.KafkaBrokers(brokers=kafka_brokers))
 
     for vKey, vVal in channel.groups[OrdererGroup].values.iteritems():
         vVal.mod_policy=BootstrapHelper.KEY_POLICY_ADMINS
@@ -666,9 +666,8 @@ def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType=
 
     #New OrdererAddress
     ordererAddress = common_dot_configuration_pb2.OrdererAddresses()
-    for ordererNodeTuple, cert in [(user_node_tuple, cert) for user_node_tuple, cert in directory.ordererAdminTuples.iteritems() if
-                      "orderer" in user_node_tuple.user and "signer" in user_node_tuple.user.lower()]:
-        ordererAddress.addresses.append("{0}:7050".format(ordererNodeTuple.nodeName))
+    for orderer_service_name in [service_name for service_name in service_names if "orderer" in service_name]:
+        ordererAddress.addresses.append("{0}:7050".format(orderer_service_name))
     assert len(ordererAddress.addresses) > 0, "No orderer nodes were found while trying to create channel ConfigGroup"
     channel.values[BootstrapHelper.KEY_ORDERER_ADDRESSES].value = toValue(ordererAddress)
 
@@ -739,13 +738,13 @@ def mergeConfigGroups(configGroupTarget, configGroupSource):
         configGroupTarget.values[k].CopyFrom(v)
 
 
-def createGenesisBlock(context, chainId, consensusType, nodeAdminTuple, signedConfigItems=[]):
+def createGenesisBlock(context, service_names, chainId, consensusType, nodeAdminTuple, signedConfigItems=[]):
     'Generates the genesis block for starting the oderers and for use in the chain config transaction by peers'
     # assert not "bootstrapGenesisBlock" in context,"Genesis block already created:\n{0}".format(context.bootstrapGenesisBlock)
     directory = getDirectory(context)
     assert len(directory.ordererAdminTuples) > 0, "No orderer admin tuples defined!!!"
 
-    channelConfig = createChannelConfigGroup(directory=directory, consensusType=consensusType)
+    channelConfig = createChannelConfigGroup(directory=directory, service_names=service_names, consensusType=consensusType)
     for configGroup in signedConfigItems:
         mergeConfigGroups(channelConfig, configGroup)
 

bddtests/steps/compose.py

@@ -82,8 +82,8 @@ def GetCompositionCallbacksFromContext(cls, context):
     def GetUUID(cls):
         return GetDockerSafeUUID()
 
-    def __init__(self, context, composeFilesYaml, projectName = None,
-                 force_recreate = True, components = []):
+    def __init__(self, context, composeFilesYaml, projectName=None,
+                 force_recreate=True, components=[], register_and_up=True):
         self.contextHelper = ContextHelper.GetHelper(context=context)
         if not projectName:
             projectName = self.contextHelper.getGuuid()
@@ -93,10 +93,11 @@ def __init__(self, context, composeFilesYaml, projectName = None,
         self.composeFilesYaml = composeFilesYaml
         self.serviceNames = []
         self.serviceNames = self._collectServiceNames()
-        # Register with contextHelper (Supports docgen)
-        self.contextHelper.registerComposition(self)
-        [callback.composing(self, context) for callback in Composition.GetCompositionCallbacksFromContext(context)]
-        self.up(context, force_recreate, components)
+        if register_and_up:
+            # Register with contextHelper (Supports docgen)
+            self.contextHelper.registerComposition(self)
+            [callback.composing(self, context) for callback in Composition.GetCompositionCallbacksFromContext(context)]
+            self.up(context, force_recreate, components)
 
     def _collectServiceNames(self):
         'First collect the services names.'

bddtests/templates/html/graph.html

@@ -12,7 +12,7 @@
 
 </style>
 <div>
-<svg width="860" height="700"></svg>
+<svg width="860" height="800"></svg>
 </div>
 <script src="https://d3js.org/d3.v4.min.js"></script>
 <script>