Skip to content

Commit 66cc54b

Browse files
Run BDD Compose files intelligently
When bringing up containers with docker compose during BDD testing a time.sleep call is not very efficient. This commit replaces this with an intelligent probing of the containers to determine when they are ready. It determines this by checking the output of the netstat command on each container for listening or connected TCP connections. Once an active connection is detected on all containers or the timeout period elapses, execution continues or aborts respectively. In combination with change 419 in this change BDDTests complete approximately 4 minutes faster. Signed-off-by: Julian Carrivick <[email protected]> Change-Id: I2c89f360d04a8e5e5daeaa2aa5027e5c191a453c
1 parent c9a0166 commit 66cc54b

File tree

6 files changed

+201
-91
lines changed

6 files changed

+201
-91
lines changed

bddtests/peer_basic.feature

+16-9
Original file line numberDiff line numberDiff line change
@@ -1161,36 +1161,43 @@ Scenario: chaincode example02 with 4 peers, two stopped
11611161
| a | 100 | b | 200 |
11621162
Then I should have received a chaincode name
11631163
Then I wait up to "60" seconds for transaction to be committed to peers:
1164-
| vp0 | vp1 | vp2 |
1164+
| vp0 | vp1 | vp2 | vp3 |
11651165

11661166
When I query chaincode "example2" function name "query" with value "a" on peers:
1167-
| vp0 | vp1 | vp2 | vp3 |
1167+
| vp0 | vp1 | vp2 | vp3 |
11681168
Then I should get a JSON response from peers with "result.message" = "100"
1169-
| vp0 | vp1 | vp2 | vp3 |
1169+
| vp0 | vp1 | vp2 | vp3 |
11701170

11711171
Given I stop peers:
1172-
| vp2 | vp3 |
1172+
| vp2 | vp3 |
11731173

11741174
When I invoke chaincode "example2" function name "invoke" on "vp0"
11751175
|arg1|arg2|arg3|
11761176
| a | b | 10 |
11771177
Then I should have received a transactionID
11781178

11791179
Given I start peers:
1180-
| vp3 |
1181-
And I wait "15" seconds
1180+
| vp3 |
1181+
1182+
# Make sure vp3 catches up first
1183+
Then I wait up to "60" seconds for transaction to be committed to peers:
1184+
| vp0 | vp1 | vp3 |
1185+
When I query chaincode "example2" function name "query" with value "a" on peers:
1186+
| vp0 | vp1 | vp3 |
1187+
Then I should get a JSON response from peers with "result.message" = "90"
1188+
| vp0 | vp1 | vp3 |
11821189

11831190
When I invoke chaincode "example2" function name "invoke" on "vp0" "9" times
11841191
|arg1|arg2|arg3|
11851192
| a | b | 10 |
11861193
Then I should have received a transactionID
11871194
Then I wait up to "60" seconds for transaction to be committed to peers:
1188-
| vp0 | vp1 | vp3 |
1195+
| vp0 | vp1 | vp3 |
11891196

11901197
When I query chaincode "example2" function name "query" with value "a" on peers:
1191-
| vp0 | vp1 | vp3 |
1198+
| vp0 | vp1 | vp3 |
11921199
Then I should get a JSON response from peers with "result.message" = "0"
1193-
| vp0 | vp1 | vp3 |
1200+
| vp0 | vp1 | vp3 |
11941201

11951202
@issue_1874b
11961203
#@doNotDecompose

bddtests/steps/bdd_compose_util.py

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#
2+
# Copyright IBM Corp. 2016 All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
import os
18+
import time
19+
import re
20+
21+
from bdd_test_util import cli_call
22+
23+
REST_PORT = "7050"
24+
25+
class ContainerData:
26+
def __init__(self, containerName, ipAddress, envFromInspect, composeService):
27+
self.containerName = containerName
28+
self.ipAddress = ipAddress
29+
self.envFromInspect = envFromInspect
30+
self.composeService = composeService
31+
32+
def getEnv(self, key):
33+
envValue = None
34+
for val in self.envFromInspect:
35+
if val.startswith(key):
36+
envValue = val[len(key):]
37+
break
38+
if envValue == None:
39+
raise Exception("ENV key not found ({0}) for container ({1})".format(key, self.containerName))
40+
return envValue
41+
42+
def getDockerComposeFileArgsFromYamlFile(compose_yaml):
43+
parts = compose_yaml.split()
44+
args = []
45+
for part in parts:
46+
args = args + ["-f"] + [part]
47+
return args
48+
49+
def parseComposeOutput(context):
50+
"""Parses the compose output results and set appropriate values into context. Merges existing with newly composed."""
51+
# Use the prefix to get the container name
52+
containerNamePrefix = os.path.basename(os.getcwd()) + "_"
53+
containerNames = []
54+
for l in context.compose_error.splitlines():
55+
tokens = l.split()
56+
print(tokens)
57+
if 1 < len(tokens):
58+
thisContainer = tokens[1]
59+
if containerNamePrefix not in thisContainer:
60+
thisContainer = containerNamePrefix + thisContainer + "_1"
61+
if thisContainer not in containerNames:
62+
containerNames.append(thisContainer)
63+
64+
print("Containers started: ")
65+
print(containerNames)
66+
# Now get the Network Address for each name, and set the ContainerData onto the context.
67+
containerDataList = []
68+
for containerName in containerNames:
69+
output, error, returncode = \
70+
cli_call(context, ["docker", "inspect", "--format", "{{ .NetworkSettings.IPAddress }}", containerName], expect_success=True)
71+
print("container {0} has address = {1}".format(containerName, output.splitlines()[0]))
72+
ipAddress = output.splitlines()[0]
73+
74+
# Get the environment array
75+
output, error, returncode = \
76+
cli_call(context, ["docker", "inspect", "--format", "{{ .Config.Env }}", containerName], expect_success=True)
77+
env = output.splitlines()[0][1:-1].split()
78+
79+
# Get the Labels to access the com.docker.compose.service value
80+
output, error, returncode = \
81+
cli_call(context, ["docker", "inspect", "--format", "{{ .Config.Labels }}", containerName], expect_success=True)
82+
labels = output.splitlines()[0][4:-1].split()
83+
dockerComposeService = [composeService[27:] for composeService in labels if composeService.startswith("com.docker.compose.service:")][0]
84+
print("dockerComposeService = {0}".format(dockerComposeService))
85+
print("container {0} has env = {1}".format(containerName, env))
86+
containerDataList.append(ContainerData(containerName, ipAddress, env, dockerComposeService))
87+
# Now merge the new containerData info with existing
88+
newContainerDataList = []
89+
if "compose_containers" in context:
90+
# Need to merge I new list
91+
newContainerDataList = context.compose_containers
92+
newContainerDataList = newContainerDataList + containerDataList
93+
94+
setattr(context, "compose_containers", newContainerDataList)
95+
print("")
96+
97+
def allContainersAreReadyWithinTimeout(context, timeout):
98+
timeoutTimestamp = time.time() + timeout
99+
formattedTime = time.strftime("%X", time.localtime(timeoutTimestamp))
100+
print("All containers should be up by {}".format(formattedTime))
101+
102+
for container in context.compose_containers:
103+
if not containerIsReadyByTimestamp(container, timeoutTimestamp):
104+
return False
105+
106+
print("All containers in ready state, ready to proceed")
107+
return True
108+
109+
def containerIsReadyByTimestamp(container, timeoutTimestamp):
110+
while containerIsNotReady(container):
111+
if timestampExceeded(timeoutTimestamp):
112+
print("Timed out waiting for {}".format(container.containerName))
113+
return False
114+
115+
print("{} not ready, waiting...".format(container.containerName))
116+
time.sleep(1)
117+
118+
print("{} now available".format(container.containerName))
119+
return True
120+
121+
def timestampExceeded(timeoutTimestamp):
122+
return time.time() > timeoutTimestamp
123+
124+
def containerIsNotReady(container):
125+
return not containerIsReady(container)
126+
127+
def containerIsReady(container):
128+
isReady = tcpPortsAreReady(container)
129+
isReady = isReady and restPortRespondsIfContainerIsPeer(container)
130+
131+
return isReady
132+
133+
def tcpPortsAreReady(container):
134+
netstatOutput = getContainerNetstatOutput(container.containerName)
135+
136+
for line in netstatOutput.splitlines():
137+
if re.search("ESTABLISHED|LISTEN", line):
138+
return True
139+
140+
print("No TCP connections are ready in container {}".format(container.containerName))
141+
return False
142+
143+
def getContainerNetstatOutput(containerName):
144+
command = ["docker", "exec", containerName, "netstat", "-atun"]
145+
stdout, stderr, returnCode = cli_call(None, command, expect_success=False)
146+
147+
return stdout
148+
149+
def restPortRespondsIfContainerIsPeer(container):
150+
containerName = container.containerName
151+
command = ["docker", "exec", containerName, "curl", "localhost:" + REST_PORT]
152+
153+
if containerIsPeer(container):
154+
stdout, stderr, returnCode = cli_call(None, command, expect_success=False)
155+
156+
if returnCode != 0:
157+
print("Connection to REST Port on {} failed".format(containerName))
158+
159+
return returnCode == 0
160+
161+
return True
162+
163+
def containerIsPeer(container):
164+
netstatOutput = getContainerNetstatOutput(container.containerName)
165+
166+
for line in netstatOutput.splitlines():
167+
if re.search(REST_PORT, line) and re.search("LISTEN", line):
168+
return True
169+
170+
return False

bddtests/steps/peer_basic_impl.py

+10-77
Original file line numberDiff line numberDiff line change
@@ -26,75 +26,11 @@
2626
import sys, requests, json
2727

2828
import bdd_test_util
29+
import bdd_compose_util
2930

3031
CORE_REST_PORT = 7050
3132
JSONRPC_VERSION = "2.0"
3233

33-
class ContainerData:
34-
def __init__(self, containerName, ipAddress, envFromInspect, composeService):
35-
self.containerName = containerName
36-
self.ipAddress = ipAddress
37-
self.envFromInspect = envFromInspect
38-
self.composeService = composeService
39-
40-
def getEnv(self, key):
41-
envValue = None
42-
for val in self.envFromInspect:
43-
if val.startswith(key):
44-
envValue = val[len(key):]
45-
break
46-
if envValue == None:
47-
raise Exception("ENV key not found ({0}) for container ({1})".format(key, self.containerName))
48-
return envValue
49-
50-
def parseComposeOutput(context):
51-
"""Parses the compose output results and set appropriate values into context. Merges existing with newly composed."""
52-
# Use the prefix to get the container name
53-
containerNamePrefix = os.path.basename(os.getcwd()) + "_"
54-
containerNames = []
55-
for l in context.compose_error.splitlines():
56-
tokens = l.split()
57-
print(tokens)
58-
if 1 < len(tokens):
59-
thisContainer = tokens[1]
60-
if containerNamePrefix not in thisContainer:
61-
thisContainer = containerNamePrefix + thisContainer + "_1"
62-
if thisContainer not in containerNames:
63-
containerNames.append(thisContainer)
64-
65-
print("Containers started: ")
66-
print(containerNames)
67-
# Now get the Network Address for each name, and set the ContainerData onto the context.
68-
containerDataList = []
69-
for containerName in containerNames:
70-
output, error, returncode = \
71-
bdd_test_util.cli_call(context, ["docker", "inspect", "--format", "{{ .NetworkSettings.IPAddress }}", containerName], expect_success=True)
72-
print("container {0} has address = {1}".format(containerName, output.splitlines()[0]))
73-
ipAddress = output.splitlines()[0]
74-
75-
# Get the environment array
76-
output, error, returncode = \
77-
bdd_test_util.cli_call(context, ["docker", "inspect", "--format", "{{ .Config.Env }}", containerName], expect_success=True)
78-
env = output.splitlines()[0][1:-1].split()
79-
80-
# Get the Labels to access the com.docker.compose.service value
81-
output, error, returncode = \
82-
bdd_test_util.cli_call(context, ["docker", "inspect", "--format", "{{ .Config.Labels }}", containerName], expect_success=True)
83-
labels = output.splitlines()[0][4:-1].split()
84-
dockerComposeService = [composeService[27:] for composeService in labels if composeService.startswith("com.docker.compose.service:")][0]
85-
print("dockerComposeService = {0}".format(dockerComposeService))
86-
print("container {0} has env = {1}".format(containerName, env))
87-
containerDataList.append(ContainerData(containerName, ipAddress, env, dockerComposeService))
88-
# Now merge the new containerData info with existing
89-
newContainerDataList = []
90-
if "compose_containers" in context:
91-
# Need to merge I new list
92-
newContainerDataList = context.compose_containers
93-
newContainerDataList = newContainerDataList + containerDataList
94-
95-
setattr(context, "compose_containers", newContainerDataList)
96-
print("")
97-
9834
def buildUrl(context, ipAddress, path):
9935
schema = "http"
10036
if 'TLS' in context.tags:
@@ -104,22 +40,19 @@ def buildUrl(context, ipAddress, path):
10440
def currentTime():
10541
return time.strftime("%H:%M:%S")
10642

107-
def getDockerComposeFileArgsFromYamlFile(compose_yaml):
108-
parts = compose_yaml.split()
109-
args = []
110-
for part in parts:
111-
args = args + ["-f"] + [part]
112-
return args
113-
11443
@given(u'we compose "{composeYamlFile}"')
11544
def step_impl(context, composeYamlFile):
11645
context.compose_yaml = composeYamlFile
117-
fileArgsToDockerCompose = getDockerComposeFileArgsFromYamlFile(context.compose_yaml)
46+
fileArgsToDockerCompose = bdd_compose_util.getDockerComposeFileArgsFromYamlFile(context.compose_yaml)
11847
context.compose_output, context.compose_error, context.compose_returncode = \
11948
bdd_test_util.cli_call(context, ["docker-compose"] + fileArgsToDockerCompose + ["up","--force-recreate", "-d"], expect_success=True)
12049
assert context.compose_returncode == 0, "docker-compose failed to bring up {0}".format(composeYamlFile)
121-
parseComposeOutput(context)
122-
time.sleep(10) # Should be replaced with a definitive interlock guaranteeing that all peers/membersrvc are ready
50+
51+
bdd_compose_util.parseComposeOutput(context)
52+
53+
timeoutSeconds = 15
54+
assert bdd_compose_util.allContainersAreReadyWithinTimeout(context, timeoutSeconds), \
55+
"Containers did not come up within {} seconds, aborting".format(timeoutSeconds)
12356

12457
@when(u'requesting "{path}" from "{containerName}"')
12558
def step_impl(context, path, containerName):
@@ -805,7 +738,7 @@ def compose_op(context, op):
805738
assert 'table' in context, "table (of peers) not found in context"
806739
assert 'compose_yaml' in context, "compose_yaml not found in context"
807740

808-
fileArgsToDockerCompose = getDockerComposeFileArgsFromYamlFile(context.compose_yaml)
741+
fileArgsToDockerCompose = bdd_compose_util.getDockerComposeFileArgsFromYamlFile(context.compose_yaml)
809742
services = context.table.headings
810743
# Loop through services and start/stop them, and modify the container data list if successful.
811744
for service in services:
@@ -815,7 +748,7 @@ def compose_op(context, op):
815748
if op == "stop" or op == "pause":
816749
context.compose_containers = [containerData for containerData in context.compose_containers if containerData.composeService != service]
817750
else:
818-
parseComposeOutput(context)
751+
bdd_compose_util.parseComposeOutput(context)
819752
print("After {0}ing, the container service list is = {1}".format(op, [containerData.composeService for containerData in context.compose_containers]))
820753

821754
def to_bytes(strlist):

tools/dbutility/bddtests/environment.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
def before_feature(context, feature):
55
print("\nRunning go build")
66
cmd = ["go", "build", "../dump_db_stats.go"]
7-
test_util.cli_call(context, cmd, expect_success=True)
7+
test_util.cli_call(cmd, expect_success=True)
88
print("go build complete")
99

1010
def after_feature(context, feature):

tools/dbutility/bddtests/steps/test.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os
22
import shutil
33

4-
import test_util
4+
from test_util import cli_call
55

66
@given(u'I create a dir "{dirPath}"')
77
def step_impl(context, dirPath):
@@ -14,14 +14,14 @@ def step_impl(contxt, dirPath):
1414
@when(u'I execute utility with no flag')
1515
def step_impl(context):
1616
cmd = ["./dump_db_stats"]
17-
context.output, context.error, context.returncode = test_util.cli_call(context, cmd, expect_success=False)
17+
context.output, context.error, context.returncode = cli_call(cmd, expect_success=False)
1818

1919
@when(u'I execute utility with flag "{flag}" and path "{path}"')
2020
def step_impl(context, flag, path):
2121
cmd = ["./dump_db_stats"]
2222
cmd.append(flag)
2323
cmd.append(path)
24-
context.output, context.error, context.returncode = test_util.cli_call(context, cmd, expect_success=False)
24+
context.output, context.error, context.returncode = cli_call(cmd, expect_success=False)
2525

2626
@then(u'I should get a process exit code "{expectedReturncode}"')
2727
def step_impl(context, expectedReturncode):

tools/dbutility/bddtests/test_util.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import subprocess
22

3-
def cli_call(context, arg_list, expect_success=True):
3+
def cli_call(arg_list, expect_success=True):
44
p = subprocess.Popen(arg_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
55
output, error = p.communicate()
66
if p.returncode != 0:

0 commit comments

Comments
 (0)