Skip to content

Commit 4958785

Browse files
adnan-cadnan-c
committed
[FAB-4425] Ledger perf. test: update logging
Put the detailed results and outputs to separate log files so that the python xml shows only pass/fail and short messages. Fixed some other typo. Change-Id: I5c12090ed1b0fe5b299c9811ef4c736c65bf9424 Signed-off-by: Adnan Choudhury <[email protected]>
1 parent 77fa0ad commit 4958785

File tree

2 files changed

+80
-48
lines changed

2 files changed

+80
-48
lines changed

‎test/regression/daily/ledger_lte.py

+78-46
Original file line numberDiff line numberDiff line change
@@ -10,30 +10,38 @@ def test_FAB_3790_VaryNumParallelTxPerChain(self):
1010
with goleveldb as the state database. We vary the number of parallel
1111
transactions per chain and observe the performance.
1212
13-
Passing criteria: all subtests (8) completed successfully
13+
Passing criteria: Underlying LTE test completed successfully with
14+
exit code 0
1415
'''
15-
result = subprocess.check_output(
16+
logfile = open("output_VaryNumParallelTxPerChain.log", "w")
17+
returncode = subprocess.call(
1618
"./runbenchmarks.sh varyNumParallelTxPerChain",
17-
shell=True, stderr=subprocess.STDOUT,
19+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
1820
cwd='../../tools/LTE/scripts')
19-
completion_count = result.count("PASS")
20-
self.assertEqual(completion_count, 8)
21+
logfile.close()
22+
self.assertEqual(returncode, 0, msg="VaryNumParallelTxPerChain "
23+
"performance test failed. \nPlease check the logfile "
24+
+logfile.name+" for more details.")
2125

22-
def test_FAB_3795_VaryNumChain(self):
26+
def test_FAB_3795_VaryNumChains(self):
2327
'''
2428
In this Performance test, we observe the performance (time to
2529
complete a set number of Ledger operations) of the Ledger component,
2630
with goleveldb as the state database. We vary the number of chains
2731
(ledgers).
2832
29-
Passing criteria: all subtests (8) completed successfully
33+
Passing criteria: Underlying LTE test completed successfully with
34+
exit code 0
3035
'''
31-
result = subprocess.check_output(
32-
"./runbenchmarks.sh varyNumChain",
33-
shell=True, stderr=subprocess.STDOUT,
36+
logfile = open("output_VaryNumChains.log", "w")
37+
returncode = subprocess.call(
38+
"./runbenchmarks.sh varyNumChains",
39+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
3440
cwd='../../tools/LTE/scripts')
35-
completion_count = result.count("PASS")
36-
self.assertEqual(completion_count, 8)
41+
logfile.close()
42+
self.assertEqual(returncode, 0, msg="VaryNumChains performance test"
43+
" failed. \nPlease check the logfile "+logfile.name+" for more "
44+
"details.")
3745

3846
def test_FAB_3798_VaryNumParallelTxWithSingleChain(self):
3947
'''
@@ -42,45 +50,57 @@ def test_FAB_3798_VaryNumParallelTxWithSingleChain(self):
4250
with goleveldb as the state database. We vary the number of parallel
4351
transactions on a single chain.
4452
45-
Passing criteria: all subtests (8) completed successfully
53+
Passing criteria: Underlying LTE test completed successfully with
54+
exit code 0
4655
'''
47-
result = subprocess.check_output(
56+
logfile = open("output_VaryNumParallelTxWithSingleChain.log", "w")
57+
returncode = subprocess.call(
4858
"./runbenchmarks.sh varyNumParallelTxWithSingleChain",
49-
shell=True, stderr=subprocess.STDOUT,
59+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
5060
cwd='../../tools/LTE/scripts')
51-
completion_count = result.count("PASS")
52-
self.assertEqual(completion_count, 8)
61+
logfile.close()
62+
self.assertEqual(returncode, 0, msg="VaryNumParallelTxWithSingleChain "
63+
"performance test failed. \nPlease check the logfile "
64+
+logfile.name+" for more details.")
5365

54-
def test_FAB_3799_VaryNumChainWithNoParallelism(self):
66+
def test_FAB_3799_VaryNumChainsWithNoParallelism(self):
5567
'''
5668
In this Performance test, we observe the performance (time to
5769
complete a set number of Ledger operations) of the Ledger component,
5870
with goleveldb as the state database. We vary the number of chains
5971
without any parallelism within a single chain.
6072
61-
Passing criteria: all subtests (8) completed successfully
73+
Passing criteria: Underlying LTE test completed successfully with
74+
exit code 0
6275
'''
63-
result = subprocess.check_output(
64-
"./runbenchmarks.sh varyNumChainWithNoParallelism",
65-
shell=True, stderr=subprocess.STDOUT,
76+
logfile = open("output_VaryNumChainsWithNoParallelism.log", "w")
77+
returncode = subprocess.call(
78+
"./runbenchmarks.sh varyNumChainsWithNoParallelism",
79+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
6680
cwd='../../tools/LTE/scripts')
67-
completion_count = result.count("PASS")
68-
self.assertEqual(completion_count, 8)
81+
logfile.close()
82+
self.assertEqual(returncode, 0, msg="varyNumChainsWithNoParallelism "
83+
"performance test failed. \nPlease check the logfile "
84+
+logfile.name+" for more details.")
6985

7086
def test_FAB_3801_VaryKVSize(self):
7187
'''
7288
In this Performance test, we observe the performance (time to
7389
complete a set number of Ledger operations) of the Ledger component,
7490
with goleveldb as the state database. We vary the size of key-value.
7591
76-
Passing criteria: all subtests (5) completed successfully
92+
Passing criteria: Underlying LTE test completed successfully with
93+
exit code 0
7794
'''
78-
result = subprocess.check_output(
95+
logfile = open("output_VaryKVSize.log", "w")
96+
returncode = subprocess.call(
7997
"./runbenchmarks.sh varyKVSize",
80-
shell=True, stderr=subprocess.STDOUT,
98+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
8199
cwd='../../tools/LTE/scripts')
82-
completion_count = result.count("PASS")
83-
self.assertEqual(completion_count, 5)
100+
logfile.close()
101+
self.assertEqual(returncode, 0, msg="varyKVSize performance test"
102+
" failed. \nPlease check the logfile "+logfile.name+" for more "
103+
"details.")
84104

85105
def test_FAB_3802_VaryBatchSize(self):
86106
'''
@@ -89,30 +109,38 @@ def test_FAB_3802_VaryBatchSize(self):
89109
with goleveldb as the state database. We vary the value of the batch
90110
size
91111
92-
Passing criteria: all subtests (4) completed successfully
112+
Passing criteria: Underlying LTE test completed successfully with
113+
exit code 0
93114
'''
94-
result = subprocess.check_output(
115+
logfile = open("output_VaryBatchSize.log", "w")
116+
returncode = subprocess.call(
95117
"./runbenchmarks.sh varyBatchSize",
96-
shell=True, stderr=subprocess.STDOUT,
118+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
97119
cwd='../../tools/LTE/scripts')
98-
completion_count = result.count("PASS")
99-
self.assertEqual(completion_count, 4)
120+
logfile.close()
121+
self.assertEqual(returncode, 0, msg="varyBatchSize performance test"
122+
" failed. \nPlease check the logfile "+logfile.name+" for more "
123+
"details.")
100124

101-
def test_FAB_3800_VaryNumKeysInEachTX(self):
125+
def test_FAB_3800_VaryNumKeysInEachTx(self):
102126
'''
103127
In this Performance test, we observe the performance (time to
104128
complete a set number of Ledger operations) of the Ledger component,
105129
with goleveldb as the state database. We vary the number of keys in
106130
each transaction.
107131
108-
Passing criteria: all subtests (5) completed successfully
132+
Passing criteria: Underlying LTE test completed successfully with
133+
exit code 0
109134
'''
110-
result = subprocess.check_output(
111-
"./runbenchmarks.sh varyNumKeysInEachTX",
112-
shell=True, stderr=subprocess.STDOUT,
135+
logfile = open("output_VaryNumKeysInEachTx.log", "w")
136+
returncode = subprocess.call(
137+
"./runbenchmarks.sh varyNumKeysInEachTx",
138+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
113139
cwd='../../tools/LTE/scripts')
114-
completion_count = result.count("PASS")
115-
self.assertEqual(completion_count, 5)
140+
logfile.close()
141+
self.assertEqual(returncode, 0, msg="varyNumKeysInEachTx performance "
142+
"test failed. \nPlease check the logfile "+logfile.name
143+
+" for more details.")
116144

117145
def test_FAB_3803_VaryNumTxs(self):
118146
'''
@@ -121,14 +149,18 @@ def test_FAB_3803_VaryNumTxs(self):
121149
with goleveldb as the state database. We vary the number of
122150
transactions carried out.
123151
124-
Passing criteria: all subtests (4) completed successfully
152+
Passing criteria: Underlying LTE test completed successfully with
153+
exit code 0
125154
'''
126-
result = subprocess.check_output(
155+
logfile = open("output_VaryNumTxs.log", "w")
156+
returncode = subprocess.call(
127157
"./runbenchmarks.sh varyNumTxs",
128-
shell=True, stderr=subprocess.STDOUT,
158+
shell=True, stderr=subprocess.STDOUT, stdout=logfile,
129159
cwd='../../tools/LTE/scripts')
130-
completion_count = result.count("PASS")
131-
self.assertEqual(completion_count, 4)
160+
logfile.close()
161+
self.assertEqual(returncode, 0, msg="varyNumTxs performance test"
162+
" failed. \nPlease check the logfile "+logfile.name+" for more "
163+
"details.")
132164

133165

134166
class perf_couchdb(unittest.TestCase):

‎test/tools/LTE/scripts/common.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ set -e
2929
#############################################################################################################################
3030

3131

32-
OUTPUT_DIR_ROOT=`echo /tmp/fabric/test/tools/LTE/results`
33-
RAW_OUTPUT_FILE="output.txt"
32+
OUTPUT_DIR_ROOT=`echo /tmp`
33+
RAW_OUTPUT_FILE="output_LTE.log"
3434
RESULTS_FILE="results.csv"
3535

3636
benchmarkLineRegex="^Benchmark.*[[:blank:]]+[[:digit:]]+[[:blank:]]+([[:digit:]]+).*$"

0 commit comments

Comments
 (0)