[FAB-4764] Rearrange Ledger test parameter setup

Increase LTE tool usability by rearranging the Ledger performance test
parameter setup so that the parameter file has more control on the test
run. Also, make the test driver take the parameter file as a (required)
option instead of being hardcoded.

Modifies the README.md and the python test wrapper to reflect these
changes.

Change-Id: I2cbdb0d082b3b8f9f143d2f346742c84556e3c3a
Signed-off-by: Adnan Choudhury <[email protected]>

Author: adnan-c

test/regression/daily/ledger_lte.py

@@ -6,6 +6,8 @@
 import unittest
 import subprocess
 
+tool_directory = '../../tools/LTE/scripts'
+
 class perf_goleveldb(unittest.TestCase):
 
     def test_FAB_3790_VaryNumParallelTxPerChain(self):
@@ -20,9 +22,9 @@ def test_FAB_3790_VaryNumParallelTxPerChain(self):
         '''
         logfile = open("output_VaryNumParallelTxPerChain.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumParallelTxPerChain",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxPerChain",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="VaryNumParallelTxPerChain "
                 "performance test failed. \nPlease check the logfile "
@@ -40,9 +42,9 @@ def test_FAB_3795_VaryNumChains(self):
         '''
         logfile = open("output_VaryNumChains.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumChains",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChains",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="VaryNumChains performance test"
                 " failed. \nPlease check the logfile "+logfile.name+" for more "
@@ -60,9 +62,9 @@ def test_FAB_3798_VaryNumParallelTxWithSingleChain(self):
         '''
         logfile = open("output_VaryNumParallelTxWithSingleChain.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumParallelTxWithSingleChain",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxWithSingleChain",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="VaryNumParallelTxWithSingleChain "
                 "performance test failed. \nPlease check the logfile "
@@ -80,9 +82,9 @@ def test_FAB_3799_VaryNumChainsWithNoParallelism(self):
         '''
         logfile = open("output_VaryNumChainsWithNoParallelism.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumChainsWithNoParallelism",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChainsWithNoParallelism",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="varyNumChainsWithNoParallelism "
                 "performance test failed. \nPlease check the logfile "
@@ -99,9 +101,9 @@ def test_FAB_3801_VaryKVSize(self):
         '''
         logfile = open("output_VaryKVSize.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyKVSize",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyKVSize",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="varyKVSize performance test"
                 " failed. \nPlease check the logfile "+logfile.name+" for more "
@@ -119,9 +121,9 @@ def test_FAB_3802_VaryBatchSize(self):
         '''
         logfile = open("output_VaryBatchSize.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyBatchSize",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyBatchSize",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="varyBatchSize performance test"
                 " failed. \nPlease check the logfile "+logfile.name+" for more "
@@ -139,9 +141,9 @@ def test_FAB_3800_VaryNumKeysInEachTx(self):
         '''
         logfile = open("output_VaryNumKeysInEachTx.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumKeysInEachTx",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumKeysInEachTx",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="varyNumKeysInEachTx performance "
                 "test failed. \nPlease check the logfile "+logfile.name
@@ -159,9 +161,9 @@ def test_FAB_3803_VaryNumTxs(self):
         '''
         logfile = open("output_VaryNumTxs.log", "w")
         returncode = subprocess.call(
-                "./runbenchmarks.sh varyNumTxs",
+                "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumTxs",
                 shell=True, stderr=subprocess.STDOUT, stdout=logfile,
-                cwd='../../tools/LTE/scripts')
+                cwd=tool_directory)
         logfile.close()
         self.assertEqual(returncode, 0, msg="varyNumTxs performance test"
                 " failed. \nPlease check the logfile "+logfile.name+" for more "

test/tools/LTE/README.md

@@ -21,12 +21,15 @@ considered to constitute a single test-run.
 
 ## How to Run The tests
 
-To run all the available tests with default parameters, run:
+To run all the available tests, run:
 ```
 cd fabric/test/tools/LTE/scripts
-./runbenchmark.sh all
+./runbenchmark.sh -f parameters_daily_CI.sh all
 ```
-you can run individual tests without running all the available tests by giving
+where the file `parameters_daily_CI.sh` has all the necessary test parameters.
+
+
+You can run individual tests without running all the available tests by giving
 the name of the test as parameter, instead of `all`. You can get the available
 test names by:
 ```
@@ -35,7 +38,7 @@ test names by:
 
 ### What the Tests Do
 
-Each test reads test parameters from the file `parameters_daily_CI.sh` and
+Each test reads test parameters from the provided parameter file and
 conducts several test-runs by varying one or two of the parameters. The
 parameters are:
 * number of chains (ledger),
@@ -95,8 +98,15 @@ transactions on chain_2 in parallel In each of the transactions executed by any
 client, the transaction expects and modifies any key(s) between Key_1 to key_50
 (because, total keys are to be 100 across two chains).
 
+### Running with Custom Parameters
+
+The tests can be run with user-defined parameters by creating a new file that
+has all the necessary parameters to run and using that file as the input (see
+the section on how to run the tests) . The names of necessary parameters can be
+found in the file `parameters_daily_CI.sh`.
+
 ## How to View the Test Results
 
 The test results can be viewed as in the stdout where it shows how long each
 single operation took to complete in a test. These results are also saved in a
-.csv file in the following directory: `/tmp/fabric/test/tools/LTE/results`
+.csv file in the following directory: `/tmp/experiments`

test/tools/LTE/scripts/parameters_daily_CI.sh

@@ -7,8 +7,30 @@
 DataDir="/tmp/fabric/test/tools/LTE/data"
 NumChains=10
 NumParallelTxPerChain=10
-NumKVs=1000000
-NumTotalTx=1000000
+NumKVs=100000
+NumTotalTx=100000
 NumKeysInEachTx=4
 BatchSize=50
 KVSize=200
+
+# Each test consists of several test-runs, where one single parameter is varied
+# between the test-runs and rest of the parameters remain same. Each array below
+# holds the values of that parameter for each corresponding test. For example,
+# the "ArrayNumChains" will be used in the test titled "varyNumChains", and the
+# 8 test-runs that exist for testing the varying number of chains will use these
+# values:
+# NumChains = (1 5 10 20 50 100 500 2000)
+# NumParallelTxPerChain=10
+# NumKVs=100000
+# NumTotalTx=100000
+# NumKeysInEachTx=4
+# BatchSize=50
+# KVSize=200
+ArrayNumParallelTxPerChain=(1 5 10 20 50 100 500 2000)
+ArrayNumChains=(1 5 10 20 50 100 500 2000)
+ArrayNumKeysInEachTx=(1 2 5 10 20)
+ArrayKVSize=(100 200 500 1000 2000)
+ArrayBatchSize=(10 20 100 500)
+ArrayNumParallelTxWithSingleChain=(1 5 10 20 50 100 500 2000)
+ArrayNumChainsWithNoParallelism=(1 5 10 20 50 100 500 2000)
+ArrayNumTxs=(100000 200000 500000 1000000)

test/tools/LTE/scripts/runbenchmarks.sh

@@ -6,66 +6,72 @@
 #
 
 source ./benchmarks.sh
-source ./parameters_daily_CI.sh
 
 ########################################################################################################
 # This shell script contains a series of benchmark tests
-# The test parameters are imported from "parameters_daily_CI.sh"
 ########################################################################################################
 
 function varyNumParallelTxPerChain {
-    for v in 1 5 10 20 50 100 500 2000; do
+    for v in "${ArrayNumParallelTxPerChain[@]}"
+    do
         NumParallelTxPerChain=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumChains {
-    for v in 1 5 10 20 50 100 500 2000; do
+    for v in "${ArrayNumChains[@]}"
+    do
         NumChains=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumKeysInEachTx {
-    for v in 1 2 5 10 20; do
+    for v in "${ArrayNumKeysInEachTx[@]}"
+    do
         NumKeysInEachTx=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyKVSize {
-    for v in 100 200 500 1000 2000; do
+    for v in "${ArrayKVSize[@]}"
+    do
         KVSize=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyBatchSize {
-    for v in 10 20 100 500; do
+    for v in "${ArrayBatchSize[@]}"
+    do
         BatchSize=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumParallelTxWithSingleChain {
     NumChains=1
-    for v in 1 5 10 20 50 100 500 2000; do
+    for v in "${ArrayNumParallelTxWithSingleChain[@]}"
+    do
         NumParallelTxPerChain=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumChainsWithNoParallelism {
     NumParallelTxPerChain=1
-    for v in 1 5 10 20 50 100 500 2000; do
+    for v in "${ArrayNumChainsWithNoParallelism[@]}"
+    do
         NumChains=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumTxs {
-    for v in 1000000 2000000 5000000 10000000; do
+    for v in "${ArrayNumTxs[@]}"
+    do
         NumTotalTx=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
@@ -77,6 +83,42 @@ function runLargeDataExperiment {
     rm -rf $DataDir;runInsertTxs;runReadWriteTxs
 }
 
+function usage () {
+    printf "Usage: ./runbenchmarks.sh [-f parameter_file_name] [test_name]\nAvailable tests (use \"all\" to run all tests):
+varyNumParallelTxPerChain
+varyNumChains
+varyNumParallelTxWithSingleChain
+varyNumChainsWithNoParallelism
+varyNumKeysInEachTx
+varyKVSize
+varyBatchSize
+varyNumTxs
+runLargeDataExperiment\n"
+}
+
+PARAM_FILE=""
+
+while getopts ":f:" opt; do
+  case $opt in
+    f)
+      printf "Parameter file: $OPTARG"
+      PARAM_FILE=$OPTARG;;
+    \?)
+      printf "Error: invalid parameter -$OPTARG!"  >> /dev/stderr
+      usage
+      exit 1;;
+  esac
+done
+
+if [ ! $PARAM_FILE ]
+then
+  printf "Error: No Parameter file given!"  >> /dev/stderr
+  usage
+  exit 1
+else
+  source $PARAM_FILE
+fi
+
 shift $(expr $OPTIND - 1 )
 
 case $1 in
@@ -99,16 +141,7 @@ case $1 in
   runLargeDataExperiment)
     runLargeDataExperiment ;;
   help)
-    printf "Usage: ./runbenchmarks.sh [test_name]\nAvailable tests (use \"all\" to run all tests):
-varyNumParallelTxPerChain
-varyNumChains
-varyNumParallelTxWithSingleChain
-varyNumChainsWithNoParallelism
-varyNumKeysInEachTx
-varyKVSize
-varyBatchSize
-varyNumTxs
-runLargeDataExperiment\n"
+    usage
     ;;
   all)
     varyNumParallelTxPerChain
@@ -122,5 +155,6 @@ runLargeDataExperiment\n"
     runLargeDataExperiment ;;
   *)
     printf "Error: test name empty/incorrect!\n"  >> /dev/stderr
+    usage
     exit 1 ;;
 esac