[FAB-4091] Update ledger benchmarking tool

Renaming the test tool to LTE (Ledger Traffic Engine), including a
readme for the tool, changing the test driver to take arguments on which
tests to run, separating the parameters to a different file.

Change-Id: I51f9d95d9979950214c75e572fccdda794b11ff3
Signed-off-by: Adnan Choudhury <[email protected]>

Author: adnan-c

test/tools/LTE/README.md

@@ -0,0 +1,102 @@
+# Ledger Traffic Engine
+
+This readme explains the working and the usage of Ledger Traffic Engine (LTE
+test tool.
+
+
+- What is the LTE Test Tool
+- How to Run the Tests
+- Where to View the Results
+
+## What is the LTE Test Tool
+
+The LTE test tool is test harness that engages the Ledger APIs and benchmarks
+the performance of the Ledger component. It contains the framework for creating
+and managing chains, for transaction submission, block creation, and block
+commit. It conducts benchmark tests for insert and read-write transactions
+(transactions per second) and contains scripts for launching the benchmarks. An
+insert benchmark followed by a readwrite benchmark on the same dataset is
+considered to constitute a single test-run.
+
+
+## How to Run The tests
+
+To run all the available tests with default parameters, run:
+```
+cd fabric/test/tools/LTE/scripts
+./runbenchmark.sh all
+```
+you can run individual tests without running all the available tests by giving
+the name of the test as parameter, instead of `all`. You can get the available
+test names by:
+```
+./runbenchmark.sh help
+```
+
+### What the Tests Do
+
+Each test reads test parameters from the file `parameters_daily_CI.sh` and
+conducts several test-runs by varying one or two of the parameters. The
+parameters are:
+* number of chains (ledger),
+* number of parallel transactions in each chain,
+* number of Key-value pairs,
+* number of transactions,
+* number of keys in each transaction,
+* size of batch for ledger,
+* size of Key-value
+
+For example, the *varyNumChains* test reads the parameters and varies the
+number of chains for each test-run while keeping the other parameters constant,
+and generate result. Varying a specific parameter for each test-runs gives
+insight into the parameter's influence on the Ledger performance.
+
+Each test-run consists of two phases: benchmarking of ledger insert phase and
+benchmarking of ledger read-write phase.
+
+#### Insert Phase
+
+The insert benchmark starts fresh chains and inserts the Key-values by
+simulating writes-only transactions For each of the chains. It launches the
+parallel clients (based on the configuration) and the clients simulate and
+commit write-only transactions. The keys are divided among clients such that
+one key is written only once and all the keys are inserted. For instance, if
+this benchmark is invoked with the following parameters:
+```
+Number of chains=2,
+Number of parallel transactions per chain=2,
+Number of key-value pairs=100
+```
+then client_1 on chain_1 will insert Key_1 to key_25 and client_2 on chain_1 will
+insert Key_26 to key_50. Similarly client_3 on chain_2 will insert Key_1 to
+key_25 and client_4 on chain_2 will insert Key_26 to key_50 where, client_1 and
+client_2 run in parallel on chain_1 and client_3 and client_4 run in parallel
+on chain_2.
+
+#### Read-write Phase
+
+Subsequently, the Read-write benchmark step opens the existing chains and
+modifies the Key-values by simulating read-write transactions For each of the
+chains. It launches the parallel clients (based on the configuration) and the
+clients simulate and commit read-write transactions. This test assumes the
+pre-populated chain by previously running insert benchmark step (described
+above). Each transaction simulated by this benchmark randomly selects a
+configurable number of keys and modifies their values. For instance, if this
+benchmark is invoked with the following test parameters:
+```
+Number of chains=2,
+Number of parallel transactions per chain=2,
+Number of key-value pairs=100,
+Number of total transactions=200
+```
+then client_1, and client_2 both execute 50 transactions on
+chain_1 in parallel.  Similarly, client_3, and client_4 both execute 50
+transactions on chain_2 in parallel In each of the transactions executed by any
+client, the transaction expects and modifies any key(s) between Key_1 to key_50
+(because, total keys are to be 100 across two chains).
+
+## How to View the Test Results
+
+The test results can be viewed as in the stdout where it shows how long each
+single operation took to complete in a test. These results are also saved in a
+.csv file in the following directory: `/tmp/fabric/test/tools/LTE/results`

test/tools/ledgerbenchmarks/chainmgmt/block_gen.go test/tools/LTE/chainmgmt/block_gen.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric/protos/common"
-	benchcommon "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common"
+	benchcommon "github.com/hyperledger/fabric/test/tools/LTE/common"
 )
 
 const (

test/tools/ledgerbenchmarks/chainmgmt/chains.go test/tools/LTE/chainmgmt/chains.go

@@ -24,7 +24,7 @@ import (
 	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/hyperledger/fabric/core/ledger/ledgermgmt"
 	"github.com/hyperledger/fabric/protos/common"
-	benchcommon "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common"
+	benchcommon "github.com/hyperledger/fabric/test/tools/LTE/common"
 )
 
 // ChainID is a type used for the ids for the chains for experiments

test/tools/ledgerbenchmarks/chainmgmt/conf.go test/tools/LTE/chainmgmt/conf.go

@@ -19,7 +19,7 @@ package experiments
 import (
 	"flag"
 
-	"github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt"
+	"github.com/hyperledger/fabric/test/tools/LTE/chainmgmt"
 )
 
 // txConf captures the transaction related configurations

test/tools/ledgerbenchmarks/chainmgmt/testenv.go test/tools/LTE/chainmgmt/testenv.go

@@ -22,8 +22,8 @@ import (
 
 	"fmt"
 
-	"github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt"
-	"github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common"
+	"github.com/hyperledger/fabric/test/tools/LTE/chainmgmt"
+	"github.com/hyperledger/fabric/test/tools/LTE/common"
 )
 
 // BenchmarkInsertTxs starts fresh chains and inserts the Key-values by simulating writes-only transactions

test/tools/ledgerbenchmarks/chainmgmt/tx_envelope_gen.go test/tools/LTE/chainmgmt/tx_envelope_gen.go

@@ -23,8 +23,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt"
-	"github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common"
+	"github.com/hyperledger/fabric/test/tools/LTE/chainmgmt"
+	"github.com/hyperledger/fabric/test/tools/LTE/common"
 )
 
 // BenchmarkReadWriteTxs opens the existing chains and modifies the Key-values by simulating read-write transactions

test/tools/ledgerbenchmarks/common/util.go test/tools/LTE/common/util.go

@@ -11,15 +11,15 @@ source ./common.sh
 # This shell script contains two functions that can be invoked to execute specific tests
 #
 # runInsertTxs - This function sets the environment variables and runs the benchmark function
-# 'BenchmarkInsertTxs' in package 'github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments'
+# 'BenchmarkInsertTxs' in package 'github.com/hyperledger/fabric/test/tools/LTE/experiments'
 #
 # runReadWriteTxs - This function sets the environment variables and runs the benchmark function
-# 'BenchmarkReadWriteTxs' in package 'github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments'
+# 'BenchmarkReadWriteTxs' in package 'github.com/hyperledger/fabric/test/tools/LTE/experiments'
 #
 # For the details of test specific parameters, refer to the documentation in 'go' files for the tests
 #######################################################################################################
 
-PKG_NAME="github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments"
+PKG_NAME="github.com/hyperledger/fabric/test/tools/LTE/experiments"
 
 function setCommonTestParams {
   TEST_PARAMS="-DataDir=$DataDir, -NumChains=$NumChains, -NumParallelTxPerChain=$NumParallelTxPerChain, -NumKeysInEachTx=$NumKeysInEachTx, -BatchSize=$BatchSize, -NumKVs=$NumKVs, -KVSize=$KVSize"

test/tools/ledgerbenchmarks/experiments/conf.go test/tools/LTE/experiments/conf.go

@@ -19,25 +19,26 @@ set -e
 # FUNCTION_NAME - Name of the Benchmark function
 # TEST_PARAMS - Parameters for the test
 # RESULTANT_DIRS - An optional list of dirs whose size needs to be captured in the results in the RESULTS_FILE
-# 
-# The default values for some of the above variables are set in this script and can be overrideen by a test specific
-# script. For remaining variables, a test specific script needs to set the appropriate values before calling the 
+#
+# The default values for some of the above variables are set in this script and can be overridden by a test specific
+# script. For remaining variables, a test specific script needs to set the appropriate values before calling the
 # function 'executeTest' of this script
-# 
+#
 # The result file for a test gets created in a csv format in the folder
 # $OUTPUT_DIR_ROOT/<last_element_of>$PKG_NAME<segment>/$FUNCTION_NAME
 #############################################################################################################################
 
 
-OUTPUT_DIR_ROOT=`echo /tmp/fabric/test/tools/ledgerbenchmarks/results`
+OUTPUT_DIR_ROOT=`echo /tmp/fabric/test/tools/LTE/results`
 RAW_OUTPUT_FILE="output.txt"
 RESULTS_FILE="results.csv"
 
 benchmarkLineRegex="^Benchmark.*[[:blank:]]+[[:digit:]]+[[:blank:]]+([[:digit:]]+).*$"
 testParamRegex="-\([^=]*\)=\([^,]*\)"
 
-ulimit -n 10000
-echo "ulimit=`ulimit -n`"
+echo "**Note**: This is a Benchmark test. Please make sure to set an appropriate value for ulimit in your OS."
+echo "Recommended value is 10000 for default parameters."
+echo "Current ulimit=`ulimit -n`"
 TESTS_SETUP_DONE=()
 
 ## Execute test and generate data file

test/tools/ledgerbenchmarks/experiments/init_test.go test/tools/LTE/experiments/init_test.go

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+DataDir="/tmp/fabric/test/tools/LTE/data"
+NumChains=10
+NumParallelTxPerChain=10
+NumKVs=1000000
+NumTotalTx=1000000
+NumKeysInEachTx=4
+BatchSize=50
+KVSize=200
+

test/tools/ledgerbenchmarks/experiments/insert_txs_test.go test/tools/LTE/experiments/insert_txs_test.go

@@ -6,64 +6,49 @@
 #
 
 source ./benchmarks.sh
+source ./parameters_daily_CI.sh
 
 ########################################################################################################
-# This shell script invokes a series of benchmark tests with desired values of test specific parameters
+# This shell script contains a series of benchmark tests
+# The test parameters are imported from "parameters_daily_CI.sh"
 ########################################################################################################
 
-function setDefaultTestParams {
-    DataDir="/tmp/fabric/test/tools/ledgerbenchmarks/data"
-    NumChains=10
-    NumParallelTxPerChain=10
-    NumKVs=1000000
-    NumTotalTx=1000000
-    NumKeysInEachTx=4
-    BatchSize=50
-    KVSize=200
-}
-
 function varyNumParallelTxPerChain {
-    setDefaultTestParams
     for v in 1 5 10 20 50 100 500 2000; do
         NumParallelTxPerChain=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumChains {
-    setDefaultTestParams
     for v in 1 5 10 20 50 100 500 2000; do
         NumChains=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumKeysInEachTx {
-    setDefaultTestParams
     for v in 1 2 5 10 20; do
         NumKeysInEachTx=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyKVSize {
-    setDefaultTestParams
     for v in 100 200 500 1000 2000; do
         KVSize=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyBatchSize {
-    setDefaultTestParams
     for v in 10 20 100 500; do
         BatchSize=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function varyNumParallelTxWithSingleChain {
-    setDefaultTestParams
     NumChains=1
     for v in 1 5 10 20 50 100 500 2000; do
         NumParallelTxPerChain=$v
@@ -72,7 +57,6 @@ function varyNumParallelTxWithSingleChain {
 }
 
 function varyNumChainsWithNoParallelism {
-    setDefaultTestParams
     NumParallelTxPerChain=1
     for v in 1 5 10 20 50 100 500 2000; do
         NumChains=$v
@@ -81,21 +65,41 @@ function varyNumChainsWithNoParallelism {
 }
 
 function varyNumTxs {
-    setDefaultTestParams
     for v in 1000000 2000000 5000000 10000000; do
         NumTotalTx=$v
         rm -rf $DataDir;runInsertTxs;runReadWriteTxs
     done
 }
 
 function runLargeDataExperiment {
-    setDefaultTestParams
     NumKVs=10000000
     NumTotalTx=10000000
     rm -rf $DataDir;runInsertTxs;runReadWriteTxs
 }
 
-### Run tests
+shift $(expr $OPTIND - 1 )
+
+case $1 in
+  varyNumParallelTxPerChain)
+    varyNumParallelTxPerChain ;;
+  varyNumChains)
+    varyNumChains ;;
+  varyNumParallelTxWithSingleChain)
+    varyNumParallelTxWithSingleChain ;;
+  varyNumChainsWithNoParallelism)
+    varyNumChainsWithNoParallelism ;;
+  varyNumKeysInEachTx)
+    varyNumKeysInEachTx ;;
+  varyKVSize)
+    varyKVSize ;;
+  varyBatchSize)
+    varyBatchSize ;;
+  varyNumTxs)
+    varyNumTxs ;;
+  runLargeDataExperiment)
+    runLargeDataExperiment ;;
+  help)
+    printf "Usage: ./runbenchmarks.sh [test_name]\nAvailable tests (use \"all\" to run all tests):
 varyNumParallelTxPerChain
 varyNumChains
 varyNumParallelTxWithSingleChain
@@ -104,4 +108,18 @@ varyNumKeysInEachTx
 varyKVSize
 varyBatchSize
 varyNumTxs
-runLargeDataExperiment
+runLargeDataExperiment\n"
+    ;;
+  all)
+    varyNumParallelTxPerChain
+    varyNumChains
+    varyNumParallelTxWithSingleChain
+    varyNumChainsWithNoParallelism
+    varyNumKeysInEachTx
+    varyKVSize
+    varyBatchSize
+    varyNumTxs
+    runLargeDataExperiment ;;
+  *)
+    printf "Error: test name empty/incorrect!\n"  >> /dev/stderr
+esac