[FAB-3013] Simple load test driver for Fabric CA server

This simple load test driver provides ability to drive
load against Fabric CA server. It can be customized using
a configuration file. Things like number of clients, number
of requests per client, test sequence each client executes
in each iteration, and etc., can be configured. For more info
please see README.md file.

Change-Id: I5080be5f596dc709e143ace09c153aa1ff8701a3
Signed-off-by: Anil Ambati <[email protected]>

Makefile

@@ -22,6 +22,11 @@
 #   - unit-tests - Performs checks first and runs the go-test based unit tests
 #   - checks - runs all check conditions (license, format, imports, lint and vet)
 #   - docker[-clean] - ensures all docker images are available[/cleaned]
+#   - bench - Runs benchmarks in all the packages and stores the results in /tmp/bench.results
+#   - bench-cpu - Runs the benchmarks in the specified package with cpu profiling
+#   - bench-mem - Runs the benchmarks in the specified package with memory profiling
+#   - bench-clean - Removes all benchmark related files
+#   - benchcmp - Compares benchmarks results of current and previous release
 #   - clean - cleans the build area
 
 PROJECT_NAME   = fabric-ca
@@ -152,8 +157,37 @@ build/%.tar.bz2:
 unit-tests: checks fabric-ca-server fabric-ca-client
 	@scripts/run_tests
 
+# Runs benchmarks in all the packages and stores the benchmarks in /tmp/bench.results
+bench: checks fabric-ca-server fabric-ca-client
+	@scripts/run_benchmarks
+
+# Runs benchmarks in the specified package with cpu profiling
+# e.g. make bench-cpu pkg=github.com/hyperledger/fabric-ca/lib
+bench-cpu: checks fabric-ca-server fabric-ca-client
+	@scripts/run_benchmarks -C -P $(pkg)
+
+# Runs benchmarks in the specified package with memory profiling
+# e.g. make bench-mem pkg=github.com/hyperledger/fabric-ca/lib
+bench-mem: checks fabric-ca-server fabric-ca-client
+	@scripts/run_benchmarks -M -P $(pkg)
+
+# Removes all benchmark related files (bench, bench-cpu, bench-mem and *.test)
+bench-clean:
+	@scripts/run_benchmarks -R
+
+# Compares benchmarks results of current and previous release
+# Previous release git tag must be specified using the prev_rel variable.
+# e.g. make benchcmp -prev_rel=v1.0.0
+benchcmp: bench
+	@scripts/compare_benchmarks $(prev_rel)
+
 container-tests: docker
 
+load-test: docker-clean docker-fvt
+	@docker run -p 8888:8888 -p 8054:8054 -v $(shell pwd):/opt/gopath/src/github.com/hyperledger/fabric-ca -e FABRIC_CA_SERVER_PROFILE_PORT=8054 --name loadTest -td hyperledger/fabric-ca-fvt test/fabric-ca-load-tester/launchServer.sh 3
+	@test/fabric-ca-load-tester/runLoad.sh -B
+	@docker kill loadTest
+
 fvt-tests:
 	@scripts/run_fvt_tests
 

scripts/run_tests

@@ -13,7 +13,7 @@ export PATH=$PATH:$GOPATH/bin
 go get github.com/axw/gocov/...
 go get github.com/AlekSi/gocov-xml
 
-PKGS=`go list github.com/hyperledger/fabric-ca/... | grep -Ev '/vendor/|/api|/dbutil|/ldap|/mocks'`
+PKGS=`go list github.com/hyperledger/fabric-ca/... | grep -Ev '/vendor/|/api|/dbutil|/ldap|/mocks|/test/fabric-ca-load-tester'`
 
 gocov test $PKGS | gocov-xml > coverage.xml
 

test/fabric-ca-load-tester/README.md

@@ -0,0 +1,13 @@
+# Simple load driver for Fabric CA
+This is a simple load driver for Fabric CA. The driver can be configured using a JSON configuration file. Things like URL of the Fabric CA server, number of clients, number of requests per client, requests per second, test sequence, Fabric CA Client config, etc can be specified in the configuration file. You can look at the default configuraton file **testConfig.yml** located in this directory.
+
+## Steps
+1. Set `registry.maxEnrollments` to at least 2 in the server configuration file
+1. Make sure Fabric CA server is running and make a note of the server URL, bootstrap user and password.
+2. Modify the **testConfig.yml** file
+    * Modify the `serverURL` property. It is of the form: `<http|https>://<bootstrap user id>:<bootstrap password>@<hostname>:<port>`. Note that the bootstrap user must have revoker authority and must be affiliated with root of the affiliation tree, which is **""** or the parent affiliation of the affiliation specified in the `affiliation` property
+    * Change load properties like `numClients`, `numReqsPerClient`, `testSeq` properties as needed. `testSeq` property specifies the sequence of tests that are run in each iteration by a client. Each test has a `name` and optional `repeat` and `req` properties. The `repeat` property specifies how many times to repeat the test in each iteration. The `req` property specifies payload for the request that is sent to the Fabric CA server.
+        * For revoke test, specify a random string for the `name` property if you need to revoke an identity. If the `name` property is empty, an ECert associated with the identity will be revoked.
+    * Change `affiliation` property. It specifies the affiliation to use in the test.
+    * If you need TLS to be used to connect to the Fabric CA server, first make sure **https** protocol is used in the `serverURL` property. Next, set `tls.enabled` to true. Then, specify root CA certificate files in the `tls.certfiles` property.
+3. Run **runLoad.sh** script to start the load test. You can invoke this script with the `-B` option to build the driver and run.

test/fabric-ca-load-tester/launchServer.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+numInstances=1
+if test -n "$1" ; then
+  numInstances=$1
+fi
+pushd scripts/fvt
+./fabric-ca_setup.sh -D -X -I -S -n$numInstances -m10 -d postgres -T
+popd
+# Docker requires your command to keep running in the foreground. Otherwise, it thinks 
+# that command has stopped and shutsdown the container. Since fabric-ca_setup.sh starts
+# fabric ca server in background and exits, we want this script to run in foreground and
+# not return so the container in daemon mode continues to run for ever until it is stopped
+tail -f /dev/null

test/fabric-ca-load-tester/main.go

@@ -0,0 +1,224 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"crypto/rand"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/hyperledger/fabric-ca/api"
+	"github.com/hyperledger/fabric-ca/lib"
+)
+
+// IdentityType represents type of identity in the fabric
+type IdentityType string
+
+const (
+	// User user identity type
+	User IdentityType = "User"
+	// Peer peer identity type
+	Peer = "Peer"
+	// Validator validator identity type
+	Validator = "Validator"
+)
+
+// TestRequest represents request properties from
+// which a request payload object can be constructed
+type TestRequest map[string]interface{}
+
+// Test represents a fabric-ca test
+type Test struct {
+	Name   string      `yaml:"name"`
+	Repeat int         `yaml:"repeat,omitempty"`
+	Req    TestRequest `yaml:"req,omitempty"`
+}
+
+type testConfig struct {
+	ServerURL      string           `yaml:"serverURL"`
+	ConfigHome     string           `yaml:"caConfigHome"`
+	NumUsers       int              `yaml:"numClients"`
+	NumReqsPerUser int              `yaml:"numReqsPerClient"`
+	TestSeq        []Test           `yaml:"testSeq"`
+	Affiliation    string           `yaml:"affiliation"`
+	CAClientConfig lib.ClientConfig `yaml:"caClientConfig"`
+}
+
+// enrollmentID encapsulates an identity's name and type
+type enrollmentID struct {
+	ID *string
+	it *IdentityType
+}
+
+var (
+	testCfg     testConfig
+	testCfgFile *string
+)
+
+func main() {
+	t0 := time.Now()
+	testCfgFile = flag.String("config", "testConfig.yml", "Fully qualified name of the test configuration file")
+	flag.Parse()
+
+	// Create CA client config
+	err := readConfig()
+	if err != nil {
+		log.Printf("Failed to create client config: %v", err)
+		return
+	}
+
+	// Enroll boostrap user
+	bootID, err1 := enrollBootstrapUser(&testCfg.ServerURL,
+		&testCfg.ConfigHome, &testCfg.CAClientConfig)
+	if err1 != nil {
+		log.Printf("Failed to enroll bootstrap user: %v", err1)
+		return
+	}
+
+	fin := make(chan testClientRes, testCfg.NumUsers)
+
+	var wg sync.WaitGroup
+	for i := 0; i < testCfg.NumUsers; i++ {
+		c := getTestClient(i, &testCfg.ConfigHome, &testCfg.CAClientConfig, bootID)
+		if err != nil {
+			log.Printf("Failed to get client: %v", err)
+			continue
+		}
+		// Increment the WaitGroup counter
+		wg.Add(1)
+		go func() {
+			// Decrement the counter when the goroutine completes
+			defer wg.Done()
+			c.runTests(fin)
+		}()
+	}
+	wg.Wait()
+	t1 := time.Now()
+	log.Printf("Load test finished in %v seconds\n", t1.Sub(t0).Seconds())
+	for i := 0; i < testCfg.NumUsers; i++ {
+		log.Println(<-fin)
+	}
+}
+
+// Enrolls bootstrap user and sets the cfg global object
+func enrollBootstrapUser(surl *string, configHome *string,
+	cfg *lib.ClientConfig) (id *lib.Identity, err error) {
+	var resp *lib.EnrollmentResponse
+	resp, err = cfg.Enroll(*surl, *configHome)
+	if err != nil {
+		log.Printf("Enrollment of boostrap user failed: %v", err)
+		return id, err
+	}
+	log.Printf("Successfully enrolled boostrap user")
+
+	id = resp.Identity
+	cfg.ID.Name = id.GetName()
+	return id, err
+}
+
+// Reads test config
+func readConfig() error {
+	tcFile, e := ioutil.ReadFile(*testCfgFile)
+	if e != nil {
+		log.Printf("Failed to read configuration file '%s': %v", *testCfgFile, e)
+		os.Exit(1)
+	}
+	yaml.Unmarshal(tcFile, &testCfg)
+
+	uo, err := url.Parse(testCfg.ServerURL)
+	if err != nil {
+		return err
+	}
+	u := fmt.Sprintf("%s://%s", uo.Scheme, uo.Host)
+	testCfg.CAClientConfig.URL = u
+
+	// Make config home absolute
+	if !filepath.IsAbs(testCfg.ConfigHome) {
+		testCfg.ConfigHome, err = filepath.Abs(testCfg.ConfigHome)
+		if err != nil {
+			log.Printf("Failed to get full path of config file: %s", err)
+		}
+	}
+
+	log.Printf("Config created: %+v", testCfg)
+	return nil
+}
+
+// Returns a random affiliation
+func getAffiliation() string {
+	return testCfg.Affiliation
+}
+
+// Returns a random enrollment ID
+func genEnrollmentID(it IdentityType) (eid *enrollmentID, err error) {
+	b := make([]byte, 16)
+	_, err = rand.Read(b)
+	if err != nil {
+		return
+	}
+	uuid := fmt.Sprintf("%s-%X-%X-%X-%X-%X", it, b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	eid = &enrollmentID{
+		ID: &uuid,
+		it: &it,
+	}
+	return
+}
+
+// Returns identity type based on the value of i
+func getIdentityType(i int) IdentityType {
+	tipe := i % 3
+	switch tipe {
+	case 0:
+		return User
+	case 1:
+		return Peer
+	case 2:
+		return Validator
+	default:
+		return User
+	}
+}
+
+func (tr TestRequest) getTCertsReq() *api.GetTCertBatchRequest {
+	count := tr["count"].(int)
+	prekey := tr["prekey"].(string)
+	disableKdf := tr["disable_kdf"].(bool)
+	encryptAttrs := tr["encrypt_attrs"].(bool)
+	return &api.GetTCertBatchRequest{
+		Count:                count,
+		PreKey:               prekey,
+		DisableKeyDerivation: disableKdf,
+		EncryptAttrs:         encryptAttrs,
+	}
+}
+
+func (tr TestRequest) getEnrollmentReq() *api.EnrollmentRequest {
+	name := tr["name"].(string)
+	pass := tr["pass"].(string)
+	return &api.EnrollmentRequest{
+		Name:   name,
+		Secret: pass,
+	}
+}

test/fabric-ca-load-tester/runLoad.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script is used to run the load driver that drives load against a
+# Fabric CA server or cluster of servers. The Fabric CA server URL and
+# load characteristics can be defined in the testConfig.yml file, which
+# must be located in the current working directory.
+#
+# When run with -B option, it will build the load driver and then runs it.
+
+pushd $GOPATH/src/github.com/hyperledger/fabric-ca/test/fabric-ca-load-tester
+if [ "$1" == "-B" ]; then
+  echo "Building fabric-ca-load-tester..."
+  if [ "$(uname)" == "Darwin" ]; then
+    # On MacOS Sierra use -ldflags -s flags to work around "Killed: 9" error
+    go build -o fabric-ca-load-tester -ldflags -s main.go testClient.go
+  else
+    go build -o fabric-ca-load-tester main.go testClient.go
+  fi
+fi
+echo "Running load"
+./fabric-ca-load-tester -config testConfig.yml
+rm -rf msp
+rm -rf fabric-ca-load-tester
+popd