[FAB-3131] Peer hangs when CouchDB unresponsive

The peer hangs when the CouchDB network connection is unresponsive.
This can be corrected by adding a timeout to the network client
connection used for connecting to CouchDB.  If the connection
times out, then the request to CouchDB will return an error and
the retry logic will take over.

A new entry is added for CouchDB in core.yaml named connectionTimeout

couchDBConfig:
   couchDBAddress: 127.0.0.1:5984
   username:
   password:
   # Number of retries for CouchDB errors
   maxRetries: 3
   # Number of retries for CouchDB errors during peer startup
   maxRetriesOnStartup: 10
   # CouchDB connection timeout (unit: duration, e.g. 60s)
   connectionTimeout: 60s

This is a duration property configurable by the peer.

Change-Id: I5028029e7f303144465c2bfeebf540c5d8e64d82
Signed-off-by: Chris Elder <[email protected]>

Author: Chris Elder

core/chaincode/chaincodetest.yaml

@@ -428,8 +428,13 @@ ledger:
        couchDBAddress: 127.0.0.1:5984
        username:
        password:
+       # Number of retries for CouchDB errors
        maxRetries: 3
+       # Number of retries for CouchDB errors during peer startup
        maxRetriesOnStartup: 10
+       # CouchDB request timeout (unit: duration, e.g. 20s)
+       requestTimeout: 20s
+
 
     # historyDatabase - options are true or false
     # Indicates if the history of key updates should be stored in goleveldb

core/chaincode/exectransaction_test.go

@@ -126,8 +126,9 @@ func finitPeer(lis net.Listener, chainIDs ...string) {
 		password := viper.GetString("ledger.state.couchDBConfig.password")
 		maxRetries := viper.GetInt("ledger.state.couchDBConfig.maxRetries")
 		maxRetriesOnStartup := viper.GetInt("ledger.state.couchDBConfig.maxRetriesOnStartup")
+		requestTimeout := viper.GetDuration("ledger.state.couchDBConfig.requestTimeout")
 
-		couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		db, _ := couchdb.CreateCouchDatabase(*couchInstance, chainID)
 		//drop the test database
 		db.DropDatabase()

core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb.go

@@ -56,7 +56,7 @@ func NewVersionedDBProvider() (*VersionedDBProvider, error) {
 	logger.Debugf("constructing CouchDB VersionedDBProvider")
 	couchDBDef := ledgerconfig.GetCouchDBDefinition()
 	couchInstance, err := couchdb.CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password,
-		couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup)
+		couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout)
 	if err != nil {
 		return nil, err
 	}

core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb_test_export.go

@@ -19,6 +19,7 @@ package statecouchdb
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb"
 	"github.com/hyperledger/fabric/core/ledger/util/couchdb"
@@ -31,6 +32,7 @@ var username = ""
 var password = ""
 var maxRetries = 3
 var maxRetriesOnStartup = 10
+var connectionTimeout = time.Second * 60
 
 // TestVDBEnv provides a couch db backed versioned db for testing
 type TestVDBEnv struct {
@@ -57,7 +59,7 @@ func (env *TestVDBEnv) Cleanup(dbName string) {
 }
 func cleanupDB(dbName string) {
 	//create a new connection
-	couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+	couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, connectionTimeout)
 	db := couchdb.CouchDatabase{CouchInstance: *couchInstance, DBName: dbName}
 	//drop the test database
 	db.DropDatabase()

core/ledger/ledgerconfig/ledger_config.go

@@ -18,6 +18,7 @@ package ledgerconfig
 
 import (
 	"path/filepath"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -29,6 +30,7 @@ type CouchDBDef struct {
 	Password            string
 	MaxRetries          int
 	MaxRetriesOnStartup int
+	RequestTimeout      time.Duration
 }
 
 //IsCouchDBEnabled exposes the useCouchDB variable
@@ -80,8 +82,9 @@ func GetCouchDBDefinition() *CouchDBDef {
 	password := viper.GetString("ledger.state.couchDBConfig.password")
 	maxRetries := viper.GetInt("ledger.state.couchDBConfig.maxRetries")
 	maxRetriesOnStartup := viper.GetInt("ledger.state.couchDBConfig.maxRetriesOnStartup")
+	requestTimeout := viper.GetDuration("ledger.state.couchDBConfig.requestTimeout")
 
-	return &CouchDBDef{couchDBAddress, username, password, maxRetries, maxRetriesOnStartup}
+	return &CouchDBDef{couchDBAddress, username, password, maxRetries, maxRetriesOnStartup, requestTimeout}
 }
 
 //GetQueryLimit exposes the queryLimit variable

core/ledger/ledgerconfig/ledger_config_test.go

@@ -18,6 +18,7 @@ package ledgerconfig
 
 import (
 	"testing"
+	"time"
 
 	"github.com/hyperledger/fabric/common/ledger/testutil"
 	ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
@@ -54,6 +55,9 @@ func TestGetCouchDBDefinition(t *testing.T) {
 	testutil.AssertEquals(t, couchDBDef.URL, "127.0.0.1:5984")
 	testutil.AssertEquals(t, couchDBDef.Username, "")
 	testutil.AssertEquals(t, couchDBDef.Password, "")
+	testutil.AssertEquals(t, couchDBDef.MaxRetries, 3)
+	testutil.AssertEquals(t, couchDBDef.MaxRetriesOnStartup, 10)
+	testutil.AssertEquals(t, couchDBDef.RequestTimeout, time.Second*20)
 }
 
 func TestIsHistoryDBEnabledDefault(t *testing.T) {

core/ledger/util/couchdb/couchdb.go

@@ -129,6 +129,7 @@ type CouchConnectionDef struct {
 	Password            string
 	MaxRetries          int
 	MaxRetriesOnStartup int
+	RequestTimeout      time.Duration
 }
 
 //CouchInstance represents a CouchDB instance
@@ -205,13 +206,11 @@ type Base64Attachment struct {
 }
 
 //CreateConnectionDefinition for a new client connection
-func CreateConnectionDefinition(couchDBAddress, username, password string, maxRetries, maxRetriesOnStartup int) (*CouchConnectionDef, error) {
+func CreateConnectionDefinition(couchDBAddress, username, password string, maxRetries,
+	maxRetriesOnStartup int, requestTimeout time.Duration) (*CouchConnectionDef, error) {
 
 	logger.Debugf("Entering CreateConnectionDefinition()")
 
-	//connectURL := fmt.Sprintf("%s//%s", "http:", couchDBAddress)
-	//connectURL := couchDBAddress
-
 	connectURL := &url.URL{
 		Host:   couchDBAddress,
 		Scheme: "http",
@@ -228,7 +227,9 @@ func CreateConnectionDefinition(couchDBAddress, username, password string, maxRe
 	logger.Debugf("Exiting CreateConnectionDefinition()")
 
 	//return an object containing the connection information
-	return &CouchConnectionDef{finalURL.String(), username, password, maxRetries, maxRetriesOnStartup}, nil
+	return &CouchConnectionDef{finalURL.String(), username, password, maxRetries,
+		maxRetriesOnStartup, requestTimeout}, nil
+
 }
 
 //CreateDatabaseIfNotExist method provides function to create database
@@ -1140,7 +1141,8 @@ func (dbclient *CouchDatabase) BatchUpdateDocuments(documents []*CouchDoc) ([]*B
 }
 
 //handleRequest method is a generic http request handler
-func (couchInstance *CouchInstance) handleRequest(method, connectURL string, data []byte, rev string, multipartBoundary string, maxRetries int) (*http.Response, *DBReturn, error) {
+func (couchInstance *CouchInstance) handleRequest(method, connectURL string, data []byte, rev string,
+	multipartBoundary string, maxRetries int) (*http.Response, *DBReturn, error) {
 
 	logger.Debugf("Entering handleRequest()  method=%s  url=%v", method, connectURL)
 
@@ -1152,6 +1154,9 @@ func (couchInstance *CouchInstance) handleRequest(method, connectURL string, dat
 	//set initial wait duration for retries
 	waitDuration := retryWaitTime * time.Millisecond
 
+	//get the connection timeout
+	requestTimeout := couchInstance.conf.RequestTimeout
+
 	//attempt the http request for the max number of retries
 	for attempts := 0; attempts < maxRetries; attempts++ {
 
@@ -1205,7 +1210,7 @@ func (couchInstance *CouchInstance) handleRequest(method, connectURL string, dat
 		}
 
 		//Create the http client
-		client := &http.Client{}
+		client := &http.Client{Timeout: requestTimeout}
 
 		transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
 		transport.DisableCompression = false

core/ledger/util/couchdb/couchdb_test.go

@@ -20,7 +20,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
+	"time"
 	"unicode/utf8"
 
 	"github.com/hyperledger/fabric/common/ledger/testutil"
@@ -36,13 +38,14 @@ var username = ""
 var password = ""
 var maxRetries = 3
 var maxRetriesOnStartup = 10
+var requestTimeout = time.Second * 20
 
 const updateDocumentConflictError = "conflict"
 const updateDocumentConflictReason = "Document update conflict."
 
 func cleanup(database string) error {
 	//create a new connection
-	couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+	couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 	if err != nil {
 		fmt.Println("Unexpected error", err)
 		return err
@@ -78,15 +81,15 @@ func TestDBConnectionDef(t *testing.T) {
 	ledgertestutil.SetupCoreYAMLConfig("./../../../../peer")
 
 	//create a new connection
-	_, err := CreateConnectionDefinition(connectURL, "", "", maxRetries, maxRetriesOnStartup)
+	_, err := CreateConnectionDefinition(connectURL, "", "", maxRetries, maxRetriesOnStartup, requestTimeout)
 	testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 }
 
 func TestDBBadConnectionDef(t *testing.T) {
 
 	//create a new connection
-	_, err := CreateConnectionDefinition("^^^localhost:5984", "", "", maxRetries, maxRetriesOnStartup)
+	_, err := CreateConnectionDefinition("^^^localhost:5984", "", "", maxRetries, maxRetriesOnStartup, requestTimeout)
 	testutil.AssertError(t, err, fmt.Sprintf("Did not receive error when trying to create database connection definition with a bad hostname"))
 
 }
@@ -102,7 +105,7 @@ func TestDBCreateSaveWithoutRevision(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -128,7 +131,7 @@ func TestDBCreateEnsureFullCommit(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -153,25 +156,25 @@ func TestDBBadDatabaseName(t *testing.T) {
 	if ledgerconfig.IsCouchDBEnabled() {
 
 		//create a new instance and database object using a valid database name mixed case
-		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		_, dberr := CreateCouchDatabase(*couchInstance, "testDB")
 		testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name"))
 
 		//create a new instance and database object using a valid database name letters and numbers
-		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		_, dberr = CreateCouchDatabase(*couchInstance, "test132")
 		testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name"))
 
 		//create a new instance and database object using a valid database name - special characters
-		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		_, dberr = CreateCouchDatabase(*couchInstance, "test1234~!@#$%^&*()[]{}.")
 		testutil.AssertNoError(t, dberr, fmt.Sprintf("Error when testing a valid database name"))
 
 		//create a new instance and database object using a invalid database name - too long	/*
-		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err = CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		_, dberr = CreateCouchDatabase(*couchInstance, "A12345678901234567890123456789012345678901234"+
 			"56789012345678901234567890123456789012345678901234567890123456789012345678901234567890"+
@@ -192,7 +195,7 @@ func TestDBBadConnection(t *testing.T) {
 	if ledgerconfig.IsCouchDBEnabled() {
 
 		//create a new instance and database object
-		_, err := CreateCouchInstance(badConnectURL, username, password, maxRetries, maxRetriesOnStartup)
+		_, err := CreateCouchInstance(badConnectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for a bad connection"))
 	}
 }
@@ -208,7 +211,7 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -286,6 +289,32 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) {
 
 }
 
+func TestDBRequestTimeout(t *testing.T) {
+
+	if ledgerconfig.IsCouchDBEnabled() {
+
+		database := "testdbrequesttimeout"
+		err := cleanup(database)
+		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to cleanup  Error: %s", err))
+		defer cleanup(database)
+
+		if err == nil {
+
+			//create an impossibly short timeout
+			impossibleTimeout := time.Microsecond * 1
+
+			//create a new instance and database object with a timeout that will fail
+			//Also use a maxRetriesOnStartup=3 to reduce the number of retries
+			_, err := CreateCouchInstance(connectURL, username, password, maxRetries, 3, impossibleTimeout)
+			testutil.AssertError(t, err, fmt.Sprintf("Error should have been thown while trying to create a couchdb instance with a connection timeout"))
+
+			//see if the error message contains the timeout error
+			testutil.AssertEquals(t, strings.Count(err.Error(), "Client.Timeout exceeded while awaiting headers"), 1)
+
+		}
+	}
+}
+
 func TestDBBadJSON(t *testing.T) {
 
 	if ledgerconfig.IsCouchDBEnabled() {
@@ -298,7 +327,7 @@ func TestDBBadJSON(t *testing.T) {
 		if err == nil {
 
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -334,7 +363,7 @@ func TestPrefixScan(t *testing.T) {
 
 	if err == nil {
 		//create a new instance and database object
-		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -406,7 +435,7 @@ func TestDBSaveAttachment(t *testing.T) {
 			attachments = append(attachments, attachment)
 
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -439,7 +468,7 @@ func TestDBDeleteDocument(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -477,7 +506,7 @@ func TestDBDeleteNonExistingDocument(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -616,7 +645,7 @@ func TestRichQuery(t *testing.T) {
 
 		if err == nil {
 			//create a new instance and database object   --------------------------------------------------------
-			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+			couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 			testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 			db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 
@@ -830,7 +859,7 @@ func TestBatchBatchOperations(t *testing.T) {
 		defer cleanup(database)
 
 		//create a new instance and database object   --------------------------------------------------------
-		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup)
+		couchInstance, err := CreateCouchInstance(connectURL, username, password, maxRetries, maxRetriesOnStartup, requestTimeout)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
 		db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
 

core/ledger/util/couchdb/couchdbutil.go

@@ -21,15 +21,18 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var validNamePattern = `^[a-z][a-z0-9_$(),+/-]+`
 var maxLength = 249
 
 //CreateCouchInstance creates a CouchDB instance
-func CreateCouchInstance(couchDBConnectURL, id, pw string, maxRetries, maxRetriesOnStartup int) (*CouchInstance, error) {
+func CreateCouchInstance(couchDBConnectURL, id, pw string, maxRetries,
+	maxRetriesOnStartup int, connectionTimeout time.Duration) (*CouchInstance, error) {
+
 	couchConf, err := CreateConnectionDefinition(couchDBConnectURL,
-		id, pw, maxRetries, maxRetriesOnStartup)
+		id, pw, maxRetries, maxRetriesOnStartup, connectionTimeout)
 	if err != nil {
 		logger.Errorf("Error during CouchDB CreateConnectionDefinition(): %s\n", err.Error())
 		return nil, err