FAB-1505 and FAB-1337 Refactor CouchDB code

FAB-1505 refactors CouchDB code to be aligned with LevelDB structure.
FAB-1337 applies all ledger tests written for LevelDB against CouchDB.

In order to test the CouchDB refactor,
had to implement FAB-1337 in same changeset to test.

Like most refactors, this changeset touches a lot of files and code,
since it is not feasible to do a partial refactor.

Tests related to versioning and deleting have been commented out,
these will be re-enabled in subsequent changeset when CouchDB
has the support, in order to keep this changeset size more reasonable.

Change-Id: I0d2d6cca89bd0252f6e84317457a9140b3a0d7a5
Signed-off-by: denyeart <[email protected]>

Author: denyeart

core/ledger/history/couchdb_histmgr.go

@@ -46,15 +46,16 @@ var compositeKeySep = []byte{0x00}
 // CouchDBHistMgr a simple implementation of interface `histmgmt.HistMgr'.
 // TODO This implementation does not currently use a lock but may need one to ensure query's are consistent
 type CouchDBHistMgr struct {
-	couchDB *couchdb.CouchDBConnectionDef // COUCHDB new properties for CouchDB
+	couchDB *couchdb.CouchDatabase // COUCHDB new properties for CouchDB
 }
 
 // NewCouchDBHistMgr constructs a new `CouchDB HistMgr`
 func NewCouchDBHistMgr(couchDBConnectURL string, dbName string, id string, pw string) *CouchDBHistMgr {
 
 	//TODO locking has not been implemented but may need some sort of locking to insure queries are valid data.
 
-	couchDB, err := couchdb.CreateCouchDBConnectionAndDB(couchDBConnectURL, dbName, id, pw)
+	couchInstance, err := couchdb.CreateCouchInstance(couchDBConnectURL, id, pw)
+	couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, dbName)
 	if err != nil {
 		logger.Errorf("===HISTORYDB=== Error during NewCouchDBHistMgr(): %s\n", err.Error())
 		return nil

core/ledger/history/pkg_test.go

@@ -46,7 +46,8 @@ func newTestEnvHistoryCouchDB(t testing.TB, dbName string) *testEnvHistoryCouchD
 func (env *testEnvHistoryCouchDB) cleanup() {
 
 	//create a new connection
-	couchDB, err := couchdb.CreateConnectionDefinition(env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
+	couchInstance, err := couchdb.CreateCouchInstance(env.couchDBAddress, env.couchUsername, env.couchPassword)
+	couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, env.couchDatabaseName)
 	if err == nil {
 		//drop the test database if it already existed
 		couchDB.DropDatabase()

core/ledger/kvledger/example/main/example.go

@@ -27,8 +27,11 @@ import (
 	"github.com/hyperledger/fabric/core/ledger/testutil"
 	"github.com/hyperledger/fabric/core/ledger/util"
 	"github.com/hyperledger/fabric/protos/common"
+	logging "github.com/op/go-logging"
 )
 
+var logger = logging.MustGetLogger("main")
+
 const (
 	ledgerID = "Default"
 )
@@ -91,6 +94,7 @@ func main() {
 }
 
 func initApp() {
+	logger.Debug("Entering initApp()")
 	tx, err := app.Init(map[string]int{
 		accounts[0]: 100,
 		accounts[1]: 100,
@@ -101,9 +105,11 @@ func initApp() {
 	err = committer.CommitBlock(rawBlock)
 	handleError(err, true)
 	printBlocksInfo(rawBlock)
+	logger.Debug("Exiting initApp()")
 }
 
 func transferFunds() {
+	logger.Debug("Entering transferFunds()")
 	tx1, err := app.TransferFunds("account1", "account2", 50)
 	handleError(err, true)
 	tx2, err := app.TransferFunds("account3", "account4", 50)
@@ -116,14 +122,18 @@ func transferFunds() {
 	err = committer.CommitBlock(rawBlock)
 	handleError(err, true)
 	printBlocksInfo(rawBlock)
+	logger.Debug("Exiting transferFunds")
 }
 
 func tryInvalidTransfer() {
+	logger.Debug("Entering tryInvalidTransfer()")
 	_, err := app.TransferFunds("account1", "account2", 60)
 	handleError(err, false)
+	logger.Debug("Exiting tryInvalidTransfer()")
 }
 
 func tryDoubleSpend() {
+	logger.Debug("Entering tryDoubleSpend()")
 	tx1, err := app.TransferFunds("account1", "account2", 50)
 	handleError(err, true)
 	tx2, err := app.TransferFunds("account1", "account4", 50)
@@ -132,9 +142,11 @@ func tryDoubleSpend() {
 	err = committer.CommitBlock(rawBlock)
 	handleError(err, true)
 	printBlocksInfo(rawBlock)
+	logger.Debug("Exiting tryDoubleSpend()")
 }
 
 func printBlocksInfo(block *common.Block) {
+	logger.Debug("Entering printBlocksInfo()")
 	// Read invalid transactions filter
 	txsFltr := util.NewFilterBitArrayFromBytes(block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER])
 	numOfInvalid := 0
@@ -146,14 +158,17 @@ func printBlocksInfo(block *common.Block) {
 	}
 	fmt.Printf("Num txs in rawBlock = [%d], num invalidTxs = [%d]\n",
 		len(block.Data.Data), numOfInvalid)
+	logger.Debug("Exiting printBlocksInfo()")
 }
 
 func printBalances() {
+	logger.Debug("Entering printBalances()")
 	balances, err := app.QueryBalances(accounts)
 	handleError(err, true)
 	for i := 0; i < len(accounts); i++ {
 		fmt.Printf("[%s] = [%d]\n", accounts[i], balances[i])
 	}
+	logger.Debug("Exiting printBalances()")
 }
 
 func handleError(err error, quit bool) {

core/ledger/kvledger/example/marble_app.go

@@ -57,7 +57,7 @@ type Marble struct {
 func (marbleApp *MarbleApp) CreateMarble(args []string) (*common.Envelope, error) {
 	//   0       1       2     3
 	// "asdf", "blue", "35", "bob"
-	logger.Debugf("===COUCHDB=== Entering ----------CreateMarble()----------")
+	logger.Debugf("Entering ----------CreateMarble()----------")
 	marbleName := args[0]
 	marbleJsonBytes, err := init_marble(args)
 	if err != nil {
@@ -76,9 +76,9 @@ func (marbleApp *MarbleApp) CreateMarble(args []string) (*common.Envelope, error
 	if txSimulationResults, err = txSimulator.GetTxSimulationResults(); err != nil {
 		return nil, err
 	}
-	logger.Debugf("===COUCHDB=== CreateMarble() simulation done, packaging into a transaction...")
+	logger.Debugf("CreateMarble() simulation done, packaging into a transaction...")
 	tx := constructTransaction(txSimulationResults)
-	logger.Debugf("===COUCHDB=== Exiting CreateMarble()")
+	logger.Debugf("Exiting CreateMarble()")
 	return tx, nil
 }
 
@@ -94,7 +94,7 @@ func init_marble(args []string) ([]byte, error) {
 		return nil, errors.New("Incorrect number of arguments. Expecting 4")
 	}
 
-	logger.Debugf("===COUCHDB=== Entering init marble")
+	logger.Debugf("Entering init marble")
 	if len(args[0]) <= 0 {
 		return nil, errors.New("1st argument must be a non-empty string")
 	}
@@ -120,7 +120,7 @@ func init_marble(args []string) ([]byte, error) {
 	marbleJson := `{"txid": "` + tx + `",  "asset_name": "` + args[0] + `", "color": "` + color + `", "size": ` + strconv.Itoa(size) + `, "owner": "` + user + `"}`
 	marbleBytes := []byte(marbleJson)
 
-	logger.Debugf("===COUCHDB=== Exiting init marble")
+	logger.Debugf("Exiting init marble")
 	return marbleBytes, nil
 }
 
@@ -134,7 +134,7 @@ func (marbleApp *MarbleApp) TransferMarble(args []string) (*common.Envelope, err
 	marbleName := args[0]
 	marbleNewOwner := args[1]
 
-	logger.Debugf("===COUCHDB=== Entering ----------TransferMarble----------")
+	logger.Debugf("Entering ----------TransferMarble----------")
 	var txSimulator ledger.TxSimulator
 	var err error
 	if txSimulator, err = marbleApp.ledger.NewTxSimulator(); err != nil {
@@ -143,25 +143,25 @@ func (marbleApp *MarbleApp) TransferMarble(args []string) (*common.Envelope, err
 	defer txSimulator.Done()
 
 	marbleBytes, err := txSimulator.GetState(marbleApp.name, marbleName)
-	logger.Debugf("===COUCHDB=== marbleBytes is: %v", marbleBytes)
+	logger.Debugf("marbleBytes is: %v", marbleBytes)
 	if marbleBytes != nil {
 		jsonString := string(marbleBytes[:])
-		logger.Debugf("===COUCHDB=== TransferMarble() Retrieved jsonString: \n   %s", jsonString)
+		logger.Debugf("TransferMarble() Retrieved jsonString: \n   %s", jsonString)
 	}
 
 	theMarble := Marble{}
 	json.Unmarshal(marbleBytes, &theMarble) //Unmarshal JSON bytes into a Marble struct
 
-	logger.Debugf("===COUCHDB===  theMarble after unmarshal: %v", theMarble)
+	logger.Debugf(" theMarble after unmarshal: %v", theMarble)
 
-	logger.Debugf("===COUCHDB===  Setting the owner to: %s", marbleNewOwner)
+	logger.Debugf(" Setting the owner to: %s", marbleNewOwner)
 	theMarble.User = marbleNewOwner      //change the user
 	theMarble.Txid = "tx000000000000002" // COUCHDB hardcode a txid for now for demo purpose
 
 	updatedMarbleBytes, _ := json.Marshal(theMarble)
 	if updatedMarbleBytes != nil {
 		updatedJsonString := string(updatedMarbleBytes[:])
-		logger.Debugf("===COUCHDB=== updatedJsonString:\n   %s", updatedJsonString)
+		logger.Debugf("updatedJsonString:\n   %s", updatedJsonString)
 	}
 	err = txSimulator.SetState(marbleApp.name, marbleName, updatedMarbleBytes)
 	if err != nil {
@@ -172,7 +172,7 @@ func (marbleApp *MarbleApp) TransferMarble(args []string) (*common.Envelope, err
 	if txSimulationResults, err = txSimulator.GetTxSimulationResults(); err != nil {
 		return nil, err
 	}
-	logger.Debugf("===COUCHDB=== TransferMarble() simulation done, packaging into a transaction...")
+	logger.Debugf("TransferMarble() simulation done, packaging into a transaction...")
 	tx := constructTransaction(txSimulationResults)
 	return tx, nil
 }

core/ledger/kvledger/kv_ledger.go

@@ -24,7 +24,6 @@ import (
 	"github.com/hyperledger/fabric/core/ledger/blkstorage"
 	"github.com/hyperledger/fabric/core/ledger/blkstorage/fsblkstorage"
 	"github.com/hyperledger/fabric/core/ledger/history"
-	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/couchdbtxmgmt"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/txmgr"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/txmgr/lockbasedtxmgr"
@@ -66,23 +65,11 @@ func NewKVLedger(versionedDBProvider statedb.VersionedDBProvider, ledgerID strin
 	var txmgmt txmgr.TxMgr
 	var historymgmt history.HistMgr
 
-	if ledgerconfig.IsCouchDBEnabled() == true {
-		//By default we can talk to CouchDB with empty id and pw (""), or you can add your own id and password to talk to a secured CouchDB
-		logger.Debugf("===COUCHDB=== NewKVLedger() Using CouchDB instead of RocksDB...hardcoding and passing connection config for now")
-
-		couchDBDef := ledgerconfig.GetCouchDBDefinition()
-
-		//create new transaction manager based on couchDB
-		txmgmt = couchdbtxmgmt.NewCouchDBTxMgr(&couchdbtxmgmt.Conf{DBPath: ""},
-			couchDBDef.URL,      //couchDB connection URL
-			"system",            //couchDB db name matches ledger name, TODO for now use system ledger, eventually allow passing in subledger name
-			couchDBDef.Username, //enter couchDB id here
-			couchDBDef.Password) //enter couchDB pw here
-	} else {
-		// Fall back to using goleveldb lockbased transaction manager
-		db := versionedDBProvider.GetDBHandle(ledgerID)
-		txmgmt = lockbasedtxmgr.NewLockBasedTxMgr(db)
+	db, err := versionedDBProvider.GetDBHandle(ledgerID)
+	if err != nil {
+		return nil, err
 	}
+	txmgmt = lockbasedtxmgr.NewLockBasedTxMgr(db)
 
 	if ledgerconfig.IsHistoryDBEnabled() == true {
 		logger.Debugf("===HISTORYDB=== NewKVLedger() Using CouchDB for transaction history database")

core/ledger/kvledger/kv_ledger_provider.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb"
+	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/statecouchdb"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/stateleveldb"
 	"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
 	"github.com/hyperledger/fabric/core/ledger/util/db"
@@ -50,7 +51,12 @@ func NewProvider() (ledger.ValidatedLedgerProvider, error) {
 		logger.Debugf("Constructing leveldb VersionedDBProvider")
 		vdbProvider = stateleveldb.NewVersionedDBProvider()
 	} else {
-		//TODO same for couchDB after refactoring of couchdb code
+		logger.Debugf("Constructing CouchDB VersionedDBProvider")
+		var err error
+		vdbProvider, err = statecouchdb.NewVersionedDBProvider()
+		if err != nil {
+			return nil, err
+		}
 	}
 	ledgerMgmtPath := ledgerconfig.GetLedgerProviderPath()
 	idStore := openIDStore(ledgerMgmtPath)

core/ledger/kvledger/marble_example/main/marble_example.go

@@ -47,7 +47,7 @@ func init() {
 	// Initialization will get a handle to the ledger at the specified path
 	// Note, if subledgers are supported in the future,
 	// the various ledgers could be created/managed at this level
-	logger.Debugf("===COUCHDB=== Marble Example main init()")
+	logger.Debugf("Marble Example main init()")
 
 	//call a helper method to load the core.yaml
 	testutil.SetupCoreYAMLConfig("./../../../../../peer")
@@ -77,7 +77,7 @@ func main() {
 }
 
 func initApp() {
-	logger.Debugf("===COUCHDB=== Marble Example initApp() to create a marble")
+	logger.Debugf("Marble Example initApp() to create a marble")
 	marble := []string{"marble1", "blue", "35", "tom"}
 	tx, err := marbleApp.CreateMarble(marble)
 	handleError(err, true)
@@ -88,7 +88,7 @@ func initApp() {
 }
 
 func transferMarble() {
-	logger.Debugf("===COUCHDB=== Marble Example transferMarble()")
+	logger.Debugf("Marble Example transferMarble()")
 	tx1, err := marbleApp.TransferMarble([]string{"marble1", "jerry"})
 	handleError(err, true)
 	rawBlock := consenter.ConstructBlock(tx1)