FAB-1585 Refactor History to enable LevelDB

The purpose of this refactor is to create a common interface and to
isolate the db implementation, so that we can enable history on
LevelDB. This change only refactors the existing code and tests.
It does not add new functionality or any new tests. There will be
future changes that add history support on LevelDB and clean up
tests to work against LevelDB.

Note: History is not enabled by default. History must be enabled in
core.yaml historyDatabase for it's unit tests to run.

Change-Id: Ia5293cd345be4499e5e3df0c2949af16f71a608d
Signed-off-by: denyeart <[email protected]>

Author: denyeart

core/ledger/history/couchdb_histmgr.go core/ledger/kvledger/history/histmgr/couchdbhistmgr/couchdb_histmgr.go

@@ -14,15 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package history
+package couchdbhistmgr
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"strconv"
 
 	"github.com/hyperledger/fabric/core/ledger"
+	helper "github.com/hyperledger/fabric/core/ledger/kvledger/history/histmgr"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/rwset"
 	"github.com/hyperledger/fabric/core/ledger/util/couchdb"
 	"github.com/hyperledger/fabric/protos/common"
@@ -39,11 +38,9 @@ type couchSavepointData struct {
 	UpdateSeq string `json:"UpdateSeq"`
 }
 
-var logger = logging.MustGetLogger("history")
+var logger = logging.MustGetLogger("couchdbhistmgr")
 
-var compositeKeySep = []byte{0x00}
-
-// CouchDBHistMgr a simple implementation of interface `histmgmt.HistMgr'.
+// CouchDBHistMgr a simple implementation of interface `histmgr.HistMgr'.
 // TODO This implementation does not currently use a lock but may need one to ensure query's are consistent
 type CouchDBHistMgr struct {
 	couchDB *couchdb.CouchDatabase // COUCHDB new properties for CouchDB
@@ -64,12 +61,12 @@ func NewCouchDBHistMgr(couchDBConnectURL string, dbName string, id string, pw st
 	return &CouchDBHistMgr{couchDB: couchDB}
 }
 
-// NewHistoryQueryExecutor implements method in interface `histmgmt.HistMgr'.
+// NewHistoryQueryExecutor implements method in interface `histr.HistMgr'.
 func (histmgr *CouchDBHistMgr) NewHistoryQueryExecutor() (ledger.HistoryQueryExecutor, error) {
 	return &CouchDBHistQueryExecutor{histmgr}, nil
 }
 
-// Commit implements method in interface `histmgmt.HistMgr`
+// Commit implements method in interface `histmgr.HistMgr`
 // This writes to a separate history database.
 // TODO dpending on how invalid transactions are handled may need to filter what history commits.
 func (histmgr *CouchDBHistMgr) Commit(block *common.Block) error {
@@ -113,7 +110,7 @@ func (histmgr *CouchDBHistMgr) Commit(block *common.Block) error {
 			for _, kvWrite := range nsRWSet.Writes {
 				writeKey := kvWrite.Key
 				writeValue := kvWrite.Value
-				compositeKey := constructCompositeKey(ns, writeKey, blockNo, tranNo)
+				compositeKey := helper.ConstructCompositeKey(ns, writeKey, blockNo, tranNo)
 				var bytesDoc []byte
 
 				logger.Debugf("===HISTORYDB=== ns (namespace or cc id) = %v, writeKey: %v, compositeKey: %v, writeValue = %v",
@@ -195,8 +192,8 @@ func (histmgr *CouchDBHistMgr) getTransactionsForNsKey(namespace string, key str
 	var compositeStartKey []byte
 	var compositeEndKey []byte
 	if key != "" {
-		compositeStartKey = constructPartialCompositeKey(namespace, key, false)
-		compositeEndKey = constructPartialCompositeKey(namespace, key, true)
+		compositeStartKey = helper.ConstructPartialCompositeKey(namespace, key, false)
+		compositeEndKey = helper.ConstructPartialCompositeKey(namespace, key, true)
 	}
 
 	//TODO the limit should not be hardcoded.  Need the config.
@@ -227,37 +224,6 @@ func (txmgr *CouchDBHistMgr) GetBlockNumFromSavepoint() (uint64, error) {
 	return savepointDoc.BlockNum, nil
 }
 
-func constructCompositeKey(ns string, key string, blocknum uint64, trannum uint64) string {
-	//History Key is:  "namespace key blocknum trannum"", with namespace being the chaincode id
-
-	// TODO - We will likely want sortable varint encoding, rather then a simple number, in order to support sorted key scans
-	var buffer bytes.Buffer
-	buffer.WriteString(ns)
-	buffer.WriteByte(0)
-	buffer.WriteString(key)
-	buffer.WriteByte(0)
-	buffer.WriteString(strconv.Itoa(int(blocknum)))
-	buffer.WriteByte(0)
-	buffer.WriteString(strconv.Itoa(int(trannum)))
-
-	return buffer.String()
-}
-
-func constructPartialCompositeKey(ns string, key string, endkey bool) []byte {
-	compositeKey := []byte(ns)
-	compositeKey = append(compositeKey, compositeKeySep...)
-	compositeKey = append(compositeKey, []byte(key)...)
-	if endkey {
-		compositeKey = append(compositeKey, []byte("1")...)
-	}
-	return compositeKey
-}
-
-func splitCompositeKey(compositePartialKey []byte, compositeKey []byte) (string, string) {
-	split := bytes.SplitN(compositeKey, compositePartialKey, 2)
-	return string(split[0]), string(split[1])
-}
-
 type histScanner struct {
 	cursor              int
 	compositePartialKey []byte
@@ -283,7 +249,7 @@ func (scanner *histScanner) next() (*historicValue, error) {
 
 	selectedValue := scanner.results[scanner.cursor]
 
-	_, blockNumTranNum := splitCompositeKey(scanner.compositePartialKey, []byte(selectedValue.ID))
+	_, blockNumTranNum := helper.SplitCompositeKey(scanner.compositePartialKey, []byte(selectedValue.ID))
 
 	return &historicValue{blockNumTranNum, selectedValue.Value}, nil
 

core/ledger/history/couchdb_histmgr_test.go core/ledger/kvledger/history/histmgr/couchdbhistmgr/couchdb_histmgr_test.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package history
+package couchdbhistmgr
 
 import (
 	"fmt"
 	"testing"
 
+	helper "github.com/hyperledger/fabric/core/ledger/kvledger/history/histmgr"
 	"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
 	"github.com/hyperledger/fabric/core/ledger/testutil"
 )
@@ -34,48 +35,48 @@ otherwise HistoryDB may not be installed and all the tests would fail
 func TestHistoryDatabaseAutoCreate(t *testing.T) {
 
 	//call a helper method to load the core.yaml
-	testutil.SetupCoreYAMLConfig("./../../../peer")
+	testutil.SetupCoreYAMLConfig("./../../../../../../peer")
 	logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
 		ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())
 
 	if ledgerconfig.IsHistoryDBEnabled() == true {
 
-		env := newTestEnvHistoryCouchDB(t, "history-test")
-		env.cleanup()       //cleanup at the beginning to ensure the database doesn't exist already
-		defer env.cleanup() //and cleanup at the end
+		env := newTestEnvCouchDB(t, "history-test")
+		env.cleanupCouchDB()       //cleanup at the beginning to ensure the database doesn't exist already
+		defer env.cleanupCouchDB() //and cleanup at the end
 
-		logger.Debugf("===HISTORYDB=== env.couchDBAddress: %v , env.couchDatabaseName: %v env.couchUsername: %v env.couchPassword: %v\n",
-			env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
+		logger.Debugf("===HISTORYDB=== env.CouchDBAddress: %v , env.CouchDatabaseName: %v env.CouchUsername: %v env.CouchPassword: %v\n",
+			env.CouchDBAddress, env.CouchDatabaseName, env.CouchUsername, env.CouchPassword)
 
 		histMgr := NewCouchDBHistMgr(
-			env.couchDBAddress,    //couchDB Address
-			env.couchDatabaseName, //couchDB db name
-			env.couchUsername,     //enter couchDB id
-			env.couchPassword)     //enter couchDB pw
+			env.CouchDBAddress,    //couchDB Address
+			env.CouchDatabaseName, //couchDB db name
+			env.CouchUsername,     //enter couchDB id
+			env.CouchPassword)     //enter couchDB pw
 
 		//NewCouchDBhistMgr should have automatically created the database, let's make sure it has been created
 		//Retrieve the info for the new database and make sure the name matches
 		dbResp, _, errdb := histMgr.couchDB.GetDatabaseInfo()
 		testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to retrieve database information"))
-		testutil.AssertEquals(t, dbResp.DbName, env.couchDatabaseName)
+		testutil.AssertEquals(t, dbResp.DbName, env.CouchDatabaseName)
 
 		//Call NewCouchDBhistMgr again, this time the database will already exist from last time
 		histMgr2 := NewCouchDBHistMgr(
-			env.couchDBAddress,    //couchDB Address
-			env.couchDatabaseName, //couchDB db name
-			env.couchUsername,     //enter couchDB id
-			env.couchPassword)     //enter couchDB pw
+			env.CouchDBAddress,    //couchDB Address
+			env.CouchDatabaseName, //couchDB db name
+			env.CouchUsername,     //enter couchDB id
+			env.CouchPassword)     //enter couchDB pw
 
 		//Retrieve the info for the database again, and make sure the name still matches
 		dbResp2, _, errdb2 := histMgr2.couchDB.GetDatabaseInfo()
 		testutil.AssertNoError(t, errdb2, fmt.Sprintf("Error when trying to retrieve database information"))
-		testutil.AssertEquals(t, dbResp2.DbName, env.couchDatabaseName)
+		testutil.AssertEquals(t, dbResp2.DbName, env.CouchDatabaseName)
 
 	}
 }
 
 func TestConstructCompositeKey(t *testing.T) {
-	compositeKey := constructCompositeKey("ns1", "key1", 1, 1)
+	compositeKey := helper.ConstructCompositeKey("ns1", "key1", 1, 1)
 
 	var compositeKeySep = []byte{0x00}
 	var strKeySep = string(compositeKeySep)
@@ -86,24 +87,29 @@ func TestConstructCompositeKey(t *testing.T) {
 //TestSavepoint tests the recordSavepoint and GetBlockNumfromSavepoint methods for recording and reading a savepoint document
 func TestSavepoint(t *testing.T) {
 
+	//call a helper method to load the core.yaml
+	testutil.SetupCoreYAMLConfig("./../../../../../../peer")
+	logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
+		ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())
+
 	if ledgerconfig.IsHistoryDBEnabled() == true {
 
-		env := newTestEnvHistoryCouchDB(t, "history-test")
-		env.cleanup()       //cleanup at the beginning to ensure the database doesn't exist already
-		defer env.cleanup() //and cleanup at the end
+		env := newTestEnvCouchDB(t, "history-test")
+		env.cleanupCouchDB()       //cleanup at the beginning to ensure the database doesn't exist already
+		defer env.cleanupCouchDB() //and cleanup at the end
 
 		logger.Debugf("===HISTORYDB=== env.couchDBAddress: %v , env.couchDatabaseName: %v env.couchUsername: %v env.couchPassword: %v\n",
-			env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
+			env.CouchDBAddress, env.CouchDatabaseName, env.CouchUsername, env.CouchPassword)
 
 		histMgr := NewCouchDBHistMgr(
-			env.couchDBAddress,    //couchDB Address
-			env.couchDatabaseName, //couchDB db name
-			env.couchUsername,     //enter couchDB id
-			env.couchPassword)     //enter couchDB pw
+			env.CouchDBAddress,    //couchDB Address
+			env.CouchDatabaseName, //couchDB db name
+			env.CouchUsername,     //enter couchDB id
+			env.CouchPassword)     //enter couchDB pw
 
 		// read the savepoint
 		blockNum, err := histMgr.GetBlockNumFromSavepoint()
-		testutil.AssertEquals(t, blockNum, 0)
+		testutil.AssertEquals(t, blockNum, uint64(0))
 
 		// record savepoint
 		blockNo := uint64(5)
@@ -117,16 +123,18 @@ func TestSavepoint(t *testing.T) {
 	}
 }
 
+/*
 //History Database commit and read is being tested with kv_ledger_test.go.
 //This test will push some of the testing down into history itself
 func TestHistoryDatabaseCommit(t *testing.T) {
-	//call a helper method to load the core.yaml
-	testutil.SetupCoreYAMLConfig("./../../../peer")
-	logger.Debugf("===HISTORYDB=== TestHistoryDatabaseCommit  IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
+
+	testutil.SetupCoreYAMLConfig("./../../../../../../peer")
+	logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
 		ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())
 
 	if ledgerconfig.IsHistoryDBEnabled() == true {
-		//TODO Build the necessary infrastructure so that history can be tested iwthout ledger
+		//TODO Build the necessary infrastructure so that history can be tested without ledger
 
 	}
 }
+*/

core/ledger/history/couchdb_history_query_executer.go core/ledger/kvledger/history/histmgr/couchdbhistmgr/couchdb_history_query_executer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package history
+package couchdbhistmgr
 
 import "github.com/hyperledger/fabric/core/ledger"
 
@@ -47,6 +47,7 @@ func (itr *qHistoryItr) Next() (ledger.QueryResult, error) {
 		return nil, nil
 	}
 	//TODO Returning blockNumTrannum as TxID for now but eventually will return txID instead
+	//TODO This includes the seperator before the blocknum and tran num:  aka(see test):   strKeySep+"1"+strKeySep+"1"
 	return &ledger.KeyModification{TxID: historicValue.blockNumTranNum, Value: historicValue.value}, nil
 }
 

core/ledger/history/pkg_test.go core/ledger/kvledger/history/histmgr/couchdbhistmgr/pkg_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package history
+package couchdbhistmgr
 
 import (
 	"testing"
@@ -23,33 +23,34 @@ import (
 	"github.com/hyperledger/fabric/core/ledger/util/couchdb"
 )
 
-//Complex setup to test the use of couch in ledger
-type testEnvHistoryCouchDB struct {
-	couchDBAddress    string
-	couchDatabaseName string
-	couchUsername     string
-	couchPassword     string
+//TestEnvCouchDB Complex setup to test the use of couchDB in ledger
+type testEnvCouchDB struct {
+	CouchDBAddress    string
+	CouchDatabaseName string
+	CouchUsername     string
+	CouchPassword     string
 }
 
-func newTestEnvHistoryCouchDB(t testing.TB, dbName string) *testEnvHistoryCouchDB {
-
-	couchDBDef := ledgerconfig.GetCouchDBDefinition()
-
-	return &testEnvHistoryCouchDB{
-		couchDBAddress:    couchDBDef.URL,
-		couchDatabaseName: dbName,
-		couchUsername:     couchDBDef.Username,
-		couchPassword:     couchDBDef.Password,
-	}
-}
-
-func (env *testEnvHistoryCouchDB) cleanup() {
-
+//CleanupCouchDB to clean up the test of couchDB in ledger
+func (env *testEnvCouchDB) cleanupCouchDB() {
 	//create a new connection
-	couchInstance, err := couchdb.CreateCouchInstance(env.couchDBAddress, env.couchUsername, env.couchPassword)
-	couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, env.couchDatabaseName)
+	couchInstance, err := couchdb.CreateCouchInstance(env.CouchDBAddress, env.CouchUsername, env.CouchPassword)
+	couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, env.CouchDatabaseName)
 	if err == nil {
 		//drop the test database if it already existed
 		couchDB.DropDatabase()
 	}
 }
+
+//NewTestEnvCouchDB to clean up the test of couchDB in ledger
+func newTestEnvCouchDB(t testing.TB, dbName string) *testEnvCouchDB {
+
+	couchDBDef := ledgerconfig.GetCouchDBDefinition()
+
+	return &testEnvCouchDB{
+		CouchDBAddress:    couchDBDef.URL,
+		CouchDatabaseName: dbName,
+		CouchUsername:     couchDBDef.Username,
+		CouchPassword:     couchDBDef.Password,
+	}
+}

core/ledger/history/histmgmt.go core/ledger/kvledger/history/histmgr/histmgr.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package history
+package histmgr
 
 import "github.com/hyperledger/fabric/protos/common"
 import "github.com/hyperledger/fabric/core/ledger"