Skip to content

Commit 0fc6c4d

Browse files
author
Chris Elder
committed
FAB-2531 Range queries fail iterating beyond 100 items
This is a temporary stabilization change until the following issue is resolved and will prevent errors in queries for the beta: FAB-2462 - Re-introduce paging for range queries and rich queries Peer sends 100 results to shim. When shim asks for 101st item, the request for the next 100 goes back to peer and it fails due to a closed iterator. This change also sets the HasMore flag on query result sets to false. This will prevent the requery from being called by the shim and throwing an exception. Purpose of this change is to stabilize the behavior for the beta by setting a query size limit in core.yaml at 10000. Then use the query size limit in the CouchDB query and in the chaincode handler. Added changes to chaincodetest.yaml to match core.yaml. Added unit test cases for the new querylimit option. Change-Id: I772d1f87beec2296db2eed68a0528181ac1ddeca Signed-off-by: Chris Elder <[email protected]>
1 parent ec1bbdf commit 0fc6c4d

File tree

10 files changed

+402
-98
lines changed

10 files changed

+402
-98
lines changed

‎core/chaincode/chaincodetest.yaml

+16-38
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ chaincode:
392392
lccc: enable
393393
escc: enable
394394
vscc: enable
395+
395396
###############################################################################
396397
#
397398
# Ledger section - ledger configuration encompases both the blockchain
@@ -402,45 +403,22 @@ ledger:
402403

403404
blockchain:
404405

405-
# Setting the deploy-system-chaincode property to false will prevent the
406-
# deploying of system chaincode at genesis time.
407-
deploy-system-chaincode: false
408-
409406
state:
410-
411-
# Control the number state deltas that are maintained. This takes additional
412-
# disk space, but allow the state to be rolled backwards and forwards
413-
# without the need to replay transactions.
414-
deltaHistorySize: 500
415-
416-
# The data structure in which the state will be stored. Different data
417-
# structures may offer different performance characteristics.
418-
# Options are 'buckettree', 'trie' and 'raw'.
419-
# ( Note:'raw' is experimental and incomplete. )
420-
# If not set, the default data structure is the 'buckettree'.
421-
# This CANNOT be changed after the DB has been created.
422-
dataStructure:
423-
# The name of the data structure is for storing the state
424-
name: buckettree
425-
# The data structure specific configurations
426-
configs:
427-
# configurations for 'bucketree'. These CANNOT be changed after the DB
428-
# has been created. 'numBuckets' defines the number of bins that the
429-
# state key-values are to be divided
430-
numBuckets: 1000003
431-
# 'maxGroupingAtEachLevel' defines the number of bins that are grouped
432-
#together to construct next level of the merkle-tree (this is applied
433-
# repeatedly for constructing the entire tree).
434-
maxGroupingAtEachLevel: 5
435-
# 'bucketCacheSize' defines the size (in MBs) of the cache that is used to keep
436-
# the buckets (from root upto secondlast level) in memory. This cache helps
437-
# in making state hash computation faster. A value less than or equals to zero
438-
# leads to disabling this caching. This caching helps more if transactions
439-
# perform significant writes.
440-
bucketCacheSize: 100
441-
442-
# configurations for 'trie'
443-
# 'tire' has no additional configurations exposed as yet
407+
# stateDatabase - options are "goleveldb", "CouchDB"
408+
# goleveldb - default state database stored in goleveldb.
409+
# CouchDB - store state database in CouchDB
410+
stateDatabase: goleveldb
411+
couchDBConfig:
412+
couchDBAddress: 127.0.0.1:5984
413+
username:
414+
password:
415+
416+
# historyDatabase - options are true or false
417+
# Indicates if the history of key updates should be stored in goleveldb
418+
historyDatabase: true
419+
420+
# Limit on the number of records to return per query
421+
queryLimit: 10000
444422

445423

446424
################################################################################

‎core/chaincode/exectransaction_test.go

+228-8
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package chaincode
1818

1919
import (
20+
"encoding/json"
2021
"fmt"
2122
"net"
2223
"os"
@@ -35,6 +36,7 @@ import (
3536
"github.com/hyperledger/fabric/core/ledger"
3637
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
3738
"github.com/hyperledger/fabric/core/ledger/ledgermgmt"
39+
"github.com/hyperledger/fabric/core/ledger/util/couchdb"
3840
"github.com/hyperledger/fabric/core/peer"
3941
"github.com/hyperledger/fabric/core/scc"
4042
"github.com/hyperledger/fabric/msp"
@@ -112,6 +114,22 @@ func finitPeer(lis net.Listener, chainIDs ...string) {
112114
ledgerPath := viper.GetString("peer.fileSystemPath")
113115
os.RemoveAll(ledgerPath)
114116
os.RemoveAll(filepath.Join(os.TempDir(), "hyperledger"))
117+
118+
//if couchdb is enabled, then cleanup the test couchdb
119+
if ledgerconfig.IsCouchDBEnabled() == true {
120+
121+
chainID := util.GetTestChainID()
122+
123+
connectURL := viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
124+
username := viper.GetString("ledger.state.couchDBConfig.username")
125+
password := viper.GetString("ledger.state.couchDBConfig.password")
126+
127+
couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password)
128+
db, _ := couchdb.CreateCouchDatabase(*couchInstance, chainID)
129+
//drop the test database
130+
db.DropDatabase()
131+
132+
}
115133
}
116134

117135
func startTxSimulation(ctxt context.Context, chainID string) (context.Context, ledger.TxSimulator, error) {
@@ -911,9 +929,10 @@ func TestQueries(t *testing.T) {
911929
return
912930
}
913931

914-
// Invoke second chaincode, which will inturn invoke the first chaincode
932+
// Add 12 marbles for testing range queries and rich queries (for capable ledgers)
933+
// The tests will test both range and rich queries and queries with query limits
915934
f = "put"
916-
args = util.ToChaincodeArgs(f, "key1", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
935+
args = util.ToChaincodeArgs(f, "marble01", "{\"docType\":\"marble\",\"name\":\"marble01\",\"color\":\"blue\",\"size\":35,\"owner\":\"tom\"}")
917936
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
918937
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
919938
nextBlockNumber++
@@ -926,7 +945,7 @@ func TestQueries(t *testing.T) {
926945
}
927946

928947
f = "put"
929-
args = util.ToChaincodeArgs(f, "key2", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
948+
args = util.ToChaincodeArgs(f, "marble02", "{\"docType\":\"marble\",\"name\":\"marble02\",\"color\":\"red\",\"size\":25,\"owner\":\"tom\"}")
930949
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
931950
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
932951
nextBlockNumber++
@@ -939,7 +958,7 @@ func TestQueries(t *testing.T) {
939958
}
940959

941960
f = "put"
942-
args = util.ToChaincodeArgs(f, "key3", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
961+
args = util.ToChaincodeArgs(f, "marble03", "{\"docType\":\"marble\",\"name\":\"marble03\",\"color\":\"green\",\"size\":15,\"owner\":\"tom\"}")
943962
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
944963
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
945964
nextBlockNumber++
@@ -950,9 +969,92 @@ func TestQueries(t *testing.T) {
950969
return
951970
}
952971

953-
f = "keys"
954-
args = util.ToChaincodeArgs(f, "key0", "key3")
972+
f = "put"
973+
args = util.ToChaincodeArgs(f, "marble04", "{\"docType\":\"marble\",\"name\":\"marble04\",\"color\":\"green\",\"size\":20,\"owner\":\"jerry\"}")
974+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
975+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
976+
nextBlockNumber++
977+
if err != nil {
978+
t.Fail()
979+
t.Logf("Error invoking <%s>: %s", ccID, err)
980+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
981+
return
982+
}
983+
984+
f = "put"
985+
args = util.ToChaincodeArgs(f, "marble05", "{\"docType\":\"marble\",\"name\":\"marble05\",\"color\":\"red\",\"size\":25,\"owner\":\"jerry\"}")
986+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
987+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
988+
nextBlockNumber++
989+
if err != nil {
990+
t.Fail()
991+
t.Logf("Error invoking <%s>: %s", ccID, err)
992+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
993+
return
994+
}
995+
996+
f = "put"
997+
args = util.ToChaincodeArgs(f, "marble06", "{\"docType\":\"marble\",\"name\":\"marble06\",\"color\":\"blue\",\"size\":35,\"owner\":\"jerry\"}")
998+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
999+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1000+
nextBlockNumber++
1001+
if err != nil {
1002+
t.Fail()
1003+
t.Logf("Error invoking <%s>: %s", ccID, err)
1004+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1005+
return
1006+
}
1007+
1008+
f = "put"
1009+
args = util.ToChaincodeArgs(f, "marble07", "{\"docType\":\"marble\",\"name\":\"marble07\",\"color\":\"yellow\",\"size\":20,\"owner\":\"jerry\"}")
1010+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1011+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1012+
nextBlockNumber++
1013+
if err != nil {
1014+
t.Fail()
1015+
t.Logf("Error invoking <%s>: %s", ccID, err)
1016+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1017+
return
1018+
}
1019+
1020+
f = "put"
1021+
args = util.ToChaincodeArgs(f, "marble08", "{\"docType\":\"marble\",\"name\":\"marble08\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
1022+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1023+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1024+
nextBlockNumber++
1025+
if err != nil {
1026+
t.Fail()
1027+
t.Logf("Error invoking <%s>: %s", ccID, err)
1028+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1029+
return
1030+
}
1031+
1032+
f = "put"
1033+
args = util.ToChaincodeArgs(f, "marble09", "{\"docType\":\"marble\",\"name\":\"marble09\",\"color\":\"yellow\",\"size\":10,\"owner\":\"jerry\"}")
1034+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1035+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1036+
nextBlockNumber++
1037+
if err != nil {
1038+
t.Fail()
1039+
t.Logf("Error invoking <%s>: %s", ccID, err)
1040+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1041+
return
1042+
}
1043+
1044+
f = "put"
1045+
args = util.ToChaincodeArgs(f, "marble10", "{\"docType\":\"marble\",\"name\":\"marble10\",\"color\":\"red\",\"size\":20,\"owner\":\"jerry\"}")
1046+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1047+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1048+
nextBlockNumber++
1049+
if err != nil {
1050+
t.Fail()
1051+
t.Logf("Error invoking <%s>: %s", ccID, err)
1052+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1053+
return
1054+
}
9551055

1056+
f = "put"
1057+
args = util.ToChaincodeArgs(f, "marble11", "{\"docType\":\"marble\",\"name\":\"marble11\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
9561058
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
9571059
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
9581060
nextBlockNumber++
@@ -963,19 +1065,137 @@ func TestQueries(t *testing.T) {
9631065
return
9641066
}
9651067

1068+
f = "put"
1069+
args = util.ToChaincodeArgs(f, "marble12", "{\"docType\":\"marble\",\"name\":\"marble12\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
1070+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1071+
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1072+
nextBlockNumber++
1073+
if err != nil {
1074+
t.Fail()
1075+
t.Logf("Error invoking <%s>: %s", ccID, err)
1076+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1077+
return
1078+
}
1079+
1080+
//TODO - the following query tests for queryLimits may change due to future designs
1081+
// for batch "paging"
1082+
1083+
//The following range query for "marble01" to "marble11" should return 10 marbles
1084+
f = "keys"
1085+
args = util.ToChaincodeArgs(f, "marble01", "marble11")
1086+
1087+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1088+
_, _, retval, err := invoke(ctxt, chainID, spec, nextBlockNumber)
1089+
nextBlockNumber++
1090+
if err != nil {
1091+
t.Fail()
1092+
t.Logf("Error invoking <%s>: %s", ccID, err)
1093+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1094+
return
1095+
}
1096+
1097+
var keys []interface{}
1098+
err = json.Unmarshal(retval, &keys)
1099+
1100+
//default query limit of 10000 is used, query should return all records that meet the criteria
1101+
if len(keys) != 10 {
1102+
t.Fail()
1103+
t.Logf("Error detected with the range query, should have returned 10 but returned %v", len(keys))
1104+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1105+
return
1106+
}
1107+
1108+
//Reset the query limit to 5
1109+
viper.Set("ledger.state.queryLimit", 5)
1110+
1111+
//The following range query for "marble01" to "marble11" should return 5 marbles due to the queryLimit
1112+
f = "keys"
1113+
args = util.ToChaincodeArgs(f, "marble01", "marble11")
1114+
1115+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1116+
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1117+
1118+
nextBlockNumber++
1119+
if err != nil {
1120+
t.Fail()
1121+
t.Logf("Error invoking <%s>: %s", ccID, err)
1122+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1123+
return
1124+
}
1125+
1126+
//unmarshal the results
1127+
err = json.Unmarshal(retval, &keys)
1128+
1129+
//check to see if there are 5 values
1130+
if len(keys) != 5 {
1131+
t.Fail()
1132+
t.Logf("Error detected with the range query, should have returned 5 but returned %v", len(keys))
1133+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1134+
return
1135+
}
1136+
1137+
//Reset the query limit to default
1138+
viper.Set("ledger.state.queryLimit", 10000)
1139+
9661140
if ledgerconfig.IsCouchDBEnabled() == true {
1141+
1142+
//The following rich query for should return 9 marbles
1143+
f = "query"
1144+
args = util.ToChaincodeArgs(f, "{\"selector\":{\"owner\":\"jerry\"}}")
1145+
1146+
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
1147+
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1148+
nextBlockNumber++
1149+
1150+
if err != nil {
1151+
t.Fail()
1152+
t.Logf("Error invoking <%s>: %s", ccID, err)
1153+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1154+
return
1155+
}
1156+
1157+
//unmarshal the results
1158+
err = json.Unmarshal(retval, &keys)
1159+
1160+
//check to see if there are 9 values
1161+
//default query limit of 10000 is used, this query is effectively unlimited
1162+
if len(keys) != 9 {
1163+
t.Fail()
1164+
t.Logf("Error detected with the rich query, should have returned 9 but returned %v", len(keys))
1165+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1166+
return
1167+
}
1168+
1169+
//Reset the query limit to 5
1170+
viper.Set("ledger.state.queryLimit", 5)
1171+
1172+
//The following rich query should return 5 marbles due to the queryLimit
9671173
f = "query"
968-
args = util.ToChaincodeArgs(f, "{\"selector\":{\"currency\":\"USD\"}}")
1174+
args = util.ToChaincodeArgs(f, "{\"selector\":{\"owner\":\"jerry\"}}")
9691175

9701176
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
971-
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1177+
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)
1178+
9721179
if err != nil {
9731180
t.Fail()
9741181
t.Logf("Error invoking <%s>: %s", ccID, err)
9751182
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
9761183
return
9771184
}
1185+
1186+
//unmarshal the results
1187+
err = json.Unmarshal(retval, &keys)
1188+
1189+
//check to see if there are 5 values
1190+
if len(keys) != 5 {
1191+
t.Fail()
1192+
t.Logf("Error detected with the rich query, should have returned 5 but returned %v", len(keys))
1193+
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
1194+
return
1195+
}
1196+
9781197
}
1198+
9791199
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
9801200
}
9811201

0 commit comments

Comments
 (0)